1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.compatibility.common.tradefed.testtype;
17 
18 import com.android.tradefed.config.OptionSetter;
19 import com.android.tradefed.device.DeviceNotAvailableException;
20 import com.android.tradefed.device.ITestDevice;
21 import com.android.tradefed.log.ITestLogger;
22 import com.android.tradefed.result.ByteArrayInputStreamSource;
23 import com.android.tradefed.result.ITestInvocationListener;
24 import com.android.tradefed.result.InputStreamSource;
25 import com.android.tradefed.result.LogDataType;
26 import com.android.tradefed.suite.checker.ISystemStatusChecker;
27 import com.android.tradefed.suite.checker.StatusCheckerResult;
28 import com.android.tradefed.suite.checker.StatusCheckerResult.CheckStatus;
29 import com.android.tradefed.testtype.IAbi;
30 import com.android.tradefed.util.AbiUtils;
31 
32 import junit.framework.TestCase;
33 
34 import org.easymock.EasyMock;
35 
36 import java.util.ArrayList;
37 import java.util.HashSet;
38 import java.util.LinkedList;
39 import java.util.List;
40 import java.util.Set;
41 
42 /**
43  * Test class for {@link CompatibilityTest}
44  */
45 public class CompatibilityTestTest extends TestCase {
46 
47     private static final String FAKE_HOST_ARCH = "arm";
48     private CompatibilityTest mTest;
49     private ITestDevice mMockDevice;
50     private ITestLogger mMockLogger;
51     private ITestInvocationListener mMockListener;
52 
53     @Override
setUp()54     public void setUp() throws Exception {
55         mTest = new CompatibilityTest() {
56             @Override
57             protected Set<String> getAbisForBuildTargetArch() {
58                 return AbiUtils.getAbisForArch(FAKE_HOST_ARCH);
59             }
60         };
61         mMockDevice = EasyMock.createMock(ITestDevice.class);
62         mTest.setDevice(mMockDevice);
63         mMockLogger = EasyMock.createMock(ITestLogger.class);
64         mMockListener = EasyMock.createMock(ITestInvocationListener.class);
65     }
66 
67     /**
68      * Test that {@link CompatibilityTest#getAbis()} is returning a proper intersection of CTS
69      * supported architectures and Device supported architectures.
70      */
testGetAbis()71     public void testGetAbis() throws DeviceNotAvailableException {
72         EasyMock.expect(mMockDevice.getProperty(EasyMock.eq("ro.product.cpu.abilist")))
73                 .andReturn("arm64-v8a,armeabi-v7a,armeabi");
74         Set<String> expectedAbis = new HashSet<>();
75         expectedAbis.add("arm64-v8a");
76         expectedAbis.add("armeabi-v7a");
77         EasyMock.replay(mMockDevice);
78         Set<IAbi> res = mTest.getAbis();
79         assertEquals(2, res.size());
80         for (IAbi abi : res) {
81             assertTrue(expectedAbis.contains(abi.getName()));
82         }
83         EasyMock.verify(mMockDevice);
84     }
85 
86     /**
87      * Test that {@link CompatibilityTest#getAbis()} is throwing an exception when none of the
88      * CTS build supported abi match the device abi.
89      */
testGetAbis_notSupported()90     public void testGetAbis_notSupported() throws DeviceNotAvailableException {
91         EasyMock.expect(mMockDevice.getProperty(EasyMock.eq("ro.product.cpu.abilist")))
92                 .andReturn("armeabi");
93         EasyMock.replay(mMockDevice);
94         try {
95             mTest.getAbis();
96             fail("Should have thrown an exception");
97         } catch (IllegalArgumentException e) {
98             assertEquals("None of the abi supported by this CTS build ('[armeabi-v7a, arm64-v8a]')"
99                     + " are supported by the device ('[armeabi]').", e.getMessage());
100         }
101         EasyMock.verify(mMockDevice);
102     }
103 
104     /**
105      * Test that {@link CompatibilityTest#getAbis()} is returning only the device primary abi.
106      */
testGetAbis_primaryAbiOnly()107     public void testGetAbis_primaryAbiOnly() throws Exception {
108         OptionSetter setter = new OptionSetter(mTest);
109         setter.setOptionValue(CompatibilityTest.PRIMARY_ABI_RUN, "true");
110         EasyMock.expect(mMockDevice.getProperty(EasyMock.eq("ro.product.cpu.abi")))
111                 .andReturn("arm64-v8a");
112         Set<String> expectedAbis = new HashSet<>();
113         expectedAbis.add("arm64-v8a");
114         EasyMock.replay(mMockDevice);
115         Set<IAbi> res = mTest.getAbis();
116         assertEquals(1, res.size());
117         for (IAbi abi : res) {
118             assertTrue(expectedAbis.contains(abi.getName()));
119         }
120         EasyMock.verify(mMockDevice);
121     }
122 
123     /**
124      * Test that {@link CompatibilityTest#getAbis()} is throwing an exception if the primary
125      * abi is not supported.
126      */
testGetAbis_primaryAbiOnly_NotSupported()127     public void testGetAbis_primaryAbiOnly_NotSupported() throws Exception {
128         OptionSetter setter = new OptionSetter(mTest);
129         setter.setOptionValue(CompatibilityTest.PRIMARY_ABI_RUN, "true");
130         EasyMock.expect(mMockDevice.getProperty(EasyMock.eq("ro.product.cpu.abi")))
131                 .andReturn("armeabi");
132         EasyMock.replay(mMockDevice);
133         try {
134             mTest.getAbis();
135             fail("Should have thrown an exception");
136         } catch (IllegalArgumentException e) {
137             assertEquals("Your CTS hasn't been built with abi 'armeabi' support, "
138                     + "this CTS currently supports '[armeabi-v7a, arm64-v8a]'.", e.getMessage());
139         }
140         EasyMock.verify(mMockDevice);
141     }
142 
143     /**
144      * Test that {@link CompatibilityTest#getAbis()} is returning the list of abi supported by
145      * Compatibility and the device, and not the particular CTS build.
146      */
testGetAbis_skipCtsArchCheck()147     public void testGetAbis_skipCtsArchCheck() throws Exception {
148         OptionSetter setter = new OptionSetter(mTest);
149         setter.setOptionValue(CompatibilityTest.SKIP_HOST_ARCH_CHECK, "true");
150         EasyMock.expect(mMockDevice.getProperty(EasyMock.eq("ro.product.cpu.abilist")))
151                 .andReturn("x86_64,x86,armeabi");
152         Set<String> expectedAbis = new HashSet<>();
153         expectedAbis.add("x86_64");
154         expectedAbis.add("x86");
155         EasyMock.replay(mMockDevice);
156         Set<IAbi> res = mTest.getAbis();
157         assertEquals(2, res.size());
158         for (IAbi abi : res) {
159             assertTrue(expectedAbis.contains(abi.getName()));
160         }
161         EasyMock.verify(mMockDevice);
162     }
163 
164     /**
165      * Test {@link CompatibilityTest#getAbis()} when we skip the Cts side architecture check and
166      * want to run x86 abi.
167      */
testGetAbis_skipCtsArchCheck_abiSpecified()168     public void testGetAbis_skipCtsArchCheck_abiSpecified() throws Exception {
169         OptionSetter setter = new OptionSetter(mTest);
170         setter.setOptionValue(CompatibilityTest.SKIP_HOST_ARCH_CHECK, "true");
171         setter.setOptionValue(CompatibilityTest.ABI_OPTION, "x86");
172         Set<String> expectedAbis = new HashSet<>();
173         expectedAbis.add("x86");
174         EasyMock.replay(mMockDevice);
175         Set<IAbi> res = mTest.getAbis();
176         assertEquals(1, res.size());
177         for (IAbi abi : res) {
178             assertTrue(expectedAbis.contains(abi.getName()));
179         }
180         EasyMock.verify(mMockDevice);
181     }
182 
183     /**
184      * Test {@link CompatibilityTest#split()} when a shard number is specified.
185      */
testSplit()186     public void testSplit() throws Exception {
187         OptionSetter setter = new OptionSetter(mTest);
188         setter.setOptionValue("shards", "4");
189         assertEquals(4, mTest.split().size());
190     }
191 
192     /**
193      * Test {@link CompatibilityTest#split()} when no shard number is specified.
194      */
testSplit_notShardable()195     public void testSplit_notShardable() throws Exception {
196         assertNull(mTest.split());
197     }
198 
199     /**
200      * Test {@link CompatibilityTest#runPreModuleCheck(String, List, ITestDevice, ITestLogger)}
201      * is successful when no system checker fails.
202      */
testRunPreModuleCheck()203     public void testRunPreModuleCheck() throws Exception {
204         List<ISystemStatusChecker> systemCheckers = new ArrayList<>();
205         // add 2 inop status checkers.
206         systemCheckers.add(new ISystemStatusChecker() {});
207         systemCheckers.add(new ISystemStatusChecker() {});
208         EasyMock.replay(mMockDevice, mMockLogger);
209         mTest.runPreModuleCheck("FAKE_MODULE", systemCheckers, mMockDevice, mMockLogger);
210         EasyMock.verify(mMockDevice, mMockLogger);
211     }
212 
213     /**
214      * Test {@link CompatibilityTest#runPreModuleCheck(String, List, ITestDevice, ITestLogger)}
215      * is failing and log the failure.
216      */
testRunPreModuleCheck_failure()217     public void testRunPreModuleCheck_failure() throws Exception {
218         List<ISystemStatusChecker> systemCheckers = new ArrayList<>();
219         // add 2 inop status checkers.
220         systemCheckers.add(new ISystemStatusChecker() {});
221         systemCheckers.add(new ISystemStatusChecker() {
222             @Override
223             public StatusCheckerResult preExecutionCheck(ITestDevice device) {
224                 // fails
225                 return new StatusCheckerResult(CheckStatus.FAILED);
226             }
227         });
228         InputStreamSource res = new ByteArrayInputStreamSource("fake bugreport".getBytes());
229         EasyMock.expect(mMockDevice.getBugreport()).andReturn(res);
230         mMockLogger.testLog(EasyMock.eq("bugreport-checker-pre-module-FAKE_MODULE"),
231                 EasyMock.eq(LogDataType.BUGREPORT),
232                 EasyMock.same(res));
233         EasyMock.replay(mMockDevice, mMockLogger);
234         mTest.runPreModuleCheck("FAKE_MODULE", systemCheckers, mMockDevice, mMockLogger);
235         EasyMock.verify(mMockDevice, mMockLogger);
236     }
237 
238     /**
239      * Test {@link CompatibilityTest#runPostModuleCheck(String, List, ITestDevice, ITestLogger)}
240      * is successful when no system checker fails.
241      */
testRunPostModuleCheck()242     public void testRunPostModuleCheck() throws Exception {
243         List<ISystemStatusChecker> systemCheckers = new ArrayList<>();
244         // add 2 inop status checkers.
245         systemCheckers.add(new ISystemStatusChecker() {});
246         systemCheckers.add(new ISystemStatusChecker() {});
247         EasyMock.replay(mMockDevice, mMockLogger);
248         mTest.runPostModuleCheck("FAKE_MODULE", systemCheckers, mMockDevice, mMockLogger);
249         EasyMock.verify(mMockDevice, mMockLogger);
250     }
251 
252     /**
253      * Test {@link CompatibilityTest#runPreModuleCheck(String, List, ITestDevice, ITestLogger)}
254      * is failing and log the failure.
255      */
testRunPostModuleCheck_failure()256     public void testRunPostModuleCheck_failure() throws Exception {
257         List<ISystemStatusChecker> systemCheckers = new ArrayList<>();
258         // add 2 inop status checkers.
259         systemCheckers.add(new ISystemStatusChecker() {});
260         systemCheckers.add(new ISystemStatusChecker() {
261             @Override
262             public StatusCheckerResult postExecutionCheck(ITestDevice device) {
263                 // fails
264                 return new StatusCheckerResult(CheckStatus.FAILED);
265             }
266         });
267         InputStreamSource res = new ByteArrayInputStreamSource("fake bugreport".getBytes());
268         EasyMock.expect(mMockDevice.getBugreport()).andReturn(res);
269         mMockLogger.testLog(EasyMock.eq("bugreport-checker-post-module-FAKE_MODULE"),
270                 EasyMock.eq(LogDataType.BUGREPORT),
271                 EasyMock.same(res));
272         EasyMock.replay(mMockDevice, mMockLogger);
273         mTest.runPostModuleCheck("FAKE_MODULE", systemCheckers, mMockDevice, mMockLogger);
274         EasyMock.verify(mMockDevice, mMockLogger);
275     }
276 
277     /**
278      * Test {@link CompatibilityTest#run(ITestInvocationListener)} returns with no further
279      * execution when there is no module to run.
280      */
testRun_noModules()281     public void testRun_noModules() throws Exception {
282         mTest = new CompatibilityTest(1, new ModuleRepo() {
283             @Override
284             public boolean isInitialized() {
285                 return true;
286             }
287             @Override
288             public LinkedList<IModuleDef> getModules(String serial, int shardIndex) {
289                 return new LinkedList<IModuleDef>();
290             }
291         }, 0);
292         mTest.setDevice(mMockDevice);
293         EasyMock.expect(mMockDevice.getSerialNumber()).andReturn("FAKE_SERIAL").times(2);
294         EasyMock.replay(mMockDevice, mMockListener);
295         mTest.run(mMockListener);
296         EasyMock.verify(mMockDevice, mMockListener);
297     }
298 
299     /**
300      * Test {@link CompatibilityTest#checkSystemStatusBlackAndWhiteList()} correctly throws
301      * if a system status is invalid.
302      */
testCheckSystemStatus_throw()303     public void testCheckSystemStatus_throw() throws Exception {
304         OptionSetter setter = new OptionSetter(mTest);
305         setter.setOptionValue("system-status-check-whitelist", "com.does.not.exit");
306         try {
307             mTest.checkSystemStatusBlackAndWhiteList();
308             fail("should have thrown an exception");
309         } catch (RuntimeException expected) {
310             // expected.
311         }
312     }
313 
314     /**
315      * Test {@link CompatibilityTest#checkSystemStatusBlackAndWhiteList()} does not throw
316      * if a system status is valid.
317      */
testCheckSystemStatus_pass()318     public void testCheckSystemStatus_pass() throws Exception {
319         OptionSetter setter = new OptionSetter(mTest);
320         setter.setOptionValue("skip-system-status-check",
321                 "com.android.tradefed.suite.checker.KeyguardStatusChecker");
322         mTest.checkSystemStatusBlackAndWhiteList();
323     }
324 }
325