1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.testtype;
18 
19 import com.android.tradefed.config.ConfigurationException;
20 import com.android.tradefed.config.OptionCopier;
21 import com.android.tradefed.device.DeviceNotAvailableException;
22 import com.android.tradefed.log.LogUtil.CLog;
23 import com.android.tradefed.result.CollectingTestListener;
24 import com.android.tradefed.result.ITestInvocationListener;
25 import com.android.tradefed.result.ResultForwarder;
26 import com.android.tradefed.result.TestDescription;
27 import com.android.tradefed.util.FileUtil;
28 
29 import com.google.common.annotations.VisibleForTesting;
30 
31 import java.io.BufferedWriter;
32 import java.io.File;
33 import java.io.FileWriter;
34 import java.io.IOException;
35 import java.util.Collection;
36 import java.util.LinkedHashMap;
37 import java.util.Map;
38 
39 /**
40  * Runs a set of instrumentation tests by specifying a list of line separated test classes and
41  * methods in a file pushed to device (expected format: com.android.foo.FooClassName#testMethodName)
42  * <p>
43  * Note: Requires a runner that supports test execution from a file. Will default to serial tests
44  * execution via {@link InstrumentationSerialTest} if any issues with file creation are encountered
45  * or if all tests in the created file fail to successfully finish execution.
46  */
47 class InstrumentationFileTest implements IRemoteTest {
48 
49     // on device test folder location where the test file should be saved
50     private static final String ON_DEVICE_TEST_DIR_LOCATION = "/data/local/tmp/";
51 
52     private InstrumentationTest mInstrumentationTest = null;
53 
54     /** the set of tests to run */
55     private final Collection<TestDescription> mTests;
56 
57     private String mFilePathOnDevice = null;
58 
59     private int mAttemps;
60 
61     private int mMaxAttemps;
62 
63     private boolean mRetrySerially;
64 
65     /**
66      * Creates a {@link InstrumentationFileTest}.
67      *
68      * @param instrumentationTest {@link InstrumentationTest} used to configure this class
69      * @param testsToRun a {@link Collection} of tests to run. Note this {@link Collection} will be
70      *     used as is (ie a reference to the testsToRun object will be kept).
71      */
InstrumentationFileTest( InstrumentationTest instrumentationTest, Collection<TestDescription> testsToRun, boolean retrySerially, int maxAttempts)72     InstrumentationFileTest(
73             InstrumentationTest instrumentationTest,
74             Collection<TestDescription> testsToRun,
75             boolean retrySerially,
76             int maxAttempts)
77             throws ConfigurationException {
78         // reuse the InstrumentationTest class to perform actual test run
79         mInstrumentationTest = createInstrumentationTest();
80         // copy all options from the original InstrumentationTest
81         OptionCopier.copyOptions(instrumentationTest, mInstrumentationTest);
82         mInstrumentationTest.setDevice(instrumentationTest.getDevice());
83         mInstrumentationTest.setForceAbi(instrumentationTest.getForceAbi());
84         mInstrumentationTest.setReRunUsingTestFile(true);
85         // no need to rerun when executing tests one by one
86         mInstrumentationTest.setRerunMode(false);
87         // keep local copy of tests to be run
88         mTests = testsToRun;
89         mAttemps = 0;
90         mRetrySerially = retrySerially;
91         mMaxAttemps = maxAttempts;
92     }
93 
94     /**
95      * {@inheritDoc}
96      */
97     @Override
run(final ITestInvocationListener listener)98     public void run(final ITestInvocationListener listener) throws DeviceNotAvailableException {
99         if (mInstrumentationTest.getDevice() == null) {
100             throw new IllegalArgumentException("Device has not been set");
101         }
102         // reuse InstrumentationTest class to perform actual test run
103         writeTestsToFileAndRun(mTests, listener);
104     }
105 
106 
107     /**
108      * Creates a file based on the {@link Collection} of tests to run. Upon successful file creation
109      * will push the file onto the test device and attempt to run them via {@link
110      * InstrumentationTest}. If something goes wrong, will default to serial test execution.
111      *
112      * @param tests a {@link Collection} of tests to run
113      * @param listener the test result listener
114      * @throws DeviceNotAvailableException
115      */
writeTestsToFileAndRun( Collection<TestDescription> tests, final ITestInvocationListener listener)116     private void writeTestsToFileAndRun(
117             Collection<TestDescription> tests, final ITestInvocationListener listener)
118             throws DeviceNotAvailableException {
119         mAttemps += 1;
120         if (mMaxAttemps > 0 && mAttemps <= mMaxAttemps) {
121             CLog.d("Try to run tests from file for the %d/%d attempts",
122                     mAttemps, mMaxAttemps);
123         } else if (mMaxAttemps > 0) {
124             if (mRetrySerially) {
125                 CLog.d("Running tests from file exceeded max attempts."
126                         + " Try to run tests serially.");
127                 reRunTestsSerially(mInstrumentationTest, listener);
128             } else {
129                 CLog.d("Running tests from file exceeded max attempts. Ignore the rest tests");
130                 return;
131             }
132         }
133         File testFile = null;
134         try {
135             // create and populate test file
136             testFile = FileUtil.createTempFile(
137                     "tf_testFile_" + InstrumentationFileTest.class.getCanonicalName(), ".txt");
138             try (BufferedWriter bw = new BufferedWriter(new FileWriter(testFile))) {
139                 // Remove parameterized tests to only re-run their base method.
140                 Collection<TestDescription> uniqueMethods = createRerunSet(tests);
141 
142                 for (TestDescription testToRun : uniqueMethods) {
143                     // We use getTestNameNoParams to avoid attempting re-running individual
144                     // parameterized tests. Instead ask the base method to re-run them all.
145                     bw.write(
146                             String.format(
147                                     "%s#%s",
148                                     testToRun.getClassName(),
149                                     testToRun.getTestNameWithoutParams()));
150                     bw.newLine();
151                 }
152                 CLog.d("Test file %s was successfully created", testFile.getAbsolutePath());
153             }
154             // push test file to the device and run
155             mFilePathOnDevice = ON_DEVICE_TEST_DIR_LOCATION + testFile.getName();
156             if (pushFileToTestDevice(testFile, mFilePathOnDevice)) {
157                 mInstrumentationTest.setTestFilePathOnDevice(mFilePathOnDevice);
158                 CLog.d("Test file %s was successfully pushed to %s on device",
159                         testFile.getAbsolutePath(), mFilePathOnDevice);
160                 runTests(mInstrumentationTest, listener);
161             } else {
162                 if (mRetrySerially) {
163                     CLog.e("Failed to push file to device, re-running tests serially");
164                     reRunTestsSerially(mInstrumentationTest, listener);
165                 } else {
166                     CLog.e("Failed to push file to device, ignore the rest of tests");
167                 }
168             }
169         } catch (IOException e) {
170             if (mRetrySerially) {
171                 CLog.e("Failed to run tests from file, re-running tests serially: %s",
172                         e.getMessage());
173                 reRunTestsSerially(mInstrumentationTest, listener);
174             } else {
175                 CLog.e("Failed to push file to device, ignore the rest of tests");
176             }
177         } finally {
178             // clean up test file, if it was created
179             FileUtil.deleteFile(testFile);
180         }
181     }
182 
183     /**
184      * Run all tests from file. Attempt to re-run not finished tests.
185      * If all tests in file fail to run default to executing them serially.
186      */
runTests(InstrumentationTest runner, ITestInvocationListener listener)187     private void runTests(InstrumentationTest runner, ITestInvocationListener listener)
188             throws DeviceNotAvailableException {
189         CollectingTestListener testTracker = new CollectingTestListener();
190         try {
191             runner.run(new ResultForwarder(listener, testTracker));
192         } finally {
193             deleteTestFileFromDevice(mFilePathOnDevice);
194             Collection<TestDescription> completedTests =
195                     testTracker.getCurrentRunResults().getCompletedTests();
196             if (mTests.removeAll(completedTests) && !mTests.isEmpty()) {
197                 // re-run remaining tests from file
198                 writeTestsToFileAndRun(mTests, listener);
199             } else if (!mTests.isEmpty()) {
200                 if (mRetrySerially) {
201                     CLog.e("all remaining tests failed to run from file, re-running tests serially");
202                     reRunTestsSerially(runner, listener);
203                 } else {
204                     CLog.e("all remaining tests failed to run from file, will be ignored");
205                 }
206             }
207         }
208     }
209 
210     /**
211      * Re-runs remaining tests one-by-one
212      */
reRunTestsSerially(InstrumentationTest runner, ITestInvocationListener listener)213     private void reRunTestsSerially(InstrumentationTest runner, ITestInvocationListener listener)
214             throws DeviceNotAvailableException {
215         // clear file path arguments to ensure it won't get used.
216         runner.setTestFilePathOnDevice(null);
217         // enforce serial re-run
218         runner.setReRunUsingTestFile(false);
219         // Set tests to run
220         runner.setTestsToRun(mTests);
221         runner.run(listener);
222     }
223 
224     /**
225      * Returns a new collection of {@link TestDescription} where only one instance of each
226      * parameterized method is in the list.
227      */
createRerunSet(Collection<TestDescription> tests)228     private Collection<TestDescription> createRerunSet(Collection<TestDescription> tests) {
229         Map<String, TestDescription> uniqueMethods = new LinkedHashMap<>();
230         for (TestDescription test : tests) {
231             uniqueMethods.put(test.getTestNameWithoutParams(), test);
232         }
233         return uniqueMethods.values();
234     }
235 
236     /**
237      * Util method to push file to a device. Exposed for unit testing.
238      *
239      * @return if file was pushed to the device successfully
240      * @throws DeviceNotAvailableException
241      */
pushFileToTestDevice(File file, String destinationPath)242     boolean pushFileToTestDevice(File file, String destinationPath)
243             throws DeviceNotAvailableException {
244         return mInstrumentationTest.getDevice().pushFile(file, destinationPath);
245     }
246 
247     /**
248      * Delete file from the device if it exists
249      */
deleteTestFileFromDevice(String pathToFile)250     void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException {
251         if (mInstrumentationTest.getDevice().doesFileExist(pathToFile)) {
252             mInstrumentationTest.getDevice()
253                     .executeShellCommand(String.format("rm %s", pathToFile));
254             CLog.d("Removed test file from device: %s", pathToFile);
255         }
256     }
257 
258     /** @return the {@link InstrumentationTest} to use. Exposed for unit testing. */
259     @VisibleForTesting
createInstrumentationTest()260     InstrumentationTest createInstrumentationTest() {
261         return new InstrumentationTest();
262     }
263 }
264