1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.testdefs;
17 
18 import com.android.ddmlib.Log;
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.Option.Importance;
21 import com.android.tradefed.config.OptionClass;
22 import com.android.tradefed.device.DeviceNotAvailableException;
23 import com.android.tradefed.device.ITestDevice;
24 import com.android.tradefed.log.LogUtil;
25 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
26 import com.android.tradefed.result.ITestInvocationListener;
27 import com.android.tradefed.testtype.IDeviceTest;
28 import com.android.tradefed.testtype.IRemoteTest;
29 import com.android.tradefed.testtype.IResumableTest;
30 import com.android.tradefed.testtype.IShardableTest;
31 import com.android.tradefed.testtype.InstrumentationTest;
32 import com.android.tradefed.util.FileUtil;
33 import com.android.tradefed.util.proto.TfMetricProtoUtil;
34 import com.android.tradefed.util.xml.AbstractXmlParser.ParseException;
35 
36 import java.io.File;
37 import java.io.FileInputStream;
38 import java.io.FileNotFoundException;
39 import java.io.IOException;
40 import java.util.ArrayList;
41 import java.util.Collection;
42 import java.util.HashMap;
43 import java.util.LinkedList;
44 import java.util.List;
45 import java.util.Queue;
46 
47 /**
48  * Runs a set of instrumentation test's defined in test_defs.xml files.
49  * <p/>
50  * The test definition files can either be one or more files on local file system, and/or one or
51  * more files stored on the device under test.
52  */
53 @OptionClass(alias = "xml-defs")
54 public class XmlDefsTest implements IDeviceTest, IResumableTest,
55         IShardableTest {
56 
57     private static final String LOG_TAG = "XmlDefsTest";
58 
59     /** the metric key name for the test coverage target value */
60     // TODO: move this to a more generic location
61     public static final String COVERAGE_TARGET_KEY = "coverage_target";
62 
63     private ITestDevice mDevice;
64 
65     /**
66      * @deprecated use shell-timeout or test-timeout instead.
67      */
68     @Deprecated
69     @Option(name = "timeout",
70             description="Deprecated - Use \"shell-timeout\" or \"test-timeout\" instead.")
71     private Integer mTimeout = null;
72 
73     @Option(name = "shell-timeout",
74             description="The defined timeout (in milliseconds) is used as a maximum waiting time "
75                     + "when expecting the command output from the device. At any time, if the "
76                     + "shell command does not output anything for a period longer than defined "
77                     + "timeout the TF run terminates. For no timeout, set to 0.")
78     private long mShellTimeout = 10 * 60 * 1000;  // default to 10 minutes
79 
80     @Option(name = "test-timeout",
81             description="Sets timeout (in milliseconds) that will be applied to each test. In the "
82                     + "event of a test timeout it will log the results and proceed with executing "
83                     + "the next test. For no timeout, set to 0.")
84     private int mTestTimeout = 10 * 60 * 1000;  // default to 10 minutes
85 
86     @Option(name = "size",
87             description = "Restrict tests to a specific test size. " +
88             "One of 'small', 'medium', 'large'",
89             importance = Importance.IF_UNSET)
90     private String mTestSize = null;
91 
92     @Option(name = "rerun",
93             description = "Rerun unexecuted tests individually on same device if test run " +
94             "fails to complete.")
95     private boolean mIsRerunMode = true;
96 
97     @Option(name = "resume",
98             description = "Schedule unexecuted tests for resumption on another device " +
99             "if first device becomes unavailable.")
100     private boolean mIsResumeMode = false;
101 
102     @Option(name = "local-file-path",
103             description = "local file path to test_defs.xml file to run.")
104     private Collection<File> mLocalFiles = new ArrayList<File>();
105 
106     @Option(name = "device-file-path",
107             description = "file path on device to test_defs.xml file to run.",
108             importance = Importance.IF_UNSET)
109     private Collection<String> mRemotePaths = new ArrayList<String>();
110 
111     @Option(name = "send-coverage",
112             description = "Send coverage target info to test listeners.")
113     private boolean mSendCoverage = true;
114 
115     @Option(name = "num-shards",
116             description = "Shard this test into given number of separately runnable chunks.")
117     private int mNumShards = 0;
118 
119     private List<InstrumentationTest> mTests = null;
120 
XmlDefsTest()121     public XmlDefsTest() {
122     }
123 
124     /**
125      * {@inheritDoc}
126      */
127     @Override
getDevice()128     public ITestDevice getDevice() {
129         return mDevice;
130     }
131 
132     /**
133      * {@inheritDoc}
134      */
135     @Override
setDevice(ITestDevice device)136     public void setDevice(ITestDevice device) {
137         mDevice = device;
138     }
139 
140     /**
141      * Adds a remote test def file path.
142      * <p/>
143      * Exposed for unit testing.
144      */
addRemoteFilePath(String path)145     void addRemoteFilePath(String path) {
146         mRemotePaths.add(path);
147     }
148 
149     /**
150      * Adds a local test def file path.
151      * <p/>
152      * Exposed for unit testing.
153      */
addLocalFilePath(File file)154     void addLocalFilePath(File file) {
155         mLocalFiles.add(file);
156     }
157 
158     /**
159      * Set the send coverage flag.
160      * <p/>
161      * Exposed for unit testing.
162      */
setSendCoverage(boolean sendCoverage)163     void setSendCoverage(boolean sendCoverage) {
164         mSendCoverage = sendCoverage;
165     }
166 
167     /**
168      * Sets the number of shards test should be split into
169      * <p/>
170      * Exposed for unit testing.
171      */
setNumShards(int shards)172     void setNumShards(int shards) {
173         mNumShards = shards;
174     }
175 
176     /**
177      * Gets the list of parsed {@link InstrumentationTest}s contained within.
178      * <p/>
179      * Exposed for unit testing.
180      */
getTests()181     List<InstrumentationTest> getTests() {
182         return mTests;
183     }
184 
185     /**
186      * {@inheritDoc}
187      */
188     @Override
run(ITestInvocationListener listener)189     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
190         if (getDevice() == null) {
191             throw new IllegalArgumentException("Device has not been set");
192         }
193         buildTests();
194         doRun(listener);
195     }
196 
197     /**
198      * Build the list of tests to run from the xml files, if not done already.
199      * @throws DeviceNotAvailableException
200      */
buildTests()201     private void buildTests() throws DeviceNotAvailableException {
202         if (mTests == null) {
203             if (mLocalFiles.isEmpty() && mRemotePaths.isEmpty()) {
204                 throw new IllegalArgumentException("No test definition files (local-file-path or " +
205                         "device-file-path) have been provided.");
206             }
207             XmlDefsParser parser = createParser();
208             for (File testDefFile : mLocalFiles) {
209                 parseFile(parser, testDefFile);
210             }
211             for (File testDefFile : getRemoteFile(mRemotePaths)) {
212                 try {
213                     parseFile(parser, testDefFile);
214                 } finally {
215                     testDefFile.delete();
216                 }
217             }
218 
219             mTests = new LinkedList<InstrumentationTest>();
220             for (InstrumentationTestDef def : parser.getTestDefs()) {
221                 // only run continuous for now. Consider making this configurable
222                 if (def.isContinuous()) {
223                     InstrumentationTest test = createInstrumentationTest();
224 
225                     test.setDevice(getDevice());
226                     test.setPackageName(def.getPackage());
227                     if (def.getRunner() != null) {
228                         test.setRunnerName(def.getRunner());
229                     }
230                     if (def.getClassName() != null) {
231                         test.setClassName(def.getClassName());
232                     }
233                     test.setRerunMode(mIsRerunMode);
234                     test.setResumeMode(mIsResumeMode);
235                     test.setTestSize(getTestSize());
236                     if (mTimeout != null) {
237                         LogUtil.CLog
238                                 .w("\"timeout\" argument is deprecated and should not be used! \"shell-timeout\""
239                                         + " argument value is overwritten with %d ms", mTimeout);
240                         setShellTimeout(mTimeout);
241                     }
242                     test.setShellTimeout(getShellTimeout());
243                     test.setTestTimeout(getTestTimeout());
244                     test.setCoverageTarget(def.getCoverageTarget());
245                     mTests.add(test);
246                 }
247             }
248         }
249     }
250 
251     /**
252      * Parse the given xml def file
253      *
254      * @param parser
255      * @param testDefFile
256      */
parseFile(XmlDefsParser parser, File testDefFile)257     private void parseFile(XmlDefsParser parser, File testDefFile) {
258         try {
259             Log.i(LOG_TAG, String.format("Parsing test def file %s",
260                     testDefFile.getAbsolutePath()));
261             parser.parse(new FileInputStream(testDefFile));
262         } catch (FileNotFoundException e) {
263             Log.e(LOG_TAG, String.format("Could not find test def file %s",
264                     testDefFile.getAbsolutePath()));
265         } catch (ParseException e) {
266             Log.e(LOG_TAG, String.format("Could not parse test def file %s: %s",
267                     testDefFile.getAbsolutePath(), e.getMessage()));
268         }
269     }
270 
271     /**
272      * Run the previously built tests.
273      *
274      * @param listener the {@link ITestInvocationListener}
275      * @throws DeviceNotAvailableException
276      */
doRun(ITestInvocationListener listener)277     private void doRun(ITestInvocationListener listener) throws DeviceNotAvailableException {
278         while (!mTests.isEmpty()) {
279             InstrumentationTest test = mTests.get(0);
280 
281             Log.d(LOG_TAG, String.format("Running test %s on %s", test.getPackageName(),
282                         getDevice().getSerialNumber()));
283 
284             if (mSendCoverage && test.getCoverageTarget() != null) {
285                 sendCoverage(test.getPackageName(), test.getCoverageTarget(), listener);
286             }
287             test.setDevice(getDevice());
288             test.run(listener);
289             // test completed, remove from list
290             mTests.remove(0);
291         }
292     }
293 
294     /**
295      * Forwards the tests coverage target info as a test metric.
296      *
297      * @param packageName
298      * @param coverageTarget
299      * @param listener
300      */
sendCoverage(String packageName, String coverageTarget, ITestInvocationListener listener)301     private void sendCoverage(String packageName, String coverageTarget,
302             ITestInvocationListener listener) {
303         HashMap<String, Metric> coverageMetric = new HashMap<>(1);
304         coverageMetric.put(COVERAGE_TARGET_KEY, TfMetricProtoUtil.stringToMetric(coverageTarget));
305         listener.testRunStarted(packageName, 0);
306         listener.testRunEnded(0, coverageMetric);
307     }
308 
309     /**
310      * Retrieves a set of files from device into temporary files on local filesystem.
311      *
312      * @param remoteFilePaths
313      */
getRemoteFile(Collection<String> remoteFilePaths)314     private Collection<File> getRemoteFile(Collection<String> remoteFilePaths)
315             throws DeviceNotAvailableException {
316         Collection<File> files = new ArrayList<File>();
317         if (getDevice() == null) {
318             Log.d(LOG_TAG, "Device not set, skipping collection of remote file");
319             return files;
320         }
321         for (String remoteFilePath : remoteFilePaths) {
322             try {
323                 File tmpFile = FileUtil.createTempFile("test_defs_", ".xml");
324                 getDevice().pullFile(remoteFilePath, tmpFile);
325                 files.add(tmpFile);
326             } catch (IOException e) {
327                 Log.e(LOG_TAG, "Failed to create temp file");
328                 Log.e(LOG_TAG, e);
329             }
330         }
331         return files;
332     }
333 
setShellTimeout(long timeout)334     void setShellTimeout(long timeout) {
335         mShellTimeout = timeout;
336     }
337 
getShellTimeout()338     long getShellTimeout() {
339         return mShellTimeout;
340     }
341 
getTestTimeout()342     int getTestTimeout() {
343         return mTestTimeout;
344     }
345 
getTestSize()346     String getTestSize() {
347         return mTestSize;
348     }
349 
350     /**
351      * Creates the {@link XmlDefsParser} to use. Exposed for unit testing.
352      */
createParser()353     XmlDefsParser createParser() {
354         return new XmlDefsParser();
355     }
356 
357     /**
358      * Creates the {@link InstrumentationTest} to use. Exposed for unit testing.
359      */
createInstrumentationTest()360     InstrumentationTest createInstrumentationTest() {
361         return new InstrumentationTest();
362     }
363 
364     /**
365      * {@inheritDoc}
366      */
367     @Override
isResumable()368     public boolean isResumable() {
369         // hack to not resume if tests were never run
370         // TODO: fix this properly in TestInvocation
371         if (mTests == null) {
372             return false;
373         }
374         return mIsResumeMode;
375     }
376 
377     /**
378      * {@inheritDoc}
379      */
380     @Override
split()381     public Collection<IRemoteTest> split() {
382         if (mLocalFiles.isEmpty()) {
383             Log.w(LOG_TAG, "sharding is only supported if local xml files have been specified");
384             return null;
385         }
386         if (mNumShards <= 1) {
387             return null;
388         }
389 
390         try {
391             buildTests();
392         } catch (DeviceNotAvailableException e) {
393             // should never happen
394         }
395         if (mTests.size() <= 1) {
396             Log.w(LOG_TAG, "no tests to shard!");
397             return null;
398         }
399 
400         // treat shardQueue as a circular queue, to sequentially distribute tests among shards
401         Queue<IRemoteTest> shardQueue = new LinkedList<IRemoteTest>();
402         // don't create more shards than the number of tests we have!
403         for (int i = 0; i < mNumShards && i < mTests.size(); i++) {
404             XmlDefsTest shard = new XmlDefsTest();
405             shard.mTests = new LinkedList<InstrumentationTest>();
406             shardQueue.add(shard);
407         }
408         while (!mTests.isEmpty()) {
409             InstrumentationTest test = mTests.remove(0);
410             XmlDefsTest shard = (XmlDefsTest)shardQueue.poll();
411             shard.mTests.add(test);
412             shardQueue.add(shard);
413         }
414         return shardQueue;
415     }
416 }
417