1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype;
17 
18 import com.android.ddmlib.FileListingService;
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.OptionClass;
21 import com.android.tradefed.device.CollectingOutputReceiver;
22 import com.android.tradefed.device.DeviceNotAvailableException;
23 import com.android.tradefed.device.ITestDevice;
24 import com.android.tradefed.log.LogUtil.CLog;
25 import com.android.tradefed.result.ITestInvocationListener;
26 import com.android.tradefed.util.proto.TfMetricProtoUtil;
27 
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.concurrent.TimeUnit;
34 
35 /**
36  * A Test that runs a Google benchmark test package on given device.
37  */
38 @OptionClass(alias = "gbenchmark")
39 public class GoogleBenchmarkTest implements IDeviceTest, IRemoteTest {
40 
41     static final String DEFAULT_TEST_PATH = "/data/benchmarktest";
42 
43     private static final String GBENCHMARK_JSON_OUTPUT_FORMAT = "--benchmark_format=json";
44     private static final String GBENCHMARK_LIST_TESTS_OPTION = "--benchmark_list_tests=true";
45     private static final String EXECUTABLE_BUILD_ID = "BuildID=";
46 
47     @Option(name = "file-exclusion-filter-regex",
48             description = "Regex to exclude certain files from executing. Can be repeated")
49     private List<String> mFileExclusionFilterRegex = new ArrayList<>();
50 
51     @Option(name = "native-benchmark-device-path",
52             description="The path on the device where native stress tests are located.")
53     private String mDeviceTestPath = DEFAULT_TEST_PATH;
54 
55     @Option(name = "benchmark-module-name",
56           description="The name of the native benchmark test module to run. " +
57           "If not specified all tests in --native-benchmark-device-path will be run.")
58     private String mTestModule = null;
59 
60     @Option(name = "benchmark-run-name",
61           description="Optional name to pass to test reporters. If unspecified, will use " +
62           "test binary as run name.")
63     private String mReportRunName = null;
64 
65     @Option(name = "max-run-time", description =
66             "The maximum time to allow for each benchmark run in ms.", isTimeVal=true)
67     private long mMaxRunTime = 15 * 60 * 1000;
68 
69     private ITestDevice mDevice = null;
70 
71     /**
72      * {@inheritDoc}
73      */
74     @Override
setDevice(ITestDevice device)75     public void setDevice(ITestDevice device) {
76         mDevice = device;
77     }
78 
79     /**
80      * {@inheritDoc}
81      */
82     @Override
getDevice()83     public ITestDevice getDevice() {
84         return mDevice;
85     }
86 
87     /**
88      * Set the Android native benchmark test module to run.
89      *
90      * @param moduleName The name of the native test module to run
91      */
setModuleName(String moduleName)92     public void setModuleName(String moduleName) {
93         mTestModule = moduleName;
94     }
95 
96     /**
97      * Get the Android native benchmark test module to run.
98      *
99      * @return the name of the native test module to run, or null if not set
100      */
getModuleName()101     public String getModuleName() {
102         return mTestModule;
103     }
104 
setReportRunName(String reportRunName)105     public void setReportRunName(String reportRunName) {
106         mReportRunName = reportRunName;
107     }
108 
109     /**
110      * Adds an exclusion file filter regex.
111      * <p/>
112      * Exposed for unit testing
113      *
114      * @param regex to exclude file.
115      */
addFileExclusionFilterRegex(String regex)116     void addFileExclusionFilterRegex(String regex) {
117         mFileExclusionFilterRegex.add(regex);
118     }
119 
120     /**
121      * Gets the path where native benchmark tests live on the device.
122      *
123      * @return The path on the device where the native tests live.
124      */
getTestPath()125     private String getTestPath() {
126         StringBuilder testPath = new StringBuilder(mDeviceTestPath);
127         if (mTestModule != null) {
128             testPath.append(FileListingService.FILE_SEPARATOR);
129             testPath.append(mTestModule);
130         }
131         return testPath.toString();
132     }
133 
134     /**
135      * Executes all native benchmark tests in a folder as well as in all subfolders recursively.
136      *
137      * @param root The root folder to begin searching for native tests
138      * @param testDevice The device to run tests on
139      * @param listener the run listener
140      * @throws DeviceNotAvailableException
141      */
doRunAllTestsInSubdirectory(String root, ITestDevice testDevice, ITestInvocationListener listener)142     private void doRunAllTestsInSubdirectory(String root, ITestDevice testDevice,
143             ITestInvocationListener listener) throws DeviceNotAvailableException {
144         if (testDevice.isDirectory(root)) {
145             // recursively run tests in all subdirectories
146             for (String child : testDevice.getChildren(root)) {
147                 doRunAllTestsInSubdirectory(root + "/" + child, testDevice, listener);
148             }
149         } else {
150             // assume every file is a valid benchmark test binary.
151             // use name of file as run name
152             String rootEntry = root.substring(root.lastIndexOf("/") + 1);
153             String runName = (mReportRunName == null ? rootEntry : mReportRunName);
154 
155             // force file to be executable
156             testDevice.executeShellCommand(String.format("chmod 755 %s", root));
157             if (shouldSkipFile(root)) {
158                 return;
159             }
160             long startTime = System.currentTimeMillis();
161 
162             // Count expected number of tests
163             int numTests = countExpectedTests(testDevice, root);
164             if (numTests == 0) {
165                 CLog.d("No tests to run.");
166                 return;
167             }
168 
169             Map<String, String> metricMap = new HashMap<String, String>();
170             CollectingOutputReceiver outputCollector = createOutputCollector();
171             GoogleBenchmarkResultParser resultParser = createResultParser(runName, listener);
172             listener.testRunStarted(runName, numTests);
173             try {
174                 String cmd = String.format("%s %s", root, GBENCHMARK_JSON_OUTPUT_FORMAT);
175                 CLog.i(String.format("Running google benchmark test on %s: %s",
176                         mDevice.getSerialNumber(), cmd));
177                 testDevice.executeShellCommand(cmd, outputCollector,
178                         mMaxRunTime, TimeUnit.MILLISECONDS, 0);
179                 metricMap = resultParser.parse(outputCollector);
180             } catch (DeviceNotAvailableException e) {
181                 listener.testRunFailed(e.getMessage());
182                 throw e;
183             } finally {
184                 final long elapsedTime = System.currentTimeMillis() - startTime;
185                 listener.testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metricMap));
186             }
187         }
188     }
189 
countExpectedTests(ITestDevice testDevice, String fullBinaryPath)190     private int countExpectedTests(ITestDevice testDevice, String fullBinaryPath)
191             throws DeviceNotAvailableException {
192         String exec = testDevice.executeShellCommand(String.format("file %s", fullBinaryPath));
193         // When inspecting our files, only the one with the marker of an executable should be ran.
194         if (!exec.contains(EXECUTABLE_BUILD_ID)) {
195             CLog.d("%s does not look like an executable", fullBinaryPath);
196             return 0;
197         }
198         String cmd = String.format("%s %s", fullBinaryPath, GBENCHMARK_LIST_TESTS_OPTION);
199         String list_output = testDevice.executeShellCommand(cmd);
200         String[] list = list_output.trim().split("\n");
201         CLog.d("List that will be used: %s", Arrays.asList(list));
202         return list.length;
203     }
204 
205     /**
206      * Helper method to determine if we should skip the execution of a given file.
207      * @param fullPath the full path of the file in question
208      * @return true if we should skip the said file.
209      */
shouldSkipFile(String fullPath)210     protected boolean shouldSkipFile(String fullPath) {
211         if (fullPath == null || fullPath.isEmpty()) {
212             return true;
213         }
214         if (mFileExclusionFilterRegex == null || mFileExclusionFilterRegex.isEmpty()) {
215             return false;
216         }
217         for (String regex : mFileExclusionFilterRegex) {
218             if (fullPath.matches(regex)) {
219                 CLog.i(String.format("File %s matches exclusion file regex %s, skipping",
220                         fullPath, regex));
221                 return true;
222             }
223         }
224         return false;
225     }
226 
227     /**
228      * Exposed for testing
229      */
createOutputCollector()230     CollectingOutputReceiver createOutputCollector() {
231         return new CollectingOutputReceiver();
232     }
233 
234     /**
235      * Exposed for testing
236      */
createResultParser(String runName, ITestInvocationListener listener)237     GoogleBenchmarkResultParser createResultParser(String runName,
238             ITestInvocationListener listener) {
239         return new GoogleBenchmarkResultParser(runName, listener);
240     }
241 
242     /**
243      * {@inheritDoc}
244      */
245     @Override
run(ITestInvocationListener listener)246     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
247         if (mDevice == null) {
248             throw new IllegalArgumentException("Device has not been set");
249         }
250         String testPath = getTestPath();
251         if (!mDevice.doesFileExist(testPath)) {
252             CLog.w(String.format("Could not find native benchmark test directory %s in %s!",
253                     testPath, mDevice.getSerialNumber()));
254             throw new RuntimeException(
255                     String.format("Could not find native benchmark test directory %s", testPath));
256         }
257         doRunAllTestsInSubdirectory(testPath, mDevice, listener);
258     }
259 }
260