1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.benchmark.app;
18 
19 import android.app.Activity;
20 import android.content.BroadcastReceiver;
21 import android.content.Context;
22 import android.content.Intent;
23 import android.content.IntentFilter;
24 import android.os.BatteryManager;
25 import android.os.Trace;
26 import android.test.ActivityInstrumentationTestCase2;
27 import android.util.Log;
28 
29 import androidx.test.InstrumentationRegistry;
30 
31 import com.android.nn.benchmark.core.BenchmarkException;
32 import com.android.nn.benchmark.core.BenchmarkResult;
33 import com.android.nn.benchmark.core.TestModels;
34 import com.android.nn.benchmark.core.TestModels.TestModelEntry;
35 
36 import org.junit.After;
37 import org.junit.Before;
38 import org.junit.runner.RunWith;
39 import org.junit.runners.Parameterized;
40 import org.junit.runners.Parameterized.Parameters;
41 
42 import java.io.IOException;
43 import java.util.List;
44 import java.util.concurrent.CountDownLatch;
45 
46 /**
47  * Benchmark test-case super-class.
48  *
49  * Helper code for managing NNAPI/NNAPI-less benchamarks.
50  */
51 @RunWith(Parameterized.class)
52 public class BenchmarkTestBase extends ActivityInstrumentationTestCase2<NNBenchmark> {
53 
54     // Only run 1 iteration now to fit the MediumTest time requirement.
55     // One iteration means running the tests continuous for 1s.
56     private NNBenchmark mActivity;
57     protected final TestModelEntry mModel;
58 
59     // The default 0.3s warmup and 1.0s runtime give reasonably repeatable results (run-to-run
60     // variability of ~20%) when run under performance settings (fixed CPU cores enabled and at
61     // fixed frequency). The continuous build is not allowed to take much more than 1s so we
62     // can't change the defaults for @MediumTest.
63     protected static final float WARMUP_SHORT_SECONDS = 0.3f;
64     protected static final float RUNTIME_SHORT_SECONDS = 1.f;
65 
66     // For running like a normal user-initiated app, the variability for 0.3s/1.0s is easily 3x.
67     // With 2s/10s it's 20-50%. This @LargeTest allows running with these timings.
68     protected static final float WARMUP_REPEATABLE_SECONDS = 2.f;
69     protected static final float RUNTIME_REPEATABLE_SECONDS = 10.f;
70 
71     // For running a complete dataset, the run should complete under 5 minutes.
72     protected static final float COMPLETE_SET_TIMEOUT_SECOND = 300.f;
73 
74     // For running compilation benchmarks.
75     protected static final float COMPILATION_WARMUP_SECONDS = 2.f;
76     protected static final float COMPILATION_RUNTIME_SECONDS = 10.f;
77     protected static final int COMPILATION_MAX_ITERATIONS = 100;
78 
BenchmarkTestBase(TestModelEntry model)79     public BenchmarkTestBase(TestModelEntry model) {
80         super(NNBenchmark.class);
81         mModel = model;
82     }
83 
setUseNNApi(boolean useNNApi)84     protected void setUseNNApi(boolean useNNApi) {
85         mActivity.setUseNNApi(useNNApi);
86     }
87 
setNnApiAcceleratorName(String acceleratorName)88     protected void setNnApiAcceleratorName(String acceleratorName) {
89         mActivity.setNnApiAcceleratorName(acceleratorName);
90     }
91 
setCompleteInputSet(boolean completeInputSet)92     protected void setCompleteInputSet(boolean completeInputSet) {
93         mActivity.setCompleteInputSet(completeInputSet);
94     }
95 
enableCompilationCachingBenchmarks()96     protected void enableCompilationCachingBenchmarks() {
97         mActivity.enableCompilationCachingBenchmarks(COMPILATION_WARMUP_SECONDS,
98                 COMPILATION_RUNTIME_SECONDS, COMPILATION_MAX_ITERATIONS);
99     }
100 
101     // Initialize the parameter for ImageProcessingActivityJB.
prepareTest()102     protected void prepareTest() {
103         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
104         mActivity = getActivity();
105         mActivity.prepareInstrumentationTest();
106         setUseNNApi(true);
107     }
108 
waitUntilCharged()109     public void waitUntilCharged() {
110         BenchmarkTestBase.waitUntilCharged(mActivity, -1);
111     }
112 
waitUntilCharged(Context context, int minChargeLevel)113     public static void waitUntilCharged(Context context, int minChargeLevel) {
114         Log.v(NNBenchmark.TAG, "Waiting for the device to charge");
115 
116         final CountDownLatch chargedLatch = new CountDownLatch(1);
117         BroadcastReceiver receiver = new BroadcastReceiver() {
118             @Override
119             public void onReceive(Context context, Intent intent) {
120                 int level = intent.getIntExtra(BatteryManager.EXTRA_LEVEL, -1);
121                 int scale = intent.getIntExtra(BatteryManager.EXTRA_SCALE, -1);
122                 int percentage = level * 100 / scale;
123                 if (minChargeLevel > 0 && minChargeLevel < 100) {
124                     if (percentage >= minChargeLevel) {
125                         Log.v(NNBenchmark.TAG,
126                                 String.format(
127                                         "Battery level: %d%% is greater than requested %d%%. "
128                                                 + "Considering the device ready.",
129                                         percentage, minChargeLevel));
130 
131                         chargedLatch.countDown();
132                         return;
133                     }
134                 }
135 
136                 Log.v(NNBenchmark.TAG, String.format("Battery level: %d%%", percentage));
137 
138                 int status = intent.getIntExtra(BatteryManager.EXTRA_STATUS, -1);
139                 if (status == BatteryManager.BATTERY_STATUS_FULL) {
140                     chargedLatch.countDown();
141                 } else if (status != BatteryManager.BATTERY_STATUS_CHARGING) {
142                     Log.e(NNBenchmark.TAG,
143                             String.format("Device is not charging, status is %d", status));
144                 }
145             }
146         };
147 
148         context.registerReceiver(receiver, new IntentFilter(Intent.ACTION_BATTERY_CHANGED));
149         try {
150             chargedLatch.await();
151         } catch (InterruptedException ignored) {
152             Thread.currentThread().interrupt();
153         }
154 
155         context.unregisterReceiver(receiver);
156     }
157 
158     @Override
159     @Before
setUp()160     public void setUp() throws Exception {
161         super.setUp();
162         prepareTest();
163         setActivityInitialTouchMode(false);
164     }
165 
166     @Override
167     @After
tearDown()168     public void tearDown() throws Exception {
169         super.tearDown();
170     }
171 
172     interface Joinable extends Runnable {
173         // Syncrhonises the caller with the completion of the current action
join()174         void join();
175     }
176 
177     class TestAction implements Joinable {
178 
179         private final TestModelEntry mTestModel;
180         private final float mMaxWarmupTimeSeconds;
181         private final float mMaxRunTimeSeconds;
182         private final CountDownLatch actionComplete;
183         private final boolean mSampleResults;
184 
185         BenchmarkResult mResult;
186         Throwable mException;
187 
TestAction(TestModelEntry testName, float maxWarmupTimeSeconds, float maxRunTimeSeconds)188         public TestAction(TestModelEntry testName, float maxWarmupTimeSeconds,
189                 float maxRunTimeSeconds) {
190             this(testName, maxWarmupTimeSeconds, maxRunTimeSeconds, false);
191         }
192 
TestAction(TestModelEntry testName, float maxWarmupTimeSeconds, float maxRunTimeSeconds, boolean sampleResults)193         public TestAction(TestModelEntry testName, float maxWarmupTimeSeconds,
194                 float maxRunTimeSeconds, boolean sampleResults) {
195             mTestModel = testName;
196             mMaxWarmupTimeSeconds = maxWarmupTimeSeconds;
197             mMaxRunTimeSeconds = maxRunTimeSeconds;
198             mSampleResults = sampleResults;
199             actionComplete = new CountDownLatch(1);
200         }
201 
run()202         public void run() {
203             Log.v(NNBenchmark.TAG, String.format(
204                     "Starting benchmark for test '%s' running for max %f seconds",
205                     mTestModel.mTestName,
206                     mMaxRunTimeSeconds));
207             try {
208                 mResult = mActivity.runSynchronously(
209                         mTestModel, mMaxWarmupTimeSeconds, mMaxRunTimeSeconds, mSampleResults);
210             } catch (BenchmarkException | IOException e) {
211                 mException = e;
212                 Log.e(NNBenchmark.TAG,
213                         String.format("Error running Benchmark for test '%s'", mTestModel), e);
214             } catch (Throwable e) {
215                 mException = e;
216                 Log.e(NNBenchmark.TAG,
217                         String.format("Failure running Benchmark for test '%s'!!", mTestModel), e);
218                 throw e;
219             } finally {
220                 actionComplete.countDown();
221             }
222         }
223 
getBenchmark()224         public BenchmarkResult getBenchmark() {
225             if (mException != null) {
226                 throw new Error("run failed", mException);
227             }
228             return mResult;
229         }
230 
231         @Override
join()232         public void join() {
233             try {
234                 actionComplete.await();
235             } catch (InterruptedException e) {
236                 Thread.currentThread().interrupt();
237                 Log.v(NNBenchmark.TAG, "Interrupted while waiting for action running", e);
238             }
239         }
240     }
241 
242     // Set the benchmark thread to run on ui thread
243     // Synchronized the thread such that the test will wait for the benchmark thread to finish
runOnUiThread(Joinable action)244     public void runOnUiThread(Joinable action) {
245         mActivity.runOnUiThread(action);
246         action.join();
247     }
248 
runTest(TestAction ta, String testName)249     public void runTest(TestAction ta, String testName) {
250         float sum = 0;
251         // For NNAPI systrace usage documentation, see
252         // frameworks/ml/nn/common/include/Tracing.h.
253         final String traceName = "[NN_LA_PO]" + testName;
254         try {
255             Trace.beginSection(traceName);
256             Log.i(NNBenchmark.TAG, "Starting test " + testName);
257             runOnUiThread(ta);
258             Log.i(NNBenchmark.TAG, "Test " + testName + " completed");
259         } finally {
260             Trace.endSection();
261         }
262         BenchmarkResult bmValue = ta.getBenchmark();
263 
264         // post result to INSTRUMENTATION_STATUS
265         getInstrumentation().sendStatus(Activity.RESULT_OK, bmValue.toBundle(testName));
266     }
267 
268     @Parameters(name = "{0}")
modelsList()269     public static List<TestModelEntry> modelsList() {
270         return TestModels.modelsList();
271     }
272 }
273