1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.crashtest.app;
18 
19 import static com.android.nn.crashtest.app.CrashTestStatus.TestResult.SUCCESS;
20 
21 import android.content.Intent;
22 import android.test.ActivityInstrumentationTestCase2;
23 import android.test.UiThreadTest;
24 import android.test.suitebuilder.annotation.LargeTest;
25 
26 import androidx.test.InstrumentationRegistry;
27 
28 import com.android.nn.benchmark.app.BenchmarkTestBase;
29 import com.android.nn.benchmark.core.NnApiDelegationFailure;
30 import com.android.nn.benchmark.core.TestModels;
31 import com.android.nn.crashtest.core.test.RandomGraphTest;
32 
33 import org.junit.Before;
34 import org.junit.Rule;
35 import org.junit.Test;
36 import org.junit.rules.TestName;
37 import org.junit.runner.RunWith;
38 import org.junit.runners.Parameterized;
39 import org.junit.runners.Parameterized.Parameters;
40 
41 import java.time.Duration;
42 import java.util.Arrays;
43 import java.util.Optional;
44 import java.util.concurrent.ExecutionException;
45 
46 @RunWith(Parameterized.class)
47 public abstract class NNRandomGraphTest
48         extends ActivityInstrumentationTestCase2<NNRandomGraphTestActivity>
49         implements AcceleratorSpecificTestSupport {
50     private static final String TAG = "NN_RAND_MODEL";
51 
52     protected final String mAcceleratorName;
53     private final int mModelCount;
54     private final int mGraphSize;
55     private final Duration mDuration;
56     private final int mDimensionRange;
57 
58     private final static int SMALL_MODEL_SIZE = 10;
59     private final static int LARGE_MODEL_SIZE = 600;
60     private final static int NARROW_DIMENSIONS_RANGE = 10;
61     private final static int WIDE_DIMENSIONS_RANGE = 1000;
62     private final static Duration MAX_TEST_DURATION = Duration.ofMinutes(15);
63     private final static int NUMBER_OF_MODELS = 600;
64     private final boolean mRunModelCompilationOnly;
65 
66     @Parameters(name = "{0} models of size {1} and dimensions range {2} for max duration of {3} "
67             + "on accelerator {4}")
testConfiguration()68     public static Iterable<Object[]> testConfiguration() {
69         return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
70                 Arrays.asList(
71                         new Object[]{NUMBER_OF_MODELS, SMALL_MODEL_SIZE, WIDE_DIMENSIONS_RANGE,
72                                 MAX_TEST_DURATION},
73                         new Object[]{NUMBER_OF_MODELS, LARGE_MODEL_SIZE, NARROW_DIMENSIONS_RANGE,
74                                 MAX_TEST_DURATION}));
75     }
76 
77     @Rule
78     public TestName mTestName = new TestName();
79 
NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange, Duration duration, String acceleratorName, boolean runModelCompilationOnly)80     public NNRandomGraphTest(int modelCount, int graphSize, int dimensionRange,
81             Duration duration, String acceleratorName, boolean runModelCompilationOnly) {
82         super(NNRandomGraphTestActivity.class);
83         mModelCount = modelCount;
84         mGraphSize = graphSize;
85         mDuration = duration;
86         mAcceleratorName = acceleratorName;
87         mDimensionRange = dimensionRange;
88         mRunModelCompilationOnly = runModelCompilationOnly;
89     }
90 
91     @Before
92     @Override
setUp()93     public void setUp() {
94         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
95         BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 60);
96         setActivityIntent(getTestModelsOfSizeAndRangeForMaxTimeIntent(mGraphSize, mDimensionRange,
97                 mModelCount, mAcceleratorName, mDuration, mTestName.getMethodName()));
98     }
99 
findModelForLivenessTest()100     protected Optional<TestModels.TestModelEntry> findModelForLivenessTest()
101             throws NnApiDelegationFailure {
102         return AcceleratorSpecificTestSupport.findTestModelRunningOnAccelerator(
103                 getInstrumentation().getTargetContext(), mAcceleratorName);
104     }
105 
106     @Test
107     @LargeTest
108     @UiThreadTest
testDriverDoesNotFailWithParallelWorkload()109     public void testDriverDoesNotFailWithParallelWorkload()
110             throws ExecutionException, InterruptedException, NnApiDelegationFailure {
111         final NNRandomGraphTestActivity activity = getActivity();
112 
113         assertEquals(SUCCESS, activity.testResult());
114     }
115 
116     /**
117      * @return the intent to use to initialise the RandomGraphTest test class
118      */
getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange, int modelsCount, String deviceName, Duration maxTestDuration, String testName)119     protected Intent getTestModelsOfSizeAndRangeForMaxTimeIntent(int graphSize, int dimensionsRange,
120             int modelsCount, String deviceName, Duration maxTestDuration, String testName) {
121         Intent result = new Intent();
122         RandomGraphTest
123                 .intentInitializer(graphSize, dimensionsRange, modelsCount,
124                         RandomGraphTest.DEFAULT_PAUSE_BETWEEN_MODELS_MILLIS,
125                         mRunModelCompilationOnly, deviceName, maxTestDuration.toMillis(), testName)
126                 .addIntentParams(result);
127         return result;
128     }
129 }