1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.crashtest.app;
18 
19 import android.content.Intent;
20 import android.os.RemoteException;
21 import android.test.ActivityInstrumentationTestCase2;
22 import android.test.UiThreadTest;
23 import android.test.suitebuilder.annotation.LargeTest;
24 
25 import androidx.test.InstrumentationRegistry;
26 
27 import com.android.nn.benchmark.core.NnApiDelegationFailure;
28 import com.android.nn.benchmark.core.TestModels;
29 
30 import junit.framework.Assert;
31 
32 import org.junit.Before;
33 import org.junit.Rule;
34 import org.junit.Test;
35 import org.junit.rules.TestName;
36 import org.junit.runner.RunWith;
37 import org.junit.runners.Parameterized;
38 import org.junit.runners.Parameterized.Parameters;
39 
40 import java.time.Duration;
41 import java.util.Optional;
42 import java.util.concurrent.ExecutionException;
43 import java.util.concurrent.ExecutorService;
44 import java.util.concurrent.Executors;
45 import java.util.concurrent.Future;
46 import java.util.stream.IntStream;
47 
48 @RunWith(Parameterized.class)
49 public class NNMemoryMappedModelCompilationTest extends
50         ActivityInstrumentationTestCase2<NNParallelTestActivity> implements
51         AcceleratorSpecificTestSupport {
52 
53     private static final String TAG = "NNMemoryMappedModelCompilation";
54     private static final Duration MAX_SEPARATE_PROCESS_EXECUTION_TIME = Duration.ofSeconds(70);
55     public static final int NNAPI_CLIENTS_COUNT = 4;
56 
57     private final ExecutorService mDriverLivenessValidationExecutor =
58             Executors.newSingleThreadExecutor();
59     private final String mAcceleratorName;
60 
61     @Rule
62     public TestName mTestName = new TestName();
63 
NNMemoryMappedModelCompilationTest(String acceleratorName)64     public NNMemoryMappedModelCompilationTest(String acceleratorName) {
65         super(NNParallelTestActivity.class);
66         mAcceleratorName = acceleratorName;
67     }
68 
69     @Parameters(name = "Accelerator({0})")
targetAccelerators()70     public static Iterable<String> targetAccelerators() {
71         return AcceleratorSpecificTestSupport.getTargetAcceleratorNames();
72     }
73 
74     @Before
75     @Override
setUp()76     public void setUp() {
77         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
78         final Intent runSomeInferencesInASeparateProcess;
79         try {
80             runSomeInferencesInASeparateProcess = compileSupportedModelsMemoryMappedOnNThreadsFor(
81                     NNAPI_CLIENTS_COUNT,
82                     MAX_SEPARATE_PROCESS_EXECUTION_TIME);
83         } catch (NnApiDelegationFailure nnApiDelegationFailure) {
84             throw new RuntimeException(
85                     "Cannot initialize test, failure looking for supported models, please check "
86                             + "the driver status",
87                     nnApiDelegationFailure);
88         }
89         setActivityIntent(runSomeInferencesInASeparateProcess);
90     }
91 
92     @Test
93     @LargeTest
94     @UiThreadTest
testDriverDoesNotFailWithParallelThreads()95     public void testDriverDoesNotFailWithParallelThreads()
96             throws ExecutionException, InterruptedException, RemoteException,
97             NnApiDelegationFailure {
98         final NNParallelTestActivity activity = getActivity();
99 
100         Optional<TestModels.TestModelEntry> modelForLivenessTest =
101                 AcceleratorSpecificTestSupport.findTestModelRunningOnAccelerator(activity, mAcceleratorName);
102         assertTrue("No model available to be run on accelerator " + mAcceleratorName,
103                 modelForLivenessTest.isPresent());
104 
105         // This should not be necessary
106         final DriverLivenessChecker driverLivenessChecker = new DriverLivenessChecker(activity,
107                 mAcceleratorName, modelForLivenessTest.get());
108         Future<Boolean> driverDidNotCrash = mDriverLivenessValidationExecutor.submit(
109                 driverLivenessChecker);
110 
111         Assert.assertEquals(CrashTestStatus.TestResult.SUCCESS, activity.testResult());
112 
113         driverLivenessChecker.stop();
114         assertTrue("Driver shouldn't have crashed", driverDidNotCrash.get());
115     }
116 
compileSupportedModelsMemoryMappedOnNThreadsFor(int threadCount, Duration testDuration)117     private Intent compileSupportedModelsMemoryMappedOnNThreadsFor(int threadCount,
118             Duration testDuration) throws NnApiDelegationFailure {
119         Intent intent = new Intent();
120         int modelsCount = TestModels.modelsList().size();
121         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_LIST, IntStream.range(0, modelsCount).toArray());
122         intent.putExtra(NNParallelTestActivity.EXTRA_THREAD_COUNT, threadCount);
123         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_DURATION_MILLIS, testDuration.toMillis());
124         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_IN_SEPARATE_PROCESS, true);
125         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_NAME, mTestName.getMethodName());
126         intent.putExtra(NNParallelTestActivity.EXTRA_ACCELERATOR_NAME, mAcceleratorName);
127         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_MODEL_COMPILATION_ONLY, true);
128         intent.putExtra(NNParallelTestActivity.EXTRA_MEMORY_MAP_MODEL, true);
129         intent.putExtra(NNParallelTestActivity.EXTRA_IGNORE_UNSUPPORTED_MODELS, true);
130         return intent;
131     }
132 }
133