1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.dogfood;
18 
19 import android.app.NotificationChannel;
20 import android.app.NotificationManager;
21 import android.app.job.JobParameters;
22 import android.app.job.JobScheduler;
23 import android.app.job.JobService;
24 import android.content.SharedPreferences;
25 import android.util.Log;
26 
27 import androidx.core.app.NotificationCompat;
28 import androidx.core.app.NotificationManagerCompat;
29 
30 import com.android.nn.benchmark.core.BenchmarkResult;
31 import com.android.nn.benchmark.core.Processor;
32 import com.android.nn.benchmark.core.TestModels;
33 import com.android.nn.benchmark.core.TfLiteBackend;
34 
35 import java.util.List;
36 import java.util.Random;
37 import java.util.concurrent.ExecutorService;
38 import java.util.concurrent.Executors;
39 
40 /** Regularly runs a random selection of the NN API benchmark models */
41 public class BenchmarkJobService extends JobService implements Processor.Callback {
42 
43     private static final String TAG = "NN_BENCHMARK";
44     private static final String CHANNEL_ID = "default";
45     private static final int NOTIFICATION_ID = 999;
46     public static final int JOB_ID = 1;
47     private NotificationManagerCompat mNotificationManager;
48     private NotificationCompat.Builder mNotification;
49     private boolean mJobStopped = false;
50     private static final int NUM_RUNS = 10;
51     private Processor mProcessor;
52     private JobParameters mJobParameters;
53     private static final String NN_API_DOGFOOD_PREF = "nn_api_dogfood";
54 
55     private static int DOGFOOD_MODELS_PER_RUN = 20;
56     private BenchmarkResult mTestResults[];
57     private final ExecutorService processorRunner = Executors.newSingleThreadExecutor();
58 
59 
60     @Override
onStartJob(JobParameters jobParameters)61     public boolean onStartJob(JobParameters jobParameters) {
62         mJobParameters = jobParameters;
63         incrementNumRuns();
64         Log.d(TAG, String.format("NN API Benchmarking job %d/%d started", getNumRuns(), NUM_RUNS));
65         showNotification();
66         doBenchmark();
67 
68         return true;
69     }
70 
71     @Override
onStopJob(JobParameters jobParameters)72     public boolean onStopJob(JobParameters jobParameters) {
73         Log.d(TAG, String.format("NN API Benchmarking job %d/%d stopped", getNumRuns(), NUM_RUNS));
74         mJobStopped = true;
75 
76         return false;
77     }
78 
doBenchmark()79     public void doBenchmark() {
80         mProcessor = new Processor(this, this, randomModelList());
81         mProcessor.setTfLiteBackend(TfLiteBackend.NNAPI);
82         mProcessor.setToggleLong(true);
83         mProcessor.setMaxRunIterations(1);
84         processorRunner.submit(mProcessor);
85     }
86 
onBenchmarkFinish(boolean ok)87     public void onBenchmarkFinish(boolean ok) {
88         mProcessor.exit();
89         if (getNumRuns() >= NUM_RUNS) {
90             mNotification
91                     .setProgress(0, 0, false)
92                     .setContentText(
93                             "Benchmarking done! please upload a bug report via BetterBug under"
94                                 + " Android > Android OS & > Apps Runtime > Machine Learning")
95                     .setOngoing(false);
96             mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
97             JobScheduler jobScheduler = getSystemService(JobScheduler.class);
98             jobScheduler.cancel(JOB_ID);
99             resetNumRuns();
100         } else {
101             mNotification
102                     .setProgress(0, 0, false)
103                     .setContentText(
104                             String.format(
105                                     "Background test %d of %d is complete", getNumRuns(), NUM_RUNS))
106                     .setOngoing(false);
107             mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
108         }
109 
110         Log.d(TAG, "NN API Benchmarking job finished");
111         jobFinished(mJobParameters, false);
112     }
113 
onStatusUpdate(int testNumber, int numTests, String modelName)114     public void onStatusUpdate(int testNumber, int numTests, String modelName) {
115         Log.d(
116                 TAG,
117                 String.format("Benchmark progress %d of %d - %s", testNumber, numTests, modelName));
118         mNotification.setProgress(numTests, testNumber, false);
119         mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
120     }
121 
showNotification()122     private void showNotification() {
123         mNotificationManager = NotificationManagerCompat.from(this);
124         NotificationChannel channel =
125                 new NotificationChannel(CHANNEL_ID, "Default", NotificationManager.IMPORTANCE_LOW);
126         // mNotificationManager.createNotificationChannel(channel);
127         mNotificationManager = NotificationManagerCompat.from(this);
128         String title = "NN API Dogfood";
129         String msg = String.format("Background test %d of %d is running", getNumRuns(), NUM_RUNS);
130 
131         mNotification =
132                 new NotificationCompat.Builder(this, CHANNEL_ID)
133                         .setSmallIcon(R.mipmap.ic_launcher)
134                         .setContentTitle(title)
135                         .setContentText("NN API Benchmarking Job")
136                         .setPriority(NotificationCompat.PRIORITY_MAX)
137                         .setOngoing(true);
138         mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
139     }
140 
randomModelList()141     private int[] randomModelList() {
142         long seed = System.currentTimeMillis();
143         List<TestModels.TestModelEntry> testList = TestModels.modelsList();
144 
145         Log.v(TAG, "Dogfood run seed " + seed);
146         Random random = new Random(seed);
147         int numModelsToSelect = Math.min(DOGFOOD_MODELS_PER_RUN, testList.size());
148         int[] randomModelIndices = new int[numModelsToSelect];
149 
150         for (int i = 0; i < numModelsToSelect; i++) {
151             randomModelIndices[i] = random.nextInt(testList.size());
152         }
153 
154         return randomModelIndices;
155     }
156 
incrementNumRuns()157     private void incrementNumRuns() {
158         SharedPreferences.Editor editor =
159                 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit();
160         editor.putInt("num_runs", getNumRuns() + 1);
161         editor.apply();
162     }
163 
resetNumRuns()164     private void resetNumRuns() {
165         SharedPreferences.Editor editor =
166                 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit();
167         editor.putInt("num_runs", 0);
168         editor.apply();
169     }
170 
getNumRuns()171     private int getNumRuns() {
172         SharedPreferences prefs = getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE);
173         return prefs.getInt("num_runs", 0);
174     }
175 }
176