1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.dogfood;
18 
19 import android.app.NotificationChannel;
20 import android.app.NotificationManager;
21 import android.app.job.JobParameters;
22 import android.app.job.JobScheduler;
23 import android.app.job.JobService;
24 import android.content.SharedPreferences;
25 import android.util.Log;
26 
27 import androidx.core.app.NotificationCompat;
28 import androidx.core.app.NotificationManagerCompat;
29 
30 import com.android.nn.benchmark.core.BenchmarkResult;
31 import com.android.nn.benchmark.core.Processor;
32 import com.android.nn.benchmark.core.TestModels;
33 
34 import java.util.List;
35 import java.util.Random;
36 import java.util.concurrent.ExecutorService;
37 import java.util.concurrent.Executors;
38 
39 /** Regularly runs a random selection of the NN API benchmark models */
40 public class BenchmarkJobService extends JobService implements Processor.Callback {
41 
42     private static final String TAG = "NN_BENCHMARK";
43     private static final String CHANNEL_ID = "default";
44     private static final int NOTIFICATION_ID = 999;
45     public static final int JOB_ID = 1;
46     private NotificationManagerCompat mNotificationManager;
47     private NotificationCompat.Builder mNotification;
48     private boolean mJobStopped = false;
49     private static final int NUM_RUNS = 10;
50     private Processor mProcessor;
51     private JobParameters mJobParameters;
52     private static final String NN_API_DOGFOOD_PREF = "nn_api_dogfood";
53 
54     private static int DOGFOOD_MODELS_PER_RUN = 20;
55     private BenchmarkResult mTestResults[];
56     private final ExecutorService processorRunner = Executors.newSingleThreadExecutor();
57 
58 
59     @Override
onStartJob(JobParameters jobParameters)60     public boolean onStartJob(JobParameters jobParameters) {
61         mJobParameters = jobParameters;
62         incrementNumRuns();
63         Log.d(TAG, String.format("NN API Benchmarking job %d/%d started", getNumRuns(), NUM_RUNS));
64         showNotification();
65         doBenchmark();
66 
67         return true;
68     }
69 
70     @Override
onStopJob(JobParameters jobParameters)71     public boolean onStopJob(JobParameters jobParameters) {
72         Log.d(TAG, String.format("NN API Benchmarking job %d/%d stopped", getNumRuns(), NUM_RUNS));
73         mJobStopped = true;
74 
75         return false;
76     }
77 
doBenchmark()78     public void doBenchmark() {
79         mProcessor = new Processor(this, this, randomModelList());
80         mProcessor.setUseNNApi(true);
81         mProcessor.setToggleLong(true);
82         processorRunner.submit(mProcessor);
83     }
84 
onBenchmarkFinish(boolean ok)85     public void onBenchmarkFinish(boolean ok) {
86         mProcessor.exit();
87         if (getNumRuns() >= NUM_RUNS) {
88             mNotification
89                     .setProgress(0, 0, false)
90                     .setContentText(
91                             "Benchmarking done! please upload a bug report via BetterBug under"
92                                 + " Android > Android OS & > Apps Runtime > Machine Learning")
93                     .setOngoing(false);
94             mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
95             JobScheduler jobScheduler = getSystemService(JobScheduler.class);
96             jobScheduler.cancel(JOB_ID);
97             resetNumRuns();
98         } else {
99             mNotification
100                     .setProgress(0, 0, false)
101                     .setContentText(
102                             String.format(
103                                     "Background test %d of %d is complete", getNumRuns(), NUM_RUNS))
104                     .setOngoing(false);
105             mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
106         }
107 
108         Log.d(TAG, "NN API Benchmarking job finished");
109         jobFinished(mJobParameters, false);
110     }
111 
onStatusUpdate(int testNumber, int numTests, String modelName)112     public void onStatusUpdate(int testNumber, int numTests, String modelName) {
113         Log.d(
114                 TAG,
115                 String.format("Benchmark progress %d of %d - %s", testNumber, numTests, modelName));
116         mNotification.setProgress(numTests, testNumber, false);
117         mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
118     }
119 
showNotification()120     private void showNotification() {
121         mNotificationManager = NotificationManagerCompat.from(this);
122         NotificationChannel channel =
123                 new NotificationChannel(CHANNEL_ID, "Default", NotificationManager.IMPORTANCE_LOW);
124         mNotificationManager.createNotificationChannel(channel);
125         mNotificationManager = NotificationManagerCompat.from(this);
126         String title = "NN API Dogfood";
127         String msg = String.format("Background test %d of %d is running", getNumRuns(), NUM_RUNS);
128 
129         mNotification =
130                 new NotificationCompat.Builder(this, CHANNEL_ID)
131                         .setSmallIcon(R.mipmap.ic_launcher)
132                         .setContentTitle(title)
133                         .setContentText("NN API Benchmarking Job")
134                         .setPriority(NotificationCompat.PRIORITY_MAX)
135                         .setOngoing(true);
136         mNotificationManager.notify(NOTIFICATION_ID, mNotification.build());
137     }
138 
randomModelList()139     private int[] randomModelList() {
140         long seed = System.currentTimeMillis();
141         List<TestModels.TestModelEntry> testList = TestModels.modelsList();
142 
143         Log.v(TAG, "Dogfood run seed " + seed);
144         Random random = new Random(seed);
145         int numModelsToSelect = Math.min(DOGFOOD_MODELS_PER_RUN, testList.size());
146         int[] randomModelIndices = new int[numModelsToSelect];
147 
148         for (int i = 0; i < numModelsToSelect; i++) {
149             randomModelIndices[i] = random.nextInt(testList.size());
150         }
151 
152         return randomModelIndices;
153     }
154 
incrementNumRuns()155     private void incrementNumRuns() {
156         SharedPreferences.Editor editor =
157                 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit();
158         editor.putInt("num_runs", getNumRuns() + 1);
159         editor.apply();
160     }
161 
resetNumRuns()162     private void resetNumRuns() {
163         SharedPreferences.Editor editor =
164                 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit();
165         editor.putInt("num_runs", 0);
166         editor.apply();
167     }
168 
getNumRuns()169     private int getNumRuns() {
170         SharedPreferences prefs = getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE);
171         return prefs.getInt("num_runs", 0);
172     }
173 }
174