1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.nn.dogfood; 18 19 import android.app.NotificationChannel; 20 import android.app.NotificationManager; 21 import android.app.job.JobParameters; 22 import android.app.job.JobScheduler; 23 import android.app.job.JobService; 24 import android.content.SharedPreferences; 25 import android.util.Log; 26 27 import androidx.core.app.NotificationCompat; 28 import androidx.core.app.NotificationManagerCompat; 29 30 import com.android.nn.benchmark.core.BenchmarkResult; 31 import com.android.nn.benchmark.core.Processor; 32 import com.android.nn.benchmark.core.TestModels; 33 import com.android.nn.benchmark.core.TfLiteBackend; 34 35 import java.util.List; 36 import java.util.Random; 37 import java.util.concurrent.ExecutorService; 38 import java.util.concurrent.Executors; 39 40 /** Regularly runs a random selection of the NN API benchmark models */ 41 public class BenchmarkJobService extends JobService implements Processor.Callback { 42 43 private static final String TAG = "NN_BENCHMARK"; 44 private static final String CHANNEL_ID = "default"; 45 private static final int NOTIFICATION_ID = 999; 46 public static final int JOB_ID = 1; 47 private NotificationManagerCompat mNotificationManager; 48 private NotificationCompat.Builder mNotification; 49 private boolean mJobStopped = false; 50 private static final int NUM_RUNS = 10; 51 private Processor mProcessor; 52 private JobParameters mJobParameters; 53 private static final String NN_API_DOGFOOD_PREF = "nn_api_dogfood"; 54 55 private static int DOGFOOD_MODELS_PER_RUN = 20; 56 private BenchmarkResult mTestResults[]; 57 private final ExecutorService processorRunner = Executors.newSingleThreadExecutor(); 58 59 60 @Override onStartJob(JobParameters jobParameters)61 public boolean onStartJob(JobParameters jobParameters) { 62 mJobParameters = jobParameters; 63 incrementNumRuns(); 64 Log.d(TAG, String.format("NN API Benchmarking job %d/%d started", getNumRuns(), NUM_RUNS)); 65 showNotification(); 66 doBenchmark(); 67 68 return true; 69 } 70 71 @Override onStopJob(JobParameters jobParameters)72 public boolean onStopJob(JobParameters jobParameters) { 73 Log.d(TAG, String.format("NN API Benchmarking job %d/%d stopped", getNumRuns(), NUM_RUNS)); 74 mJobStopped = true; 75 76 return false; 77 } 78 doBenchmark()79 public void doBenchmark() { 80 mProcessor = new Processor(this, this, randomModelList()); 81 mProcessor.setTfLiteBackend(TfLiteBackend.NNAPI); 82 mProcessor.setToggleLong(true); 83 mProcessor.setMaxRunIterations(1); 84 processorRunner.submit(mProcessor); 85 } 86 onBenchmarkFinish(boolean ok)87 public void onBenchmarkFinish(boolean ok) { 88 mProcessor.exit(); 89 if (getNumRuns() >= NUM_RUNS) { 90 mNotification 91 .setProgress(0, 0, false) 92 .setContentText( 93 "Benchmarking done! please upload a bug report via BetterBug under" 94 + " Android > Android OS & > Apps Runtime > Machine Learning") 95 .setOngoing(false); 96 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 97 JobScheduler jobScheduler = getSystemService(JobScheduler.class); 98 jobScheduler.cancel(JOB_ID); 99 resetNumRuns(); 100 } else { 101 mNotification 102 .setProgress(0, 0, false) 103 .setContentText( 104 String.format( 105 "Background test %d of %d is complete", getNumRuns(), NUM_RUNS)) 106 .setOngoing(false); 107 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 108 } 109 110 Log.d(TAG, "NN API Benchmarking job finished"); 111 jobFinished(mJobParameters, false); 112 } 113 onStatusUpdate(int testNumber, int numTests, String modelName)114 public void onStatusUpdate(int testNumber, int numTests, String modelName) { 115 Log.d( 116 TAG, 117 String.format("Benchmark progress %d of %d - %s", testNumber, numTests, modelName)); 118 mNotification.setProgress(numTests, testNumber, false); 119 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 120 } 121 showNotification()122 private void showNotification() { 123 mNotificationManager = NotificationManagerCompat.from(this); 124 NotificationChannel channel = 125 new NotificationChannel(CHANNEL_ID, "Default", NotificationManager.IMPORTANCE_LOW); 126 // mNotificationManager.createNotificationChannel(channel); 127 mNotificationManager = NotificationManagerCompat.from(this); 128 String title = "NN API Dogfood"; 129 String msg = String.format("Background test %d of %d is running", getNumRuns(), NUM_RUNS); 130 131 mNotification = 132 new NotificationCompat.Builder(this, CHANNEL_ID) 133 .setSmallIcon(R.mipmap.ic_launcher) 134 .setContentTitle(title) 135 .setContentText("NN API Benchmarking Job") 136 .setPriority(NotificationCompat.PRIORITY_MAX) 137 .setOngoing(true); 138 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 139 } 140 randomModelList()141 private int[] randomModelList() { 142 long seed = System.currentTimeMillis(); 143 List<TestModels.TestModelEntry> testList = TestModels.modelsList(); 144 145 Log.v(TAG, "Dogfood run seed " + seed); 146 Random random = new Random(seed); 147 int numModelsToSelect = Math.min(DOGFOOD_MODELS_PER_RUN, testList.size()); 148 int[] randomModelIndices = new int[numModelsToSelect]; 149 150 for (int i = 0; i < numModelsToSelect; i++) { 151 randomModelIndices[i] = random.nextInt(testList.size()); 152 } 153 154 return randomModelIndices; 155 } 156 incrementNumRuns()157 private void incrementNumRuns() { 158 SharedPreferences.Editor editor = 159 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit(); 160 editor.putInt("num_runs", getNumRuns() + 1); 161 editor.apply(); 162 } 163 resetNumRuns()164 private void resetNumRuns() { 165 SharedPreferences.Editor editor = 166 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit(); 167 editor.putInt("num_runs", 0); 168 editor.apply(); 169 } 170 getNumRuns()171 private int getNumRuns() { 172 SharedPreferences prefs = getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE); 173 return prefs.getInt("num_runs", 0); 174 } 175 } 176