1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.nn.dogfood; 18 19 import android.app.NotificationChannel; 20 import android.app.NotificationManager; 21 import android.app.job.JobParameters; 22 import android.app.job.JobScheduler; 23 import android.app.job.JobService; 24 import android.content.SharedPreferences; 25 import android.util.Log; 26 27 import androidx.core.app.NotificationCompat; 28 import androidx.core.app.NotificationManagerCompat; 29 30 import com.android.nn.benchmark.core.BenchmarkResult; 31 import com.android.nn.benchmark.core.Processor; 32 import com.android.nn.benchmark.core.TestModels; 33 34 import java.util.List; 35 import java.util.Random; 36 import java.util.concurrent.ExecutorService; 37 import java.util.concurrent.Executors; 38 39 /** Regularly runs a random selection of the NN API benchmark models */ 40 public class BenchmarkJobService extends JobService implements Processor.Callback { 41 42 private static final String TAG = "NN_BENCHMARK"; 43 private static final String CHANNEL_ID = "default"; 44 private static final int NOTIFICATION_ID = 999; 45 public static final int JOB_ID = 1; 46 private NotificationManagerCompat mNotificationManager; 47 private NotificationCompat.Builder mNotification; 48 private boolean mJobStopped = false; 49 private static final int NUM_RUNS = 10; 50 private Processor mProcessor; 51 private JobParameters mJobParameters; 52 private static final String NN_API_DOGFOOD_PREF = "nn_api_dogfood"; 53 54 private static int DOGFOOD_MODELS_PER_RUN = 20; 55 private BenchmarkResult mTestResults[]; 56 private final ExecutorService processorRunner = Executors.newSingleThreadExecutor(); 57 58 59 @Override onStartJob(JobParameters jobParameters)60 public boolean onStartJob(JobParameters jobParameters) { 61 mJobParameters = jobParameters; 62 incrementNumRuns(); 63 Log.d(TAG, String.format("NN API Benchmarking job %d/%d started", getNumRuns(), NUM_RUNS)); 64 showNotification(); 65 doBenchmark(); 66 67 return true; 68 } 69 70 @Override onStopJob(JobParameters jobParameters)71 public boolean onStopJob(JobParameters jobParameters) { 72 Log.d(TAG, String.format("NN API Benchmarking job %d/%d stopped", getNumRuns(), NUM_RUNS)); 73 mJobStopped = true; 74 75 return false; 76 } 77 doBenchmark()78 public void doBenchmark() { 79 mProcessor = new Processor(this, this, randomModelList()); 80 mProcessor.setUseNNApi(true); 81 mProcessor.setToggleLong(true); 82 processorRunner.submit(mProcessor); 83 } 84 onBenchmarkFinish(boolean ok)85 public void onBenchmarkFinish(boolean ok) { 86 mProcessor.exit(); 87 if (getNumRuns() >= NUM_RUNS) { 88 mNotification 89 .setProgress(0, 0, false) 90 .setContentText( 91 "Benchmarking done! please upload a bug report via BetterBug under" 92 + " Android > Android OS & > Apps Runtime > Machine Learning") 93 .setOngoing(false); 94 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 95 JobScheduler jobScheduler = getSystemService(JobScheduler.class); 96 jobScheduler.cancel(JOB_ID); 97 resetNumRuns(); 98 } else { 99 mNotification 100 .setProgress(0, 0, false) 101 .setContentText( 102 String.format( 103 "Background test %d of %d is complete", getNumRuns(), NUM_RUNS)) 104 .setOngoing(false); 105 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 106 } 107 108 Log.d(TAG, "NN API Benchmarking job finished"); 109 jobFinished(mJobParameters, false); 110 } 111 onStatusUpdate(int testNumber, int numTests, String modelName)112 public void onStatusUpdate(int testNumber, int numTests, String modelName) { 113 Log.d( 114 TAG, 115 String.format("Benchmark progress %d of %d - %s", testNumber, numTests, modelName)); 116 mNotification.setProgress(numTests, testNumber, false); 117 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 118 } 119 showNotification()120 private void showNotification() { 121 mNotificationManager = NotificationManagerCompat.from(this); 122 NotificationChannel channel = 123 new NotificationChannel(CHANNEL_ID, "Default", NotificationManager.IMPORTANCE_LOW); 124 mNotificationManager.createNotificationChannel(channel); 125 mNotificationManager = NotificationManagerCompat.from(this); 126 String title = "NN API Dogfood"; 127 String msg = String.format("Background test %d of %d is running", getNumRuns(), NUM_RUNS); 128 129 mNotification = 130 new NotificationCompat.Builder(this, CHANNEL_ID) 131 .setSmallIcon(R.mipmap.ic_launcher) 132 .setContentTitle(title) 133 .setContentText("NN API Benchmarking Job") 134 .setPriority(NotificationCompat.PRIORITY_MAX) 135 .setOngoing(true); 136 mNotificationManager.notify(NOTIFICATION_ID, mNotification.build()); 137 } 138 randomModelList()139 private int[] randomModelList() { 140 long seed = System.currentTimeMillis(); 141 List<TestModels.TestModelEntry> testList = TestModels.modelsList(); 142 143 Log.v(TAG, "Dogfood run seed " + seed); 144 Random random = new Random(seed); 145 int numModelsToSelect = Math.min(DOGFOOD_MODELS_PER_RUN, testList.size()); 146 int[] randomModelIndices = new int[numModelsToSelect]; 147 148 for (int i = 0; i < numModelsToSelect; i++) { 149 randomModelIndices[i] = random.nextInt(testList.size()); 150 } 151 152 return randomModelIndices; 153 } 154 incrementNumRuns()155 private void incrementNumRuns() { 156 SharedPreferences.Editor editor = 157 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit(); 158 editor.putInt("num_runs", getNumRuns() + 1); 159 editor.apply(); 160 } 161 resetNumRuns()162 private void resetNumRuns() { 163 SharedPreferences.Editor editor = 164 getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE).edit(); 165 editor.putInt("num_runs", 0); 166 editor.apply(); 167 } 168 getNumRuns()169 private int getNumRuns() { 170 SharedPreferences prefs = getSharedPreferences(NN_API_DOGFOOD_PREF, MODE_PRIVATE); 171 return prefs.getInt("num_runs", 0); 172 } 173 } 174