1#!/bin/bash 2# 3# Build benchmark app and run it, mimicking a user-initiated run 4# 5# Output is logged to a temporary folder and summarized in txt and JSON formats. 6# parallel-inference-stress tests produce no output except for the success or failure notification, 7# which is not logged. 8 9if [[ "$OSTYPE" == "darwin"* ]]; then 10 OPTS="$(getopt f:rb -- "$*")" 11else 12 OPTS="$(getopt -o f:rb -l filter-driver:,include-nnapi-reference,nnapi-reference-only,skip-build -- "$@")" 13fi 14 15if [ $? -ne 0 ]; then 16 echo "Invalid arguments, accepted options are" 17 if [[ "$OSTYPE" == "darwin"* ]]; then 18 echo " -f <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression" 19 echo " -r : to include nnapi-reference in target drivers" 20 echo " -b : skip build and installation of tests" 21 else 22 echo " -f <regex> | --filter-driver <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression" 23 echo " -r | --include-nnapi-reference : to include nnapi-reference in target drivers" 24 echo " --nnapi-reference-only : to run tests only vs nnapi-reference" 25 echo " -b | --skip-build : skip build and installation of tests" 26 fi 27 exit 28fi 29 30eval set -- "$OPTS" 31 32DRIVER_FILTER_OPT="" 33INCLUDE_NNAPI_REF_OPT="" 34BUILD_AND_INSTALL=true 35while [ $# -gt 0 ] ; do 36 case "$1" in 37 -f|--filter-driver) 38 DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter $2" 39 shift 2 40 ;; 41 -r|--include-nnapi-reference) 42 INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true" 43 shift 44 ;; 45 --nnapi-reference-only) 46 DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter no-device" 47 INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true" 48 shift 49 ;; 50 -b|--skip-build) 51 BUILD_AND_INSTALL=false 52 shift 53 ;; 54 --) 55 shift 56 break 57 ;; 58 *) 59 echo "Unsupported arg $1" 60 exit 1 61 esac 62done 63 64MODE="${1:-scoring}" 65INSTALL_NATIVE_TESTS=false 66CRASH_TEST_APP="NeuralNetworksApiCrashTest" 67APP="NeuralNetworksApiBenchmark" 68case "$MODE" in 69 scoring) 70 CLASS=com.android.nn.benchmark.app.NNScoringTest 71 ;; 72 inference-stress) 73 CLASS=com.android.nn.benchmark.app.NNInferenceStressTest 74 ;; 75 model-loading-stress) 76 CLASS=com.android.nn.benchmark.app.NNModelLoadingStressTest 77 ;; 78 parallel-inference-stress) 79 CLASS=com.android.nn.crashtest.app.NNParallelCrashResistantInferenceTest 80 APP="$CRASH_TEST_APP" 81 ;; 82 parallel-inference-stress-in-process) 83 CLASS=com.android.nn.crashtest.app.NNParallelInProcessInferenceTest 84 APP="$CRASH_TEST_APP" 85 ;; 86 client-early-termination-stress) 87 CLASS=com.android.nn.crashtest.app.NNClientEarlyTerminationTest 88 APP="$CRASH_TEST_APP" 89 ;; 90 multi-process-inference-stress) 91 CLASS=com.android.nn.crashtest.app.NNMultipleProcessInferenceTest 92 APP="$CRASH_TEST_APP" 93 INSTALL_NATIVE_TESTS=true 94 ;; 95 multi-process-model-load-stress) 96 CLASS=com.android.nn.crashtest.app.NNMultipleProcessModelLoadTest 97 APP="$CRASH_TEST_APP" 98 INSTALL_NATIVE_TESTS=true 99 ;; 100 memory-mapped-model-load-stress) 101 CLASS=com.android.nn.crashtest.app.NNMemoryMappedModelCompilationTest 102 APP="$CRASH_TEST_APP" 103 ;; 104 model-load-random-stress) 105 APP="$CRASH_TEST_APP" 106 CLASS=com.android.nn.crashtest.app.NNRandomGraphLoadTest 107 ;; 108 inference-random-stress) 109 APP="$CRASH_TEST_APP" 110 CLASS=com.android.nn.crashtest.app.NNRandomGraphExecutionTest 111 ;; 112 performance-degradation-stress) 113 APP="$CRASH_TEST_APP" 114 CLASS=com.android.nn.crashtest.app.NNPerformanceDegradationTest 115 ;; 116 *) 117 echo "Unknown execution mode: $1" 118 echo "Known modes: scoring (default), inference-stress, model-loading-stress, " \ 119 "parallel-inference-stress, parallel-inference-stress-in-process, " \ 120 "client-early-termination-stress, multi-process-inference-stress, " \ 121 "multi-process-model-load-stress memory-mapped-model-load-stress, " \ 122 "model-load-random-stress, inference-random-stress, performance-degradation-stress" 123 exit 1 124 ;; 125esac 126 127if [[ -z "$ANDROID_BUILD_TOP" ]]; then 128 echo ANDROID_BUILD_TOP not set, bailing out 129 echo you must run lunch before running this script 130 exit 1 131fi 132 133set -e 134cd $ANDROID_BUILD_TOP 135 136if [ "$BUILD_AND_INSTALL" = true ]; then 137 # Build and install benchmark app 138 TMPFILE=$(mktemp) 139 build/soong/soong_ui.bash --make-mode ${APP} 2>&1 | tee ${TMPFILE} 140 TARGET_ARCH=$(cat ${TMPFILE} | grep TARGET_ARCH= | sed -e 's/TARGET_ARCH=//') 141 if [ "${TARGET_ARCH}" = "aarch64" ]; then 142 APK_DIR=arm64 143 else 144 APK_DIR=${TARGET_ARCH} 145 fi 146 if ! adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk; then 147 adb uninstall com.android.nn.benchmark.app 148 adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk 149 fi 150 151 if [ "$INSTALL_NATIVE_TESTS" = true ]; then 152 build/soong/soong_ui.bash --make-mode nn_stress_test 153 adb push $OUT/system/bin/nn_stress_test /bin/ 154 fi 155fi 156 157# Should we figure out if we run on release device 158if [ -z "$MLTS_RELEASE_DEVICE" ]; then 159 BUILD_DESCRIPTION=`adb shell getprop ro.build.description` 160 if [[ $BUILD_DESCRIPTION =~ .*release.* ]] 161 then 162 MLTS_RELEASE_DEVICE=True 163 else 164 MLTS_RELEASE_DEVICE=False 165 fi 166fi 167 168# Pass --no-isolated-storage to am instrument? 169BUILD_VERSION_RELEASE=`adb shell getprop ro.build.version.release` 170AM_INSTRUMENT_FLAGS="$DRIVER_FILTER_OPT $INCLUDE_NNAPI_REF_OPT" 171if [[ $BUILD_VERSION_RELEASE == "Q" ]]; then 172 AM_INSTRUMENT_FLAGS+=" --no-isolated-storage" 173fi 174 175if [[ "$MODE" == "scoring" ]]; then 176 if [[ "$MLTS_RELEASE_DEVICE" == "True" ]]; then 177 TEST_EXTENRAL_STORAGE="com.android.nn.benchmark.app/com.android.nn.benchmark.util.TestExternalStorageActivity" 178 while ! adb shell "am start -W $TEST_EXTENRAL_STORAGE && rm /sdcard/mlts_write_external_storage" > /dev/null 2>&1; do 179 echo "************************************************************" 180 echo "Grant External storage write permissions to MLTS to proceed!" 181 echo "************************************************************" 182 read -n 1 -r -p "Continue? (press any key)" 183 echo 184 done 185 else 186 adb root 187 adb shell "pm grant com.android.nn.benchmark.app android.permission.WRITE_EXTERNAL_STORAGE" 188 # Skip setup wizard and remount (read-write) 189 if ! adb shell test -f /data/local.prop; then 190 adb shell 'echo ro.setupwizard.mode=DISABLED > /data/local.prop' 191 adb shell 'chmod 644 /data/local.prop' 192 adb shell 'settings put global device_provisioned 1*' 193 adb shell 'settings put secure user_setup_complete 1' 194 adb disable-verity 195 adb reboot 196 sleep 5 197 adb wait-for-usb-device root 198 adb wait-for-usb-device remount 199 sleep 5 200 fi 201 set +e 202 # Enable menu key press through adb 203 adb shell 'echo testing > /data/local/enable_menu_key' 204 # Leave screen on (affects scheduling) 205 adb shell settings put system screen_off_timeout 86400000 206 # Stop background apps, seem to take ~10% CPU otherwise 207 adb shell 'pm disable com.google.android.googlequicksearchbox' 208 adb shell 'pm list packages -f' | sed -e 's/.*=//' | sed 's/\r//g' | grep "com.breel.wallpapers" | while read pkg; do adb shell "pm disable $pkg"; done; 209 set -e 210 fi 211fi 212 213adb shell setprop debug.nn.cpuonly 0 214adb shell setprop debug.nn.vlog "''" 215 216# Menukey - make sure screen is on 217adb shell "input keyevent 82" 218# Show homescreen 219adb shell wm dismiss-keyguard 220 221if [[ "$MODE" == "scoring" ]]; then 222 LOGDIR=$(mktemp -d)/mlts-logs 223 HOST_CSV=$LOGDIR/benchmark.csv 224 RESULT_HTML=$LOGDIR/result.html 225 DEVICE_CSV=/sdcard/mlts_benchmark.csv 226 227 mkdir -p $LOGDIR 228 echo Creating logs in $LOGDIR 229 230 # Remove old benchmark csv data 231 adb shell rm -f ${DEVICE_CSV} 232fi 233 234# Set the shell pid as a top-app and run tests 235time adb shell "echo $$ > /dev/stune/top-app/tasks; am instrument ${AM_INSTRUMENT_FLAGS} -w -e class $CLASS com.android.nn.benchmark.app/androidx.test.runner.AndroidJUnitRunner" 236 237if [[ "$MODE" == "scoring" ]]; then 238 adb pull $DEVICE_CSV $HOST_CSV 239 echo Benchmark data saved in $HOST_CSV 240 241 $ANDROID_BUILD_TOP/test/mlts/benchmark/results/generate_result.py $HOST_CSV $RESULT_HTML 242 echo Results stored in $RESULT_HTML 243 xdg-open $RESULT_HTML 244fi 245