1#!/bin/bash
2#
3# Build benchmark app and run it, mimicking a user-initiated run
4#
5# Output is logged to a temporary folder and summarized in txt and JSON formats.
6# parallel-inference-stress tests produce no output except for the success or failure notification,
7# which is not logged.
8
9if [[ "$OSTYPE" == "darwin"* ]]; then
10  OPTS="$(getopt f:rb -- "$*")"
11else
12  OPTS="$(getopt -o f:rb -l filter-driver:,include-nnapi-reference,nnapi-reference-only,skip-build -- "$@")"
13fi
14
15if [ $? -ne 0 ]; then
16  echo "Invalid arguments, accepted options are"
17  if [[ "$OSTYPE" == "darwin"* ]]; then
18    echo " -f <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression"
19    echo " -r : to include nnapi-reference in target drivers"
20    echo " -b : skip build and installation of tests"
21  else
22    echo " -f <regex> | --filter-driver <regex> : to run crash tests only on the drivers (ignoring nnapi-reference) matching the specified regular expression"
23    echo " -r | --include-nnapi-reference : to include nnapi-reference in target drivers"
24    echo " --nnapi-reference-only : to run tests only vs nnapi-reference"
25    echo " -b | --skip-build : skip build and installation of tests"
26  fi
27  exit
28fi
29
30eval set -- "$OPTS"
31
32DRIVER_FILTER_OPT=""
33INCLUDE_NNAPI_REF_OPT=""
34BUILD_AND_INSTALL=true
35while [ $# -gt 0 ] ; do
36  case "$1" in
37    -f|--filter-driver)
38      DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter $2"
39      shift 2
40      ;;
41    -r|--include-nnapi-reference)
42      INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true"
43      shift
44      ;;
45    --nnapi-reference-only)
46      DRIVER_FILTER_OPT="-e nnCrashtestDeviceFilter no-device"
47      INCLUDE_NNAPI_REF_OPT="-e nnCrashtestIncludeNnapiReference true"
48      shift
49      ;;
50    -b|--skip-build)
51      BUILD_AND_INSTALL=false
52      shift
53      ;;
54    --)
55      shift
56      break
57      ;;
58    *)
59      echo "Unsupported arg $1"
60      exit 1
61  esac
62done
63
64MODE="${1:-scoring}"
65INSTALL_NATIVE_TESTS=false
66CRASH_TEST_APP="NeuralNetworksApiCrashTest"
67APP="NeuralNetworksApiBenchmark"
68case "$MODE" in
69  scoring)
70    CLASS=com.android.nn.benchmark.app.NNScoringTest
71    ;;
72  inference-stress)
73    CLASS=com.android.nn.benchmark.app.NNInferenceStressTest
74    ;;
75  model-loading-stress)
76    CLASS=com.android.nn.benchmark.app.NNModelLoadingStressTest
77    ;;
78  parallel-inference-stress)
79    CLASS=com.android.nn.crashtest.app.NNParallelCrashResistantInferenceTest
80    APP="$CRASH_TEST_APP"
81    ;;
82  parallel-inference-stress-in-process)
83    CLASS=com.android.nn.crashtest.app.NNParallelInProcessInferenceTest
84    APP="$CRASH_TEST_APP"
85    ;;
86  client-early-termination-stress)
87    CLASS=com.android.nn.crashtest.app.NNClientEarlyTerminationTest
88    APP="$CRASH_TEST_APP"
89    ;;
90  multi-process-inference-stress)
91    CLASS=com.android.nn.crashtest.app.NNMultipleProcessInferenceTest
92    APP="$CRASH_TEST_APP"
93    INSTALL_NATIVE_TESTS=true
94    ;;
95  multi-process-model-load-stress)
96    CLASS=com.android.nn.crashtest.app.NNMultipleProcessModelLoadTest
97    APP="$CRASH_TEST_APP"
98    INSTALL_NATIVE_TESTS=true
99    ;;
100  memory-mapped-model-load-stress)
101    CLASS=com.android.nn.crashtest.app.NNMemoryMappedModelCompilationTest
102    APP="$CRASH_TEST_APP"
103    ;;
104  model-load-random-stress)
105    APP="$CRASH_TEST_APP"
106    CLASS=com.android.nn.crashtest.app.NNRandomGraphLoadTest
107    ;;
108  inference-random-stress)
109    APP="$CRASH_TEST_APP"
110    CLASS=com.android.nn.crashtest.app.NNRandomGraphExecutionTest
111    ;;
112  performance-degradation-stress)
113    APP="$CRASH_TEST_APP"
114    CLASS=com.android.nn.crashtest.app.NNPerformanceDegradationTest
115    ;;
116  *)
117    echo "Unknown execution mode: $1"
118    echo "Known modes: scoring (default), inference-stress, model-loading-stress, " \
119      "parallel-inference-stress, parallel-inference-stress-in-process, " \
120      "client-early-termination-stress, multi-process-inference-stress, " \
121      "multi-process-model-load-stress memory-mapped-model-load-stress, " \
122      "model-load-random-stress, inference-random-stress, performance-degradation-stress"
123    exit 1
124    ;;
125esac
126
127if [[ -z "$ANDROID_BUILD_TOP" ]]; then
128  echo ANDROID_BUILD_TOP not set, bailing out
129  echo you must run lunch before running this script
130  exit 1
131fi
132
133set -e
134cd $ANDROID_BUILD_TOP
135
136if [ "$BUILD_AND_INSTALL" = true ]; then
137  # Build and install benchmark app
138  TMPFILE=$(mktemp)
139  build/soong/soong_ui.bash --make-mode ${APP} 2>&1 | tee ${TMPFILE}
140  TARGET_ARCH=$(cat ${TMPFILE} | grep TARGET_ARCH= | sed -e 's/TARGET_ARCH=//')
141  if [ "${TARGET_ARCH}" = "aarch64" ]; then
142      APK_DIR=arm64
143  else
144      APK_DIR=${TARGET_ARCH}
145  fi
146  if ! adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk; then
147    adb uninstall com.android.nn.benchmark.app
148    adb install -r $OUT/testcases/${APP}/${APK_DIR}/${APP}.apk
149  fi
150
151  if [ "$INSTALL_NATIVE_TESTS" = true ]; then
152    build/soong/soong_ui.bash --make-mode nn_stress_test
153    adb push $OUT/system/bin/nn_stress_test /bin/
154  fi
155fi
156
157# Should we figure out if we run on release device
158if [ -z "$MLTS_RELEASE_DEVICE" ]; then
159  BUILD_DESCRIPTION=`adb shell getprop ro.build.description`
160  if [[ $BUILD_DESCRIPTION =~ .*release.* ]]
161  then
162    MLTS_RELEASE_DEVICE=True
163  else
164    MLTS_RELEASE_DEVICE=False
165  fi
166fi
167
168# Pass --no-isolated-storage to am instrument?
169BUILD_VERSION_RELEASE=`adb shell getprop ro.build.version.release`
170AM_INSTRUMENT_FLAGS="$DRIVER_FILTER_OPT $INCLUDE_NNAPI_REF_OPT"
171if [[ $BUILD_VERSION_RELEASE == "Q" ]]; then
172  AM_INSTRUMENT_FLAGS+=" --no-isolated-storage"
173fi
174
175if [[ "$MODE" == "scoring" ]]; then
176  if [[ "$MLTS_RELEASE_DEVICE" == "True" ]]; then
177    TEST_EXTENRAL_STORAGE="com.android.nn.benchmark.app/com.android.nn.benchmark.util.TestExternalStorageActivity"
178    while ! adb shell "am start -W $TEST_EXTENRAL_STORAGE && rm /sdcard/mlts_write_external_storage" > /dev/null 2>&1; do
179       echo "************************************************************"
180       echo "Grant External storage write permissions to MLTS to proceed!"
181       echo "************************************************************"
182       read -n 1 -r -p "Continue? (press any key)"
183       echo
184    done
185  else
186    adb root
187    adb shell "pm grant com.android.nn.benchmark.app android.permission.WRITE_EXTERNAL_STORAGE"
188    # Skip setup wizard and remount (read-write)
189    if ! adb shell test -f /data/local.prop; then
190      adb shell 'echo ro.setupwizard.mode=DISABLED > /data/local.prop'
191      adb shell 'chmod 644 /data/local.prop'
192      adb shell 'settings put global device_provisioned 1*'
193      adb shell 'settings put secure user_setup_complete 1'
194      adb disable-verity
195      adb reboot
196      sleep 5
197      adb wait-for-usb-device root
198      adb wait-for-usb-device remount
199      sleep 5
200    fi
201    set +e
202    # Enable menu key press through adb
203    adb shell 'echo testing > /data/local/enable_menu_key'
204    # Leave screen on (affects scheduling)
205    adb shell settings put system screen_off_timeout 86400000
206    # Stop background apps, seem to take ~10% CPU otherwise
207    adb shell 'pm disable com.google.android.googlequicksearchbox'
208    adb shell 'pm list packages -f' | sed -e 's/.*=//' | sed 's/\r//g' | grep "com.breel.wallpapers" | while read pkg; do adb shell "pm disable $pkg"; done;
209    set -e
210  fi
211fi
212
213adb shell setprop debug.nn.cpuonly 0
214adb shell setprop debug.nn.vlog "''"
215
216# Menukey - make sure screen is on
217adb shell "input keyevent 82"
218# Show homescreen
219adb shell wm dismiss-keyguard
220
221if [[ "$MODE" == "scoring" ]]; then
222  LOGDIR=$(mktemp -d)/mlts-logs
223  HOST_CSV=$LOGDIR/benchmark.csv
224  RESULT_HTML=$LOGDIR/result.html
225  DEVICE_CSV=/sdcard/mlts_benchmark.csv
226
227  mkdir -p $LOGDIR
228  echo Creating logs in $LOGDIR
229
230  # Remove old benchmark csv data
231  adb shell rm -f ${DEVICE_CSV}
232fi
233
234# Set the shell pid as a top-app and run tests
235time adb shell "echo $$ > /dev/stune/top-app/tasks; am instrument ${AM_INSTRUMENT_FLAGS} -w -e class $CLASS com.android.nn.benchmark.app/androidx.test.runner.AndroidJUnitRunner"
236
237if [[ "$MODE" == "scoring" ]]; then
238  adb pull $DEVICE_CSV $HOST_CSV
239  echo Benchmark data saved in $HOST_CSV
240
241  $ANDROID_BUILD_TOP/test/mlts/benchmark/results/generate_result.py $HOST_CSV $RESULT_HTML
242  echo Results stored  in $RESULT_HTML
243  xdg-open $RESULT_HTML
244fi
245