1#!/bin/bash
2#
3# Build benchmark app and run it, mimicking a user-initiated run
4#
5# Output is logged to a temporary folder and summarized in txt and JSON formats.
6
7MODE="${1:-scoring}"
8
9case "$MODE" in
10  scoring)
11    CLASS=com.android.nn.benchmark.app.NNScoringTest
12    ;;
13  inference-stress)
14    CLASS=com.android.nn.benchmark.app.NNInferenceStressTest
15    ;;
16  model-loading-stress)
17    CLASS=com.android.nn.benchmark.app.NNModelLoadingStressTest
18    ;;
19  *)
20    echo "Unknown execution mode: $1"
21    echo "Known modes: scoring (default), inference-stress, model-loading-stress"
22    exit 1
23    ;;
24esac
25
26if [[ -z "$ANDROID_BUILD_TOP" ]]; then
27  echo ANDROID_BUILD_TOP not set, bailing out
28  echo you must run lunch before running this script
29  exit 1
30fi
31
32set -e
33cd $ANDROID_BUILD_TOP
34
35# Build and install benchmark app
36build/soong/soong_ui.bash --make-mode NeuralNetworksApiBenchmark
37if ! adb install -r $OUT/testcases/NeuralNetworksApiBenchmark/arm64/NeuralNetworksApiBenchmark.apk; then
38  adb uninstall com.android.nn.benchmark.app
39  adb install -r $OUT/testcases/NeuralNetworksApiBenchmark/arm64/NeuralNetworksApiBenchmark.apk
40fi
41
42
43# Should we figure out if we run on release device
44if [ -z "$MLTS_RELEASE_DEVICE" ]; then
45  BUILD_DESCRIPTION=`adb shell getprop ro.build.description`
46  if [[ $BUILD_DESCRIPTION =~ .*release.* ]]
47  then
48    MLTS_RELEASE_DEVICE=True
49  else
50    MLTS_RELEASE_DEVICE=False
51  fi
52fi
53
54# Pass --no-isolated-storage to am instrument?
55BUILD_VERSION_RELEASE=`adb shell getprop ro.build.version.release`
56AM_INSTRUMENT_FLAGS=""
57if [[ $BUILD_VERSION_RELEASE == "Q" ]]; then
58  AM_INSTRUMENT_FLAGS+="--no-isolated-storage"
59fi
60
61if [[ "$MLTS_RELEASE_DEVICE" == "True" ]]; then
62  TEST_EXTENRAL_STORAGE="com.android.nn.benchmark.app/com.android.nn.benchmark.util.TestExternalStorageActivity"
63  while ! adb shell "am start -W $TEST_EXTENRAL_STORAGE && rm /sdcard/mlts_write_external_storage" > /dev/null 2>&1; do
64     echo "************************************************************"
65     echo "Grant External storage write permissions to MLTS to proceed!"
66     echo "************************************************************"
67     read -n 1 -r -p "Continue? (press any key)"
68     echo
69  done
70else
71  adb root
72  adb shell "pm grant com.android.nn.benchmark.app android.permission.WRITE_EXTERNAL_STORAGE"
73  # Skip setup wizard and remount (read-write)
74  if ! adb shell test -f /data/local.prop; then
75    adb shell 'echo ro.setupwizard.mode=DISABLED > /data/local.prop'
76    adb shell 'chmod 644 /data/local.prop'
77    adb shell 'settings put global device_provisioned 1*'
78    adb shell 'settings put secure user_setup_complete 1'
79    adb disable-verity
80    adb reboot
81    sleep 5
82    adb wait-for-usb-device root
83    adb wait-for-usb-device remount
84    sleep 5
85  fi
86  set +e
87  # Enable menu key press through adb
88  adb shell 'echo testing > /data/local/enable_menu_key'
89  # Leave screen on (affects scheduling)
90  adb shell settings put system screen_off_timeout 86400000
91  # Stop background apps, seem to take ~10% CPU otherwise
92  adb shell 'pm disable com.google.android.googlequicksearchbox'
93  adb shell 'pm list packages -f' | sed -e 's/.*=//' | sed 's/\r//g' | grep "com.breel.wallpapers" | while read pkg; do adb shell "pm disable $pkg"; done;
94  set -e
95fi
96
97adb shell setprop debug.nn.cpuonly 0
98adb shell setprop debug.nn.vlog 0
99
100# Menukey - make sure screen is on
101adb shell "input keyevent 82"
102# Show homescreen
103adb shell wm dismiss-keyguard
104
105if [[ "$MODE" == "scoring" ]]; then
106  LOGDIR=$(mktemp -d)/mlts-logs
107  HOST_CSV=$LOGDIR/benchmark.csv
108  RESULT_HTML=$LOGDIR/result.html
109  DEVICE_CSV=/sdcard/mlts_benchmark.csv
110
111  mkdir -p $LOGDIR
112  echo Creating logs in $LOGDIR
113
114  # Remove old benchmark csv data
115  adb shell rm -f ${DEVICE_CSV}
116fi
117
118# Set the shell pid as a top-app and run tests
119time adb shell "echo $$ > /dev/stune/top-app/tasks; am instrument ${AM_INSTRUMENT_FLAGS} -w -e class $CLASS com.android.nn.benchmark.app/androidx.test.runner.AndroidJUnitRunner"
120
121if [[ "$MODE" == "scoring" ]]; then
122  adb pull $DEVICE_CSV $HOST_CSV
123  echo Benchmark data saved in $HOST_CSV
124
125  $ANDROID_BUILD_TOP/test/mlts/benchmark/results/generate_result.py $HOST_CSV $RESULT_HTML
126  echo Results stored  in $RESULT_HTML
127  xdg-open $RESULT_HTML
128fi
129