1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6from recipe_engine import recipe_api
7
8from . import default
9import subprocess  # TODO(borenet): No! Remove this.
10
11
12"""Android flavor, used for running code on Android."""
13
14
15class AndroidFlavor(default.DefaultFlavor):
16  def __init__(self, m):
17    super(AndroidFlavor, self).__init__(m)
18    self._ever_ran_adb = False
19    self.ADB_BINARY = '/usr/bin/adb.1.0.35'
20    self.ADB_PUB_KEY = '/home/chrome-bot/.android/adbkey'
21    if 'skia' not in self.m.vars.swarming_bot_id:
22      self.ADB_BINARY = '/opt/infra-android/tools/adb'
23      self.ADB_PUB_KEY = ('/home/chrome-bot/.android/'
24                          'chrome_infrastructure_adbkey')
25
26    # Data should go in android_data_dir, which may be preserved across runs.
27    android_data_dir = '/sdcard/revenge_of_the_skiabot/'
28    self.device_dirs = default.DeviceDirs(
29        bin_dir       = '/data/local/tmp/',
30        dm_dir        = android_data_dir + 'dm_out',
31        perf_data_dir = android_data_dir + 'perf',
32        resource_dir  = android_data_dir + 'resources',
33        images_dir    = android_data_dir + 'images',
34        lotties_dir   = android_data_dir + 'lotties',
35        skp_dir       = android_data_dir + 'skps',
36        svg_dir       = android_data_dir + 'svgs',
37        tmp_dir       = android_data_dir)
38
39    # A list of devices we can't root.  If rooting fails and a device is not
40    # on the list, we fail the task to avoid perf inconsistencies.
41    self.rootable_blacklist = ['GalaxyS6', 'GalaxyS7_G930FD', 'GalaxyS9',
42                               'MotoG4', 'NVIDIA_Shield']
43
44    # Maps device type -> CPU ids that should be scaled for nanobench.
45    # Many devices have two (or more) different CPUs (e.g. big.LITTLE
46    # on Nexus5x). The CPUs listed are the biggest cpus on the device.
47    # The CPUs are grouped together, so we only need to scale one of them
48    # (the one listed) in order to scale them all.
49    # E.g. Nexus5x has cpu0-3 as one chip and cpu4-5 as the other. Thus,
50    # if one wants to run a single-threaded application (e.g. nanobench), one
51    # can disable cpu0-3 and scale cpu 4 to have only cpu4 and 5 at the same
52    # frequency.  See also disable_for_nanobench.
53    self.cpus_to_scale = {
54      'Nexus5x': [4],
55      'NexusPlayer': [0, 2], # has 2 identical chips, so scale them both.
56      'Pixel': [2],
57      'Pixel2XL': [4]
58    }
59
60    # Maps device type -> CPU ids that should be turned off when running
61    # single-threaded applications like nanobench. The devices listed have
62    # multiple, differnt CPUs. We notice a lot of noise that seems to be
63    # caused by nanobench running on the slow CPU, then the big CPU. By
64    # disabling this, we see less of that noise by forcing the same CPU
65    # to be used for the performance testing every time.
66    self.disable_for_nanobench = {
67      'Nexus5x': range(0, 4),
68      'Pixel': range(0, 2),
69      'Pixel2XL': range(0, 4)
70    }
71
72    self.gpu_scaling = {
73      "Nexus5":  450000000,
74      "Nexus5x": 600000000,
75    }
76
77  def _run(self, title, *cmd, **kwargs):
78    with self.m.context(cwd=self.m.path['start_dir'].join('skia')):
79      return self.m.run(self.m.step, title, cmd=list(cmd), **kwargs)
80
81  def _adb(self, title, *cmd, **kwargs):
82    # The only non-infra adb steps (dm / nanobench) happen to not use _adb().
83    if 'infra_step' not in kwargs:
84      kwargs['infra_step'] = True
85
86    self._ever_ran_adb = True
87    # ADB seems to be occasionally flaky on every device, so always retry.
88    attempts = 3
89
90    def wait_for_device(attempt):
91      self.m.run(self.m.step,
92                 'kill adb server after failure of \'%s\' (attempt %d)' % (
93                     title, attempt),
94                 cmd=[self.ADB_BINARY, 'kill-server'],
95                 infra_step=True, timeout=30, abort_on_failure=False,
96                 fail_build_on_failure=False)
97      self.m.run(self.m.step,
98                 'wait for device after failure of \'%s\' (attempt %d)' % (
99                     title, attempt),
100                 cmd=[self.ADB_BINARY, 'wait-for-device'], infra_step=True,
101                 timeout=180, abort_on_failure=False,
102                 fail_build_on_failure=False)
103
104    with self.m.context(cwd=self.m.path['start_dir'].join('skia')):
105      with self.m.env({'ADB_VENDOR_KEYS': self.ADB_PUB_KEY}):
106        return self.m.run.with_retry(self.m.step, title, attempts,
107                                     cmd=[self.ADB_BINARY]+list(cmd),
108                                     between_attempts_fn=wait_for_device,
109                                     **kwargs)
110
111  def _scale_for_dm(self):
112    device = self.m.vars.builder_cfg.get('model')
113    if (device in self.rootable_blacklist or
114        self.m.vars.internal_hardware_label):
115      return
116
117    # This is paranoia... any CPUs we disabled while running nanobench
118    # ought to be back online now that we've restarted the device.
119    for i in self.disable_for_nanobench.get(device, []):
120      self._set_cpu_online(i, 1) # enable
121
122    scale_up = self.cpus_to_scale.get(device, [0])
123    # For big.LITTLE devices, make sure we scale the LITTLE cores up;
124    # there is a chance they are still in powersave mode from when
125    # swarming slows things down for cooling down and charging.
126    if 0 not in scale_up:
127      scale_up.append(0)
128    for i in scale_up:
129      # AndroidOne doesn't support ondemand governor. hotplug is similar.
130      if device == 'AndroidOne':
131        self._set_governor(i, 'hotplug')
132      else:
133        self._set_governor(i, 'ondemand')
134
135  def _scale_for_nanobench(self):
136    device = self.m.vars.builder_cfg.get('model')
137    if (device in self.rootable_blacklist or
138      self.m.vars.internal_hardware_label):
139      return
140
141    for i in self.cpus_to_scale.get(device, [0]):
142      self._set_governor(i, 'userspace')
143      self._scale_cpu(i, 0.6)
144
145    for i in self.disable_for_nanobench.get(device, []):
146      self._set_cpu_online(i, 0) # disable
147
148    if device in self.gpu_scaling:
149      #https://developer.qualcomm.com/qfile/28823/lm80-p0436-11_adb_commands.pdf
150      # Section 3.2.1 Commands to put the GPU in performance mode
151      # Nexus 5 is  320000000 by default
152      # Nexus 5x is 180000000 by default
153      gpu_freq = self.gpu_scaling[device]
154      self.m.run.with_retry(self.m.python.inline,
155        "Lock GPU to %d (and other perf tweaks)" % gpu_freq,
156        3, # attempts
157        program="""
158import os
159import subprocess
160import sys
161import time
162ADB = sys.argv[1]
163freq = sys.argv[2]
164idle_timer = "10000"
165
166log = subprocess.check_output([ADB, 'root'])
167# check for message like 'adbd cannot run as root in production builds'
168print log
169if 'cannot' in log:
170  raise Exception('adb root failed')
171
172subprocess.check_output([ADB, 'shell', 'stop', 'thermald'])
173
174subprocess.check_output([ADB, 'shell', 'echo "%s" > '
175    '/sys/class/kgsl/kgsl-3d0/gpuclk' % freq])
176
177actual_freq = subprocess.check_output([ADB, 'shell', 'cat '
178    '/sys/class/kgsl/kgsl-3d0/gpuclk']).strip()
179if actual_freq != freq:
180  raise Exception('Frequency (actual, expected) (%s, %s)'
181                  % (actual_freq, freq))
182
183subprocess.check_output([ADB, 'shell', 'echo "%s" > '
184    '/sys/class/kgsl/kgsl-3d0/idle_timer' % idle_timer])
185
186actual_timer = subprocess.check_output([ADB, 'shell', 'cat '
187    '/sys/class/kgsl/kgsl-3d0/idle_timer']).strip()
188if actual_timer != idle_timer:
189  raise Exception('idle_timer (actual, expected) (%s, %s)'
190                  % (actual_timer, idle_timer))
191
192for s in ['force_bus_on', 'force_rail_on', 'force_clk_on']:
193  subprocess.check_output([ADB, 'shell', 'echo "1" > '
194      '/sys/class/kgsl/kgsl-3d0/%s' % s])
195  actual_set = subprocess.check_output([ADB, 'shell', 'cat '
196      '/sys/class/kgsl/kgsl-3d0/%s' % s]).strip()
197  if actual_set != "1":
198    raise Exception('%s (actual, expected) (%s, 1)'
199                    % (s, actual_set))
200""",
201        args = [self.ADB_BINARY, gpu_freq],
202        infra_step=True,
203        timeout=30)
204
205  def _set_governor(self, cpu, gov):
206    self._ever_ran_adb = True
207    self.m.run.with_retry(self.m.python.inline,
208        "Set CPU %d's governor to %s" % (cpu, gov),
209        3, # attempts
210        program="""
211import os
212import subprocess
213import sys
214import time
215ADB = sys.argv[1]
216cpu = int(sys.argv[2])
217gov = sys.argv[3]
218
219log = subprocess.check_output([ADB, 'root'])
220# check for message like 'adbd cannot run as root in production builds'
221print log
222if 'cannot' in log:
223  raise Exception('adb root failed')
224
225subprocess.check_output([ADB, 'shell', 'echo "%s" > '
226    '/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor' % (gov, cpu)])
227actual_gov = subprocess.check_output([ADB, 'shell', 'cat '
228    '/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor' % cpu]).strip()
229if actual_gov != gov:
230  raise Exception('(actual, expected) (%s, %s)'
231                  % (actual_gov, gov))
232""",
233        args = [self.ADB_BINARY, cpu, gov],
234        infra_step=True,
235        timeout=30)
236
237
238  def _set_cpu_online(self, cpu, value):
239    """Set /sys/devices/system/cpu/cpu{N}/online to value (0 or 1)."""
240    self._ever_ran_adb = True
241    msg = 'Disabling'
242    if value:
243      msg = 'Enabling'
244    self.m.run.with_retry(self.m.python.inline,
245        '%s CPU %d' % (msg, cpu),
246        3, # attempts
247        program="""
248import os
249import subprocess
250import sys
251import time
252ADB = sys.argv[1]
253cpu = int(sys.argv[2])
254value = int(sys.argv[3])
255
256log = subprocess.check_output([ADB, 'root'])
257# check for message like 'adbd cannot run as root in production builds'
258print log
259if 'cannot' in log:
260  raise Exception('adb root failed')
261
262# If we try to echo 1 to an already online cpu, adb returns exit code 1.
263# So, check the value before trying to write it.
264prior_status = subprocess.check_output([ADB, 'shell', 'cat '
265    '/sys/devices/system/cpu/cpu%d/online' % cpu]).strip()
266if prior_status == str(value):
267  print 'CPU %d online already %d' % (cpu, value)
268  sys.exit()
269
270subprocess.check_output([ADB, 'shell', 'echo %s > '
271    '/sys/devices/system/cpu/cpu%d/online' % (value, cpu)])
272actual_status = subprocess.check_output([ADB, 'shell', 'cat '
273    '/sys/devices/system/cpu/cpu%d/online' % cpu]).strip()
274if actual_status != str(value):
275  raise Exception('(actual, expected) (%s, %d)'
276                  % (actual_status, value))
277""",
278        args = [self.ADB_BINARY, cpu, value],
279        infra_step=True,
280        timeout=30)
281
282
283  def _scale_cpu(self, cpu, target_percent):
284    self._ever_ran_adb = True
285    self.m.run.with_retry(self.m.python.inline,
286        'Scale CPU %d to %f' % (cpu, target_percent),
287        3, # attempts
288        program="""
289import os
290import subprocess
291import sys
292import time
293ADB = sys.argv[1]
294target_percent = float(sys.argv[2])
295cpu = int(sys.argv[3])
296log = subprocess.check_output([ADB, 'root'])
297# check for message like 'adbd cannot run as root in production builds'
298print log
299if 'cannot' in log:
300  raise Exception('adb root failed')
301
302root = '/sys/devices/system/cpu/cpu%d/cpufreq' %cpu
303
304# All devices we test on give a list of their available frequencies.
305available_freqs = subprocess.check_output([ADB, 'shell',
306    'cat %s/scaling_available_frequencies' % root])
307
308# Check for message like '/system/bin/sh: file not found'
309if available_freqs and '/system/bin/sh' not in available_freqs:
310  available_freqs = sorted(
311      int(i) for i in available_freqs.strip().split())
312else:
313  raise Exception('Could not get list of available frequencies: %s' %
314                  available_freqs)
315
316maxfreq = available_freqs[-1]
317target = int(round(maxfreq * target_percent))
318freq = maxfreq
319for f in reversed(available_freqs):
320  if f <= target:
321    freq = f
322    break
323
324print 'Setting frequency to %d' % freq
325
326# If scaling_max_freq is lower than our attempted setting, it won't take.
327# We must set min first, because if we try to set max to be less than min
328# (which sometimes happens after certain devices reboot) it returns a
329# perplexing permissions error.
330subprocess.check_output([ADB, 'shell', 'echo 0 > '
331    '%s/scaling_min_freq' % root])
332subprocess.check_output([ADB, 'shell', 'echo %d > '
333    '%s/scaling_max_freq' % (freq, root)])
334subprocess.check_output([ADB, 'shell', 'echo %d > '
335    '%s/scaling_setspeed' % (freq, root)])
336time.sleep(5)
337actual_freq = subprocess.check_output([ADB, 'shell', 'cat '
338    '%s/scaling_cur_freq' % root]).strip()
339if actual_freq != str(freq):
340  raise Exception('(actual, expected) (%s, %d)'
341                  % (actual_freq, freq))
342""",
343        args = [self.ADB_BINARY, str(target_percent), cpu],
344        infra_step=True,
345        timeout=30)
346
347
348  def install(self):
349    self._adb('mkdir ' + self.device_dirs.resource_dir,
350              'shell', 'mkdir', '-p', self.device_dirs.resource_dir)
351    if 'ASAN' in self.m.vars.extra_tokens:
352      self._ever_ran_adb = True
353      asan_setup = self.m.vars.slave_dir.join(
354            'android_ndk_linux', 'toolchains', 'llvm', 'prebuilt',
355            'linux-x86_64', 'lib64', 'clang', '8.0.2', 'bin',
356            'asan_device_setup')
357      self.m.run(self.m.python.inline, 'Setting up device to run ASAN',
358        program="""
359import os
360import subprocess
361import sys
362import time
363ADB = sys.argv[1]
364ASAN_SETUP = sys.argv[2]
365
366def wait_for_device():
367  while True:
368    time.sleep(5)
369    print 'Waiting for device'
370    subprocess.check_output([ADB, 'wait-for-device'])
371    bit1 = subprocess.check_output([ADB, 'shell', 'getprop',
372                                   'dev.bootcomplete'])
373    bit2 = subprocess.check_output([ADB, 'shell', 'getprop',
374                                   'sys.boot_completed'])
375    if '1' in bit1 and '1' in bit2:
376      print 'Device detected'
377      break
378
379log = subprocess.check_output([ADB, 'root'])
380# check for message like 'adbd cannot run as root in production builds'
381print log
382if 'cannot' in log:
383  raise Exception('adb root failed')
384
385output = subprocess.check_output([ADB, 'disable-verity'])
386print output
387
388if 'already disabled' not in output:
389  print 'Rebooting device'
390  subprocess.check_output([ADB, 'reboot'])
391  wait_for_device()
392
393def installASAN(revert=False):
394  # ASAN setup script is idempotent, either it installs it or
395  # says it's installed.  Returns True on success, false otherwise.
396  out = subprocess.check_output([ADB, 'wait-for-device'])
397  print out
398  cmd = [ASAN_SETUP]
399  if revert:
400    cmd = [ASAN_SETUP, '--revert']
401  process = subprocess.Popen(cmd, env={'ADB': ADB},
402                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
403
404  # this also blocks until command finishes
405  (stdout, stderr) = process.communicate()
406  print stdout
407  print 'Stderr: %s' % stderr
408  return process.returncode == 0
409
410if not installASAN():
411  print 'Trying to revert the ASAN install and then re-install'
412  # ASAN script sometimes has issues if it was interrupted or partially applied
413  # Try reverting it, then re-enabling it
414  if not installASAN(revert=True):
415    raise Exception('reverting ASAN install failed')
416
417  # Sleep because device does not reboot instantly
418  time.sleep(10)
419
420  if not installASAN():
421    raise Exception('Tried twice to setup ASAN and failed.')
422
423# Sleep because device does not reboot instantly
424time.sleep(10)
425wait_for_device()
426# Sleep again to hopefully avoid error "secure_mkdirs failed: No such file or
427# directory" when pushing resources to the device.
428time.sleep(60)
429""",
430        args = [self.ADB_BINARY, asan_setup],
431          infra_step=True,
432          timeout=300,
433          abort_on_failure=True)
434
435
436  def cleanup_steps(self):
437    if 'ASAN' in self.m.vars.extra_tokens:
438      self._ever_ran_adb = True
439      # Remove ASAN.
440      asan_setup = self.m.vars.slave_dir.join(
441            'android_ndk_linux', 'toolchains', 'llvm', 'prebuilt',
442            'linux-x86_64', 'lib64', 'clang', '8.0.2', 'bin',
443            'asan_device_setup')
444      self.m.run(self.m.step,
445                 'wait for device before uninstalling ASAN',
446                 cmd=[self.ADB_BINARY, 'wait-for-device'], infra_step=True,
447                 timeout=180, abort_on_failure=False,
448                 fail_build_on_failure=False)
449      self.m.run(self.m.step, 'uninstall ASAN',
450                 cmd=[asan_setup, '--revert'], infra_step=True, timeout=300,
451                 abort_on_failure=False, fail_build_on_failure=False)
452
453    if self._ever_ran_adb:
454      self.m.run(self.m.python.inline, 'dump log', program="""
455          import os
456          import subprocess
457          import sys
458          out = sys.argv[1]
459          log = subprocess.check_output(['%s', 'logcat', '-d'])
460          for line in log.split('\\n'):
461            tokens = line.split()
462            if len(tokens) == 11 and tokens[-7] == 'F' and tokens[-3] == 'pc':
463              addr, path = tokens[-2:]
464              local = os.path.join(out, os.path.basename(path))
465              if os.path.exists(local):
466                sym = subprocess.check_output(['addr2line', '-Cfpe', local, addr])
467                line = line.replace(addr, addr + ' ' + sym.strip())
468            print line
469          """ % self.ADB_BINARY,
470          args=[self.host_dirs.bin_dir],
471          infra_step=True,
472          timeout=300,
473          abort_on_failure=False)
474
475    # Only quarantine the bot if the first failed step
476    # is an infra step. If, instead, we did this for any infra failures, we
477    # would do this too much. For example, if a Nexus 10 died during dm
478    # and the following pull step would also fail "device not found" - causing
479    # us to run the shutdown command when the device was probably not in a
480    # broken state; it was just rebooting.
481    if (self.m.run.failed_steps and
482        isinstance(self.m.run.failed_steps[0], recipe_api.InfraFailure)):
483      bot_id = self.m.vars.swarming_bot_id
484      self.m.file.write_text('Quarantining Bot',
485                             '/home/chrome-bot/%s.force_quarantine' % bot_id,
486                             ' ')
487
488    if self._ever_ran_adb:
489      self._adb('kill adb server', 'kill-server')
490
491  def step(self, name, cmd, **kwargs):
492    if (cmd[0] == 'nanobench'):
493      self._scale_for_nanobench()
494    else:
495      self._scale_for_dm()
496    app = self.host_dirs.bin_dir.join(cmd[0])
497    self._adb('push %s' % cmd[0],
498              'push', app, self.device_dirs.bin_dir)
499
500    sh = '%s.sh' % cmd[0]
501    self.m.run.writefile(self.m.vars.tmp_dir.join(sh),
502        'set -x; %s%s; echo $? >%src' % (
503            self.device_dirs.bin_dir, subprocess.list2cmdline(map(str, cmd)),
504            self.device_dirs.bin_dir))
505    self._adb('push %s' % sh,
506              'push', self.m.vars.tmp_dir.join(sh), self.device_dirs.bin_dir)
507
508    self._adb('clear log', 'logcat', '-c')
509    self.m.python.inline('%s' % cmd[0], """
510    import subprocess
511    import sys
512    bin_dir = sys.argv[1]
513    sh      = sys.argv[2]
514    subprocess.check_call(['%s', 'shell', 'sh', bin_dir + sh])
515    try:
516      sys.exit(int(subprocess.check_output(['%s', 'shell', 'cat',
517                                            bin_dir + 'rc'])))
518    except ValueError:
519      print "Couldn't read the return code.  Probably killed for OOM."
520      sys.exit(1)
521    """ % (self.ADB_BINARY, self.ADB_BINARY),
522      args=[self.device_dirs.bin_dir, sh])
523
524  def copy_file_to_device(self, host, device):
525    self._adb('push %s %s' % (host, device), 'push', host, device)
526
527  def copy_directory_contents_to_device(self, host, device):
528    # Copy the tree, avoiding hidden directories and resolving symlinks.
529    self.m.run(self.m.python.inline, 'push %s/* %s' % (host, device),
530               program="""
531    import os
532    import subprocess
533    import sys
534    host   = sys.argv[1]
535    device = sys.argv[2]
536    for d, _, fs in os.walk(host):
537      p = os.path.relpath(d, host)
538      if p != '.' and p.startswith('.'):
539        continue
540      for f in fs:
541        print os.path.join(p,f)
542        subprocess.check_call(['%s', 'push',
543                               os.path.realpath(os.path.join(host, p, f)),
544                               os.path.join(device, p, f)])
545    """ % self.ADB_BINARY, args=[host, device], infra_step=True)
546
547  def copy_directory_contents_to_host(self, device, host):
548    # TODO(borenet): When all of our devices are on Android 6.0 and up, we can
549    # switch to using tar to zip up the results before pulling.
550    with self.m.step.nest('adb pull'):
551      with self.m.tempfile.temp_dir('adb_pull') as tmp:
552        self._adb('pull %s' % device, 'pull', device, tmp)
553        paths = self.m.file.glob_paths(
554            'list pulled files',
555            tmp,
556            self.m.path.basename(device) + self.m.path.sep + '*',
557            test_data=['%d.png' % i for i in (1, 2)])
558        for p in paths:
559          self.m.file.copy('copy %s' % self.m.path.basename(p), p, host)
560
561  def read_file_on_device(self, path, **kwargs):
562    rv = self._adb('read %s' % path,
563                   'shell', 'cat', path, stdout=self.m.raw_io.output(),
564                   **kwargs)
565    return rv.stdout.rstrip() if rv and rv.stdout else None
566
567  def remove_file_on_device(self, path):
568    self._adb('rm %s' % path, 'shell', 'rm', '-f', path)
569
570  def create_clean_device_dir(self, path):
571    self._adb('rm %s' % path, 'shell', 'rm', '-rf', path)
572    self._adb('mkdir %s' % path, 'shell', 'mkdir', '-p', path)
573