1#!/usr/bin/env python
2# Copyright (c) 2018 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Script that triggers and waits for tasks on android-compile.skia.org"""
7
8import base64
9import hashlib
10import json
11import math
12import optparse
13import os
14import requests
15import subprocess
16import sys
17import time
18
19INFRA_BOTS_DIR = os.path.abspath(os.path.realpath(os.path.join(
20    os.path.dirname(os.path.abspath(__file__)), os.pardir)))
21sys.path.insert(0, INFRA_BOTS_DIR)
22import utils
23
24
25ANDROID_COMPILE_BUCKET = 'android-compile-tasks'
26
27GS_RETRIES = 5
28GS_RETRY_WAIT_BASE = 15
29
30POLLING_FREQUENCY_SECS = 10
31DEADLINE_SECS = 60 * 60  # 60 minutes.
32
33INFRA_FAILURE_ERROR_MSG = (
34      '\n\n'
35      'Your run failed due to infra failures. '
36      'It could be due to any of the following:\n\n'
37      '* Need to rebase\n\n'
38      '* Failure when running "python -c from gn import gn_to_bp"\n\n'
39      '* Problem with syncing Android repository.\n\n'
40      'See go/skia-android-framework-compile-bot-cloud-logs-errors for the '
41      'compile server\'s logs.'
42)
43
44
45class AndroidCompileException(Exception):
46  pass
47
48
49def _create_task_dict(options):
50  """Creates a dict representation of the requested task."""
51  params = {}
52  params['lunch_target'] = options.lunch_target
53  params['mmma_targets'] = options.mmma_targets
54  params['issue'] = options.issue
55  params['patchset'] = options.patchset
56  params['hash'] = options.hash
57  return params
58
59
60def _get_gs_bucket():
61  """Returns the Google storage bucket with the gs:// prefix."""
62  return 'gs://%s' % ANDROID_COMPILE_BUCKET
63
64
65def _write_to_storage(task):
66  """Writes the specified compile task to Google storage."""
67  with utils.tmp_dir():
68    json_file = os.path.join(os.getcwd(), _get_task_file_name(task))
69    with open(json_file, 'w') as f:
70      json.dump(task, f)
71    subprocess.check_call(['gsutil', 'cp', json_file, '%s/' % _get_gs_bucket()])
72    print 'Created %s/%s' % (_get_gs_bucket(), os.path.basename(json_file))
73
74
75def _get_task_file_name(task):
76  """Returns the file name of the compile task. Eg: ${issue}-${patchset}.json"""
77  return '%s-%s-%s.json' % (task['lunch_target'], task['issue'],
78                            task['patchset'])
79
80
81# Checks to see if task already exists in Google storage.
82# If the task has completed then the Google storage file is deleted.
83def _does_task_exist_in_storage(task):
84  """Checks to see if the corresponding file of the task exists in storage.
85
86  If the file exists and the task has already completed then the storage file is
87  deleted and False is returned.
88  """
89  gs_file = '%s/%s' % (_get_gs_bucket(), _get_task_file_name(task))
90  try:
91    output = subprocess.check_output(['gsutil', 'cat', gs_file])
92  except subprocess.CalledProcessError:
93    print 'Task does not exist in Google storage'
94    return False
95  taskJSON = json.loads(output)
96  if taskJSON.get('done'):
97    print 'Task exists in Google storage and has completed.'
98    print 'Deleting it so that a new run can be scheduled.'
99    subprocess.check_call(['gsutil', 'rm', gs_file])
100    return False
101  else:
102    print 'Tasks exists in Google storage and is still running.'
103    return True
104
105
106def _trigger_task(options):
107  """Triggers a task on the compile server by creating a file in storage."""
108  task = _create_task_dict(options)
109  # Check to see if file already exists in Google Storage.
110  if not _does_task_exist_in_storage(task):
111    _write_to_storage(task)
112  return task
113
114
115def trigger_and_wait(options):
116  """Triggers a task on the compile server and waits for it to complete."""
117  task = _trigger_task(options)
118  print 'Android Compile Task for %d/%d has been successfully added to %s.' % (
119      options.issue, options.patchset, ANDROID_COMPILE_BUCKET)
120  print '%s will be polled every %d seconds.' % (ANDROID_COMPILE_BUCKET,
121                                                 POLLING_FREQUENCY_SECS)
122
123  # Now poll the Google storage file till the task completes or till deadline
124  # is hit.
125  time_started_polling = time.time()
126  while True:
127    if (time.time() - time_started_polling) > DEADLINE_SECS:
128      raise AndroidCompileException(
129          'Task did not complete in the deadline of %s seconds.' % (
130              DEADLINE_SECS))
131
132    # Get the status of the task.
133    gs_file = '%s/%s' % (_get_gs_bucket(), _get_task_file_name(task))
134
135    for retry in range(GS_RETRIES):
136      try:
137        output = subprocess.check_output(['gsutil', 'cat', gs_file])
138      except subprocess.CalledProcessError:
139        raise AndroidCompileException('The %s file no longer exists.' % gs_file)
140      try:
141        ret = json.loads(output)
142        break
143      except ValueError, e:
144        print 'Received output that could not be converted to json: %s' % output
145        print e
146        if retry == (GS_RETRIES-1):
147          print '%d retries did not help' % GS_RETRIES
148          raise
149        waittime = GS_RETRY_WAIT_BASE * math.pow(2, retry)
150        print 'Retry in %d seconds.' % waittime
151        time.sleep(waittime)
152
153    if ret.get('infra_failure'):
154      raise AndroidCompileException(INFRA_FAILURE_ERROR_MSG)
155
156    if ret.get('done'):
157      if not ret.get('is_master_branch', True):
158        print 'The Android Framework Compile bot only works for patches and'
159        print 'hashes from the master branch.'
160        return 0
161      elif ret['withpatch_success']:
162        print 'Your run was successfully completed.'
163        print 'With patch logs are here: %s' % ret['withpatch_log']
164        return 0
165      elif ret['nopatch_success']:
166        raise AndroidCompileException('The build with the patch failed and the '
167               'build without the patch succeeded. This means that the patch '
168               'causes Android to fail compilation.\n\n'
169               'With patch logs are here: %s\n\n'
170               'No patch logs are here: %s\n\n' % (
171                   ret['withpatch_log'], ret['nopatch_log']))
172      else:
173        print ('Both with patch and no patch builds failed. This means that the'
174               ' Android tree is currently broken. Marking this bot as '
175               'successful')
176        print 'With patch logs are here: %s' % ret['withpatch_log']
177        print 'No patch logs are here: %s' % ret['nopatch_log']
178        return 0
179
180    # Print status of the task.
181    print 'Task: %s\n' % pretty_task_str(ret)
182    time.sleep(POLLING_FREQUENCY_SECS)
183
184
185def pretty_task_str(task):
186  status = 'Not picked up by server yet'
187  if task.get('task_id'):
188    status = 'Running withpatch compilation'
189  if task.get('withpatch_log'):
190    status = 'Running nopatch compilation'
191  return '[id: %s, checkout: %s, status: %s]' % (
192      task.get('task_id'), task.get('checkout'), status)
193
194
195def main():
196  option_parser = optparse.OptionParser()
197  option_parser.add_option(
198      '', '--lunch_target', type=str, default='',
199      help='The lunch target the android compile bot should build with.')
200  option_parser.add_option(
201      '', '--mmma_targets', type=str, default='',
202      help='The comma-separated mmma targets the android compile bot should '
203           'build.')
204  option_parser.add_option(
205      '', '--issue', type=int, default=0,
206      help='The Gerrit change number to get the patch from.')
207  option_parser.add_option(
208      '', '--patchset', type=int, default=0,
209      help='The Gerrit change patchset to use.')
210  option_parser.add_option(
211      '', '--hash', type=str, default='',
212      help='The Skia repo hash to compile against.')
213  options, _ = option_parser.parse_args()
214  sys.exit(trigger_and_wait(options))
215
216
217if __name__ == '__main__':
218  main()
219