1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6from recipe_engine import recipe_api
7import shlex
8
9
10DEFAULT_TASK_EXPIRATION = 20*60*60
11DEFAULT_TASK_TIMEOUT = 4*60*60
12DEFAULT_IO_TIMEOUT = 40*60
13
14MILO_LOG_LINK = 'https://luci-milo.appspot.com/swarming/task/%s'
15
16
17class SkiaSwarmingApi(recipe_api.RecipeApi):
18  """Provides steps to run Skia tasks on swarming bots."""
19
20  @property
21  def swarming_temp_dir(self):
22    """Path where artifacts like isolate file and json output will be stored."""
23    return self.m.path['start_dir'].join('swarming_temp_dir')
24
25  @property
26  def tasks_output_dir(self):
27    """Directory where the outputs of the swarming tasks will be stored."""
28    return self.swarming_temp_dir.join('outputs')
29
30  def isolated_file_path(self, task_name):
31    """Get the path to the given task's .isolated file."""
32    return self.swarming_temp_dir.join('skia-task-%s.isolated' % task_name)
33
34  def setup(self, luci_go_dir, swarming_rev=None):
35    """Performs setup steps for swarming."""
36    self.m.swarming_client.checkout(revision=swarming_rev)
37    self.m.swarming.check_client_version(step_test_data=(0, 8, 6))
38    self.setup_go_isolate(luci_go_dir)
39    self.m.swarming.add_default_tag('allow_milo:1')
40
41  # TODO(rmistry): Remove once the Go binaries are moved to recipes or buildbot.
42  def setup_go_isolate(self, luci_go_dir):
43    """Generates and puts in place the isolate Go binary."""
44    depot_tools_path = self.m.depot_tools.package_repo_resource()
45    env = {'PATH': self.m.path.pathsep.join([
46                       str(depot_tools_path), '%(PATH)s'])}
47    with self.m.step.context({'env': env}):
48      self.m.step('download luci-go linux',
49                  ['download_from_google_storage', '--no_resume',
50                   '--platform=linux*', '--no_auth',
51                   '--bucket', 'chromium-luci',
52                   '-d', luci_go_dir.join('linux64')])
53      self.m.step('download luci-go mac',
54                  ['download_from_google_storage', '--no_resume',
55                   '--platform=darwin', '--no_auth',
56                   '--bucket', 'chromium-luci',
57                   '-d', luci_go_dir.join('mac64')])
58      self.m.step('download luci-go win',
59                  ['download_from_google_storage', '--no_resume',
60                   '--platform=win32', '--no_auth', '--bucket',
61                   'chromium-luci',
62                   '-d', luci_go_dir.join('win64')])
63    # Copy binaries to the expected location.
64    dest = self.m.path['start_dir'].join('luci-go')
65    self.m.run.rmtree(dest)
66    self.m.file.copytree('Copy Go binary',
67                         source=luci_go_dir,
68                         dest=dest)
69
70  def create_isolated_gen_json(self, isolate_path, base_dir, os_type,
71                               task_name, extra_variables, blacklist=None):
72    """Creates an isolated.gen.json file (used by the isolate recipe module).
73
74    Args:
75      isolate_path: path obj. Path to the isolate file.
76      base_dir: path obj. Dir that is the base of all paths in the isolate file.
77      os_type: str. The OS type to use when archiving the isolate file.
78          Eg: linux.
79      task_name: str. The isolated.gen.json file will be suffixed by this str.
80      extra_variables: dict of str to str. The extra vars to pass to isolate.
81          Eg: {'SLAVE_NUM': '1', 'MASTER': 'ChromiumPerfFYI'}
82      blacklist: list of regular expressions indicating which files/directories
83          not to archive.
84    """
85    self.m.file.makedirs('swarming tmp dir', self.swarming_temp_dir)
86    isolated_path = self.isolated_file_path(task_name)
87    isolate_args = [
88      '--isolate', isolate_path,
89      '--isolated', isolated_path,
90      '--config-variable', 'OS', os_type,
91    ]
92    if blacklist:  # pragma: no cover
93      for b in blacklist:
94        isolate_args.extend(['--blacklist', b])
95    for k, v in extra_variables.iteritems():
96      isolate_args.extend(['--extra-variable', k, v])
97    isolated_gen_dict = {
98      'version': 1,
99      'dir': base_dir,
100      'args': isolate_args,
101    }
102    isolated_gen_json = self.swarming_temp_dir.join(
103        '%s.isolated.gen.json' % task_name)
104    self.m.file.write(
105        'Write %s.isolated.gen.json' % task_name,
106        isolated_gen_json,
107        self.m.json.dumps(isolated_gen_dict, indent=4),
108    )
109
110  def batcharchive(self, targets):
111    """Calls batcharchive on the skia.isolated.gen.json file.
112
113    Args:
114      targets: list of str. The suffixes of the isolated.gen.json files to
115               archive.
116
117    Returns:
118      list of tuples containing (task_name, swarming_hash).
119    """
120    return self.m.isolate.isolate_tests(
121        verbose=True,  # To avoid no output timeouts.
122        build_dir=self.swarming_temp_dir,
123        targets=targets).presentation.properties['swarm_hashes'].items()
124
125  def trigger_swarming_tasks(
126      self, swarm_hashes, dimensions, idempotent=False, store_output=True,
127      extra_args=None, expiration=None, hard_timeout=None, io_timeout=None,
128      cipd_packages=None):
129    """Triggers swarming tasks using swarm hashes.
130
131    Args:
132      swarm_hashes: list of str. List of swarm hashes from the isolate server.
133      dimensions: dict of str to str. The dimensions to run the task on.
134                  Eg: {'os': 'Ubuntu', 'gpu': '10de', 'pool': 'Skia'}
135      idempotent: bool. Whether or not to de-duplicate tasks.
136      store_output: bool. Whether task output should be stored.
137      extra_args: list of str. Extra arguments to pass to the task.
138      expiration: int. Task will expire if not picked up within this time.
139                  DEFAULT_TASK_EXPIRATION is used if this argument is None.
140      hard_timeout: int. Task will timeout if not completed within this time.
141                    DEFAULT_TASK_TIMEOUT is used if this argument is None.
142      io_timeout: int. Task will timeout if there is no output within this time.
143                  DEFAULT_IO_TIMEOUT is used if this argument is None.
144      cipd_packages: CIPD packages which these tasks depend on.
145
146    Returns:
147      List of swarming.SwarmingTask instances.
148    """
149    swarming_tasks = []
150    for task_name, swarm_hash in swarm_hashes:
151      swarming_task = self.m.swarming.task(
152          title=task_name,
153          cipd_packages=cipd_packages,
154          isolated_hash=swarm_hash)
155      if store_output:
156        swarming_task.task_output_dir = self.tasks_output_dir.join(task_name)
157      swarming_task.dimensions = dimensions
158      swarming_task.idempotent = idempotent
159      swarming_task.priority = 90
160      swarming_task.expiration = (
161          expiration if expiration else DEFAULT_TASK_EXPIRATION)
162      swarming_task.hard_timeout = (
163          hard_timeout if hard_timeout else DEFAULT_TASK_TIMEOUT)
164      swarming_task.io_timeout = (
165          io_timeout if io_timeout else DEFAULT_IO_TIMEOUT)
166      if extra_args:  # pragma: no cover
167        swarming_task.extra_args = extra_args
168      revision = self.m.properties.get('revision')
169      if revision:
170        swarming_task.tags.add('revision:%s' % revision)
171      swarming_tasks.append(swarming_task)
172    step_results = self.m.swarming.trigger(swarming_tasks)
173    for step_result in step_results:
174      self._add_log_links(step_result, step_result.json.output)
175    return swarming_tasks
176
177  def collect_swarming_task(self, swarming_task):
178    """Collects the specified swarming task.
179
180    Args:
181      swarming_task: An instance of swarming.SwarmingTask.
182    """
183    try:
184      rv = self.m.swarming.collect_task(swarming_task)
185    except self.m.step.StepFailure as e:  # pragma: no cover
186      step_result = self.m.step.active_result
187      # Change step result to Infra failure if the swarming task failed due to
188      # expiration, time outs, bot crashes or task cancelations.
189      # Infra failures have step.EXCEPTION.
190      states_infra_failure = (
191          self.m.swarming.State.EXPIRED, self.m.swarming.State.TIMED_OUT,
192          self.m.swarming.State.BOT_DIED, self.m.swarming.State.CANCELED)
193      summary = step_result.swarming.summary
194      if summary['shards'][0]['state'] in states_infra_failure:
195        step_result.presentation.status = self.m.step.EXCEPTION
196        raise self.m.step.InfraFailure(e.name, step_result)
197      raise
198    finally:
199      step_result = self.m.step.active_result
200      # Add log link.
201      self._add_log_links(step_result, step_result.swarming.summary)
202    return rv
203
204  def _add_log_links(self, step_result, summary):
205    """Add Milo log links to all shards in the step."""
206    ids = []
207    shards = summary.get('shards')
208    if shards:
209      for shard in shards:
210        ids.append(shard['id'])
211    else:
212      for _, task in summary.get('tasks', {}).iteritems():
213        ids.append(task['task_id'])
214    for idx, task_id in enumerate(ids):
215      link = MILO_LOG_LINK % task_id
216      k = 'view steps on Milo'
217      if len(ids) > 1:  # pragma: nocover
218        k += ' (shard index %d, %d total)' % (idx, len(ids))
219      step_result.presentation.links[k] = link
220
221