1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import collections
6import cPickle
7import json
8import logging
9import os
10import re
11import socket
12import time
13import urllib
14import urllib2
15
16
17PENDING = None
18SUCCESS = 0
19WARNING = 1
20FAILURE = 2
21EXCEPTION = 4
22SLAVE_LOST = 5
23
24
25BASE_URL = 'http://build.chromium.org/p'
26CACHE_FILE_NAME = 'cache.dat'
27
28
29StackTraceLine = collections.namedtuple(
30    'StackTraceLine', ('file', 'function', 'line', 'source'))
31
32
33def _FetchData(master, url):
34  url = '%s/%s/json/%s' % (BASE_URL, master, url)
35  try:
36    logging.info('Retrieving ' + url)
37    return json.load(urllib2.urlopen(url))
38  except (urllib2.HTTPError, socket.error):
39    # Could be intermittent; try again.
40    try:
41      return json.load(urllib2.urlopen(url))
42    except (urllib2.HTTPError, socket.error):
43      logging.error('Error retrieving URL ' + url)
44      raise
45  except:
46    logging.error('Error retrieving URL ' + url)
47    raise
48
49
50def Builders(master):
51  builders = {}
52
53  # Load builders from cache file.
54  if os.path.exists(master):
55    start_time = time.time()
56    for builder_name in os.listdir(master):
57      cache_file_path = os.path.join(master, builder_name, CACHE_FILE_NAME)
58      if os.path.exists(cache_file_path):
59        with open(cache_file_path, 'rb') as cache_file:
60          try:
61            builders[builder_name] = cPickle.load(cache_file)
62          except EOFError:
63            logging.error('File is corrupted: %s', cache_file_path)
64            raise
65    logging.info('Loaded builder caches in %0.2f seconds.',
66                 time.time() - start_time)
67
68  return builders
69
70
71def Update(master, builders):
72  # Update builders with latest information.
73  builder_data = _FetchData(master, 'builders')
74  for builder_name, builder_info in builder_data.iteritems():
75    if builder_name in builders:
76      builders[builder_name].Update(builder_info)
77    else:
78      builders[builder_name] = Builder(master, builder_name, builder_info)
79
80  return builders
81
82
83class Builder(object):
84  # pylint: disable=too-many-instance-attributes
85
86  def __init__(self, master, name, data):
87    self._master = master
88    self._name = name
89
90    self.Update(data)
91
92    self._builds = {}
93
94  def __setstate__(self, state):
95    self.__dict__ = state  # pylint: disable=attribute-defined-outside-init
96    if not hasattr(self, '_builds'):
97      self._builds = {}
98
99  def __lt__(self, other):
100    return self.name < other.name
101
102  def __str__(self):
103    return self.name
104
105  def __getitem__(self, key):
106    if not isinstance(key, int):
107      raise TypeError('build numbers must be integers, not %s' %
108                      type(key).__name__)
109
110    self._FetchBuilds(key)
111    return self._builds[key]
112
113  def _FetchBuilds(self, *build_numbers):
114    """Download build details, if not already cached.
115
116    Returns:
117      A tuple of downloaded build numbers.
118    """
119    build_numbers = tuple(build_number for build_number in build_numbers
120                          if not (build_number in self._builds and
121                                  self._builds[build_number].complete))
122    if not build_numbers:
123      return ()
124
125    for build_number in build_numbers:
126      if build_number < 0:
127        raise ValueError('Invalid build number: %d' % build_number)
128
129    build_query = urllib.urlencode(
130        [('select', build) for build in build_numbers])
131    url = 'builders/%s/builds/?%s' % (urllib.quote(self.name), build_query)
132    builds = _FetchData(self.master, url)
133    for build_info in builds.itervalues():
134      self._builds[build_info['number']] = Build(self.master, build_info)
135
136    self._Cache()
137
138    return build_numbers
139
140  def FetchRecentBuilds(self, number_of_builds):
141    min_build = max(self.last_build - number_of_builds, -1)
142    return self._FetchBuilds(*xrange(self.last_build, min_build, -1))
143
144  def Update(self, data=None):
145    if not data:
146      data = _FetchData(self.master, 'builders/%s' % urllib.quote(self.name))
147    self._state = data['state']
148    self._pending_build_count = data['pendingBuilds']
149    self._current_builds = tuple(data['currentBuilds'])
150    self._cached_builds = tuple(data['cachedBuilds'])
151    self._slaves = tuple(data['slaves'])
152
153    self._Cache()
154
155  def _Cache(self):
156    cache_dir_path = os.path.join(self.master, self.name)
157    if not os.path.exists(cache_dir_path):
158      os.makedirs(cache_dir_path)
159    cache_file_path = os.path.join(cache_dir_path, CACHE_FILE_NAME)
160    with open(cache_file_path, 'wb') as cache_file:
161      cPickle.dump(self, cache_file, -1)
162
163  def LastBuilds(self, count):
164    min_build = max(self.last_build - count, -1)
165    for build_number in xrange(self.last_build, min_build, -1):
166      yield self._builds[build_number]
167
168  @property
169  def master(self):
170    return self._master
171
172  @property
173  def name(self):
174    return self._name
175
176  @property
177  def state(self):
178    return self._state
179
180  @property
181  def pending_build_count(self):
182    return self._pending_build_count
183
184  @property
185  def current_builds(self):
186    """List of build numbers currently building.
187
188    There may be multiple entries if there are multiple build slaves."""
189    return self._current_builds
190
191  @property
192  def cached_builds(self):
193    """Builds whose data are visible on the master in increasing order.
194
195    More builds may be available than this."""
196    return self._cached_builds
197
198  @property
199  def last_build(self):
200    """Last completed build."""
201    for build_number in reversed(self.cached_builds):
202      if build_number not in self.current_builds:
203        return build_number
204    return None
205
206  @property
207  def slaves(self):
208    return self._slaves
209
210
211class Build(object):
212  def __init__(self, master, data):
213    self._master = master
214    self._builder_name = data['builderName']
215    self._number = data['number']
216    self._complete = not ('currentStep' in data and data['currentStep'])
217    self._start_time, self._end_time = data['times']
218
219    self._steps = {
220        step_info['name']:
221            Step(self._master, self._builder_name, self._number, step_info)
222        for step_info in data['steps']
223    }
224
225  def __str__(self):
226    return str(self.number)
227
228  def __lt__(self, other):
229    return self.number < other.number
230
231  @property
232  def builder_name(self):
233    return self._builder_name
234
235  @property
236  def number(self):
237    return self._number
238
239  @property
240  def complete(self):
241    return self._complete
242
243  @property
244  def start_time(self):
245    return self._start_time
246
247  @property
248  def end_time(self):
249    return self._end_time
250
251  @property
252  def steps(self):
253    return self._steps
254
255
256def _ParseTraceFromLog(log):
257  """Search the log for a stack trace and return a structured representation.
258
259  This function supports both default Python-style stacks and Telemetry-style
260  stacks. It returns the first stack trace found in the log - sometimes a bug
261  leads to a cascade of failures, so the first one is usually the root cause.
262  """
263  log_iterator = iter(log.splitlines())
264  for line in log_iterator:
265    if line == 'Traceback (most recent call last):':
266      break
267  else:
268    return (None, None)
269
270  stack_trace = []
271  while True:
272    line = log_iterator.next()
273    match1 = re.match(r'\s*File "(?P<file>.+)", line (?P<line>[0-9]+), '
274                      'in (?P<function>.+)', line)
275    match2 = re.match(r'\s*(?P<function>.+) at '
276                      '(?P<file>.+):(?P<line>[0-9]+)', line)
277    match = match1 or match2
278    if not match:
279      exception = line
280      break
281    trace_line = match.groupdict()
282    # Use the base name, because the path will be different
283    # across platforms and configurations.
284    file_base_name = trace_line['file'].split('/')[-1].split('\\')[-1]
285    source = log_iterator.next().strip()
286    stack_trace.append(StackTraceLine(
287        file_base_name, trace_line['function'], trace_line['line'], source))
288
289  return tuple(stack_trace), exception
290
291
292class Step(object):
293  # pylint: disable=too-many-instance-attributes
294
295  def __init__(self, master, builder_name, build_number, data):
296    self._master = master
297    self._builder_name = builder_name
298    self._build_number = build_number
299    self._name = data['name']
300    self._result = data['results'][0]
301    self._start_time, self._end_time = data['times']
302
303    self._log_link = None
304    self._results_link = None
305    for link_name, link_url in data['logs']:
306      if link_name == 'stdio':
307        self._log_link = link_url + '/text'
308      elif link_name == 'json.output':
309        self._results_link = link_url + '/text'
310
311    self._log = None
312    self._results = None
313    self._stack_trace = None
314
315  def __getstate__(self):
316    return {
317        '_master': self._master,
318        '_builder_name': self._builder_name,
319        '_build_number': self._build_number,
320        '_name': self._name,
321        '_result': self._result,
322        '_start_time': self._start_time,
323        '_end_time': self._end_time,
324        '_log_link': self._log_link,
325        '_results_link': self._results_link,
326    }
327
328  def __setstate__(self, state):
329    self.__dict__ = state  # pylint: disable=attribute-defined-outside-init
330    self._log = None
331    self._results = None
332    self._stack_trace = None
333
334  def __str__(self):
335    return self.name
336
337  @property
338  def name(self):
339    return self._name
340
341  @property
342  def result(self):
343    return self._result
344
345  @property
346  def start_time(self):
347    return self._start_time
348
349  @property
350  def end_time(self):
351    return self._end_time
352
353  @property
354  def log_link(self):
355    return self._log_link
356
357  @property
358  def results_link(self):
359    return self._results_link
360
361  @property
362  def log(self):
363    if self._log is None:
364      if not self.log_link:
365        return None
366      cache_file_path = os.path.join(
367          self._master, self._builder_name,
368          str(self._build_number), self._name, 'log')
369      if os.path.exists(cache_file_path):
370        # Load cache file, if it exists.
371        with open(cache_file_path, 'rb') as cache_file:
372          self._log = cache_file.read()
373      else:
374        # Otherwise, download it.
375        logging.info('Retrieving ' + self.log_link)
376        try:
377          data = urllib2.urlopen(self.log_link).read()
378        except (urllib2.HTTPError, socket.error):
379          # Could be intermittent; try again.
380          try:
381            data = urllib2.urlopen(self.log_link).read()
382          except (urllib2.HTTPError, socket.error):
383            logging.error('Error retrieving URL ' + self.log_link)
384            raise
385        except:
386          logging.error('Error retrieving URL ' + self.log_link)
387          raise
388        # And cache the newly downloaded data.
389        cache_dir_path = os.path.dirname(cache_file_path)
390        if not os.path.exists(cache_dir_path):
391          os.makedirs(cache_dir_path)
392        with open(cache_file_path, 'wb') as cache_file:
393          cache_file.write(data)
394        self._log = data
395    return self._log
396
397  @property
398  def results(self):
399    if self._results is None:
400      if not self.results_link:
401        return None
402      cache_file_path = os.path.join(
403          self._master, self._builder_name,
404          str(self._build_number), self._name, 'results')
405      if os.path.exists(cache_file_path):
406        # Load cache file, if it exists.
407        try:
408          with open(cache_file_path, 'rb') as cache_file:
409            self._results = cPickle.load(cache_file)
410        except EOFError:
411          os.remove(cache_file_path)
412          return self.results
413      else:
414        # Otherwise, download it.
415        logging.info('Retrieving ' + self.results_link)
416        try:
417          data = json.load(urllib2.urlopen(self.results_link))
418        except (urllib2.HTTPError, socket.error):
419          # Could be intermittent; try again.
420          try:
421            data = json.load(urllib2.urlopen(self.results_link))
422          except (urllib2.HTTPError, socket.error):
423            logging.error('Error retrieving URL ' + self.results_link)
424            raise
425        except ValueError:
426          # If the build had an exception, the results might not be valid.
427          data = None
428        except:
429          logging.error('Error retrieving URL ' + self.results_link)
430          raise
431        # And cache the newly downloaded data.
432        cache_dir_path = os.path.dirname(cache_file_path)
433        if not os.path.exists(cache_dir_path):
434          os.makedirs(cache_dir_path)
435        with open(cache_file_path, 'wb') as cache_file:
436          cPickle.dump(data, cache_file, -1)
437        self._results = data
438    return self._results
439
440  @property
441  def stack_trace(self):
442    if self._stack_trace is None:
443      self._stack_trace = _ParseTraceFromLog(self.log)
444    return self._stack_trace
445