1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""General functions which are useful throughout this project."""
6
7import base64
8import binascii
9import json
10import logging
11import os
12import re
13import time
14
15from apiclient import discovery
16from apiclient import errors
17from google.appengine.api import memcache
18from google.appengine.api import urlfetch
19from google.appengine.api import urlfetch_errors
20from google.appengine.api import users
21from google.appengine.ext import ndb
22from oauth2client import client
23
24from dashboard import stored_object
25
26SHERIFF_DOMAINS_KEY = 'sheriff_domains_key'
27IP_WHITELIST_KEY = 'ip_whitelist'
28SERVICE_ACCOUNT_KEY = 'service_account'
29EMAIL_SCOPE = 'https://www.googleapis.com/auth/userinfo.email'
30_PROJECT_ID_KEY = 'project_id'
31_DEFAULT_CUSTOM_METRIC_VAL = 1
32
33
34def _GetNowRfc3339():
35  """Returns the current time formatted per RFC 3339."""
36  return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
37
38
39def TickMonitoringCustomMetric(metric_name):
40  """Increments the stackdriver custom metric with the given name.
41
42  This is used for cron job monitoring; if these metrics stop being received
43  an alert mail is sent. For more information on custom metrics, see
44  https://cloud.google.com/monitoring/custom-metrics/using-custom-metrics
45
46  Args:
47    metric_name: The name of the metric being monitored.
48  """
49  credentials = client.GoogleCredentials.get_application_default()
50  monitoring = discovery.build(
51      'cloudmonitoring', 'v2beta2', credentials=credentials)
52  now = _GetNowRfc3339()
53  project_id = stored_object.Get(_PROJECT_ID_KEY)
54  desc = {
55      'project': project_id,
56      'metric': 'custom.cloudmonitoring.googleapis.com/%s' % metric_name
57  }
58  point = {
59      'start': now,
60      'end': now,
61      'int64Value': _DEFAULT_CUSTOM_METRIC_VAL
62  }
63  write_request = monitoring.timeseries().write(
64      project=project_id,
65      body={'timeseries': [{'timeseriesDesc': desc, 'point': point}]})
66  write_request.execute()
67
68
69def TestPath(key):
70  """Returns the test path for a Test from an ndb.Key.
71
72  A "test path" is just a convenient string representation of an ndb.Key.
73  Each test path corresponds to one ndb.Key, which can be used to get an
74  entity.
75
76  Args:
77    key: An ndb.Key where all IDs are string IDs.
78
79  Returns:
80    A test path string.
81  """
82  return '/'.join(key.flat()[1::2])
83
84
85def TestSuiteName(test_key):
86  """Returns the test suite name for a given Test key."""
87  pairs = test_key.pairs()
88  if len(pairs) < 3:
89    return None
90  return pairs[2][1]
91
92
93def TestKey(test_path):
94  """Returns the ndb.Key that corresponds to a test path."""
95  if test_path is None:
96    return None
97  path_parts = test_path.split('/')
98  if path_parts is None:
99    return None
100  key_list = [('Master', path_parts[0])]
101  if len(path_parts) > 1:
102    key_list += [('Bot', path_parts[1])]
103  if len(path_parts) > 2:
104    key_list += [('Test', x) for x in path_parts[2:]]
105  return ndb.Key(pairs=key_list)
106
107
108def TestMatchesPattern(test, pattern):
109  """Checks whether a test matches a test path pattern.
110
111  Args:
112    test: A Test entity or a Test key.
113    pattern: A test path which can include wildcard characters (*).
114
115  Returns:
116    True if it matches, False otherwise.
117  """
118  if not test:
119    return False
120  if type(test) is ndb.Key:
121    test_path = TestPath(test)
122  else:
123    test_path = test.test_path
124  test_path_parts = test_path.split('/')
125  pattern_parts = pattern.split('/')
126  if len(test_path_parts) != len(pattern_parts):
127    return False
128  for test_path_part, pattern_part in zip(test_path_parts, pattern_parts):
129    if not _MatchesPatternPart(pattern_part, test_path_part):
130      return False
131  return True
132
133
134def _MatchesPatternPart(pattern_part, test_path_part):
135  """Checks whether a pattern (possibly with a *) matches the given string.
136
137  Args:
138    pattern_part: A string which may contain a wildcard (*).
139    test_path_part: Another string.
140
141  Returns:
142    True if it matches, False otherwise.
143  """
144  if pattern_part == '*' or pattern_part == test_path_part:
145    return True
146  if '*' not in pattern_part:
147    return False
148  # Escape any other special non-alphanumeric characters.
149  pattern_part = re.escape(pattern_part)
150  # There are not supposed to be any other asterisk characters, so all
151  # occurrences of backslash-asterisk can now be replaced with dot-asterisk.
152  re_pattern = re.compile('^' + pattern_part.replace('\\*', '.*') + '$')
153  return re_pattern.match(test_path_part)
154
155
156def TimestampMilliseconds(datetime):
157  """Returns the number of milliseconds since the epoch."""
158  return int(time.mktime(datetime.timetuple()) * 1000)
159
160
161def GetTestContainerKey(test):
162  """Gets the TestContainer key for the given Test.
163
164  Args:
165    test: Either a Test entity or its ndb.Key.
166
167  Returns:
168    ndb.Key('TestContainer', test path)
169  """
170  test_path = None
171  if type(test) is ndb.Key:
172    test_path = TestPath(test)
173  else:
174    test_path = test.test_path
175  return ndb.Key('TestContainer', test_path)
176
177
178def GetMulti(keys):
179  """Gets a list of entities from a list of keys.
180
181  If this user is logged in, this is the same as ndb.get_multi. However, if the
182  user is logged out and any of the data is internal only, an AssertionError
183  will be raised.
184
185  Args:
186    keys: A list of ndb entity keys.
187
188  Returns:
189    A list of entities, but no internal_only ones if the user is not logged in.
190  """
191  if IsInternalUser():
192    return ndb.get_multi(keys)
193  # Not logged in. Check each key individually.
194  entities = []
195  for key in keys:
196    try:
197      entities.append(key.get())
198    except AssertionError:
199      continue
200  return entities
201
202
203def MinimumAlertRange(alerts):
204  """Returns the intersection of the revision ranges for a set of alerts.
205
206  Args:
207    alerts: An iterable of Alerts (Anomaly or StoppageAlert entities).
208
209  Returns:
210    A pair (start, end) if there is a valid minimum range,
211    or None if the ranges are not overlapping.
212  """
213  ranges = [(a.start_revision, a.end_revision) for a in alerts if a]
214  return MinimumRange(ranges)
215
216
217def MinimumRange(ranges):
218  """Returns the intersection of the given ranges, or None."""
219  if not ranges:
220    return None
221  starts, ends = zip(*ranges)
222  start, end = (max(starts), min(ends))
223  if start > end:
224    return None
225  return start, end
226
227
228def IsInternalUser():
229  """Checks whether the user should be able to see internal-only data."""
230  username = users.get_current_user()
231  if not username:
232    return False
233  cached = GetCachedIsInternalUser(username)
234  if cached is not None:
235    return cached
236  is_internal_user = IsGroupMember(identity=username, group='googlers')
237  SetCachedIsInternalUser(username, is_internal_user)
238  return is_internal_user
239
240
241def GetCachedIsInternalUser(username):
242  return memcache.get(_IsInternalUserCacheKey(username))
243
244
245def SetCachedIsInternalUser(username, value):
246  memcache.add(_IsInternalUserCacheKey(username), value, time=60*60*24)
247
248
249def _IsInternalUserCacheKey(username):
250  return 'is_internal_user_%s' % username
251
252
253def IsGroupMember(identity, group):
254  """Checks if a user is a group member of using chrome-infra-auth.appspot.com.
255
256  Args:
257    identity: User email address.
258    group: Group name.
259
260  Returns:
261    True if confirmed to be a member, False otherwise.
262  """
263  try:
264    discovery_url = ('https://chrome-infra-auth.appspot.com'
265                     '/_ah/api/discovery/v1/apis/{api}/{apiVersion}/rest')
266    service = discovery.build(
267        'auth', 'v1', discoveryServiceUrl=discovery_url,
268        credentials=ServiceAccountCredentials())
269    request = service.membership(identity=identity, group=group)
270    response = request.execute()
271    return response['is_member']
272  except (errors.HttpError, KeyError, AttributeError) as e:
273    logging.error('Failed to check membership of %s: %s', identity, e)
274    return False
275
276
277def ServiceAccountCredentials():
278  """Returns the Credentials of the service account if available."""
279  account_details = stored_object.Get(SERVICE_ACCOUNT_KEY)
280  if not account_details:
281    logging.error('Service account credentials not found.')
282    return None
283  return client.SignedJwtAssertionCredentials(
284      service_account_name=account_details['client_email'],
285      private_key=account_details['private_key'],
286      scope=EMAIL_SCOPE)
287
288
289def IsValidSheriffUser():
290  """Checks whether the user should be allowed to triage alerts."""
291  user = users.get_current_user()
292  sheriff_domains = stored_object.Get(SHERIFF_DOMAINS_KEY)
293  return user and sheriff_domains and any(
294      user.email().endswith('@' + domain) for domain in sheriff_domains)
295
296
297def GetIpWhitelist():
298  """Returns a list of IP address strings in the whitelist."""
299  return stored_object.Get(IP_WHITELIST_KEY)
300
301
302def BisectConfigPythonString(config):
303  """Turns a bisect config dict into a properly formatted Python string.
304
305  Args:
306    config: A bisect config dict (see start_try_job.GetBisectConfig)
307
308  Returns:
309    A config string suitable to store in a TryJob entity.
310  """
311  return 'config = %s\n' % json.dumps(
312      config, sort_keys=True, indent=2, separators=(',', ': '))
313
314
315def DownloadChromiumFile(path):
316  """Downloads a file in the chromium/src repository.
317
318  This function uses gitiles to fetch files. As of September 2015,
319  gitiles supports fetching base-64 encoding of files. If it supports
320  fetching plain text in the future, that may be simpler.
321
322  Args:
323    path: Path to a file in src repository, without a leading slash or "src/".
324
325  Returns:
326    The contents of the file as a string, or None.
327  """
328  base_url = 'https://chromium.googlesource.com/chromium/src/+/master/'
329  url = '%s%s?format=TEXT' % (base_url, path)
330  response = urlfetch.fetch(url)
331  if response.status_code != 200:
332    logging.error('Got %d fetching "%s".', response.status_code, url)
333    return None
334  try:
335    plaintext_content = base64.decodestring(response.content)
336  except binascii.Error:
337    logging.error('Failed to decode "%s" from "%s".', response.content, url)
338    return None
339  return plaintext_content
340
341
342def GetRequestId():
343  """Returns the request log ID which can be used to find a specific log."""
344  return os.environ.get('REQUEST_LOG_ID')
345
346
347def Validate(expected, actual):
348  """Generic validator for expected keys, values, and types.
349
350  Values are also considered equal if |actual| can be converted to |expected|'s
351  type.  For instance:
352    _Validate([3], '3')  # Returns True.
353
354  See utils_test.py for more examples.
355
356  Args:
357    expected: Either a list of expected values or a dictionary of expected
358        keys and type.  A dictionary can contain a list of expected values.
359    actual: A value.
360  """
361  def IsValidType(expected, actual):
362    if type(expected) is type and type(actual) is not expected:
363      try:
364        expected(actual)
365      except ValueError:
366        return False
367    return True
368
369  def IsInList(expected, actual):
370    for value in expected:
371      try:
372        if type(value)(actual) == value:
373          return True
374      except ValueError:
375        pass
376    return False
377
378  if not expected:
379    return
380  expected_type = type(expected)
381  actual_type = type(actual)
382  if expected_type is list:
383    if not IsInList(expected, actual):
384      raise ValueError('Invalid value. Expected one of the following: '
385                       '%s. Actual: %s.' % (','.join(expected), actual))
386  elif expected_type is dict:
387    if actual_type is not dict:
388      raise ValueError('Invalid type. Expected: %s. Actual: %s.'
389                       % (expected_type, actual_type))
390    missing = set(expected.keys()) - set(actual.keys())
391    if missing:
392      raise ValueError('Missing the following properties: %s'
393                       % ','.join(missing))
394    for key in expected:
395      Validate(expected[key], actual[key])
396  elif not IsValidType(expected, actual):
397    raise ValueError('Invalid type. Expected: %s. Actual: %s.' %
398                     (expected, actual_type))
399
400
401def FetchURL(request_url, skip_status_code=False):
402  """Wrapper around URL fetch service to make request.
403
404  Args:
405    request_url: URL of request.
406    skip_status_code: Skips return code check when True, default is False.
407
408  Returns:
409    Response object return by URL fetch, otherwise None when there's an error.
410  """
411  logging.info('URL being fetched: ' + request_url)
412  try:
413    response = urlfetch.fetch(request_url)
414  except urlfetch_errors.DeadlineExceededError:
415    logging.error('Deadline exceeded error checking %s', request_url)
416    return None
417  except urlfetch_errors.DownloadError as err:
418    # DownloadError is raised to indicate a non-specific failure when there
419    # was not a 4xx or 5xx status code.
420    logging.error(err)
421    return None
422  if skip_status_code:
423    return response
424  elif response.status_code != 200:
425    logging.error(
426        'ERROR %s checking %s', response.status_code, request_url)
427    return None
428  return response
429