1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Defines common functionality used for interacting with Rietveld."""
6
7import json
8import logging
9import mimetypes
10import urllib
11
12import httplib2
13
14from google.appengine.ext import ndb
15
16from dashboard import utils
17
18_DESCRIPTION = """This patch was automatically uploaded by the Chrome Perf
19Dashboard (https://chromeperf.appspot.com). It is being used to run a perf
20bisect try job. It should not be submitted."""
21
22
23class ResponseObject(object):
24  """Class for Response Object.
25
26  This class holds attributes similar to response object returned by
27  google.appengine.api.urlfetch. This is used to convert response object
28  returned by httplib2.Http.request.
29  """
30
31  def __init__(self, status_code, content):
32    self.status_code = int(status_code)
33    self.content = content
34
35
36class RietveldConfig(ndb.Model):
37  """Configuration info for a Rietveld service account.
38
39  The data is stored only in the App Engine datastore (and the cloud console)
40  and not the code because it contains sensitive information like private keys.
41  """
42  # TODO(qyearsley): Remove RietveldConfig and store the server URL in
43  # datastore.
44  client_email = ndb.TextProperty()
45  service_account_key = ndb.TextProperty()
46
47  # The protocol and domain of the Rietveld host. Should not contain path.
48  server_url = ndb.TextProperty()
49
50  # The protocol and domain of the Internal Rietveld host which is used
51  # to create issues for internal only tests.
52  internal_server_url = ndb.TextProperty()
53
54
55def GetDefaultRietveldConfig():
56  """Returns the default rietveld config entity from the datastore."""
57  return ndb.Key(RietveldConfig, 'default_rietveld_config').get()
58
59
60class RietveldService(object):
61  """Implements a Python API to Rietveld via HTTP.
62
63  Authentication is handled via an OAuth2 access token minted from an RSA key
64  associated with a service account (which can be created via the Google API
65  console). For this to work, the Rietveld instance to talk to must be
66  configured to allow the service account client ID as OAuth2 audience (see
67  Rietveld source). Both the RSA key and the server URL are provided via static
68  application configuration.
69  """
70
71  def __init__(self, internal_only=False):
72    self.internal_only = internal_only
73    self._config = None
74    self._http = None
75
76  def Config(self):
77    if not self._config:
78      self._config = GetDefaultRietveldConfig()
79    return self._config
80
81  def MakeRequest(self, path, *args, **kwargs):
82    """Makes a request to the Rietveld server."""
83    if self.internal_only:
84      server_url = self.Config().internal_server_url
85    else:
86      server_url = self.Config().server_url
87    url = '%s/%s' % (server_url, path)
88    response, content = self._Http().request(url, *args, **kwargs)
89    return ResponseObject(response.get('status'), content)
90
91  def _Http(self):
92    if not self._http:
93      self._http = httplib2.Http()
94      credentials = utils.ServiceAccountCredentials()
95      credentials.authorize(self._http)
96    return self._http
97
98  def _XsrfToken(self):
99    """Requests a XSRF token from Rietveld."""
100    return self.MakeRequest(
101        'xsrf_token', headers={'X-Requesting-XSRF-Token': 1}).content
102
103  def _EncodeMultipartFormData(self, fields, files):
104    """Encode form fields for multipart/form-data.
105
106    Args:
107      fields: A sequence of (name, value) elements for regular form fields.
108      files: A sequence of (name, filename, value) elements for data to be
109             uploaded as files.
110    Returns:
111      (content_type, body) ready for httplib.HTTP instance.
112
113    Source:
114      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
115    """
116    boundary = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
117    crlf = '\r\n'
118    lines = []
119    for (key, value) in fields:
120      lines.append('--' + boundary)
121      lines.append('Content-Disposition: form-data; name="%s"' % key)
122      lines.append('')
123      if isinstance(value, unicode):
124        value = value.encode('utf-8')
125      lines.append(value)
126    for (key, filename, value) in files:
127      lines.append('--' + boundary)
128      lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
129                   (key, filename))
130      content_type = (mimetypes.guess_type(filename)[0] or
131                      'application/octet-stream')
132      lines.append('Content-Type: %s' % content_type)
133      lines.append('')
134      if isinstance(value, unicode):
135        value = value.encode('utf-8')
136      lines.append(value)
137    lines.append('--' + boundary + '--')
138    lines.append('')
139    body = crlf.join(lines)
140    content_type = 'multipart/form-data; boundary=%s' % boundary
141    return content_type, body
142
143  def UploadPatch(self, subject, patch, base_checksum, base_hashes,
144                  base_content, config_path):
145    """Uploads the given patch file contents to Rietveld.
146
147    The process of creating an issue and uploading the patch requires several
148    HTTP requests to Rietveld.
149
150    Rietveld API documentation: https://code.google.com/p/rietveld/wiki/APIs
151    For specific implementation in Rietveld codebase, see http://goo.gl/BW205J.
152
153    Args:
154      subject: Title of the job, as it will appear in rietveld.
155      patch: The patch, which is a specially-formatted string.
156      base_checksum: Base md5 checksum to send.
157      base_hashes: "Base hashes" string to send.
158      base_content: Base config file contents.
159      config_path: Path to the config file.
160
161    Returns:
162      A (issue ID, patchset ID) pair. These are strings that contain numerical
163      IDs. If the patch upload was unsuccessful, then (None, None) is returned.
164    """
165    base = 'https://chromium.googlesource.com/chromium/src.git@master'
166    repo_guid = 'c14d891d44f0afff64e56ed7c9702df1d807b1ee'
167    form_fields = [
168        ('subject', subject),
169        ('description', _DESCRIPTION),
170        ('base', base),
171        ('xsrf_token', self._XsrfToken()),
172        ('repo_guid', repo_guid),
173        ('content_upload', '1'),
174        ('base_hashes', base_hashes),
175    ]
176    uploaded_diff_file = [('data', 'data.diff', patch)]
177    ctype, body = self._EncodeMultipartFormData(
178        form_fields, uploaded_diff_file)
179    response = self.MakeRequest(
180        'upload', method='POST', body=body, headers={'content-type': ctype})
181    if response.status_code != 200:
182      logging.error('Error %s uploading to /upload', response.status_code)
183      logging.error(response.content)
184      return (None, None)
185
186    # There should always be 3 lines in the request, but sometimes Rietveld
187    # returns 2 lines. Log the content so we can debug further.
188    logging.info('Response from Rietveld /upload:\n%s', response.content)
189    if not response.content.startswith('Issue created.'):
190      logging.error('Unexpected response: %s', response.content)
191      return (None, None)
192    lines = response.content.splitlines()
193    if len(lines) < 2:
194      logging.error('Unexpected response %s', response.content)
195      return (None, None)
196
197    msg = lines[0]
198    issue_id = msg[msg.rfind('/') + 1:]
199    patchset_id = lines[1].strip()
200    patches = [x.split(' ', 1) for x in lines[2:]]
201    request_path = '%d/upload_content/%d/%d' % (
202        int(issue_id), int(patchset_id), int(patches[0][0]))
203    form_fields = [
204        ('filename', config_path),
205        ('status', 'M'),
206        ('checksum', base_checksum),
207        ('is_binary', str(False)),
208        ('is_current', str(False)),
209    ]
210    uploaded_diff_file = [('data', config_path, base_content)]
211    ctype, body = self._EncodeMultipartFormData(form_fields, uploaded_diff_file)
212    response = self.MakeRequest(
213        request_path, method='POST', body=body, headers={'content-type': ctype})
214    if response.status_code != 200:
215      logging.error(
216          'Error %s uploading to %s', response.status_code, request_path)
217      logging.error(response.content)
218      return (None, None)
219
220    request_path = '%s/upload_complete/%s' % (issue_id, patchset_id)
221    response = self.MakeRequest(request_path, method='POST')
222    if response.status_code != 200:
223      logging.error(
224          'Error %s uploading to %s', response.status_code, request_path)
225      logging.error(response.content)
226      return (None, None)
227    return issue_id, patchset_id
228
229  def TryPatch(self, tryserver_master, issue_id, patchset_id, bot):
230    """Sends a request to try the given patchset on the given trybot.
231
232    To see exactly how this request is handled, you can see the try_patchset
233    handler in the Chromium branch of Rietveld: http://goo.gl/U6tJQZ
234
235    Args:
236      tryserver_master: Master name, e.g. "tryserver.chromium.perf".
237      issue_id: Rietveld issue ID.
238      patchset_id: Patchset ID (returned when a patch is uploaded).
239      bot: Bisect bot name.
240
241    Returns:
242      True if successful, False otherwise.
243    """
244    args = {
245        'xsrf_token': self._XsrfToken(),
246        'builders': json.dumps({bot: ['defaulttests']}),
247        'master': tryserver_master,
248        'reason': 'Perf bisect',
249        'clobber': 'False',
250    }
251    request_path = '%s/try/%s' % (issue_id, patchset_id)
252    response = self.MakeRequest(
253        request_path, method='POST', body=urllib.urlencode(args))
254    if response.status_code != 200:
255      logging.error(
256          'Error %s POSTing to /%s/try/%s', response.status_code, issue_id,
257          patchset_id)
258      logging.error(response.content)
259      return False
260    return True
261