1# Copyright 2020 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#
15################################################################################
16"""Utility module for Google Cloud Build scripts."""
17import base64
18import collections
19import os
20import six.moves.urllib.parse as urlparse
21import sys
22import time
23
24import requests
25
26import google.auth
27import googleapiclient.discovery
28from oauth2client.service_account import ServiceAccountCredentials
29
30BUILD_TIMEOUT = 12 * 60 * 60
31
32# Needed for reading public target.list.* files.
33GCS_URL_BASENAME = 'https://storage.googleapis.com/'
34
35GCS_UPLOAD_URL_FORMAT = '/{0}/{1}/{2}'
36
37# Where corpus backups can be downloaded from.
38CORPUS_BACKUP_URL = ('/{project}-backup.clusterfuzz-external.appspot.com/'
39                     'corpus/libFuzzer/{fuzzer}/latest.zip')
40
41# Cloud Builder has a limit of 100 build steps and 100 arguments for each step.
42CORPUS_DOWNLOAD_BATCH_SIZE = 100
43
44TARGETS_LIST_BASENAME = 'targets.list'
45
46EngineInfo = collections.namedtuple(
47    'EngineInfo',
48    ['upload_bucket', 'supported_sanitizers', 'supported_architectures'])
49
50ENGINE_INFO = {
51    'libfuzzer':
52        EngineInfo(upload_bucket='clusterfuzz-builds',
53                   supported_sanitizers=['address', 'memory', 'undefined'],
54                   supported_architectures=['x86_64', 'i386']),
55    'afl':
56        EngineInfo(upload_bucket='clusterfuzz-builds-afl',
57                   supported_sanitizers=['address'],
58                   supported_architectures=['x86_64']),
59    'honggfuzz':
60        EngineInfo(upload_bucket='clusterfuzz-builds-honggfuzz',
61                   supported_sanitizers=['address'],
62                   supported_architectures=['x86_64']),
63    'dataflow':
64        EngineInfo(upload_bucket='clusterfuzz-builds-dataflow',
65                   supported_sanitizers=['dataflow'],
66                   supported_architectures=['x86_64']),
67    'none':
68        EngineInfo(upload_bucket='clusterfuzz-builds-no-engine',
69                   supported_sanitizers=['address'],
70                   supported_architectures=['x86_64']),
71}
72
73
74def get_targets_list_filename(sanitizer):
75  """Returns target list filename."""
76  return TARGETS_LIST_BASENAME + '.' + sanitizer
77
78
79def get_targets_list_url(bucket, project, sanitizer):
80  """Returns target list url."""
81  filename = get_targets_list_filename(sanitizer)
82  url = GCS_UPLOAD_URL_FORMAT.format(bucket, project, filename)
83  return url
84
85
86def _get_targets_list(project_name):
87  """Returns target list."""
88  # libFuzzer ASan is the default configuration, get list of targets from it.
89  url = get_targets_list_url(ENGINE_INFO['libfuzzer'].upload_bucket,
90                             project_name, 'address')
91
92  url = urlparse.urljoin(GCS_URL_BASENAME, url)
93  response = requests.get(url)
94  if not response.status_code == 200:
95    sys.stderr.write('Failed to get list of targets from "%s".\n' % url)
96    sys.stderr.write('Status code: %d \t\tText:\n%s\n' %
97                     (response.status_code, response.text))
98    return None
99
100  return response.text.split()
101
102
103# pylint: disable=no-member
104def get_signed_url(path, method='PUT', content_type=''):
105  """Returns signed url."""
106  timestamp = int(time.time() + BUILD_TIMEOUT)
107  blob = '{0}\n\n{1}\n{2}\n{3}'.format(method, content_type, timestamp, path)
108
109  service_account_path = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
110  if service_account_path:
111    creds = ServiceAccountCredentials.from_json_keyfile_name(
112        os.environ['GOOGLE_APPLICATION_CREDENTIALS'])
113    client_id = creds.service_account_email
114    signature = base64.b64encode(creds.sign_blob(blob)[1])
115  else:
116    credentials, project = google.auth.default()
117    iam = googleapiclient.discovery.build('iamcredentials',
118                                          'v1',
119                                          credentials=credentials,
120                                          cache_discovery=False)
121    client_id = project + '@appspot.gserviceaccount.com'
122    service_account = 'projects/-/serviceAccounts/{0}'.format(client_id)
123    response = iam.projects().serviceAccounts().signBlob(
124        name=service_account,
125        body={
126            'delegates': [],
127            'payload': base64.b64encode(blob.encode('utf-8')).decode('utf-8'),
128        }).execute()
129    signature = response['signedBlob']
130
131  values = {
132      'GoogleAccessId': client_id,
133      'Expires': timestamp,
134      'Signature': signature,
135  }
136  return ('https://storage.googleapis.com{0}?'.format(path) +
137          urlparse.urlencode(values))
138
139
140def download_corpora_steps(project_name):
141  """Returns GCB steps for downloading corpora backups for the given project.
142  """
143  fuzz_targets = _get_targets_list(project_name)
144  if not fuzz_targets:
145    sys.stderr.write('No fuzz targets found for project "%s".\n' % project_name)
146    return None
147
148  steps = []
149  # Split fuzz targets into batches of CORPUS_DOWNLOAD_BATCH_SIZE.
150  for i in range(0, len(fuzz_targets), CORPUS_DOWNLOAD_BATCH_SIZE):
151    download_corpus_args = []
152    for binary_name in fuzz_targets[i:i + CORPUS_DOWNLOAD_BATCH_SIZE]:
153      qualified_name = binary_name
154      qualified_name_prefix = '%s_' % project_name
155      if not binary_name.startswith(qualified_name_prefix):
156        qualified_name = qualified_name_prefix + binary_name
157
158      url = get_signed_url(CORPUS_BACKUP_URL.format(project=project_name,
159                                                    fuzzer=qualified_name),
160                           method='GET')
161
162      corpus_archive_path = os.path.join('/corpus', binary_name + '.zip')
163      download_corpus_args.append('%s %s' % (corpus_archive_path, url))
164
165    steps.append({
166        'name': 'gcr.io/oss-fuzz-base/base-runner',
167        'entrypoint': 'download_corpus',
168        'args': download_corpus_args,
169        'volumes': [{
170            'name': 'corpus',
171            'path': '/corpus'
172        }],
173    })
174
175  return steps
176
177
178def http_upload_step(data, signed_url, content_type):
179  """Returns a GCB step to upload data to the given URL via GCS HTTP API."""
180  step = {
181      'name':
182          'gcr.io/cloud-builders/curl',
183      'args': [
184          '-H',
185          'Content-Type: ' + content_type,
186          '-X',
187          'PUT',
188          '-d',
189          data,
190          signed_url,
191      ],
192  }
193  return step
194
195
196def gsutil_rm_rf_step(url):
197  """Returns a GCB step to recursively delete the object with given GCS url."""
198  step = {
199      'name': 'gcr.io/cloud-builders/gsutil',
200      'entrypoint': 'sh',
201      'args': [
202          '-c',
203          'gsutil -m rm -rf %s || exit 0' % url,
204      ],
205  }
206  return step
207
208
209def project_image_steps(name, image, language):
210  """Returns GCB steps to build OSS-Fuzz project image."""
211  steps = [{
212      'args': [
213          'clone',
214          'https://github.com/google/oss-fuzz.git',
215      ],
216      'name': 'gcr.io/cloud-builders/git',
217  }, {
218      'name': 'gcr.io/cloud-builders/docker',
219      'args': [
220          'build',
221          '-t',
222          image,
223          '.',
224      ],
225      'dir': 'oss-fuzz/projects/' + name,
226  }, {
227      'name':
228          image,
229      'args': [
230          'bash', '-c',
231          'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
232      ],
233      'env': [
234          'OSSFUZZ_REVISION=$REVISION_ID',
235          'FUZZING_LANGUAGE=%s' % language,
236      ],
237  }]
238
239  return steps
240