1#!/usr/bin/env python
2# Copyright 2019 Google Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16################################################################################
17"""Build modified projects."""
18
19from __future__ import print_function
20
21import enum
22import os
23import re
24import sys
25import subprocess
26import yaml
27
28CANARY_PROJECT = 'skcms'
29
30DEFAULT_ARCHITECTURES = ['x86_64']
31DEFAULT_ENGINES = ['afl', 'honggfuzz', 'libfuzzer']
32DEFAULT_SANITIZERS = ['address', 'undefined']
33
34# Languages from project.yaml that have code coverage support.
35LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust']
36
37
38def get_changed_files_output():
39  """Returns the output of a git command that discovers changed files."""
40  branch_commit_hash = subprocess.check_output(
41      ['git', 'merge-base', 'FETCH_HEAD', 'origin/HEAD']).strip().decode()
42
43  return subprocess.check_output(
44      ['git', 'diff', '--name-only', branch_commit_hash + '..']).decode()
45
46
47def get_modified_buildable_projects():
48  """Returns a list of all the projects modified in this commit that have a
49  build.sh file."""
50  git_output = get_changed_files_output()
51  projects_regex = '.*projects/(?P<name>.*)/.*\n'
52  modified_projects = set(re.findall(projects_regex, git_output))
53  projects_dir = os.path.join(get_oss_fuzz_root(), 'projects')
54  # Filter out projects without Dockerfile files since new projects and reverted
55  # projects frequently don't have them. In these cases we don't want Travis's
56  # builds to fail.
57  modified_buildable_projects = []
58  for project in modified_projects:
59    if not os.path.exists(os.path.join(projects_dir, project, 'Dockerfile')):
60      print('Project {0} does not have Dockerfile. skipping build.'.format(
61          project))
62      continue
63    modified_buildable_projects.append(project)
64  return modified_buildable_projects
65
66
67def get_oss_fuzz_root():
68  """Get the absolute path of the root of the oss-fuzz checkout."""
69  script_path = os.path.realpath(__file__)
70  return os.path.abspath(
71      os.path.dirname(os.path.dirname(os.path.dirname(script_path))))
72
73
74def execute_helper_command(helper_command):
75  """Execute |helper_command| using helper.py."""
76  root = get_oss_fuzz_root()
77  script_path = os.path.join(root, 'infra', 'helper.py')
78  command = ['python', script_path] + helper_command
79  print('Running command: %s' % ' '.join(command))
80  subprocess.check_call(command)
81
82
83def build_fuzzers(project, engine, sanitizer, architecture):
84  """Execute helper.py's build_fuzzers command on |project|. Build the fuzzers
85  with |engine| and |sanitizer| for |architecture|."""
86  execute_helper_command([
87      'build_fuzzers', project, '--engine', engine, '--sanitizer', sanitizer,
88      '--architecture', architecture
89  ])
90
91
92def check_build(project, engine, sanitizer, architecture):
93  """Execute helper.py's check_build command on |project|, assuming it was most
94  recently built with |engine| and |sanitizer| for |architecture|."""
95  execute_helper_command([
96      'check_build', project, '--engine', engine, '--sanitizer', sanitizer,
97      '--architecture', architecture
98  ])
99
100
101def should_build_coverage(project_yaml):
102  """Returns True if a coverage build should be done based on project.yaml
103  contents."""
104  # Enable coverage builds on projects that use engines. Those that don't use
105  # engines shouldn't get coverage builds.
106  engines = project_yaml.get('fuzzing_engines', DEFAULT_ENGINES)
107  engineless = 'none' in engines
108  if engineless:
109    assert_message = ('Forbidden to specify multiple engines for '
110                      '"fuzzing_engines" if "none" is specified.')
111    assert len(engines) == 1, assert_message
112    return False
113
114  language = project_yaml.get('language')
115  if language not in LANGUAGES_WITH_COVERAGE_SUPPORT:
116    print(('Project is written in "{language}", '
117           'coverage is not supported yet.').format(language=language))
118    return False
119
120  return True
121
122
123def should_build(project_yaml):
124  """Returns True on if the build specified is enabled in the project.yaml."""
125
126  if os.getenv('SANITIZER') == 'coverage':
127    # This assumes we only do coverage builds with libFuzzer on x86_64.
128    return should_build_coverage(project_yaml)
129
130  def is_enabled(env_var, yaml_name, defaults):
131    """Is the value of |env_var| enabled in |project_yaml| (in the |yaml_name|
132    section)? Uses |defaults| if |yaml_name| section is unspecified."""
133    return os.getenv(env_var) in project_yaml.get(yaml_name, defaults)
134
135  return (is_enabled('ENGINE', 'fuzzing_engines', DEFAULT_ENGINES) and
136          is_enabled('SANITIZER', 'sanitizers', DEFAULT_SANITIZERS) and
137          is_enabled('ARCHITECTURE', 'architectures', DEFAULT_ARCHITECTURES))
138
139
140def build_project(project):
141  """Do the build of |project| that is specified by the environment variables -
142  SANITIZER, ENGINE, and ARCHITECTURE."""
143  root = get_oss_fuzz_root()
144  project_yaml_path = os.path.join(root, 'projects', project, 'project.yaml')
145  with open(project_yaml_path) as file_handle:
146    project_yaml = yaml.safe_load(file_handle)
147
148  if project_yaml.get('disabled', False):
149    print('Project {0} is disabled, skipping build.'.format(project))
150    return
151
152  engine = os.getenv('ENGINE')
153  sanitizer = os.getenv('SANITIZER')
154  architecture = os.getenv('ARCHITECTURE')
155
156  if not should_build(project_yaml):
157    print(('Specified build: engine: {0}, sanitizer: {1}, architecture: {2} '
158           'not enabled for this project: {3}. Skipping build.').format(
159               engine, sanitizer, architecture, project))
160
161    return
162
163  print('Building project', project)
164  build_fuzzers(project, engine, sanitizer, architecture)
165
166  if engine != 'none' and sanitizer != 'coverage':
167    check_build(project, engine, sanitizer, architecture)
168
169
170class BuildModifiedProjectsResult(enum.Enum):
171  """Enum containing the return values of build_modified_projects()."""
172  NONE_BUILT = 0
173  BUILD_SUCCESS = 1
174  BUILD_FAIL = 2
175
176
177def build_modified_projects():
178  """Build modified projects. Returns BuildModifiedProjectsResult.NONE_BUILT if
179  no builds were attempted. Returns BuildModifiedProjectsResult.BUILD_SUCCESS if
180  all attempts succeed, otherwise returns
181  BuildModifiedProjectsResult.BUILD_FAIL."""
182  projects = get_modified_buildable_projects()
183  if not projects:
184    return BuildModifiedProjectsResult.NONE_BUILT
185
186  failed_projects = []
187  for project in projects:
188    try:
189      build_project(project)
190    except subprocess.CalledProcessError:
191      failed_projects.append(project)
192
193  if failed_projects:
194    print('Failed projects:', ' '.join(failed_projects))
195    return BuildModifiedProjectsResult.BUILD_FAIL
196
197  return BuildModifiedProjectsResult.BUILD_SUCCESS
198
199
200def is_infra_changed():
201  """Returns True if the infra directory was changed."""
202  git_output = get_changed_files_output()
203  infra_code_regex = '.*infra/.*\n'
204  return re.search(infra_code_regex, git_output) is not None
205
206
207def build_base_images():
208  """Builds base images."""
209  # TODO(jonathanmetzman): Investigate why caching fails so often and
210  # when we improve it, build base-clang as well. Also, move this function
211  # to a helper command when we can support base-clang.
212  execute_helper_command(['pull_images'])
213  images = [
214      'base-image',
215      'base-builder',
216      'base-runner',
217  ]
218  for image in images:
219    try:
220      execute_helper_command(['build_image', image, '--no-pull'])
221    except subprocess.CalledProcessError:
222      return 1
223
224  return 0
225
226
227def build_canary_project():
228  """Builds a specific project when infra/ is changed to verify that infra/
229  changes don't break things. Returns False if build was attempted but
230  failed."""
231
232  try:
233    build_project('skcms')
234  except subprocess.CalledProcessError:
235    return False
236
237  return True
238
239
240def main():
241  """Build modified projects or canary project."""
242  infra_changed = is_infra_changed()
243  if infra_changed:
244    print('Pulling and building base images first.')
245    if build_base_images():
246      return 1
247
248  result = build_modified_projects()
249  if result == BuildModifiedProjectsResult.BUILD_FAIL:
250    return 1
251
252  # It's unnecessary to build the canary if we've built any projects already.
253  no_projects_built = result == BuildModifiedProjectsResult.NONE_BUILT
254  should_build_canary = no_projects_built and infra_changed
255  if should_build_canary and not build_canary_project():
256    return 1
257
258  return 0
259
260
261if __name__ == '__main__':
262  sys.exit(main())
263