1#!/usr/bin/env python
2# Copyright 2016 Google Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16################################################################################
17"""Helper script for OSS-Fuzz users. Can do common tasks like building
18projects/fuzzers, running them etc."""
19
20from __future__ import print_function
21from multiprocessing.dummy import Pool as ThreadPool
22import argparse
23import datetime
24import errno
25import os
26import pipes
27import re
28import subprocess
29import sys
30import templates
31
32OSS_FUZZ_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
33BUILD_DIR = os.path.join(OSS_FUZZ_DIR, 'build')
34
35BASE_IMAGES = [
36    'gcr.io/oss-fuzz-base/base-image',
37    'gcr.io/oss-fuzz-base/base-clang',
38    'gcr.io/oss-fuzz-base/base-builder',
39    'gcr.io/oss-fuzz-base/base-runner',
40    'gcr.io/oss-fuzz-base/base-runner-debug',
41    'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder',
42    'gcr.io/oss-fuzz-base/msan-libs-builder',
43]
44
45VALID_PROJECT_NAME_REGEX = re.compile(r'^[a-zA-Z0-9_-]+$')
46MAX_PROJECT_NAME_LENGTH = 26
47
48if sys.version_info[0] >= 3:
49  raw_input = input  # pylint: disable=invalid-name
50
51CORPUS_URL_FORMAT = (
52    'gs://{project_name}-corpus.clusterfuzz-external.appspot.com/libFuzzer/'
53    '{fuzz_target}/')
54CORPUS_BACKUP_URL_FORMAT = (
55    'gs://{project_name}-backup.clusterfuzz-external.appspot.com/corpus/'
56    'libFuzzer/{fuzz_target}/')
57
58PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)')
59
60# Languages from project.yaml that have code coverage support.
61LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust']
62
63# pylint: disable=too-many-lines
64
65
66def main():  # pylint: disable=too-many-branches,too-many-return-statements
67  """Get subcommand from program arguments and do it."""
68  os.chdir(OSS_FUZZ_DIR)
69  if not os.path.exists(BUILD_DIR):
70    os.mkdir(BUILD_DIR)
71
72  args = parse_args()
73
74  # We have different default values for `sanitizer` depending on the `engine`.
75  # Some commands do not have `sanitizer` argument, so `hasattr` is necessary.
76  if hasattr(args, 'sanitizer') and not args.sanitizer:
77    if args.engine == 'dataflow':
78      args.sanitizer = 'dataflow'
79    else:
80      args.sanitizer = 'address'
81
82  if args.command == 'generate':
83    return generate(args)
84  if args.command == 'build_image':
85    return build_image(args)
86  if args.command == 'build_fuzzers':
87    return build_fuzzers(args)
88  if args.command == 'check_build':
89    return check_build(args)
90  if args.command == 'download_corpora':
91    return download_corpora(args)
92  if args.command == 'run_fuzzer':
93    return run_fuzzer(args)
94  if args.command == 'coverage':
95    return coverage(args)
96  if args.command == 'reproduce':
97    return reproduce(args)
98  if args.command == 'shell':
99    return shell(args)
100  if args.command == 'pull_images':
101    return pull_images(args)
102
103  return 0
104
105
106def parse_args(args=None):
107  """Parses args using argparser and returns parsed args."""
108  # Use default argument None for args so that in production, argparse does its
109  # normal behavior, but unittesting is easier.
110  parser = get_parser()
111  return parser.parse_args(args)
112
113
114def get_parser():  # pylint: disable=too-many-statements
115  """Returns an argparse parser."""
116  parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
117  subparsers = parser.add_subparsers(dest='command')
118
119  generate_parser = subparsers.add_parser(
120      'generate', help='Generate files for new project.')
121  generate_parser.add_argument('project_name')
122
123  build_image_parser = subparsers.add_parser('build_image',
124                                             help='Build an image.')
125  build_image_parser.add_argument('project_name')
126  build_image_parser.add_argument('--pull',
127                                  action='store_true',
128                                  help='Pull latest base image.')
129  build_image_parser.add_argument('--no-pull',
130                                  action='store_true',
131                                  help='Do not pull latest base image.')
132
133  build_fuzzers_parser = subparsers.add_parser(
134      'build_fuzzers', help='Build fuzzers for a project.')
135  _add_architecture_args(build_fuzzers_parser)
136  _add_engine_args(build_fuzzers_parser)
137  _add_sanitizer_args(build_fuzzers_parser)
138  _add_environment_args(build_fuzzers_parser)
139  build_fuzzers_parser.add_argument('project_name')
140  build_fuzzers_parser.add_argument('source_path',
141                                    help='path of local source',
142                                    nargs='?')
143  build_fuzzers_parser.add_argument('--clean',
144                                    dest='clean',
145                                    action='store_true',
146                                    help='clean existing artifacts.')
147  build_fuzzers_parser.add_argument('--no-clean',
148                                    dest='clean',
149                                    action='store_false',
150                                    help='do not clean existing artifacts '
151                                    '(default).')
152  build_fuzzers_parser.set_defaults(clean=False)
153
154  check_build_parser = subparsers.add_parser(
155      'check_build', help='Checks that fuzzers execute without errors.')
156  _add_architecture_args(check_build_parser)
157  _add_engine_args(
158      check_build_parser,
159      choices=['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none'])
160  _add_sanitizer_args(
161      check_build_parser,
162      choices=['address', 'memory', 'undefined', 'dataflow', 'thread'])
163  _add_environment_args(check_build_parser)
164  check_build_parser.add_argument('project_name', help='name of the project')
165  check_build_parser.add_argument('fuzzer_name',
166                                  help='name of the fuzzer',
167                                  nargs='?')
168
169  run_fuzzer_parser = subparsers.add_parser(
170      'run_fuzzer', help='Run a fuzzer in the emulated fuzzing environment.')
171  _add_engine_args(run_fuzzer_parser)
172  _add_sanitizer_args(run_fuzzer_parser)
173  _add_environment_args(run_fuzzer_parser)
174  run_fuzzer_parser.add_argument(
175      '--corpus-dir', help='directory to store corpus for the fuzz target')
176  run_fuzzer_parser.add_argument('project_name', help='name of the project')
177  run_fuzzer_parser.add_argument('fuzzer_name', help='name of the fuzzer')
178  run_fuzzer_parser.add_argument('fuzzer_args',
179                                 help='arguments to pass to the fuzzer',
180                                 nargs=argparse.REMAINDER)
181
182  coverage_parser = subparsers.add_parser(
183      'coverage', help='Generate code coverage report for the project.')
184  coverage_parser.add_argument('--no-corpus-download',
185                               action='store_true',
186                               help='do not download corpus backup from '
187                               'OSS-Fuzz; use corpus located in '
188                               'build/corpus/<project>/<fuzz_target>/')
189  coverage_parser.add_argument('--port',
190                               default='8008',
191                               help='specify port for'
192                               ' a local HTTP server rendering coverage report')
193  coverage_parser.add_argument('--fuzz-target',
194                               help='specify name of a fuzz '
195                               'target to be run for generating coverage '
196                               'report')
197  coverage_parser.add_argument('--corpus-dir',
198                               help='specify location of corpus'
199                               ' to be used (requires --fuzz-target argument)')
200  coverage_parser.add_argument('project_name', help='name of the project')
201  coverage_parser.add_argument('extra_args',
202                               help='additional arguments to '
203                               'pass to llvm-cov utility.',
204                               nargs='*')
205
206  download_corpora_parser = subparsers.add_parser(
207      'download_corpora', help='Download all corpora for a project.')
208  download_corpora_parser.add_argument('--fuzz-target',
209                                       help='specify name of a fuzz target')
210  download_corpora_parser.add_argument('project_name',
211                                       help='name of the project')
212
213  reproduce_parser = subparsers.add_parser('reproduce',
214                                           help='Reproduce a crash.')
215  reproduce_parser.add_argument('--valgrind',
216                                action='store_true',
217                                help='run with valgrind')
218  reproduce_parser.add_argument('project_name', help='name of the project')
219  reproduce_parser.add_argument('fuzzer_name', help='name of the fuzzer')
220  reproduce_parser.add_argument('testcase_path', help='path of local testcase')
221  reproduce_parser.add_argument('fuzzer_args',
222                                help='arguments to pass to the fuzzer',
223                                nargs=argparse.REMAINDER)
224  _add_environment_args(reproduce_parser)
225
226  shell_parser = subparsers.add_parser(
227      'shell', help='Run /bin/bash within the builder container.')
228  shell_parser.add_argument('project_name', help='name of the project')
229  shell_parser.add_argument('source_path',
230                            help='path of local source',
231                            nargs='?')
232  _add_architecture_args(shell_parser)
233  _add_engine_args(shell_parser)
234  _add_sanitizer_args(shell_parser)
235  _add_environment_args(shell_parser)
236
237  subparsers.add_parser('pull_images', help='Pull base images.')
238  return parser
239
240
241def is_base_image(image_name):
242  """Checks if the image name is a base image."""
243  return os.path.exists(os.path.join('infra', 'base-images', image_name))
244
245
246def check_project_exists(project_name):
247  """Checks if a project exists."""
248  if not os.path.exists(_get_project_dir(project_name)):
249    print(project_name, 'does not exist', file=sys.stderr)
250    return False
251
252  return True
253
254
255def _check_fuzzer_exists(project_name, fuzzer_name):
256  """Checks if a fuzzer exists."""
257  command = ['docker', 'run', '--rm']
258  command.extend(['-v', '%s:/out' % _get_output_dir(project_name)])
259  command.append('ubuntu:16.04')
260
261  command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name])
262
263  try:
264    subprocess.check_call(command)
265  except subprocess.CalledProcessError:
266    print(fuzzer_name,
267          'does not seem to exist. Please run build_fuzzers first.',
268          file=sys.stderr)
269    return False
270
271  return True
272
273
274def _get_absolute_path(path):
275  """Returns absolute path with user expansion."""
276  return os.path.abspath(os.path.expanduser(path))
277
278
279def _get_command_string(command):
280  """Returns a shell escaped command string."""
281  return ' '.join(pipes.quote(part) for part in command)
282
283
284def _get_project_dir(project_name):
285  """Returns path to the project."""
286  return os.path.join(OSS_FUZZ_DIR, 'projects', project_name)
287
288
289def get_dockerfile_path(project_name):
290  """Returns path to the project Dockerfile."""
291  return os.path.join(_get_project_dir(project_name), 'Dockerfile')
292
293
294def _get_corpus_dir(project_name=''):
295  """Creates and returns path to /corpus directory for the given project (if
296  specified)."""
297  directory = os.path.join(BUILD_DIR, 'corpus', project_name)
298  if not os.path.exists(directory):
299    os.makedirs(directory)
300
301  return directory
302
303
304def _get_output_dir(project_name=''):
305  """Creates and returns path to /out directory for the given project (if
306  specified)."""
307  directory = os.path.join(BUILD_DIR, 'out', project_name)
308  if not os.path.exists(directory):
309    os.makedirs(directory)
310
311  return directory
312
313
314def _get_work_dir(project_name=''):
315  """Creates and returns path to /work directory for the given project (if
316  specified)."""
317  directory = os.path.join(BUILD_DIR, 'work', project_name)
318  if not os.path.exists(directory):
319    os.makedirs(directory)
320
321  return directory
322
323
324def _get_project_language(project_name):
325  """Returns project language."""
326  project_yaml_path = os.path.join(OSS_FUZZ_DIR, 'projects', project_name,
327                                   'project.yaml')
328  with open(project_yaml_path) as file_handle:
329    content = file_handle.read()
330    for line in content.splitlines():
331      match = PROJECT_LANGUAGE_REGEX.match(line)
332      if match:
333        return match.group(1)
334
335  return None
336
337
338def _add_architecture_args(parser, choices=('x86_64', 'i386')):
339  """Add common architecture args."""
340  parser.add_argument('--architecture', default='x86_64', choices=choices)
341
342
343def _add_engine_args(parser,
344                     choices=('libfuzzer', 'afl', 'honggfuzz', 'dataflow',
345                              'none')):
346  """Add common engine args."""
347  parser.add_argument('--engine', default='libfuzzer', choices=choices)
348
349
350def _add_sanitizer_args(parser,
351                        choices=('address', 'memory', 'undefined', 'coverage',
352                                 'dataflow', 'thread')):
353  """Add common sanitizer args."""
354  parser.add_argument(
355      '--sanitizer',
356      default=None,
357      choices=choices,
358      help='the default is "address"; "dataflow" for "dataflow" engine')
359
360
361def _add_environment_args(parser):
362  """Add common environment args."""
363  parser.add_argument('-e',
364                      action='append',
365                      help="set environment variable e.g. VAR=value")
366
367
368def build_image_impl(image_name, no_cache=False, pull=False):
369  """Build image."""
370
371  proj_is_base_image = is_base_image(image_name)
372  if proj_is_base_image:
373    image_project = 'oss-fuzz-base'
374    dockerfile_dir = os.path.join('infra', 'base-images', image_name)
375  else:
376    image_project = 'oss-fuzz'
377    if not check_project_exists(image_name):
378      return False
379
380    dockerfile_dir = os.path.join('projects', image_name)
381
382  build_args = []
383  if no_cache:
384    build_args.append('--no-cache')
385
386  build_args += [
387      '-t', 'gcr.io/%s/%s' % (image_project, image_name), dockerfile_dir
388  ]
389
390  return docker_build(build_args, pull=pull)
391
392
393def _env_to_docker_args(env_list):
394  """Turn envirnoment variable list into docker arguments."""
395  return sum([['-e', v] for v in env_list], [])
396
397
398WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
399
400
401def workdir_from_lines(lines, default='/src'):
402  """Get the WORKDIR from the given lines."""
403  for line in reversed(lines):  # reversed to get last WORKDIR.
404    match = re.match(WORKDIR_REGEX, line)
405    if match:
406      workdir = match.group(1)
407      workdir = workdir.replace('$SRC', '/src')
408
409      if not os.path.isabs(workdir):
410        workdir = os.path.join('/src', workdir)
411
412      return os.path.normpath(workdir)
413
414  return default
415
416
417def _workdir_from_dockerfile(project_name):
418  """Parse WORKDIR from the Dockerfile for the given project."""
419  dockerfile_path = get_dockerfile_path(project_name)
420
421  with open(dockerfile_path) as file_handle:
422    lines = file_handle.readlines()
423
424  return workdir_from_lines(lines, default=os.path.join('/src', project_name))
425
426
427def docker_run(run_args, print_output=True):
428  """Call `docker run`."""
429  command = ['docker', 'run', '--rm', '--privileged']
430
431  # Support environments with a TTY.
432  if sys.stdin.isatty():
433    command.append('-i')
434
435  command.extend(run_args)
436
437  print('Running:', _get_command_string(command))
438  stdout = None
439  if not print_output:
440    stdout = open(os.devnull, 'w')
441
442  try:
443    subprocess.check_call(command, stdout=stdout, stderr=subprocess.STDOUT)
444  except subprocess.CalledProcessError as error:
445    return error.returncode
446
447  return 0
448
449
450def docker_build(build_args, pull=False):
451  """Call `docker build`."""
452  command = ['docker', 'build']
453  if pull:
454    command.append('--pull')
455
456  command.extend(build_args)
457  print('Running:', _get_command_string(command))
458
459  try:
460    subprocess.check_call(command)
461  except subprocess.CalledProcessError:
462    print('docker build failed.', file=sys.stderr)
463    return False
464
465  return True
466
467
468def docker_pull(image):
469  """Call `docker pull`."""
470  command = ['docker', 'pull', image]
471  print('Running:', _get_command_string(command))
472
473  try:
474    subprocess.check_call(command)
475  except subprocess.CalledProcessError:
476    print('docker pull failed.', file=sys.stderr)
477    return False
478
479  return True
480
481
482def build_image(args):
483  """Build docker image."""
484  if args.pull and args.no_pull:
485    print('Incompatible arguments --pull and --no-pull.')
486    return 1
487
488  if args.pull:
489    pull = True
490  elif args.no_pull:
491    pull = False
492  else:
493    y_or_n = raw_input('Pull latest base images (compiler/runtime)? (y/N): ')
494    pull = y_or_n.lower() == 'y'
495
496  if pull:
497    print('Pulling latest base images...')
498  else:
499    print('Using cached base images...')
500
501  # If build_image is called explicitly, don't use cache.
502  if build_image_impl(args.project_name, no_cache=True, pull=pull):
503    return 0
504
505  return 1
506
507
508def build_fuzzers_impl(  # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
509    project_name,
510    clean,
511    engine,
512    sanitizer,
513    architecture,
514    env_to_add,
515    source_path,
516    no_cache=False,
517    mount_location=None):
518  """Build fuzzers."""
519  if not build_image_impl(project_name, no_cache=no_cache):
520    return 1
521
522  project_out_dir = _get_output_dir(project_name)
523  project_work_dir = _get_work_dir(project_name)
524  project_language = _get_project_language(project_name)
525  if not project_language:
526    print('WARNING: language not specified in project.yaml. Build may fail.')
527
528  if clean:
529    print('Cleaning existing build artifacts.')
530
531    # Clean old and possibly conflicting artifacts in project's out directory.
532    docker_run([
533        '-v',
534        '%s:/out' % project_out_dir, '-t',
535        'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /out/*'
536    ])
537
538    docker_run([
539        '-v',
540        '%s:/work' % project_work_dir, '-t',
541        'gcr.io/oss-fuzz/%s' % project_name, '/bin/bash', '-c', 'rm -rf /work/*'
542    ])
543
544  else:
545    print('Keeping existing build artifacts as-is (if any).')
546  env = [
547      'FUZZING_ENGINE=' + engine,
548      'SANITIZER=' + sanitizer,
549      'ARCHITECTURE=' + architecture,
550  ]
551
552  if project_language:
553    env.append('FUZZING_LANGUAGE=' + project_language)
554
555  if env_to_add:
556    env += env_to_add
557
558  # Copy instrumented libraries.
559  if sanitizer == 'memory':
560    docker_run([
561        '-v',
562        '%s:/work' % project_work_dir, 'gcr.io/oss-fuzz-base/msan-libs-builder',
563        'bash', '-c', 'cp -r /msan /work'
564    ])
565    env.append('MSAN_LIBS_PATH=' + '/work/msan')
566
567  command = ['--cap-add', 'SYS_PTRACE'] + _env_to_docker_args(env)
568  if source_path:
569    workdir = _workdir_from_dockerfile(project_name)
570    if mount_location:
571      command += [
572          '-v',
573          '%s:%s' % (_get_absolute_path(source_path), mount_location),
574      ]
575    else:
576      if workdir == '/src':
577        print('Cannot use local checkout with "WORKDIR: /src".',
578              file=sys.stderr)
579        return 1
580
581      command += [
582          '-v',
583          '%s:%s' % (_get_absolute_path(source_path), workdir),
584      ]
585
586  command += [
587      '-v',
588      '%s:/out' % project_out_dir, '-v',
589      '%s:/work' % project_work_dir, '-t',
590      'gcr.io/oss-fuzz/%s' % project_name
591  ]
592
593  result_code = docker_run(command)
594  if result_code:
595    print('Building fuzzers failed.', file=sys.stderr)
596    return result_code
597
598  # Patch MSan builds to use instrumented shared libraries.
599  if sanitizer == 'memory':
600    docker_run([
601        '-v',
602        '%s:/out' % project_out_dir, '-v',
603        '%s:/work' % project_work_dir
604    ] + _env_to_docker_args(env) + [
605        'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', 'patch_build.py',
606        '/out'
607    ])
608
609  return 0
610
611
612def build_fuzzers(args):
613  """Build fuzzers."""
614  return build_fuzzers_impl(args.project_name, args.clean, args.engine,
615                            args.sanitizer, args.architecture, args.e,
616                            args.source_path)
617
618
619def check_build(args):
620  """Checks that fuzzers in the container execute without errors."""
621  if not check_project_exists(args.project_name):
622    return 1
623
624  if (args.fuzzer_name and
625      not _check_fuzzer_exists(args.project_name, args.fuzzer_name)):
626    return 1
627
628  fuzzing_language = _get_project_language(args.project_name)
629  if fuzzing_language is None:
630    print('WARNING: language not specified in project.yaml. Defaulting to C++.')
631    fuzzing_language = 'c++'
632
633  env = [
634      'FUZZING_ENGINE=' + args.engine,
635      'SANITIZER=' + args.sanitizer,
636      'ARCHITECTURE=' + args.architecture,
637      'FUZZING_LANGUAGE=' + fuzzing_language,
638  ]
639  if args.e:
640    env += args.e
641
642  run_args = _env_to_docker_args(env) + [
643      '-v',
644      '%s:/out' % _get_output_dir(args.project_name), '-t',
645      'gcr.io/oss-fuzz-base/base-runner'
646  ]
647
648  if args.fuzzer_name:
649    run_args += ['test_one.py', args.fuzzer_name]
650  else:
651    run_args.append('test_all.py')
652
653  exit_code = docker_run(run_args)
654  if exit_code == 0:
655    print('Check build passed.')
656  else:
657    print('Check build failed.')
658
659  return exit_code
660
661
662def _get_fuzz_targets(project_name):
663  """Return names of fuzz targest build in the project's /out directory."""
664  fuzz_targets = []
665  for name in os.listdir(_get_output_dir(project_name)):
666    if name.startswith('afl-'):
667      continue
668
669    path = os.path.join(_get_output_dir(project_name), name)
670    if os.path.isfile(path) and os.access(path, os.X_OK):
671      fuzz_targets.append(name)
672
673  return fuzz_targets
674
675
676def _get_latest_corpus(project_name, fuzz_target, base_corpus_dir):
677  """Download the latest corpus for the given fuzz target."""
678  corpus_dir = os.path.join(base_corpus_dir, fuzz_target)
679  if not os.path.exists(corpus_dir):
680    os.makedirs(corpus_dir)
681
682  if not fuzz_target.startswith(project_name + '_'):
683    fuzz_target = '%s_%s' % (project_name, fuzz_target)
684
685  corpus_backup_url = CORPUS_BACKUP_URL_FORMAT.format(project_name=project_name,
686                                                      fuzz_target=fuzz_target)
687  command = ['gsutil', 'ls', corpus_backup_url]
688
689  # Don't capture stderr. We want it to print in real time, in case gsutil is
690  # asking for two-factor authentication.
691  corpus_listing = subprocess.Popen(command, stdout=subprocess.PIPE)
692  output, _ = corpus_listing.communicate()
693
694  # Some fuzz targets (e.g. new ones) may not have corpus yet, just skip those.
695  if corpus_listing.returncode:
696    print('WARNING: corpus for {0} not found:\n'.format(fuzz_target),
697          file=sys.stderr)
698    return
699
700  if output:
701    latest_backup_url = output.splitlines()[-1]
702    archive_path = corpus_dir + '.zip'
703    command = ['gsutil', '-q', 'cp', latest_backup_url, archive_path]
704    subprocess.check_call(command)
705
706    command = ['unzip', '-q', '-o', archive_path, '-d', corpus_dir]
707    subprocess.check_call(command)
708    os.remove(archive_path)
709  else:
710    # Sync the working corpus copy if a minimized backup is not available.
711    corpus_url = CORPUS_URL_FORMAT.format(project_name=project_name,
712                                          fuzz_target=fuzz_target)
713    command = ['gsutil', '-m', '-q', 'rsync', '-R', corpus_url, corpus_dir]
714    subprocess.check_call(command)
715
716
717def download_corpora(args):
718  """Download most recent corpora from GCS for the given project."""
719  if not check_project_exists(args.project_name):
720    return 1
721
722  try:
723    with open(os.devnull, 'w') as stdout:
724      subprocess.check_call(['gsutil', '--version'], stdout=stdout)
725  except OSError:
726    print(
727        'ERROR: gsutil not found. Please install it from '
728        'https://cloud.google.com/storage/docs/gsutil_install',
729        file=sys.stderr)
730    return False
731
732  if args.fuzz_target:
733    fuzz_targets = [args.fuzz_target]
734  else:
735    fuzz_targets = _get_fuzz_targets(args.project_name)
736
737  corpus_dir = _get_corpus_dir(args.project_name)
738  if not os.path.exists(corpus_dir):
739    os.makedirs(corpus_dir)
740
741  def _download_for_single_target(fuzz_target):
742    try:
743      _get_latest_corpus(args.project_name, fuzz_target, corpus_dir)
744      return True
745    except Exception as error:  # pylint:disable=broad-except
746      print('ERROR: corpus download for %s failed: %s' %
747            (fuzz_target, str(error)),
748            file=sys.stderr)
749      return False
750
751  print('Downloading corpora for %s project to %s' %
752        (args.project_name, corpus_dir))
753  thread_pool = ThreadPool()
754  return all(thread_pool.map(_download_for_single_target, fuzz_targets))
755
756
757def coverage(args):
758  """Generate code coverage using clang source based code coverage."""
759  if args.corpus_dir and not args.fuzz_target:
760    print(
761        'ERROR: --corpus-dir requires specifying a particular fuzz target '
762        'using --fuzz-target',
763        file=sys.stderr)
764    return 1
765
766  if not check_project_exists(args.project_name):
767    return 1
768
769  project_language = _get_project_language(args.project_name)
770  if project_language not in LANGUAGES_WITH_COVERAGE_SUPPORT:
771    print(
772        'ERROR: Project is written in %s, coverage for it is not supported yet.'
773        % project_language,
774        file=sys.stderr)
775    return 1
776
777  if not args.no_corpus_download and not args.corpus_dir:
778    if not download_corpora(args):
779      return 1
780
781  env = [
782      'FUZZING_ENGINE=libfuzzer',
783      'FUZZING_LANGUAGE=%s' % project_language,
784      'PROJECT=%s' % args.project_name,
785      'SANITIZER=coverage',
786      'HTTP_PORT=%s' % args.port,
787      'COVERAGE_EXTRA_ARGS=%s' % ' '.join(args.extra_args),
788  ]
789
790  run_args = _env_to_docker_args(env)
791
792  if args.port:
793    run_args.extend([
794        '-p',
795        '%s:%s' % (args.port, args.port),
796    ])
797
798  if args.corpus_dir:
799    if not os.path.exists(args.corpus_dir):
800      print('ERROR: the path provided in --corpus-dir argument does not exist',
801            file=sys.stderr)
802      return 1
803    corpus_dir = os.path.realpath(args.corpus_dir)
804    run_args.extend(['-v', '%s:/corpus/%s' % (corpus_dir, args.fuzz_target)])
805  else:
806    run_args.extend(['-v', '%s:/corpus' % _get_corpus_dir(args.project_name)])
807
808  run_args.extend([
809      '-v',
810      '%s:/out' % _get_output_dir(args.project_name),
811      '-t',
812      'gcr.io/oss-fuzz-base/base-runner',
813  ])
814
815  run_args.append('coverage')
816  if args.fuzz_target:
817    run_args.append(args.fuzz_target)
818
819  exit_code = docker_run(run_args)
820  if exit_code == 0:
821    print('Successfully generated clang code coverage report.')
822  else:
823    print('Failed to generate clang code coverage report.')
824
825  return exit_code
826
827
828def run_fuzzer(args):
829  """Runs a fuzzer in the container."""
830  if not check_project_exists(args.project_name):
831    return 1
832
833  if not _check_fuzzer_exists(args.project_name, args.fuzzer_name):
834    return 1
835
836  env = [
837      'FUZZING_ENGINE=' + args.engine,
838      'SANITIZER=' + args.sanitizer,
839      'RUN_FUZZER_MODE=interactive',
840  ]
841
842  if args.e:
843    env += args.e
844
845  run_args = _env_to_docker_args(env)
846
847  if args.corpus_dir:
848    if not os.path.exists(args.corpus_dir):
849      print('ERROR: the path provided in --corpus-dir argument does not exist',
850            file=sys.stderr)
851      return 1
852    corpus_dir = os.path.realpath(args.corpus_dir)
853    run_args.extend([
854        '-v',
855        '{corpus_dir}:/tmp/{fuzzer}_corpus'.format(corpus_dir=corpus_dir,
856                                                   fuzzer=args.fuzzer_name)
857    ])
858
859  run_args.extend([
860      '-v',
861      '%s:/out' % _get_output_dir(args.project_name),
862      '-t',
863      'gcr.io/oss-fuzz-base/base-runner',
864      'run_fuzzer',
865      args.fuzzer_name,
866  ] + args.fuzzer_args)
867
868  return docker_run(run_args)
869
870
871def reproduce(args):
872  """Reproduce a specific test case from a specific project."""
873  return reproduce_impl(args.project_name, args.fuzzer_name, args.valgrind,
874                        args.e, args.fuzzer_args, args.testcase_path)
875
876
877def reproduce_impl(  # pylint: disable=too-many-arguments
878    project_name,
879    fuzzer_name,
880    valgrind,
881    env_to_add,
882    fuzzer_args,
883    testcase_path,
884    runner=docker_run,
885    err_result=1):
886  """Reproduces a testcase in the container."""
887  if not check_project_exists(project_name):
888    return err_result
889
890  if not _check_fuzzer_exists(project_name, fuzzer_name):
891    return err_result
892
893  debugger = ''
894  env = []
895  image_name = 'base-runner'
896
897  if valgrind:
898    debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full'
899
900  if debugger:
901    image_name = 'base-runner-debug'
902    env += ['DEBUGGER=' + debugger]
903
904  if env_to_add:
905    env += env_to_add
906
907  run_args = _env_to_docker_args(env) + [
908      '-v',
909      '%s:/out' % _get_output_dir(project_name),
910      '-v',
911      '%s:/testcase' % _get_absolute_path(testcase_path),
912      '-t',
913      'gcr.io/oss-fuzz-base/%s' % image_name,
914      'reproduce',
915      fuzzer_name,
916      '-runs=100',
917  ] + fuzzer_args
918
919  return runner(run_args)
920
921
922def generate(args):
923  """Generate empty project files."""
924  if len(args.project_name) > MAX_PROJECT_NAME_LENGTH:
925    print('Project name needs to be less than or equal to %d characters.' %
926          MAX_PROJECT_NAME_LENGTH,
927          file=sys.stderr)
928    return 1
929
930  if not VALID_PROJECT_NAME_REGEX.match(args.project_name):
931    print('Invalid project name.', file=sys.stderr)
932    return 1
933
934  directory = os.path.join('projects', args.project_name)
935
936  try:
937    os.mkdir(directory)
938  except OSError as error:
939    if error.errno != errno.EEXIST:
940      raise
941    print(directory, 'already exists.', file=sys.stderr)
942    return 1
943
944  print('Writing new files to', directory)
945
946  template_args = {
947      'project_name': args.project_name,
948      'year': datetime.datetime.now().year
949  }
950  with open(os.path.join(directory, 'project.yaml'), 'w') as file_handle:
951    file_handle.write(templates.PROJECT_YAML_TEMPLATE % template_args)
952
953  with open(os.path.join(directory, 'Dockerfile'), 'w') as file_handle:
954    file_handle.write(templates.DOCKER_TEMPLATE % template_args)
955
956  build_sh_path = os.path.join(directory, 'build.sh')
957  with open(build_sh_path, 'w') as file_handle:
958    file_handle.write(templates.BUILD_TEMPLATE % template_args)
959
960  os.chmod(build_sh_path, 0o755)
961  return 0
962
963
964def shell(args):
965  """Runs a shell within a docker image."""
966  if not build_image_impl(args.project_name):
967    return 1
968
969  env = [
970      'FUZZING_ENGINE=' + args.engine,
971      'SANITIZER=' + args.sanitizer,
972      'ARCHITECTURE=' + args.architecture,
973  ]
974
975  if args.project_name != 'base-runner-debug':
976    env.append('FUZZING_LANGUAGE=' + _get_project_language(args.project_name))
977
978  if args.e:
979    env += args.e
980
981  if is_base_image(args.project_name):
982    image_project = 'oss-fuzz-base'
983    out_dir = _get_output_dir()
984  else:
985    image_project = 'oss-fuzz'
986    out_dir = _get_output_dir(args.project_name)
987
988  run_args = _env_to_docker_args(env)
989  if args.source_path:
990    run_args.extend([
991        '-v',
992        '%s:%s' % (_get_absolute_path(args.source_path), '/src'),
993    ])
994
995  run_args.extend([
996      '-v',
997      '%s:/out' % out_dir, '-v',
998      '%s:/work' % _get_work_dir(args.project_name), '-t',
999      'gcr.io/%s/%s' % (image_project, args.project_name), '/bin/bash'
1000  ])
1001
1002  docker_run(run_args)
1003  return 0
1004
1005
1006def pull_images(_):
1007  """Pull base images."""
1008  for base_image in BASE_IMAGES:
1009    if not docker_pull(base_image):
1010      return 1
1011
1012  return 0
1013
1014
1015if __name__ == '__main__':
1016  sys.exit(main())
1017