1#!/usr/bin/env python
2# Copyright 2017 Google Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16################################################################################
17
18from __future__ import print_function
19import argparse
20import imp
21import os
22import multiprocessing
23import resource
24import shutil
25import subprocess
26import tempfile
27
28import apt
29from apt import debfile
30
31from packages import package
32import wrapper_utils
33
34SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
35PACKAGES_DIR = os.path.join(SCRIPT_DIR, 'packages')
36
37TRACK_ORIGINS_ARG = '-fsanitize-memory-track-origins='
38
39INJECTED_ARGS = [
40    '-fsanitize=memory',
41    '-fsanitize-recover=memory',
42    '-fPIC',
43    '-fno-omit-frame-pointer',
44]
45
46
47class MSanBuildException(Exception):
48  """Base exception."""
49
50
51def GetTrackOriginsFlag():
52  """Get the track origins flag."""
53  if os.getenv('MSAN_NO_TRACK_ORIGINS'):
54    return TRACK_ORIGINS_ARG + '0'
55
56  return TRACK_ORIGINS_ARG + '2'
57
58
59def GetInjectedFlags():
60  return INJECTED_ARGS + [GetTrackOriginsFlag()]
61
62
63def SetUpEnvironment(work_dir):
64  """Set up build environment."""
65  env = {}
66  env['REAL_CLANG_PATH'] = subprocess.check_output(['which', 'clang']).strip()
67  print('Real clang at', env['REAL_CLANG_PATH'])
68  compiler_wrapper_path = os.path.join(SCRIPT_DIR, 'compiler_wrapper.py')
69
70  # Symlink binaries into TMP/bin
71  bin_dir = os.path.join(work_dir, 'bin')
72  os.mkdir(bin_dir)
73
74  dpkg_host_architecture = wrapper_utils.DpkgHostArchitecture()
75  wrapper_utils.CreateSymlinks(
76      compiler_wrapper_path,
77      bin_dir,
78      [
79          'clang',
80          'clang++',
81          # Not all build rules respect $CC/$CXX, so make additional symlinks.
82          'gcc',
83          'g++',
84          'cc',
85          'c++',
86          dpkg_host_architecture + '-gcc',
87          dpkg_host_architecture + '-g++',
88      ])
89
90  env['CC'] = os.path.join(bin_dir, 'clang')
91  env['CXX'] = os.path.join(bin_dir, 'clang++')
92
93  MSAN_OPTIONS = ' '.join(GetInjectedFlags())
94
95  # We don't use nostrip because some build rules incorrectly break when it is
96  # passed. Instead we install our own no-op strip binaries.
97  env['DEB_BUILD_OPTIONS'] = ('nocheck parallel=%d' %
98                              multiprocessing.cpu_count())
99  env['DEB_CFLAGS_APPEND'] = MSAN_OPTIONS
100  env['DEB_CXXFLAGS_APPEND'] = MSAN_OPTIONS + ' -stdlib=libc++'
101  env['DEB_CPPFLAGS_APPEND'] = MSAN_OPTIONS
102  env['DEB_LDFLAGS_APPEND'] = MSAN_OPTIONS
103  env['DPKG_GENSYMBOLS_CHECK_LEVEL'] = '0'
104
105  # debian/rules can set DPKG_GENSYMBOLS_CHECK_LEVEL explicitly, so override it.
106  gen_symbols_wrapper = ('#!/bin/sh\n'
107                         'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n'
108                         '/usr/bin/dpkg-gensymbols "$@"\n')
109
110  wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols', gen_symbols_wrapper)
111
112  # Install no-op strip binaries.
113  no_op_strip = ('#!/bin/sh\n' 'exit 0\n')
114  wrapper_utils.InstallWrapper(bin_dir, 'strip', no_op_strip,
115                               [dpkg_host_architecture + '-strip'])
116
117  env['PATH'] = bin_dir + ':' + os.environ['PATH']
118
119  # nocheck doesn't disable override_dh_auto_test. So we have this hack to try
120  # to disable "make check" or "make test" invocations.
121  make_wrapper = ('#!/bin/bash\n'
122                  'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n'
123                  '  exit 0\n'
124                  'fi\n'
125                  '/usr/bin/make "$@"\n')
126  wrapper_utils.InstallWrapper(bin_dir, 'make', make_wrapper)
127
128  # Prevent entire build from failing because of bugs/uninstrumented in tools
129  # that are part of the build.
130  msan_log_dir = os.path.join(work_dir, 'msan')
131  os.mkdir(msan_log_dir)
132  msan_log_path = os.path.join(msan_log_dir, 'log')
133  env['MSAN_OPTIONS'] = ('halt_on_error=0:exitcode=0:report_umrs=0:log_path=' +
134                         msan_log_path)
135
136  # Increase maximum stack size to prevent tests from failing.
137  limit = 128 * 1024 * 1024
138  resource.setrlimit(resource.RLIMIT_STACK, (limit, limit))
139  return env
140
141
142def FindPackageDebs(package_name, work_directory):
143  """Find package debs."""
144  deb_paths = []
145  cache = apt.Cache()
146
147  for filename in os.listdir(work_directory):
148    file_path = os.path.join(work_directory, filename)
149    if not file_path.endswith('.deb'):
150      continue
151
152    # Matching package name.
153    deb = debfile.DebPackage(file_path)
154    if deb.pkgname == package_name:
155      deb_paths.append(file_path)
156      continue
157
158    # Also include -dev packages that depend on the runtime package.
159    pkg = cache[deb.pkgname]
160    if pkg.section != 'libdevel' and pkg.section != 'universe/libdevel':
161      continue
162
163    # But ignore -dbg packages.
164    if deb.pkgname.endswith('-dbg'):
165      continue
166
167    for dependency in deb.depends:
168      if any(dep[0] == package_name for dep in dependency):
169        deb_paths.append(file_path)
170        break
171
172  return deb_paths
173
174
175def ExtractLibraries(deb_paths, work_directory, output_directory):
176  """Extract libraries from .deb packages."""
177  extract_directory = os.path.join(work_directory, 'extracted')
178  if os.path.exists(extract_directory):
179    shutil.rmtree(extract_directory, ignore_errors=True)
180
181  os.mkdir(extract_directory)
182
183  for deb_path in deb_paths:
184    subprocess.check_call(['dpkg-deb', '-x', deb_path, extract_directory])
185
186  extracted = []
187  for root, _, filenames in os.walk(extract_directory):
188    if 'libx32' in root or 'lib32' in root:
189      continue
190
191    for filename in filenames:
192      if (not filename.endswith('.so') and '.so.' not in filename and
193          not filename.endswith('.a') and '.a' not in filename):
194        continue
195
196      file_path = os.path.join(root, filename)
197      rel_file_path = os.path.relpath(file_path, extract_directory)
198      rel_directory = os.path.dirname(rel_file_path)
199
200      target_dir = os.path.join(output_directory, rel_directory)
201      if not os.path.exists(target_dir):
202        os.makedirs(target_dir)
203
204      target_file_path = os.path.join(output_directory, rel_file_path)
205      extracted.append(target_file_path)
206
207      if os.path.lexists(target_file_path):
208        os.remove(target_file_path)
209
210      if os.path.islink(file_path):
211        link_path = os.readlink(file_path)
212        if os.path.isabs(link_path):
213          # Make absolute links relative.
214          link_path = os.path.relpath(link_path,
215                                      os.path.join('/', rel_directory))
216
217        os.symlink(link_path, target_file_path)
218      else:
219        shutil.copy2(file_path, target_file_path)
220
221  return extracted
222
223
224def GetPackage(package_name):
225  apt_cache = apt.Cache()
226  version = apt_cache[package_name].candidate
227  source_name = version.source_name
228  local_source_name = source_name.replace('.', '_')
229
230  custom_package_path = os.path.join(PACKAGES_DIR, local_source_name) + '.py'
231  if not os.path.exists(custom_package_path):
232    print('Using default package build steps.')
233    return package.Package(source_name, version)
234
235  print('Using custom package build steps.')
236  module = imp.load_source('packages.' + local_source_name, custom_package_path)
237  return module.Package(version)
238
239
240def PatchRpath(path, output_directory):
241  """Patch rpath to be relative to $ORIGIN."""
242  try:
243    rpaths = subprocess.check_output(['patchelf', '--print-rpath',
244                                      path]).strip()
245  except subprocess.CalledProcessError:
246    return
247
248  if not rpaths:
249    return
250
251  processed_rpath = []
252  rel_directory = os.path.join(
253      '/', os.path.dirname(os.path.relpath(path, output_directory)))
254
255  for rpath in rpaths.split(':'):
256    if '$ORIGIN' in rpath:
257      # Already relative.
258      processed_rpath.append(rpath)
259      continue
260
261    processed_rpath.append(
262        os.path.join('$ORIGIN', os.path.relpath(rpath, rel_directory)))
263
264  processed_rpath = ':'.join(processed_rpath)
265  print('Patching rpath for', path, 'to', processed_rpath)
266  subprocess.check_call(
267      ['patchelf', '--force-rpath', '--set-rpath', processed_rpath, path])
268
269
270def _CollectDependencies(apt_cache, pkg, cache, dependencies):
271  """Collect dependencies that need to be built."""
272  C_OR_CXX_DEPS = [
273      'libc++1',
274      'libc6',
275      'libc++abi1',
276      'libgcc1',
277      'libstdc++6',
278  ]
279
280  BLACKLISTED_PACKAGES = [
281      'libcapnp-0.5.3',  # fails to compile on newer clang.
282      'libllvm5.0',
283      'libmircore1',
284      'libmircommon7',
285      'libmirclient9',
286      'libmirprotobuf3',
287      'multiarch-support',
288  ]
289
290  if pkg.name in BLACKLISTED_PACKAGES:
291    return False
292
293  if pkg.section != 'libs' and pkg.section != 'universe/libs':
294    return False
295
296  if pkg.name in C_OR_CXX_DEPS:
297    return True
298
299  is_c_or_cxx = False
300  for dependency in pkg.candidate.dependencies:
301    dependency = dependency[0]
302
303    if dependency.name in cache:
304      is_c_or_cxx |= cache[dependency.name]
305    else:
306      is_c_or_cxx |= _CollectDependencies(apt_cache, apt_cache[dependency.name],
307                                          cache, dependencies)
308  if is_c_or_cxx:
309    dependencies.append(pkg.name)
310
311  cache[pkg.name] = is_c_or_cxx
312  return is_c_or_cxx
313
314
315def GetBuildList(package_name):
316  """Get list of packages that need to be built including dependencies."""
317  apt_cache = apt.Cache()
318  pkg = apt_cache[package_name]
319
320  dependencies = []
321  _CollectDependencies(apt_cache, pkg, {}, dependencies)
322  return dependencies
323
324
325class MSanBuilder(object):
326  """MSan builder."""
327
328  def __init__(self,
329               debug=False,
330               log_path=None,
331               work_dir=None,
332               no_track_origins=False):
333    self.debug = debug
334    self.log_path = log_path
335    self.work_dir = work_dir
336    self.no_track_origins = no_track_origins
337    self.env = None
338
339  def __enter__(self):
340    if not self.work_dir:
341      self.work_dir = tempfile.mkdtemp(dir=self.work_dir)
342
343    if os.path.exists(self.work_dir):
344      shutil.rmtree(self.work_dir, ignore_errors=True)
345
346    os.makedirs(self.work_dir)
347    self.env = SetUpEnvironment(self.work_dir)
348
349    if self.debug and self.log_path:
350      self.env['WRAPPER_DEBUG_LOG_PATH'] = self.log_path
351
352    if self.no_track_origins:
353      self.env['MSAN_NO_TRACK_ORIGINS'] = '1'
354
355    return self
356
357  def __exit__(self, exc_type, exc_value, traceback):
358    if not self.debug:
359      shutil.rmtree(self.work_dir, ignore_errors=True)
360
361  def Build(self, package_name, output_directory, create_subdirs=False):
362    """Build the package and write results into the output directory."""
363    deb_paths = FindPackageDebs(package_name, self.work_dir)
364    if deb_paths:
365      print('Source package already built for', package_name)
366    else:
367      pkg = GetPackage(package_name)
368
369      pkg.InstallBuildDeps()
370      source_directory = pkg.DownloadSource(self.work_dir)
371      print('Source downloaded to', source_directory)
372
373      # custom bin directory for custom build scripts to write wrappers.
374      custom_bin_dir = os.path.join(self.work_dir, package_name + '_bin')
375      os.mkdir(custom_bin_dir)
376      env = self.env.copy()
377      env['PATH'] = custom_bin_dir + ':' + env['PATH']
378
379      pkg.Build(source_directory, env, custom_bin_dir)
380      shutil.rmtree(custom_bin_dir, ignore_errors=True)
381
382      deb_paths = FindPackageDebs(package_name, self.work_dir)
383
384    if not deb_paths:
385      raise MSanBuildException('Failed to find .deb packages.')
386
387    print('Extracting', ' '.join(deb_paths))
388
389    if create_subdirs:
390      extract_directory = os.path.join(output_directory, package_name)
391    else:
392      extract_directory = output_directory
393
394    extracted_paths = ExtractLibraries(deb_paths, self.work_dir,
395                                       extract_directory)
396    for extracted_path in extracted_paths:
397      if os.path.islink(extracted_path):
398        continue
399      if os.path.basename(extracted_path) == 'llvm-symbolizer':
400        continue
401      PatchRpath(extracted_path, extract_directory)
402
403
404def main():
405  parser = argparse.ArgumentParser('msan_build.py', description='MSan builder.')
406  parser.add_argument('package_names', nargs='+', help='Name of the packages.')
407  parser.add_argument('output_dir', help='Output directory.')
408  parser.add_argument('--create-subdirs',
409                      action='store_true',
410                      help=('Create subdirectories in the output '
411                            'directory for each package.'))
412  parser.add_argument('--work-dir', help='Work directory.')
413  parser.add_argument('--no-build-deps',
414                      action='store_true',
415                      help='Don\'t build dependencies.')
416  parser.add_argument('--debug', action='store_true', help='Enable debug mode.')
417  parser.add_argument('--log-path', help='Log path for debugging.')
418  parser.add_argument('--no-track-origins',
419                      action='store_true',
420                      help='Build with -fsanitize-memory-track-origins=0.')
421  args = parser.parse_args()
422
423  if args.no_track_origins:
424    os.environ['MSAN_NO_TRACK_ORIGINS'] = '1'
425
426  if not os.path.exists(args.output_dir):
427    os.makedirs(args.output_dir)
428
429  if args.no_build_deps:
430    package_names = args.package_names
431  else:
432    all_packages = set()
433    package_names = []
434
435    # Get list of packages to build, including all dependencies.
436    for package_name in args.package_names:
437      for dep in GetBuildList(package_name):
438        if dep in all_packages:
439          continue
440
441        if args.create_subdirs:
442          os.mkdir(os.path.join(args.output_dir, dep))
443
444        all_packages.add(dep)
445        package_names.append(dep)
446
447  print('Going to build:')
448  for package_name in package_names:
449    print('\t', package_name)
450
451  with MSanBuilder(debug=args.debug,
452                   log_path=args.log_path,
453                   work_dir=args.work_dir,
454                   no_track_origins=args.no_track_origins) as builder:
455    for package_name in package_names:
456      builder.Build(package_name, args.output_dir, args.create_subdirs)
457
458
459if __name__ == '__main__':
460  main()
461