1#!/usr/bin/env python3
2# Copyright (C) 2019 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This tool uses a collection of BUILD.gn files and build targets to generate
17# an "amalgamated" C++ header and source file pair which compiles to an
18# equivalent program. The tool also outputs the necessary compiler and linker
19# flags needed to compile the resulting source code.
20
21from __future__ import print_function
22import argparse
23import os
24import re
25import shutil
26import subprocess
27import sys
28import tempfile
29
30import gn_utils
31
32# Default targets to include in the result.
33# TODO(primiano): change this script to recurse into target deps when generating
34# headers, but only for proto targets. .pbzero.h files don't include each other
35# and we need to list targets here individually, which is unmaintainable.
36default_targets = [
37    '//:libperfetto_client_experimental',
38    '//include/perfetto/protozero:protozero',
39    '//protos/perfetto/config:zero',
40    '//protos/perfetto/trace:zero',
41]
42
43# Arguments for the GN output directory (unless overridden from the command
44# line).
45gn_args = ' '.join([
46    'is_debug=false',
47    'is_perfetto_build_generator=true',
48    'is_perfetto_embedder=true',
49    'use_custom_libcxx=false',
50    'enable_perfetto_ipc=true',
51])
52
53# By default, the amalgamated .h only recurses in #includes but not in the
54# target deps. In the case of protos we want to follow deps even in lieu of
55# direct #includes. This is because, by design, protozero headers don't
56# include each other but rely on forward declarations. The alternative would
57# be adding each proto sub-target individually (e.g. //proto/trace/gpu:zero),
58# but doing that is unmaintainable. We also do this for cpp bindings since some
59# tracing SDK functions depend on them (and the system tracing IPC mechanism
60# does so too).
61recurse_in_header_deps = '^//protos/.*(cpp|zero)$'
62
63# Compiler flags which aren't filtered out.
64cflag_allowlist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$'
65
66# Linker flags which aren't filtered out.
67ldflag_allowlist = r'^-()$'
68
69# Libraries which are filtered out.
70lib_denylist = r'^(c|gcc_eh)$'
71
72# Macros which aren't filtered out.
73define_allowlist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$'
74
75# Includes which will be removed from the generated source.
76includes_to_remove = r'^(gtest).*$'
77
78default_cflags = [
79    # Since we're expanding header files into the generated source file, some
80    # constant may remain unused.
81    '-Wno-unused-const-variable'
82]
83
84# Build flags to satisfy a protobuf (lite or full) dependency.
85protobuf_cflags = [
86    # Note that these point to the local copy of protobuf in buildtools. In
87    # reality the user of the amalgamated result will have to provide a path to
88    # an installed copy of the exact same version of protobuf which was used to
89    # generate the amalgamated build.
90    '-isystembuildtools/protobuf/src',
91    '-Lbuildtools/protobuf/src/.libs',
92    # We also need to disable some warnings for protobuf.
93    '-Wno-missing-prototypes',
94    '-Wno-missing-variable-declarations',
95    '-Wno-sign-conversion',
96    '-Wno-unknown-pragmas',
97    '-Wno-unused-macros',
98]
99
100# A mapping of dependencies to system libraries. Libraries in this map will not
101# be built statically but instead added as dependencies of the amalgamated
102# project.
103system_library_map = {
104    '//buildtools:protobuf_full': {
105        'libs': ['protobuf'],
106        'cflags': protobuf_cflags,
107    },
108    '//buildtools:protobuf_lite': {
109        'libs': ['protobuf-lite'],
110        'cflags': protobuf_cflags,
111    },
112    '//buildtools:protoc_lib': {
113        'libs': ['protoc']
114    },
115}
116
117# ----------------------------------------------------------------------------
118# End of configuration.
119# ----------------------------------------------------------------------------
120
121tool_name = os.path.basename(__file__)
122project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
123preamble = """// Copyright (C) 2019 The Android Open Source Project
124//
125// Licensed under the Apache License, Version 2.0 (the "License");
126// you may not use this file except in compliance with the License.
127// You may obtain a copy of the License at
128//
129//      http://www.apache.org/licenses/LICENSE-2.0
130//
131// Unless required by applicable law or agreed to in writing, software
132// distributed under the License is distributed on an "AS IS" BASIS,
133// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
134// See the License for the specific language governing permissions and
135// limitations under the License.
136//
137// This file is automatically generated by %s. Do not edit.
138""" % tool_name
139
140
141def apply_denylist(denylist, items):
142  return [item for item in items if not re.match(denylist, item)]
143
144
145def apply_allowlist(allowlist, items):
146  return [item for item in items if re.match(allowlist, item)]
147
148
149def normalize_path(path):
150  path = os.path.relpath(path, project_root)
151  path = re.sub(r'^out/[^/]+/', '', path)
152  return path
153
154
155class Error(Exception):
156  pass
157
158
159class DependencyNode(object):
160  """A target in a GN build description along with its dependencies."""
161
162  def __init__(self, target_name):
163    self.target_name = target_name
164    self.dependencies = set()
165
166  def add_dependency(self, target_node):
167    if target_node in self.dependencies:
168      return
169    self.dependencies.add(target_node)
170
171  def iterate_depth_first(self):
172    for node in sorted(self.dependencies, key=lambda n: n.target_name):
173      for node in node.iterate_depth_first():
174        yield node
175    if self.target_name:
176      yield self
177
178
179class DependencyTree(object):
180  """A tree of GN build target dependencies."""
181
182  def __init__(self):
183    self.target_to_node_map = {}
184    self.root = self._get_or_create_node(None)
185
186  def _get_or_create_node(self, target_name):
187    if target_name in self.target_to_node_map:
188      return self.target_to_node_map[target_name]
189    node = DependencyNode(target_name)
190    self.target_to_node_map[target_name] = node
191    return node
192
193  def add_dependency(self, from_target, to_target):
194    from_node = self._get_or_create_node(from_target)
195    to_node = self._get_or_create_node(to_target)
196    assert from_node is not to_node
197    from_node.add_dependency(to_node)
198
199  def iterate_depth_first(self):
200    for node in self.root.iterate_depth_first():
201      yield node
202
203
204class AmalgamatedProject(object):
205  """In-memory representation of an amalgamated source/header pair."""
206
207  def __init__(self, desc, source_deps, compute_deps_only=False):
208    """Constructor.
209
210        Args:
211            desc: JSON build description.
212            source_deps: A map of (source file, [dependency header]) which is
213                to detect which header files are included by each source file.
214            compute_deps_only: If True, the project will only be used to compute
215                dependency information. Use |get_source_files()| to retrieve
216                the result.
217        """
218    self.desc = desc
219    self.source_deps = source_deps
220    self.header = []
221    self.source = []
222    self.source_defines = []
223    # Note that we don't support multi-arg flags.
224    self.cflags = set(default_cflags)
225    self.ldflags = set()
226    self.defines = set()
227    self.libs = set()
228    self._dependency_tree = DependencyTree()
229    self._processed_sources = set()
230    self._processed_headers = set()
231    self._processed_header_deps = set()
232    self._processed_source_headers = set()  # Header files included from .cc
233    self._include_re = re.compile(r'#include "(.*)"')
234    self._compute_deps_only = compute_deps_only
235
236  def add_target(self, target_name):
237    """Include |target_name| in the amalgamated result."""
238    self._dependency_tree.add_dependency(None, target_name)
239    self._add_target_dependencies(target_name)
240    self._add_target_flags(target_name)
241    self._add_target_headers(target_name)
242
243    # Recurse into target deps, but only for protos. This generates headers
244    # for all the .{pbzero,gen}.h files, even if they don't #include each other.
245    for _, dep in self._iterate_dep_edges(target_name):
246      if (dep not in self._processed_header_deps and
247          re.match(recurse_in_header_deps, dep)):
248        self._processed_header_deps.add(dep)
249        self.add_target(dep)
250
251  def _iterate_dep_edges(self, target_name):
252    target = self.desc[target_name]
253    for dep in target.get('deps', []):
254      # Ignore system libraries since they will be added as build-time
255      # dependencies.
256      if dep in system_library_map:
257        continue
258      # Don't descend into build action dependencies.
259      if self.desc[dep]['type'] == 'action':
260        continue
261      for sub_target, sub_dep in self._iterate_dep_edges(dep):
262        yield sub_target, sub_dep
263      yield target_name, dep
264
265  def _iterate_target_and_deps(self, target_name):
266    yield target_name
267    for _, dep in self._iterate_dep_edges(target_name):
268      yield dep
269
270  def _add_target_dependencies(self, target_name):
271    for target, dep in self._iterate_dep_edges(target_name):
272      self._dependency_tree.add_dependency(target, dep)
273
274    def process_dep(dep):
275      if dep in system_library_map:
276        self.libs.update(system_library_map[dep].get('libs', []))
277        self.cflags.update(system_library_map[dep].get('cflags', []))
278        self.defines.update(system_library_map[dep].get('defines', []))
279        return True
280
281    def walk_all_deps(target_name):
282      target = self.desc[target_name]
283      for dep in target.get('deps', []):
284        if process_dep(dep):
285          return
286        walk_all_deps(dep)
287
288    walk_all_deps(target_name)
289
290  def _filter_cflags(self, cflags):
291    # Since we want to deduplicate flags, combine two-part switches (e.g.,
292    # "-foo bar") into one value ("-foobar") so we can store the result as
293    # a set.
294    result = []
295    for flag in cflags:
296      if flag.startswith('-'):
297        result.append(flag)
298      else:
299        result[-1] += flag
300    return apply_allowlist(cflag_allowlist, result)
301
302  def _add_target_flags(self, target_name):
303    for target_name in self._iterate_target_and_deps(target_name):
304      target = self.desc[target_name]
305      self.cflags.update(self._filter_cflags(target.get('cflags', [])))
306      self.cflags.update(self._filter_cflags(target.get('cflags_cc', [])))
307      self.ldflags.update(
308          apply_allowlist(ldflag_allowlist, target.get('ldflags', [])))
309      self.libs.update(apply_denylist(lib_denylist, target.get('libs', [])))
310      self.defines.update(
311          apply_allowlist(define_allowlist, target.get('defines', [])))
312
313  def _add_target_headers(self, target_name):
314    target = self.desc[target_name]
315    if not 'sources' in target:
316      return
317    headers = [
318        gn_utils.label_to_path(s) for s in target['sources'] if s.endswith('.h')
319    ]
320    for header in headers:
321      self._add_header(target_name, header)
322
323  def _get_include_dirs(self, target_name):
324    include_dirs = set()
325    for target_name in self._iterate_target_and_deps(target_name):
326      target = self.desc[target_name]
327      if 'include_dirs' in target:
328        include_dirs.update(
329            [gn_utils.label_to_path(d) for d in target['include_dirs']])
330    return include_dirs
331
332  def _add_source_included_header(self, include_dirs, allowed_files,
333                                  header_name):
334    if header_name in self._processed_headers:
335      return
336    if header_name in self._processed_source_headers:
337      return
338    self._processed_source_headers.add(header_name)
339    for include_dir in include_dirs:
340      rel_path = os.path.join(include_dir, header_name)
341      full_path = os.path.join(gn_utils.repo_root(), rel_path)
342      if os.path.exists(full_path):
343        if not rel_path in allowed_files:
344          return
345        with open(full_path) as f:
346          self.source.append(
347              '// %s begin header: %s' % (tool_name, normalize_path(full_path)))
348          self.source.extend(
349              self._process_source_includes(include_dirs, allowed_files, f))
350        return
351    if self._compute_deps_only:
352      return
353    msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs)
354    raise Error('Header file %s not found. %s' % (header_name, msg))
355
356  def _add_source(self, target_name, source_name):
357    if source_name in self._processed_sources:
358      return
359    self._processed_sources.add(source_name)
360    include_dirs = self._get_include_dirs(target_name)
361    deps = self.source_deps[source_name]
362    full_path = os.path.join(gn_utils.repo_root(), source_name)
363    if not os.path.exists(full_path):
364      raise Error('Source file %s not found' % source_name)
365    with open(full_path) as f:
366      self.source.append(
367          '// %s begin source: %s' % (tool_name, normalize_path(full_path)))
368      try:
369        self.source.extend(
370            self._patch_source(
371                source_name, self._process_source_includes(
372                    include_dirs, deps, f)))
373      except Error as e:
374        raise Error('Failed adding source %s: %s' % (source_name, e.message))
375
376  def _add_header_included_header(self, include_dirs, header_name):
377    if header_name in self._processed_headers:
378      return
379    self._processed_headers.add(header_name)
380    for include_dir in include_dirs:
381      full_path = os.path.join(gn_utils.repo_root(), include_dir, header_name)
382      if os.path.exists(full_path):
383        with open(full_path) as f:
384          self.header.append(
385              '// %s begin header: %s' % (tool_name, normalize_path(full_path)))
386          self.header.extend(self._process_header_includes(include_dirs, f))
387        return
388    if self._compute_deps_only:
389      return
390    msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs)
391    raise Error('Header file %s not found. %s' % (header_name, msg))
392
393  def _add_header(self, target_name, header_name):
394    if header_name in self._processed_headers:
395      return
396    self._processed_headers.add(header_name)
397    include_dirs = self._get_include_dirs(target_name)
398    full_path = os.path.join(gn_utils.repo_root(), header_name)
399    if not os.path.exists(full_path):
400      if self._compute_deps_only:
401        return
402      raise Error('Header file %s not found' % header_name)
403    with open(full_path) as f:
404      self.header.append(
405          '// %s begin header: %s' % (tool_name, normalize_path(full_path)))
406      try:
407        self.header.extend(self._process_header_includes(include_dirs, f))
408      except Error as e:
409        raise Error('Failed adding header %s: %s' % (header_name, e.message))
410
411  def _patch_source(self, source_name, lines):
412    result = []
413    namespace = re.sub(r'[^a-z]', '_',
414                       os.path.splitext(os.path.basename(source_name))[0])
415    for line in lines:
416      # Protobuf generates an identical anonymous function into each
417      # message description. Rename all but the first occurrence to avoid
418      # duplicate symbol definitions.
419      line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace)
420      result.append(line)
421    return result
422
423  def _process_source_includes(self, include_dirs, allowed_files, file):
424    result = []
425    for line in file:
426      line = line.rstrip('\n')
427      m = self._include_re.match(line)
428      if not m:
429        result.append(line)
430        continue
431      elif re.match(includes_to_remove, m.group(1)):
432        result.append('// %s removed: %s' % (tool_name, line))
433      else:
434        result.append('// %s expanded: %s' % (tool_name, line))
435        self._add_source_included_header(include_dirs, allowed_files,
436                                         m.group(1))
437    return result
438
439  def _process_header_includes(self, include_dirs, file):
440    result = []
441    for line in file:
442      line = line.rstrip('\n')
443      m = self._include_re.match(line)
444      if not m:
445        result.append(line)
446        continue
447      elif re.match(includes_to_remove, m.group(1)):
448        result.append('// %s removed: %s' % (tool_name, line))
449      else:
450        result.append('// %s expanded: %s' % (tool_name, line))
451        self._add_header_included_header(include_dirs, m.group(1))
452    return result
453
454  def generate(self):
455    """Prepares the output for this amalgamated project.
456
457        Call save() to persist the result.
458        """
459    assert not self._compute_deps_only
460    self.source_defines.append('// %s: predefined macros' % tool_name)
461
462    def add_define(name):
463      # Valued macros aren't supported for now.
464      assert '=' not in name
465      self.source_defines.append('#if !defined(%s)' % name)
466      self.source_defines.append('#define %s' % name)
467      self.source_defines.append('#endif')
468
469    for name in self.defines:
470      add_define(name)
471    for target_name, source_name in self.get_source_files():
472      self._add_source(target_name, source_name)
473
474  def get_source_files(self):
475    """Return a list of (target, [source file]) that describes the source
476           files pulled in by each target which is a dependency of this project.
477        """
478    source_files = []
479    for node in self._dependency_tree.iterate_depth_first():
480      target = self.desc[node.target_name]
481      if not 'sources' in target:
482        continue
483      sources = [(node.target_name, gn_utils.label_to_path(s))
484                 for s in target['sources']
485                 if s.endswith('.cc')]
486      source_files.extend(sources)
487    return source_files
488
489  def _get_nice_path(self, prefix, format):
490    basename = os.path.basename(prefix)
491    return os.path.join(
492        os.path.relpath(os.path.dirname(prefix)), format % basename)
493
494  def _make_directories(self, directory):
495    if not os.path.isdir(directory):
496      os.makedirs(directory)
497
498  def save(self, output_prefix):
499    """Save the generated header and source file pair.
500
501        Returns a message describing the output with build instructions.
502        """
503    header_file = self._get_nice_path(output_prefix, '%s.h')
504    source_file = self._get_nice_path(output_prefix, '%s.cc')
505    self._make_directories(os.path.dirname(header_file))
506    self._make_directories(os.path.dirname(source_file))
507    with open(header_file, 'w') as f:
508      f.write('\n'.join([preamble] + self.header + ['\n']))
509    with open(source_file, 'w') as f:
510      include_stmt = '#include "%s"' % os.path.basename(header_file)
511      f.write('\n'.join([preamble] + self.source_defines + [include_stmt] +
512                        self.source + ['\n']))
513    build_cmd = self.get_build_command(output_prefix)
514    return """Amalgamated project written to %s and %s.
515
516Build settings:
517 - cflags:    %s
518 - ldflags:   %s
519 - libs:      %s
520
521Example build command:
522
523%s
524""" % (header_file, source_file, ' '.join(self.cflags), ' '.join(self.ldflags),
525       ' '.join(self.libs), ' '.join(build_cmd))
526
527  def get_build_command(self, output_prefix):
528    """Returns an example command line for building the output source."""
529    source = self._get_nice_path(output_prefix, '%s.cc')
530    library = self._get_nice_path(output_prefix, 'lib%s.so')
531
532    if sys.platform.startswith('linux'):
533      llvm_script = os.path.join(gn_utils.repo_root(), 'gn',
534                                 'standalone', 'toolchain',
535                                 'linux_find_llvm.py')
536      cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode()
537    else:
538      cxx = 'clang++'
539
540    build_cmd = [cxx, source, '-o', library, '-shared'] + \
541        sorted(self.cflags) + sorted(self.ldflags)
542    for lib in sorted(self.libs):
543      build_cmd.append('-l%s' % lib)
544    return build_cmd
545
546
547def main():
548  parser = argparse.ArgumentParser(
549      description='Generate an amalgamated header/source pair from a GN '
550      'build description.')
551  parser.add_argument(
552      '--out',
553      help='The name of the temporary build folder in \'out\'',
554      default='tmp.gen_amalgamated.%u' % os.getpid())
555  parser.add_argument(
556      '--output',
557      help='Base name of files to create. A .cc/.h extension will be added',
558      default=os.path.join(gn_utils.repo_root(), 'out/amalgamated/perfetto'))
559  parser.add_argument(
560      '--gn_args',
561      help='GN arguments used to prepare the output directory',
562      default=gn_args)
563  parser.add_argument(
564      '--keep',
565      help='Don\'t delete the GN output directory at exit',
566      action='store_true')
567  parser.add_argument(
568      '--build', help='Also compile the generated files', action='store_true')
569  parser.add_argument(
570      '--check', help='Don\'t keep the generated files', action='store_true')
571  parser.add_argument('--quiet', help='Only report errors', action='store_true')
572  parser.add_argument(
573      '--dump-deps',
574      help='List all source files that the amalgamated output depends on',
575      action='store_true')
576  parser.add_argument(
577      'targets',
578      nargs=argparse.REMAINDER,
579      help='Targets to include in the output (e.g., "//:libperfetto")')
580  args = parser.parse_args()
581  targets = args.targets or default_targets
582
583  # The CHANGELOG mtime triggers the the perfetto_version.gen.h genrule. This is
584  # to avoid emitting a stale version information in the remote case of somebody
585  # running gen_amalgamated incrementally after having moved to another commit.
586  changelog_path = os.path.join(project_root, 'CHANGELOG')
587  assert(os.path.exists(changelog_path))
588  subprocess.check_call(['touch', '-c', changelog_path])
589
590  output = args.output
591  if args.check:
592    output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated')
593
594  out = gn_utils.prepare_out_directory(args.gn_args, args.out)
595  if not args.quiet:
596    print('Building project...')
597  try:
598    desc = gn_utils.load_build_description(out)
599
600    # We need to build everything first so that the necessary header
601    # dependencies get generated. However if we are just dumping dependency
602    # information this can be skipped, allowing cross-platform operation.
603    if not args.dump_deps:
604      gn_utils.build_targets(out, targets)
605    source_deps = gn_utils.compute_source_dependencies(out)
606    project = AmalgamatedProject(
607        desc, source_deps, compute_deps_only=args.dump_deps)
608
609    for target in targets:
610      project.add_target(target)
611
612    if args.dump_deps:
613      source_files = [
614          source_file for _, source_file in project.get_source_files()
615      ]
616      print('\n'.join(sorted(set(source_files))))
617      return
618
619    project.generate()
620    result = project.save(output)
621    if not args.quiet:
622      print(result)
623    if args.build:
624      if not args.quiet:
625        sys.stdout.write('Building amalgamated project...')
626        sys.stdout.flush()
627      subprocess.check_call(project.get_build_command(output))
628      if not args.quiet:
629        print('done')
630  finally:
631    if not args.keep:
632      shutil.rmtree(out)
633    if args.check:
634      shutil.rmtree(os.path.dirname(output))
635
636
637if __name__ == '__main__':
638  sys.exit(main())
639