1#!/usr/bin/env python3 2# Copyright (C) 2019 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# This tool uses a collection of BUILD.gn files and build targets to generate 17# an "amalgamated" C++ header and source file pair which compiles to an 18# equivalent program. The tool also outputs the necessary compiler and linker 19# flags needed to compile the resulting source code. 20 21from __future__ import print_function 22import argparse 23import os 24import re 25import shutil 26import subprocess 27import sys 28import tempfile 29 30import gn_utils 31 32# Default targets to include in the result. 33# TODO(primiano): change this script to recurse into target deps when generating 34# headers, but only for proto targets. .pbzero.h files don't include each other 35# and we need to list targets here individually, which is unmaintainable. 36default_targets = [ 37 '//:libperfetto_client_experimental', 38 '//include/perfetto/protozero:protozero', 39 '//protos/perfetto/config:zero', 40 '//protos/perfetto/trace:zero', 41] 42 43# Arguments for the GN output directory (unless overridden from the command 44# line). 45gn_args = ' '.join([ 46 'is_debug=false', 47 'is_perfetto_build_generator=true', 48 'is_perfetto_embedder=true', 49 'use_custom_libcxx=false', 50 'enable_perfetto_ipc=true', 51]) 52 53# By default, the amalgamated .h only recurses in #includes but not in the 54# target deps. In the case of protos we want to follow deps even in lieu of 55# direct #includes. This is because, by design, protozero headers don't 56# include each other but rely on forward declarations. The alternative would 57# be adding each proto sub-target individually (e.g. //proto/trace/gpu:zero), 58# but doing that is unmaintainable. We also do this for cpp bindings since some 59# tracing SDK functions depend on them (and the system tracing IPC mechanism 60# does so too). 61recurse_in_header_deps = '^//protos/.*(cpp|zero)$' 62 63# Compiler flags which aren't filtered out. 64cflag_allowlist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$' 65 66# Linker flags which aren't filtered out. 67ldflag_allowlist = r'^-()$' 68 69# Libraries which are filtered out. 70lib_denylist = r'^(c|gcc_eh)$' 71 72# Macros which aren't filtered out. 73define_allowlist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$' 74 75# Includes which will be removed from the generated source. 76includes_to_remove = r'^(gtest).*$' 77 78default_cflags = [ 79 # Since we're expanding header files into the generated source file, some 80 # constant may remain unused. 81 '-Wno-unused-const-variable' 82] 83 84# Build flags to satisfy a protobuf (lite or full) dependency. 85protobuf_cflags = [ 86 # Note that these point to the local copy of protobuf in buildtools. In 87 # reality the user of the amalgamated result will have to provide a path to 88 # an installed copy of the exact same version of protobuf which was used to 89 # generate the amalgamated build. 90 '-isystembuildtools/protobuf/src', 91 '-Lbuildtools/protobuf/src/.libs', 92 # We also need to disable some warnings for protobuf. 93 '-Wno-missing-prototypes', 94 '-Wno-missing-variable-declarations', 95 '-Wno-sign-conversion', 96 '-Wno-unknown-pragmas', 97 '-Wno-unused-macros', 98] 99 100# A mapping of dependencies to system libraries. Libraries in this map will not 101# be built statically but instead added as dependencies of the amalgamated 102# project. 103system_library_map = { 104 '//buildtools:protobuf_full': { 105 'libs': ['protobuf'], 106 'cflags': protobuf_cflags, 107 }, 108 '//buildtools:protobuf_lite': { 109 'libs': ['protobuf-lite'], 110 'cflags': protobuf_cflags, 111 }, 112 '//buildtools:protoc_lib': { 113 'libs': ['protoc'] 114 }, 115} 116 117# ---------------------------------------------------------------------------- 118# End of configuration. 119# ---------------------------------------------------------------------------- 120 121tool_name = os.path.basename(__file__) 122project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 123preamble = """// Copyright (C) 2019 The Android Open Source Project 124// 125// Licensed under the Apache License, Version 2.0 (the "License"); 126// you may not use this file except in compliance with the License. 127// You may obtain a copy of the License at 128// 129// http://www.apache.org/licenses/LICENSE-2.0 130// 131// Unless required by applicable law or agreed to in writing, software 132// distributed under the License is distributed on an "AS IS" BASIS, 133// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 134// See the License for the specific language governing permissions and 135// limitations under the License. 136// 137// This file is automatically generated by %s. Do not edit. 138""" % tool_name 139 140 141def apply_denylist(denylist, items): 142 return [item for item in items if not re.match(denylist, item)] 143 144 145def apply_allowlist(allowlist, items): 146 return [item for item in items if re.match(allowlist, item)] 147 148 149def normalize_path(path): 150 path = os.path.relpath(path, project_root) 151 path = re.sub(r'^out/[^/]+/', '', path) 152 return path 153 154 155class Error(Exception): 156 pass 157 158 159class DependencyNode(object): 160 """A target in a GN build description along with its dependencies.""" 161 162 def __init__(self, target_name): 163 self.target_name = target_name 164 self.dependencies = set() 165 166 def add_dependency(self, target_node): 167 if target_node in self.dependencies: 168 return 169 self.dependencies.add(target_node) 170 171 def iterate_depth_first(self): 172 for node in sorted(self.dependencies, key=lambda n: n.target_name): 173 for node in node.iterate_depth_first(): 174 yield node 175 if self.target_name: 176 yield self 177 178 179class DependencyTree(object): 180 """A tree of GN build target dependencies.""" 181 182 def __init__(self): 183 self.target_to_node_map = {} 184 self.root = self._get_or_create_node(None) 185 186 def _get_or_create_node(self, target_name): 187 if target_name in self.target_to_node_map: 188 return self.target_to_node_map[target_name] 189 node = DependencyNode(target_name) 190 self.target_to_node_map[target_name] = node 191 return node 192 193 def add_dependency(self, from_target, to_target): 194 from_node = self._get_or_create_node(from_target) 195 to_node = self._get_or_create_node(to_target) 196 assert from_node is not to_node 197 from_node.add_dependency(to_node) 198 199 def iterate_depth_first(self): 200 for node in self.root.iterate_depth_first(): 201 yield node 202 203 204class AmalgamatedProject(object): 205 """In-memory representation of an amalgamated source/header pair.""" 206 207 def __init__(self, desc, source_deps, compute_deps_only=False): 208 """Constructor. 209 210 Args: 211 desc: JSON build description. 212 source_deps: A map of (source file, [dependency header]) which is 213 to detect which header files are included by each source file. 214 compute_deps_only: If True, the project will only be used to compute 215 dependency information. Use |get_source_files()| to retrieve 216 the result. 217 """ 218 self.desc = desc 219 self.source_deps = source_deps 220 self.header = [] 221 self.source = [] 222 self.source_defines = [] 223 # Note that we don't support multi-arg flags. 224 self.cflags = set(default_cflags) 225 self.ldflags = set() 226 self.defines = set() 227 self.libs = set() 228 self._dependency_tree = DependencyTree() 229 self._processed_sources = set() 230 self._processed_headers = set() 231 self._processed_header_deps = set() 232 self._processed_source_headers = set() # Header files included from .cc 233 self._include_re = re.compile(r'#include "(.*)"') 234 self._compute_deps_only = compute_deps_only 235 236 def add_target(self, target_name): 237 """Include |target_name| in the amalgamated result.""" 238 self._dependency_tree.add_dependency(None, target_name) 239 self._add_target_dependencies(target_name) 240 self._add_target_flags(target_name) 241 self._add_target_headers(target_name) 242 243 # Recurse into target deps, but only for protos. This generates headers 244 # for all the .{pbzero,gen}.h files, even if they don't #include each other. 245 for _, dep in self._iterate_dep_edges(target_name): 246 if (dep not in self._processed_header_deps and 247 re.match(recurse_in_header_deps, dep)): 248 self._processed_header_deps.add(dep) 249 self.add_target(dep) 250 251 def _iterate_dep_edges(self, target_name): 252 target = self.desc[target_name] 253 for dep in target.get('deps', []): 254 # Ignore system libraries since they will be added as build-time 255 # dependencies. 256 if dep in system_library_map: 257 continue 258 # Don't descend into build action dependencies. 259 if self.desc[dep]['type'] == 'action': 260 continue 261 for sub_target, sub_dep in self._iterate_dep_edges(dep): 262 yield sub_target, sub_dep 263 yield target_name, dep 264 265 def _iterate_target_and_deps(self, target_name): 266 yield target_name 267 for _, dep in self._iterate_dep_edges(target_name): 268 yield dep 269 270 def _add_target_dependencies(self, target_name): 271 for target, dep in self._iterate_dep_edges(target_name): 272 self._dependency_tree.add_dependency(target, dep) 273 274 def process_dep(dep): 275 if dep in system_library_map: 276 self.libs.update(system_library_map[dep].get('libs', [])) 277 self.cflags.update(system_library_map[dep].get('cflags', [])) 278 self.defines.update(system_library_map[dep].get('defines', [])) 279 return True 280 281 def walk_all_deps(target_name): 282 target = self.desc[target_name] 283 for dep in target.get('deps', []): 284 if process_dep(dep): 285 return 286 walk_all_deps(dep) 287 288 walk_all_deps(target_name) 289 290 def _filter_cflags(self, cflags): 291 # Since we want to deduplicate flags, combine two-part switches (e.g., 292 # "-foo bar") into one value ("-foobar") so we can store the result as 293 # a set. 294 result = [] 295 for flag in cflags: 296 if flag.startswith('-'): 297 result.append(flag) 298 else: 299 result[-1] += flag 300 return apply_allowlist(cflag_allowlist, result) 301 302 def _add_target_flags(self, target_name): 303 for target_name in self._iterate_target_and_deps(target_name): 304 target = self.desc[target_name] 305 self.cflags.update(self._filter_cflags(target.get('cflags', []))) 306 self.cflags.update(self._filter_cflags(target.get('cflags_cc', []))) 307 self.ldflags.update( 308 apply_allowlist(ldflag_allowlist, target.get('ldflags', []))) 309 self.libs.update(apply_denylist(lib_denylist, target.get('libs', []))) 310 self.defines.update( 311 apply_allowlist(define_allowlist, target.get('defines', []))) 312 313 def _add_target_headers(self, target_name): 314 target = self.desc[target_name] 315 if not 'sources' in target: 316 return 317 headers = [ 318 gn_utils.label_to_path(s) for s in target['sources'] if s.endswith('.h') 319 ] 320 for header in headers: 321 self._add_header(target_name, header) 322 323 def _get_include_dirs(self, target_name): 324 include_dirs = set() 325 for target_name in self._iterate_target_and_deps(target_name): 326 target = self.desc[target_name] 327 if 'include_dirs' in target: 328 include_dirs.update( 329 [gn_utils.label_to_path(d) for d in target['include_dirs']]) 330 return include_dirs 331 332 def _add_source_included_header(self, include_dirs, allowed_files, 333 header_name): 334 if header_name in self._processed_headers: 335 return 336 if header_name in self._processed_source_headers: 337 return 338 self._processed_source_headers.add(header_name) 339 for include_dir in include_dirs: 340 rel_path = os.path.join(include_dir, header_name) 341 full_path = os.path.join(gn_utils.repo_root(), rel_path) 342 if os.path.exists(full_path): 343 if not rel_path in allowed_files: 344 return 345 with open(full_path) as f: 346 self.source.append( 347 '// %s begin header: %s' % (tool_name, normalize_path(full_path))) 348 self.source.extend( 349 self._process_source_includes(include_dirs, allowed_files, f)) 350 return 351 if self._compute_deps_only: 352 return 353 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 354 raise Error('Header file %s not found. %s' % (header_name, msg)) 355 356 def _add_source(self, target_name, source_name): 357 if source_name in self._processed_sources: 358 return 359 self._processed_sources.add(source_name) 360 include_dirs = self._get_include_dirs(target_name) 361 deps = self.source_deps[source_name] 362 full_path = os.path.join(gn_utils.repo_root(), source_name) 363 if not os.path.exists(full_path): 364 raise Error('Source file %s not found' % source_name) 365 with open(full_path) as f: 366 self.source.append( 367 '// %s begin source: %s' % (tool_name, normalize_path(full_path))) 368 try: 369 self.source.extend( 370 self._patch_source( 371 source_name, self._process_source_includes( 372 include_dirs, deps, f))) 373 except Error as e: 374 raise Error('Failed adding source %s: %s' % (source_name, e.message)) 375 376 def _add_header_included_header(self, include_dirs, header_name): 377 if header_name in self._processed_headers: 378 return 379 self._processed_headers.add(header_name) 380 for include_dir in include_dirs: 381 full_path = os.path.join(gn_utils.repo_root(), include_dir, header_name) 382 if os.path.exists(full_path): 383 with open(full_path) as f: 384 self.header.append( 385 '// %s begin header: %s' % (tool_name, normalize_path(full_path))) 386 self.header.extend(self._process_header_includes(include_dirs, f)) 387 return 388 if self._compute_deps_only: 389 return 390 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) 391 raise Error('Header file %s not found. %s' % (header_name, msg)) 392 393 def _add_header(self, target_name, header_name): 394 if header_name in self._processed_headers: 395 return 396 self._processed_headers.add(header_name) 397 include_dirs = self._get_include_dirs(target_name) 398 full_path = os.path.join(gn_utils.repo_root(), header_name) 399 if not os.path.exists(full_path): 400 if self._compute_deps_only: 401 return 402 raise Error('Header file %s not found' % header_name) 403 with open(full_path) as f: 404 self.header.append( 405 '// %s begin header: %s' % (tool_name, normalize_path(full_path))) 406 try: 407 self.header.extend(self._process_header_includes(include_dirs, f)) 408 except Error as e: 409 raise Error('Failed adding header %s: %s' % (header_name, e.message)) 410 411 def _patch_source(self, source_name, lines): 412 result = [] 413 namespace = re.sub(r'[^a-z]', '_', 414 os.path.splitext(os.path.basename(source_name))[0]) 415 for line in lines: 416 # Protobuf generates an identical anonymous function into each 417 # message description. Rename all but the first occurrence to avoid 418 # duplicate symbol definitions. 419 line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace) 420 result.append(line) 421 return result 422 423 def _process_source_includes(self, include_dirs, allowed_files, file): 424 result = [] 425 for line in file: 426 line = line.rstrip('\n') 427 m = self._include_re.match(line) 428 if not m: 429 result.append(line) 430 continue 431 elif re.match(includes_to_remove, m.group(1)): 432 result.append('// %s removed: %s' % (tool_name, line)) 433 else: 434 result.append('// %s expanded: %s' % (tool_name, line)) 435 self._add_source_included_header(include_dirs, allowed_files, 436 m.group(1)) 437 return result 438 439 def _process_header_includes(self, include_dirs, file): 440 result = [] 441 for line in file: 442 line = line.rstrip('\n') 443 m = self._include_re.match(line) 444 if not m: 445 result.append(line) 446 continue 447 elif re.match(includes_to_remove, m.group(1)): 448 result.append('// %s removed: %s' % (tool_name, line)) 449 else: 450 result.append('// %s expanded: %s' % (tool_name, line)) 451 self._add_header_included_header(include_dirs, m.group(1)) 452 return result 453 454 def generate(self): 455 """Prepares the output for this amalgamated project. 456 457 Call save() to persist the result. 458 """ 459 assert not self._compute_deps_only 460 self.source_defines.append('// %s: predefined macros' % tool_name) 461 462 def add_define(name): 463 # Valued macros aren't supported for now. 464 assert '=' not in name 465 self.source_defines.append('#if !defined(%s)' % name) 466 self.source_defines.append('#define %s' % name) 467 self.source_defines.append('#endif') 468 469 for name in self.defines: 470 add_define(name) 471 for target_name, source_name in self.get_source_files(): 472 self._add_source(target_name, source_name) 473 474 def get_source_files(self): 475 """Return a list of (target, [source file]) that describes the source 476 files pulled in by each target which is a dependency of this project. 477 """ 478 source_files = [] 479 for node in self._dependency_tree.iterate_depth_first(): 480 target = self.desc[node.target_name] 481 if not 'sources' in target: 482 continue 483 sources = [(node.target_name, gn_utils.label_to_path(s)) 484 for s in target['sources'] 485 if s.endswith('.cc')] 486 source_files.extend(sources) 487 return source_files 488 489 def _get_nice_path(self, prefix, format): 490 basename = os.path.basename(prefix) 491 return os.path.join( 492 os.path.relpath(os.path.dirname(prefix)), format % basename) 493 494 def _make_directories(self, directory): 495 if not os.path.isdir(directory): 496 os.makedirs(directory) 497 498 def save(self, output_prefix): 499 """Save the generated header and source file pair. 500 501 Returns a message describing the output with build instructions. 502 """ 503 header_file = self._get_nice_path(output_prefix, '%s.h') 504 source_file = self._get_nice_path(output_prefix, '%s.cc') 505 self._make_directories(os.path.dirname(header_file)) 506 self._make_directories(os.path.dirname(source_file)) 507 with open(header_file, 'w') as f: 508 f.write('\n'.join([preamble] + self.header + ['\n'])) 509 with open(source_file, 'w') as f: 510 include_stmt = '#include "%s"' % os.path.basename(header_file) 511 f.write('\n'.join([preamble] + self.source_defines + [include_stmt] + 512 self.source + ['\n'])) 513 build_cmd = self.get_build_command(output_prefix) 514 return """Amalgamated project written to %s and %s. 515 516Build settings: 517 - cflags: %s 518 - ldflags: %s 519 - libs: %s 520 521Example build command: 522 523%s 524""" % (header_file, source_file, ' '.join(self.cflags), ' '.join(self.ldflags), 525 ' '.join(self.libs), ' '.join(build_cmd)) 526 527 def get_build_command(self, output_prefix): 528 """Returns an example command line for building the output source.""" 529 source = self._get_nice_path(output_prefix, '%s.cc') 530 library = self._get_nice_path(output_prefix, 'lib%s.so') 531 532 if sys.platform.startswith('linux'): 533 llvm_script = os.path.join(gn_utils.repo_root(), 'gn', 534 'standalone', 'toolchain', 535 'linux_find_llvm.py') 536 cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode() 537 else: 538 cxx = 'clang++' 539 540 build_cmd = [cxx, source, '-o', library, '-shared'] + \ 541 sorted(self.cflags) + sorted(self.ldflags) 542 for lib in sorted(self.libs): 543 build_cmd.append('-l%s' % lib) 544 return build_cmd 545 546 547def main(): 548 parser = argparse.ArgumentParser( 549 description='Generate an amalgamated header/source pair from a GN ' 550 'build description.') 551 parser.add_argument( 552 '--out', 553 help='The name of the temporary build folder in \'out\'', 554 default='tmp.gen_amalgamated.%u' % os.getpid()) 555 parser.add_argument( 556 '--output', 557 help='Base name of files to create. A .cc/.h extension will be added', 558 default=os.path.join(gn_utils.repo_root(), 'out/amalgamated/perfetto')) 559 parser.add_argument( 560 '--gn_args', 561 help='GN arguments used to prepare the output directory', 562 default=gn_args) 563 parser.add_argument( 564 '--keep', 565 help='Don\'t delete the GN output directory at exit', 566 action='store_true') 567 parser.add_argument( 568 '--build', help='Also compile the generated files', action='store_true') 569 parser.add_argument( 570 '--check', help='Don\'t keep the generated files', action='store_true') 571 parser.add_argument('--quiet', help='Only report errors', action='store_true') 572 parser.add_argument( 573 '--dump-deps', 574 help='List all source files that the amalgamated output depends on', 575 action='store_true') 576 parser.add_argument( 577 'targets', 578 nargs=argparse.REMAINDER, 579 help='Targets to include in the output (e.g., "//:libperfetto")') 580 args = parser.parse_args() 581 targets = args.targets or default_targets 582 583 # The CHANGELOG mtime triggers the the perfetto_version.gen.h genrule. This is 584 # to avoid emitting a stale version information in the remote case of somebody 585 # running gen_amalgamated incrementally after having moved to another commit. 586 changelog_path = os.path.join(project_root, 'CHANGELOG') 587 assert(os.path.exists(changelog_path)) 588 subprocess.check_call(['touch', '-c', changelog_path]) 589 590 output = args.output 591 if args.check: 592 output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated') 593 594 out = gn_utils.prepare_out_directory(args.gn_args, args.out) 595 if not args.quiet: 596 print('Building project...') 597 try: 598 desc = gn_utils.load_build_description(out) 599 600 # We need to build everything first so that the necessary header 601 # dependencies get generated. However if we are just dumping dependency 602 # information this can be skipped, allowing cross-platform operation. 603 if not args.dump_deps: 604 gn_utils.build_targets(out, targets) 605 source_deps = gn_utils.compute_source_dependencies(out) 606 project = AmalgamatedProject( 607 desc, source_deps, compute_deps_only=args.dump_deps) 608 609 for target in targets: 610 project.add_target(target) 611 612 if args.dump_deps: 613 source_files = [ 614 source_file for _, source_file in project.get_source_files() 615 ] 616 print('\n'.join(sorted(set(source_files)))) 617 return 618 619 project.generate() 620 result = project.save(output) 621 if not args.quiet: 622 print(result) 623 if args.build: 624 if not args.quiet: 625 sys.stdout.write('Building amalgamated project...') 626 sys.stdout.flush() 627 subprocess.check_call(project.get_build_command(output)) 628 if not args.quiet: 629 print('done') 630 finally: 631 if not args.keep: 632 shutil.rmtree(out) 633 if args.check: 634 shutil.rmtree(os.path.dirname(output)) 635 636 637if __name__ == '__main__': 638 sys.exit(main()) 639