1# Copyright (C) 2019 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15# A collection of utilities for extracting build rule information from GN
16# projects.
17
18from __future__ import print_function
19import collections
20import errno
21import filecmp
22import json
23import os
24import re
25import shutil
26import subprocess
27import sys
28from compat import iteritems
29
30BUILDFLAGS_TARGET = '//gn:gen_buildflags'
31GEN_VERSION_TARGET = '//src/base:version_gen_h'
32TARGET_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host'
33HOST_TOOLCHAIN = '//gn/standalone/toolchain:gcc_like_host'
34LINKER_UNIT_TYPES = ('executable', 'shared_library', 'static_library')
35
36# TODO(primiano): investigate these, they require further componentization.
37ODR_VIOLATION_IGNORE_TARGETS = {
38    '//test/cts:perfetto_cts_deps',
39    '//:perfetto_integrationtests',
40}
41
42
43def _check_command_output(cmd, cwd):
44  try:
45    output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, cwd=cwd)
46  except subprocess.CalledProcessError as e:
47    print(
48        'Command "{}" failed in {}:'.format(' '.join(cmd), cwd),
49        file=sys.stderr)
50    print(e.output.decode(), file=sys.stderr)
51    sys.exit(1)
52  else:
53    return output.decode()
54
55
56def repo_root():
57  """Returns an absolute path to the repository root."""
58  return os.path.join(
59      os.path.realpath(os.path.dirname(__file__)), os.path.pardir)
60
61
62def _tool_path(name):
63  return os.path.join(repo_root(), 'tools', name)
64
65
66def prepare_out_directory(gn_args, name, root=repo_root()):
67  """Creates the JSON build description by running GN.
68
69    Returns (path, desc) where |path| is the location of the output directory
70    and |desc| is the JSON build description.
71    """
72  out = os.path.join(root, 'out', name)
73  try:
74    os.makedirs(out)
75  except OSError as e:
76    if e.errno != errno.EEXIST:
77      raise
78  _check_command_output([_tool_path('gn'), 'gen', out,
79                         '--args=%s' % gn_args],
80                        cwd=repo_root())
81  return out
82
83
84def load_build_description(out):
85  """Creates the JSON build description by running GN."""
86  desc = _check_command_output([
87      _tool_path('gn'), 'desc', out, '--format=json', '--all-toolchains', '//*'
88  ],
89                               cwd=repo_root())
90  return json.loads(desc)
91
92
93def create_build_description(gn_args, root=repo_root()):
94  """Prepares a GN out directory and loads the build description from it.
95
96    The temporary out directory is automatically deleted.
97    """
98  out = prepare_out_directory(gn_args, 'tmp.gn_utils', root=root)
99  try:
100    return load_build_description(out)
101  finally:
102    shutil.rmtree(out)
103
104
105def build_targets(out, targets, quiet=False):
106  """Runs ninja to build a list of GN targets in the given out directory.
107
108    Compiling these targets is required so that we can include any generated
109    source files in the amalgamated result.
110    """
111  targets = [t.replace('//', '') for t in targets]
112  with open(os.devnull, 'w') as devnull:
113    stdout = devnull if quiet else None
114    subprocess.check_call(
115        [_tool_path('ninja')] + targets, cwd=out, stdout=stdout)
116
117
118def compute_source_dependencies(out):
119  """For each source file, computes a set of headers it depends on."""
120  ninja_deps = _check_command_output([_tool_path('ninja'), '-t', 'deps'],
121                                     cwd=out)
122  deps = {}
123  current_source = None
124  for line in ninja_deps.split('\n'):
125    filename = os.path.relpath(os.path.join(out, line.strip()), repo_root())
126    if not line or line[0] != ' ':
127      current_source = None
128      continue
129    elif not current_source:
130      # We're assuming the source file is always listed before the
131      # headers.
132      assert os.path.splitext(line)[1] in ['.c', '.cc', '.cpp', '.S']
133      current_source = filename
134      deps[current_source] = []
135    else:
136      assert current_source
137      deps[current_source].append(filename)
138  return deps
139
140
141def label_to_path(label):
142  """Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
143  assert label.startswith('//')
144  return label[2:]
145
146
147def label_without_toolchain(label):
148  """Strips the toolchain from a GN label.
149
150    Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
151    gcc_like_host) without the parenthesised toolchain part.
152    """
153  return label.split('(')[0]
154
155
156def label_to_target_name_with_path(label):
157  """
158  Turn a GN label into a target name involving the full path.
159  e.g., //src/perfetto:tests -> src_perfetto_tests
160  """
161  name = re.sub(r'^//:?', '', label)
162  name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
163  return name
164
165
166def gen_buildflags(gn_args, target_file):
167  """Generates the perfetto_build_flags.h for the given config.
168
169    target_file: the path, relative to the repo root, where the generated
170        buildflag header will be copied into.
171    """
172  tmp_out = prepare_out_directory(gn_args, 'tmp.gen_buildflags')
173  build_targets(tmp_out, [BUILDFLAGS_TARGET], quiet=True)
174  src = os.path.join(tmp_out, 'gen', 'build_config', 'perfetto_build_flags.h')
175  shutil.copy(src, os.path.join(repo_root(), target_file))
176  shutil.rmtree(tmp_out)
177
178
179def check_or_commit_generated_files(tmp_files, check):
180  """Checks that gen files are unchanged or renames them to the final location
181
182    Takes in input a list of 'xxx.swp' files that have been written.
183    If check == False, it renames xxx.swp -> xxx.
184    If check == True, it just checks that the contents of 'xxx.swp' == 'xxx'.
185    Returns 0 if no diff was detected, 1 otherwise (to be used as exit code).
186    """
187  res = 0
188  for tmp_file in tmp_files:
189    assert (tmp_file.endswith('.swp'))
190    target_file = os.path.relpath(tmp_file[:-4])
191    if check:
192      if not filecmp.cmp(tmp_file, target_file):
193        sys.stderr.write('%s needs to be regenerated\n' % target_file)
194        res = 1
195      os.unlink(tmp_file)
196    else:
197      os.rename(tmp_file, target_file)
198  return res
199
200
201class ODRChecker(object):
202  """Detects ODR violations in linker units
203
204  When we turn GN source sets into Soong & Bazel file groups, there is the risk
205  to create ODR violations by including the same file group into different
206  linker unit (this is because other build systems don't have a concept
207  equivalent to GN's source_set). This class navigates the transitive
208  dependencies (mostly static libraries) of a target and detects if multiple
209  paths end up including the same file group. This is to avoid situations like:
210
211  traced.exe -> base(file group)
212  traced.exe -> libperfetto(static lib) -> base(file group)
213  """
214
215  def __init__(self, gn, target_name):
216    self.gn = gn
217    self.root = gn.get_target(target_name)
218    self.source_sets = collections.defaultdict(set)
219    self.deps_visited = set()
220    self.source_set_hdr_only = {}
221
222    self._visit(target_name)
223    num_violations = 0
224    if target_name in ODR_VIOLATION_IGNORE_TARGETS:
225      return
226    for sset, paths in self.source_sets.items():
227      if self.is_header_only(sset):
228        continue
229      if len(paths) != 1:
230        num_violations += 1
231        print(
232            'ODR violation in target %s, multiple paths include %s:\n  %s' %
233            (target_name, sset, '\n  '.join(paths)),
234            file=sys.stderr)
235    if num_violations > 0:
236      raise Exception('%d ODR violations detected. Build generation aborted' %
237                      num_violations)
238
239  def _visit(self, target_name, parent_path=''):
240    target = self.gn.get_target(target_name)
241    path = ((parent_path + ' > ') if parent_path else '') + target_name
242    if not target:
243      raise Exception('Cannot find target %s' % target_name)
244    for ssdep in target.source_set_deps:
245      name_and_path = '%s (via %s)' % (target_name, path)
246      self.source_sets[ssdep].add(name_and_path)
247    deps = set(target.deps).union(target.proto_deps) - self.deps_visited
248    for dep_name in deps:
249      dep = self.gn.get_target(dep_name)
250      if dep.type == 'executable':
251        continue  # Execs are strong boundaries and don't cause ODR violations.
252      # static_library dependencies should reset the path. It doesn't matter if
253      # we get to a source file via:
254      # source_set1 > static_lib > source.cc OR
255      # source_set1 > source_set2 > static_lib > source.cc
256      # This is NOT an ODR violation because source.cc is linked from the same
257      # static library
258      next_parent_path = path if dep.type != 'static_library' else ''
259      self.deps_visited.add(dep_name)
260      self._visit(dep_name, next_parent_path)
261
262  def is_header_only(self, source_set_name):
263    cached = self.source_set_hdr_only.get(source_set_name)
264    if cached is not None:
265      return cached
266    target = self.gn.get_target(source_set_name)
267    if target.type != 'source_set':
268      raise TypeError('%s is not a source_set' % source_set_name)
269    res = all(src.endswith('.h') for src in target.sources)
270    self.source_set_hdr_only[source_set_name] = res
271    return res
272
273
274class GnParser(object):
275  """A parser with some cleverness for GN json desc files
276
277    The main goals of this parser are:
278    1) Deal with the fact that other build systems don't have an equivalent
279       notion to GN's source_set. Conversely to Bazel's and Soong's filegroups,
280       GN source_sets expect that dependencies, cflags and other source_set
281       properties propagate up to the linker unit (static_library, executable or
282       shared_library). This parser simulates the same behavior: when a
283       source_set is encountered, some of its variables (cflags and such) are
284       copied up to the dependent targets. This is to allow gen_xxx to create
285       one filegroup for each source_set and then squash all the other flags
286       onto the linker unit.
287    2) Detect and special-case protobuf targets, figuring out the protoc-plugin
288       being used.
289    """
290
291  class Target(object):
292    """Reperesents A GN target.
293
294        Maked properties are propagated up the dependency chain when a
295        source_set dependency is encountered.
296        """
297
298    def __init__(self, name, type):
299      self.name = name  # e.g. //src/ipc:ipc
300
301      VALID_TYPES = ('static_library', 'shared_library', 'executable', 'group',
302                     'action', 'source_set', 'proto_library')
303      assert (type in VALID_TYPES)
304      self.type = type
305      self.testonly = False
306      self.toolchain = None
307
308      # These are valid only for type == proto_library.
309      # This is typically: 'proto', 'protozero', 'ipc'.
310      self.proto_plugin = None
311      self.proto_paths = set()
312
313      self.sources = set()
314      # TODO(primiano): consider whether the public section should be part of
315      # bubbled-up sources.
316      self.public_headers = set()  # 'public'
317
318      # These are valid only for type == 'action'
319      self.inputs = set()
320      self.outputs = set()
321      self.script = None
322      self.args = []
323
324      # These variables are propagated up when encountering a dependency
325      # on a source_set target.
326      self.cflags = set()
327      self.defines = set()
328      self.deps = set()
329      self.libs = set()
330      self.include_dirs = set()
331      self.ldflags = set()
332      self.source_set_deps = set()  # Transitive set of source_set deps.
333      self.proto_deps = set()  # Transitive set of protobuf deps.
334
335      # Deps on //gn:xxx have this flag set to True. These dependencies
336      # are special because they pull third_party code from buildtools/.
337      # We don't want to keep recursing into //buildtools in generators,
338      # this flag is used to stop the recursion and create an empty
339      # placeholder target once we hit //gn:protoc or similar.
340      self.is_third_party_dep_ = False
341
342    def __lt__(self, other):
343      if isinstance(other, self.__class__):
344        return self.name < other.name
345      raise TypeError(
346          '\'<\' not supported between instances of \'%s\' and \'%s\'' %
347          (type(self).__name__, type(other).__name__))
348
349    def __repr__(self):
350      return json.dumps({
351          k: (list(sorted(v)) if isinstance(v, set) else v)
352          for (k, v) in iteritems(self.__dict__)
353      },
354                        indent=4,
355                        sort_keys=True)
356
357    def update(self, other):
358      for key in ('cflags', 'defines', 'deps', 'include_dirs', 'ldflags',
359                  'source_set_deps', 'proto_deps', 'libs', 'proto_paths'):
360        self.__dict__[key].update(other.__dict__.get(key, []))
361
362  def __init__(self, gn_desc):
363    self.gn_desc_ = gn_desc
364    self.all_targets = {}
365    self.linker_units = {}  # Executables, shared or static libraries.
366    self.source_sets = {}
367    self.actions = {}
368    self.proto_libs = {}
369
370  def get_target(self, gn_target_name):
371    """Returns a Target object from the fully qualified GN target name.
372
373        It bubbles up variables from source_set dependencies as described in the
374        class-level comments.
375        """
376    target = self.all_targets.get(gn_target_name)
377    if target is not None:
378      return target  # Target already processed.
379
380    desc = self.gn_desc_[gn_target_name]
381    target = GnParser.Target(gn_target_name, desc['type'])
382    target.testonly = desc.get('testonly', False)
383    target.toolchain = desc.get('toolchain', None)
384    self.all_targets[gn_target_name] = target
385
386    # We should never have GN targets directly depend on buidtools. They
387    # should hop via //gn:xxx, so we can give generators an opportunity to
388    # override them.
389    assert (not gn_target_name.startswith('//buildtools'))
390
391    # Don't descend further into third_party targets. Genrators are supposed
392    # to either ignore them or route to other externally-provided targets.
393    if gn_target_name.startswith('//gn'):
394      target.is_third_party_dep_ = True
395      return target
396
397    proto_target_type, proto_desc = self.get_proto_target_type_(target)
398    if proto_target_type is not None:
399      self.proto_libs[target.name] = target
400      target.type = 'proto_library'
401      target.proto_plugin = proto_target_type
402      target.proto_paths.update(self.get_proto_paths(proto_desc))
403      target.sources.update(proto_desc.get('sources', []))
404      assert (all(x.endswith('.proto') for x in target.sources))
405    elif target.type == 'source_set':
406      self.source_sets[gn_target_name] = target
407      target.sources.update(desc.get('sources', []))
408    elif target.type in LINKER_UNIT_TYPES:
409      self.linker_units[gn_target_name] = target
410      target.sources.update(desc.get('sources', []))
411    elif target.type == 'action':
412      self.actions[gn_target_name] = target
413      target.inputs.update(desc.get('inputs', []))
414      target.sources.update(desc.get('sources', []))
415      outs = [re.sub('^//out/.+?/gen/', '', x) for x in desc['outputs']]
416      target.outputs.update(outs)
417      target.script = desc['script']
418      # Args are typically relative to the root build dir (../../xxx)
419      # because root build dir is typically out/xxx/).
420      target.args = [re.sub('^../../', '//', x) for x in desc['args']]
421
422    # Default for 'public' is //* - all headers in 'sources' are public.
423    # TODO(primiano): if a 'public' section is specified (even if empty), then
424    # the rest of 'sources' is considered inaccessible by gn. Consider
425    # emulating that, so that generated build files don't end up with overly
426    # accessible headers.
427    public_headers = [x for x in desc.get('public', []) if x != '*']
428    target.public_headers.update(public_headers)
429
430    target.cflags.update(desc.get('cflags', []) + desc.get('cflags_cc', []))
431    target.libs.update(desc.get('libs', []))
432    target.ldflags.update(desc.get('ldflags', []))
433    target.defines.update(desc.get('defines', []))
434    target.include_dirs.update(desc.get('include_dirs', []))
435
436    # Recurse in dependencies.
437    for dep_name in desc.get('deps', []):
438      dep = self.get_target(dep_name)
439      if dep.is_third_party_dep_:
440        target.deps.add(dep_name)
441      elif dep.type == 'proto_library':
442        target.proto_deps.add(dep_name)
443        target.proto_paths.update(dep.proto_paths)
444
445        # Don't bubble deps for action targets
446        if target.type != 'action':
447          target.proto_deps.update(dep.proto_deps)  # Bubble up deps.
448      elif dep.type == 'source_set':
449        target.source_set_deps.add(dep_name)
450        target.update(dep)  # Bubble up source set's cflags/ldflags etc.
451      elif dep.type == 'group':
452        target.update(dep)  # Bubble up groups's cflags/ldflags etc.
453      elif dep.type == 'action':
454        if proto_target_type is None:
455          target.deps.add(dep_name)
456      elif dep.type in LINKER_UNIT_TYPES:
457        target.deps.add(dep_name)
458
459    return target
460
461  def get_proto_paths(self, proto_desc):
462    # import_dirs in metadata will be available for source_set targets.
463    metadata = proto_desc.get('metadata', {})
464    import_dirs = metadata.get('import_dirs', [])
465    if import_dirs:
466      return import_dirs
467
468    # For all non-source-set targets, we need to parse the command line
469    # of the protoc invocation.
470    proto_paths = []
471    args = proto_desc.get('args', [])
472    for i, arg in enumerate(args):
473      if arg != '--proto_path':
474        continue
475      proto_paths.append(re.sub('^../../', '//', args[i + 1]))
476    return proto_paths
477
478  def get_proto_target_type_(self, target):
479    """ Checks if the target is a proto library and return the plugin.
480
481        Returns:
482            (None, None): if the target is not a proto library.
483            (plugin, proto_desc) where |plugin| is 'proto' in the default (lite)
484            case or 'protozero' or 'ipc' or 'descriptor'; |proto_desc| is the GN
485            json desc of the target with the .proto sources (_gen target for
486            non-descriptor types or the target itself for descriptor type).
487        """
488    parts = target.name.split('(', 1)
489    name = parts[0]
490    toolchain = '(' + parts[1] if len(parts) > 1 else ''
491
492    # Descriptor targets don't have a _gen target; instead we look for the
493    # characteristic flag in the args of the target itself.
494    desc = self.gn_desc_.get(target.name)
495    if '--descriptor_set_out' in desc.get('args', []):
496      return 'descriptor', desc
497
498    # Source set proto targets have a non-empty proto_library_sources in the
499    # metadata of the descirption.
500    metadata = desc.get('metadata', {})
501    if 'proto_library_sources' in metadata:
502      return 'source_set', desc
503
504    # In all other cases, we want to look at the _gen target as that has the
505    # important information.
506    gen_desc = self.gn_desc_.get('%s_gen%s' % (name, toolchain))
507    if gen_desc is None or gen_desc['type'] != 'action':
508      return None, None
509    args = gen_desc.get('args', [])
510    if '/protoc' not in args[0]:
511      return None, None
512    plugin = 'proto'
513    for arg in (arg for arg in args if arg.startswith('--plugin=')):
514      # |arg| at this point looks like:
515      #  --plugin=protoc-gen-plugin=gcc_like_host/protozero_plugin
516      # or
517      #  --plugin=protoc-gen-plugin=protozero_plugin
518      plugin = arg.split('=')[-1].split('/')[-1].replace('_plugin', '')
519    return plugin, gen_desc
520