1#!/usr/bin/env python3
2# Copyright (C) 2018 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This tool translates a collection of BUILD.gn files into a mostly equivalent
17# BUILD file for the Bazel build system. The input to the tool is a
18# JSON description of the GN build definition generated with the following
19# command:
20#
21#   gn desc out --format=json --all-toolchains "//*" > desc.json
22#
23# The tool is then given a list of GN labels for which to generate Bazel
24# build rules.
25
26from __future__ import print_function
27import argparse
28import json
29import os
30import re
31import sys
32
33import gn_utils
34
35from compat import itervalues, iteritems, basestring
36
37# Arguments for the GN output directory.
38# host_os="linux" is to generate the right build files from Mac OS.
39gn_args = ' '.join([
40    'host_os="linux"',
41    'is_debug=false',
42    'is_perfetto_build_generator=true',
43    'enable_perfetto_watchdog=true',
44    'monolithic_binaries=true',
45    'target_os="linux"',
46])
47
48# Default targets to translate to the blueprint file.
49
50# These targets will be exported with public visibility in the generated BUILD.
51public_targets = [
52    '//:libperfetto_client_experimental',
53    '//src/perfetto_cmd:perfetto',
54    '//src/traced/probes:traced_probes',
55    '//src/traced/service:traced',
56    '//src/trace_processor:trace_processor_shell',
57    '//src/trace_processor:trace_processor',
58    '//tools/trace_to_text:trace_to_text',
59    '//tools/trace_to_text:libpprofbuilder',
60]
61
62# These targets are required by internal build rules but don't need to be
63# exported publicly.
64default_targets = [
65    '//test:client_api_example',
66    '//src/ipc:perfetto_ipc',
67    '//src/ipc/protoc_plugin:ipc_plugin',
68    '//src/protozero:protozero',
69    '//src/protozero/protoc_plugin:protozero_plugin',
70    '//src/protozero/protoc_plugin:cppgen_plugin',
71] + public_targets
72
73# Root proto targets (to force discovery of intermediate proto targets).
74# These targets are marked public.
75proto_targets = [
76    '//protos/perfetto/trace:merged_trace',
77    '//protos/perfetto/trace:non_minimal_lite',
78    '//protos/perfetto/config:merged_config',
79    '//protos/perfetto/metrics:lite',
80    '//protos/perfetto/metrics/android:lite',
81    '//protos/perfetto/trace:lite',
82    '//protos/perfetto/config:lite',
83]
84
85# Path for the protobuf sources in the standalone build.
86buildtools_protobuf_src = '//buildtools/protobuf/src'
87
88# The directory where the generated perfetto_build_flags.h will be copied into.
89buildflags_dir = 'include/perfetto/base/build_configs/bazel'
90
91# Internal equivalents for third-party libraries that the upstream project
92# depends on.
93external_deps = {
94    '//gn:default_deps': [],
95    '//gn:jsoncpp': ['PERFETTO_CONFIG.deps.jsoncpp'],
96    '//gn:linenoise': ['PERFETTO_CONFIG.deps.linenoise'],
97    '//gn:protobuf_full': ['PERFETTO_CONFIG.deps.protobuf_full'],
98    '//gn:protobuf_lite': ['PERFETTO_CONFIG.deps.protobuf_lite'],
99    '//gn:protoc_lib': ['PERFETTO_CONFIG.deps.protoc_lib'],
100    '//gn:protoc': ['PERFETTO_CONFIG.deps.protoc'],
101    '//gn:sqlite': [
102        'PERFETTO_CONFIG.deps.sqlite',
103        'PERFETTO_CONFIG.deps.sqlite_ext_percentile'
104    ],
105    '//gn:zlib': ['PERFETTO_CONFIG.deps.zlib'],
106    '//src/trace_processor/metrics:gen_merged_sql_metrics': [[
107        ':cc_merged_sql_metrics'
108    ]],
109    gn_utils.GEN_VERSION_TARGET: ['PERFETTO_CONFIG.deps.version_header'],
110}
111
112
113def gen_sql_metrics(target):
114  label = BazelLabel(get_bazel_label_name(target.name), 'genrule')
115  label.srcs += [re.sub('^//', '', x) for x in sorted(target.inputs)]
116  label.outs += target.outputs
117  label.cmd = r'$(location gen_merged_sql_metrics_py) --cpp_out=$@ $(SRCS)'
118  label.exec_tools += [':gen_merged_sql_metrics_py']
119  return [label]
120
121
122def gen_version_header(target):
123  label = BazelLabel(get_bazel_label_name(target.name), 'genrule')
124  label.srcs += [re.sub('^//', '', x) for x in sorted(target.inputs)]
125  label.outs += target.outputs
126  label.cmd = r'$(location gen_version_header_py)'
127  label.cmd += r' --cpp_out=$@ --changelog=$(location CHANGELOG)'
128  label.exec_tools += [':gen_version_header_py']
129  return [label]
130
131
132def gen_cc_metrics_descriptor(target):
133  label = BazelLabel(
134      get_bazel_label_name(target.name), 'perfetto_cc_proto_descriptor')
135  label.deps += [':' + get_bazel_label_name(x) for x in target.proto_deps]
136  label.outs += target.outputs
137  return [label]
138
139
140custom_actions = {
141    gn_utils.GEN_VERSION_TARGET: gen_version_header,
142    '//src/trace_processor/metrics:gen_merged_sql_metrics': gen_sql_metrics,
143}
144
145# ------------------------------------------------------------------------------
146# End of configuration.
147# ------------------------------------------------------------------------------
148
149
150class Error(Exception):
151  pass
152
153
154class BazelLabel(object):
155
156  def __init__(self, name, type):
157    self.comment = None
158    self.name = name
159    self.type = type
160    self.visibility = []
161    self.srcs = []
162    self.hdrs = []
163    self.deps = []
164    self.external_deps = []
165    self.tools = []
166    self.exec_tools = []
167    self.outs = []
168
169  def __lt__(self, other):
170    if isinstance(other, self.__class__):
171      return self.name < other.name
172    raise TypeError('\'<\' not supported between instances of \'%s\' and \'%s\''
173                    % (type(self).__name__, type(other).__name__))
174
175  def __str__(self):
176    """Converts the object into a Bazel Starlark label."""
177    res = ''
178    res += ('# GN target: %s\n' % self.comment) if self.comment else ''
179    res += '%s(\n' % self.type
180    any_deps = len(self.deps) + len(self.external_deps) > 0
181    ORD = [
182      'name','srcs', 'hdrs', 'visibility', 'deps', 'outs', 'cmd', 'tools',
183      'exec_tools'
184    ]
185    hasher = lambda x: sum((99,) + tuple(ord(c) for c in x))
186    key_sorter = lambda kv: ORD.index(kv[0]) if kv[0] in ORD else hasher(kv[0])
187    for k, v in sorted(iteritems(self.__dict__), key=key_sorter):
188      if k in ('type', 'comment',
189               'external_deps') or v is None or (v == [] and
190                                                 (k != 'deps' or not any_deps)):
191        continue
192      res += '    %s = ' % k
193      if isinstance(v, basestring):
194        if v.startswith('PERFETTO_CONFIG.'):
195          res += '%s,\n' % v
196        else:
197          res += '"%s",\n' % v
198      elif isinstance(v, bool):
199        res += '%s,\n' % v
200      elif isinstance(v, list):
201        res += '[\n'
202        if k == 'deps' and len(self.external_deps) > 1:
203          indent = '           '
204        else:
205          indent = '    '
206        for entry in sorted(v):
207          if entry.startswith('PERFETTO_CONFIG.'):
208            res += '%s    %s,\n' % (indent, entry)
209          else:
210            res += '%s    "%s",\n' % (indent, entry)
211        res += '%s]' % indent
212        if k == 'deps' and self.external_deps:
213          res += ' + %s' % self.external_deps[0]
214          for edep in self.external_deps[1:]:
215            if isinstance(edep, list):
216              res += ' + [\n'
217              for inner_dep in edep:
218                res += '        "%s",\n' % inner_dep
219              res += '    ]'
220            else:
221              res += ' +\n%s%s' % (indent, edep)
222        res += ',\n'
223      else:
224        raise Error('Unsupported value %s', type(v))
225    res += ')\n\n'
226    return res
227
228
229# Public visibility for targets in Bazel.
230PUBLIC_VISIBILITY = 'PERFETTO_CONFIG.public_visibility'
231
232
233def get_bazel_label_name(gn_name):
234  """Converts a GN target name into a Bazel label name.
235
236  If target is in the public target list, returns only the GN target name,
237  e.g.: //src/ipc:perfetto_ipc -> perfetto_ipc
238
239  Otherwise, in the case of an intermediate target, returns a mangled path.
240  e.g.:  //include/perfetto/base:base -> include_perfetto_base_base.
241  """
242  if gn_name in default_targets:
243    return gn_utils.label_without_toolchain(gn_name).split(':')[1]
244  return gn_utils.label_to_target_name_with_path(gn_name)
245
246
247def gen_proto_labels(target):
248  """ Generates the xx_proto_library label for proto targets.
249
250  Bazel requires that each protobuf-related target is modeled with two labels:
251  1. A plugin-agnostic target that defines only the .proto sources and their
252     dependencies.
253  2. A plugin-dependent target (e.g. cc_library, cc_protozero_library) that has
254     only a dependency on 1 and does NOT refer to any .proto sources.
255  """
256  assert (target.type == 'proto_library')
257
258  def get_sources_label(target_name):
259    return re.sub('_(lite|zero|cpp|ipc|source_set|descriptor)$', '',
260                  get_bazel_label_name(target_name)) + '_protos'
261
262  sources_label_name = get_sources_label(target.name)
263
264  # Generates 1.
265  sources_label = BazelLabel(sources_label_name, 'perfetto_proto_library')
266  sources_label.comment = target.name
267  assert (all(x.startswith('//') for x in target.sources))
268  assert (all(x.endswith('.proto') for x in target.sources))
269  sources_label.srcs = sorted([x[2:] for x in target.sources])  # Strip //.
270
271  deps = [
272      ':' + get_sources_label(x)
273      for x in target.proto_deps
274
275      # This is to avoid a dependency-on-self in the case where
276      # protos/perfetto/ipc:ipc depends on protos/perfetto/ipc:cpp and both
277      # targets resolve to "protos_perfetto_ipc_protos".
278      if get_sources_label(x) != sources_label_name
279  ]
280  sources_label.deps = sorted(deps)
281
282  # In Bazel, proto_paths are not a supported concept becauase strong dependency
283  # checking is enabled. Instead, we need to depend on the target which includes
284  # the proto we want to depend on.
285  # For example, we include the proto_path |buildtools_protobuf_src| because we
286  # want to depend on the "google/protobuf/descriptor.proto" proto file. This
287  # will be exposed by the |protobuf_descriptor_proto| dep.
288  if buildtools_protobuf_src in target.proto_paths:
289    sources_label.external_deps = [
290        'PERFETTO_CONFIG.deps.protobuf_descriptor_proto'
291    ]
292
293  if target.name in proto_targets:
294    sources_label.visibility = PUBLIC_VISIBILITY
295  else:
296    sources_label.visibility = ['PERFETTO_CONFIG.proto_library_visibility']
297
298  # For 'source_set' plugins, we don't want to generate any plugin-dependent
299  # targets so just return the label of the proto sources only.
300  if target.proto_plugin == 'source_set':
301    return [sources_label]
302
303  # Generates 2.
304  if target.proto_plugin == 'proto':
305    plugin_label_type = 'perfetto_cc_proto_library'
306  elif target.proto_plugin == 'protozero':
307    plugin_label_type = 'perfetto_cc_protozero_library'
308  elif target.proto_plugin == 'cppgen':
309    plugin_label_type = 'perfetto_cc_protocpp_library'
310  elif target.proto_plugin == 'ipc':
311    plugin_label_type = 'perfetto_cc_ipc_library'
312  elif target.proto_plugin == 'descriptor':
313    plugin_label_type = 'perfetto_proto_descriptor'
314  else:
315    raise Error('Unknown proto plugin: %s' % target.proto_plugin)
316  plugin_label_name = get_bazel_label_name(target.name)
317  plugin_label = BazelLabel(plugin_label_name, plugin_label_type)
318  plugin_label.comment = target.name
319  plugin_label.deps += [':' + sources_label_name]
320
321  # When using the plugins we need to pass down also the transitive deps.
322  # For instance consider foo.proto including common.proto. The generated
323  # foo.cc will #include "common.gen.h". Hence the generated cc_protocpp_library
324  # rule need to pass down the dependency on the target that generates
325  # common.gen.{cc,h}.
326  if target.proto_deps and target.proto_plugin in (
327      'cppgen', 'ipc', 'protozero'):
328    plugin_label.deps += [
329        ':' + get_bazel_label_name(x) for x in target.proto_deps
330    ]
331
332  if target.proto_plugin == 'descriptor':
333    plugin_label.outs = [plugin_label_name + '.bin']
334
335  return [sources_label, plugin_label]
336
337
338def gen_target(gn_target):
339  if gn_target.type == 'proto_library':
340    return gen_proto_labels(gn_target)
341  elif gn_target.type == 'action':
342    if gn_target.name in custom_actions:
343      return custom_actions[gn_target.name](gn_target)
344    elif re.match('.*gen_cc_.*_descriptor$', gn_target.name):
345      return gen_cc_metrics_descriptor(gn_target)
346    return []
347  elif gn_target.type == 'group':
348    return []
349  elif gn_target.type == 'executable':
350    bazel_type = 'perfetto_cc_binary'
351  elif gn_target.type == 'shared_library':
352    bazel_type = 'perfetto_cc_binary'
353    vars['linkshared'] = True
354  elif gn_target.type == 'static_library':
355    bazel_type = 'perfetto_cc_library'
356  elif gn_target.type == 'source_set':
357    bazel_type = 'filegroup'
358  else:
359    raise Error('target type not supported: %s' % gn_target.type)
360
361  label = BazelLabel(get_bazel_label_name(gn_target.name), bazel_type)
362  label.comment = gn_target.name
363
364  # Supporting 'public' on source_sets would require not converting them to
365  # filegroups in bazel.
366  if gn_target.public_headers:
367    if bazel_type == 'perfetto_cc_library':
368      label.hdrs += [x[2:] for x in gn_target.public_headers]
369    else:
370      raise Error('%s: \'public\' currently supported only for cc_library' %
371              gn_target.name)
372
373  raw_srcs = [x[2:] for x in gn_target.sources]
374  if bazel_type == 'perfetto_cc_library':
375    label.srcs += [x for x in raw_srcs if not x.startswith('include')]
376    label.hdrs += [x for x in raw_srcs if x.startswith('include')]
377
378    # Most Perfetto libraries cannot by dynamically linked as they would
379    # cause ODR violations.
380    label.__dict__['linkstatic'] = True
381  else:
382    label.srcs = raw_srcs
383
384  if gn_target.name in public_targets:
385    label.visibility = ['//visibility:public']
386
387  if gn_target.type in gn_utils.LINKER_UNIT_TYPES:
388    # |source_sets| contains the transitive set of source_set deps.
389    for trans_dep in gn_target.source_set_deps:
390      name = ':' + get_bazel_label_name(trans_dep)
391      if name.startswith(
392          ':include_perfetto_') and gn_target.type != 'executable':
393        label.hdrs += [name]
394      else:
395        label.srcs += [name]
396    for dep in sorted(gn_target.deps):
397      if dep.startswith('//gn:'):
398        assert (dep in external_deps), dep
399      if dep in external_deps:
400        assert (isinstance(external_deps[dep], list))
401        label.external_deps += external_deps[dep]
402      else:
403        label.deps += [':' + get_bazel_label_name(dep)]
404    label.deps += [':' + get_bazel_label_name(x) for x in gn_target.proto_deps]
405
406  # All items starting with : need to be sorted to the end of the list.
407  # However, Python makes specifying a comparator function hard so cheat
408  # instead and make everything start with : sort as if it started with |
409  # As | > all other normal ASCII characters, this will lead to all : targets
410  # starting with : to be sorted to the end.
411  label.srcs = sorted(label.srcs, key=lambda x: x.replace(':', '|'))
412
413  label.deps = sorted(label.deps)
414  label.hdrs = sorted(label.hdrs)
415  return [label]
416
417
418def gen_target_str(gn_target):
419  return ''.join(str(x) for x in gen_target(gn_target))
420
421
422def generate_build(gn_desc, targets, extras):
423  gn = gn_utils.GnParser(gn_desc)
424  project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
425  tool_name = os.path.relpath(os.path.abspath(__file__), project_root)
426  res = '''
427# Copyright (C) 2019 The Android Open Source Project
428#
429# Licensed under the Apache License, Version 2.0 (the "License");
430# you may not use this file except in compliance with the License.
431# You may obtain a copy of the License at
432#
433#      http://www.apache.org/licenses/LICENSE-2.0
434#
435# Unless required by applicable law or agreed to in writing, software
436# distributed under the License is distributed on an "AS IS" BASIS,
437# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
438# See the License for the specific language governing permissions and
439# limitations under the License.
440#
441# This file is automatically generated by {}. Do not edit.
442
443load("@perfetto_cfg//:perfetto_cfg.bzl", "PERFETTO_CONFIG")
444load(
445    "@perfetto//bazel:rules.bzl",
446    "perfetto_cc_binary",
447    "perfetto_cc_ipc_library",
448    "perfetto_cc_library",
449    "perfetto_cc_proto_descriptor",
450    "perfetto_cc_proto_library",
451    "perfetto_cc_protocpp_library",
452    "perfetto_cc_protozero_library",
453    "perfetto_java_proto_library",
454    "perfetto_java_lite_proto_library",
455    "perfetto_proto_library",
456    "perfetto_proto_descriptor",
457    "perfetto_py_binary",
458    "perfetto_py_library",
459    "perfetto_gensignature_internal_only",
460)
461
462package(default_visibility = ["//visibility:private"])
463
464licenses(["notice"])
465
466exports_files(["NOTICE"])
467
468'''.format(tool_name).lstrip()
469
470  # Public targets need to be computed at the beginning (to figure out the
471  # intermediate deps) but printed at the end (because declaration order matters
472  # in Bazel).
473  public_str = ''
474  for target_name in sorted(public_targets):
475    target = gn.get_target(target_name)
476    public_str += gen_target_str(target)
477
478  res += '''
479# ##############################################################################
480# Internal targets
481# ##############################################################################
482
483'''.lstrip()
484  # Generate the other non-public targets.
485  for target_name in sorted(set(default_targets) - set(public_targets)):
486    target = gn.get_target(target_name)
487    res += gen_target_str(target)
488
489  # Generate all the intermediate targets.
490  for target in sorted(itervalues(gn.all_targets)):
491    if target.name in default_targets or target.name in gn.proto_libs:
492      continue
493    res += gen_target_str(target)
494
495  res += '''
496# ##############################################################################
497# Proto libraries
498# ##############################################################################
499
500'''.lstrip()
501  # Force discovery of explicilty listed root proto targets.
502  for target_name in sorted(proto_targets):
503    gn.get_target(target_name)
504
505  # Generate targets for the transitive set of proto targets.
506  # TODO explain deduping here.
507  labels = {}
508  for target in sorted(itervalues(gn.proto_libs)):
509    for label in gen_target(target):
510      # Ensure that if the existing target has public visibility, we preserve
511      # that in the new label; this ensures that we don't accidentaly reduce
512      # the visibility of targets which are meant to be public.
513      existing_label = labels.get(label.name)
514      if existing_label and existing_label.visibility == PUBLIC_VISIBILITY:
515        label.visibility = PUBLIC_VISIBILITY
516      labels[label.name] = label
517
518  res += ''.join(str(x) for x in sorted(itervalues(labels)))
519
520  res += '''
521# ##############################################################################
522# Public targets
523# ##############################################################################
524
525'''.lstrip()
526  res += public_str
527  res += '# Content from BUILD.extras\n\n'
528  res += extras
529
530  # Check for ODR violations
531  for target_name in default_targets + proto_targets:
532    checker = gn_utils.ODRChecker(gn, target_name)
533
534  return res
535
536def main():
537  parser = argparse.ArgumentParser(
538      description='Generate BUILD from a GN description.')
539  parser.add_argument(
540      '--check-only',
541      help='Don\'t keep the generated files',
542      action='store_true')
543  parser.add_argument(
544      '--desc',
545      help='GN description ' +
546      '(e.g., gn desc out --format=json --all-toolchains "//*"')
547  parser.add_argument(
548      '--repo-root',
549      help='Standalone Perfetto repository to generate a GN description',
550      default=gn_utils.repo_root(),
551  )
552  parser.add_argument(
553      '--extras',
554      help='Extra targets to include at the end of the BUILD file',
555      default=os.path.join(gn_utils.repo_root(), 'BUILD.extras'),
556  )
557  parser.add_argument(
558      '--output',
559      help='BUILD file to create',
560      default=os.path.join(gn_utils.repo_root(), 'BUILD'),
561  )
562  parser.add_argument(
563      '--output-proto',
564      help='Proto BUILD file to create',
565      default=os.path.join(gn_utils.repo_root(), 'protos', 'BUILD'),
566  )
567  parser.add_argument(
568      'targets',
569      nargs=argparse.REMAINDER,
570      help='Targets to include in the BUILD file (e.g., "//:perfetto_tests")')
571  args = parser.parse_args()
572
573  if args.desc:
574    with open(args.desc) as f:
575      desc = json.load(f)
576  else:
577    desc = gn_utils.create_build_description(gn_args, args.repo_root)
578
579  out_files = []
580
581  # Generate the main BUILD file.
582  with open(args.extras, 'r') as extra_f:
583    extras = extra_f.read()
584
585  contents = generate_build(desc, args.targets or default_targets, extras)
586  out_files.append(args.output + '.swp')
587  with open(out_files[-1], 'w') as out_f:
588    out_f.write(contents)
589
590  # Generate the build flags file.
591  out_files.append(os.path.join(buildflags_dir, 'perfetto_build_flags.h.swp'))
592  gn_utils.gen_buildflags(gn_args, out_files[-1])
593
594  return gn_utils.check_or_commit_generated_files(out_files, args.check_only)
595
596
597if __name__ == '__main__':
598  sys.exit(main())
599