1#!/usr/bin/env python
2# Copyright (C) 2018 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This tool translates a collection of BUILD.gn files into a mostly equivalent
17# BUILD file for the Bazel build system. The input to the tool is a
18# JSON description of the GN build definition generated with the following
19# command:
20#
21#   gn desc out --format=json --all-toolchains "//*" > desc.json
22#
23# The tool is then given a list of GN labels for which to generate Bazel
24# build rules.
25
26from __future__ import print_function
27import argparse
28import errno
29import functools
30import json
31import os
32import re
33import shutil
34import subprocess
35import sys
36import textwrap
37
38# Copyright header for generated code.
39header = """# Copyright (C) 2019 The Android Open Source Project
40#
41# Licensed under the Apache License, Version 2.0 (the "License");
42# you may not use this file except in compliance with the License.
43# You may obtain a copy of the License at
44#
45#      http://www.apache.org/licenses/LICENSE-2.0
46#
47# Unless required by applicable law or agreed to in writing, software
48# distributed under the License is distributed on an "AS IS" BASIS,
49# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
50# See the License for the specific language governing permissions and
51# limitations under the License.
52#
53# This file is automatically generated by {}. Do not edit.
54""".format(__file__)
55
56# Arguments for the GN output directory.
57# host_os="linux" is to generate the right build files from Mac OS.
58gn_args = 'target_os="linux" is_debug=false host_os="linux"'
59
60# Default targets to translate to the blueprint file.
61default_targets = [
62  '//src/protozero:libprotozero',
63  '//src/trace_processor:trace_processor',
64  '//src/trace_processor:trace_processor_shell_host(//gn/standalone/toolchain:gcc_like_host)',
65  '//tools/trace_to_text:trace_to_text_host(//gn/standalone/toolchain:gcc_like_host)',
66  '//protos/perfetto/config:merged_config_gen',
67  '//protos/perfetto/trace:merged_trace_gen',
68]
69
70# Aliases to add to the BUILD file
71alias_targets = {
72  '//src/protozero:libprotozero': 'libprotozero',
73  '//src/trace_processor:trace_processor': 'trace_processor',
74  '//src/trace_processor:trace_processor_shell_host': 'trace_processor_shell',
75  '//tools/trace_to_text:trace_to_text_host': 'trace_to_text',
76}
77
78
79def enable_sqlite(module):
80  module.deps.add(Label('//third_party/sqlite'))
81  module.deps.add(Label('//third_party/sqlite:sqlite_ext_percentile'))
82
83
84def enable_jsoncpp(module):
85  module.deps.add(Label('//third_party/perfetto/google:jsoncpp'))
86
87
88def enable_linenoise(module):
89  module.deps.add(Label('//third_party/perfetto/google:linenoise'))
90
91
92def enable_gtest_prod(module):
93  module.deps.add(Label('//third_party/perfetto/google:gtest_prod'))
94
95
96def enable_protobuf_full(module):
97  module.deps.add(Label('//third_party/protobuf:libprotoc'))
98  module.deps.add(Label('//third_party/protobuf'))
99
100
101def enable_perfetto_version(module):
102  module.deps.add(Label('//third_party/perfetto/google:perfetto_version'))
103
104
105def disable_module(module):
106  pass
107
108
109# Internal equivalents for third-party libraries that the upstream project
110# depends on.
111builtin_deps = {
112    '//gn:jsoncpp_deps': enable_jsoncpp,
113    '//buildtools:linenoise': enable_linenoise,
114    '//buildtools:protobuf_lite': disable_module,
115    '//buildtools:protobuf_full': enable_protobuf_full,
116    '//buildtools:protoc': disable_module,
117    '//buildtools:sqlite': enable_sqlite,
118    '//gn:default_deps': disable_module,
119    '//gn:gtest_prod_config': enable_gtest_prod,
120    '//gn:protoc_lib_deps': enable_protobuf_full,
121    '//gn/standalone:gen_git_revision': enable_perfetto_version,
122}
123
124# ----------------------------------------------------------------------------
125# End of configuration.
126# ----------------------------------------------------------------------------
127
128
129def check_output(cmd, cwd):
130  try:
131    output = subprocess.check_output(
132        cmd, stderr=subprocess.STDOUT, cwd=cwd)
133  except subprocess.CalledProcessError as e:
134    print('Cmd "{}" failed in {}:'.format(
135        ' '.join(cmd), cwd), file=sys.stderr)
136    print(e.output)
137    exit(1)
138  else:
139    return output
140
141
142class Error(Exception):
143  pass
144
145
146def repo_root():
147  """Returns an absolute path to the repository root."""
148  return os.path.join(
149      os.path.realpath(os.path.dirname(__file__)), os.path.pardir)
150
151
152def create_build_description(repo_root):
153  """Creates the JSON build description by running GN."""
154
155  out = os.path.join(repo_root, 'out', 'tmp.gen_build')
156  try:
157    try:
158      os.makedirs(out)
159    except OSError as e:
160      if e.errno != errno.EEXIST:
161        raise
162    check_output(
163        ['gn', 'gen', out, '--args=%s' % gn_args], repo_root)
164    desc = check_output(
165        ['gn', 'desc', out, '--format=json', '--all-toolchains', '//*'],
166        repo_root)
167    return json.loads(desc)
168  finally:
169    shutil.rmtree(out)
170
171
172def label_to_path(label):
173  """Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
174  assert label.startswith('//')
175  return label[2:]
176
177
178def label_to_target_name_with_path(label):
179  """
180  Turn a GN label into a target name involving the full path.
181  e.g., //src/perfetto:tests -> src_perfetto_tests
182  """
183  name = re.sub(r'^//:?', '', label)
184  name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
185  return name
186
187
188def label_without_toolchain(label):
189  """Strips the toolchain from a GN label.
190
191  Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
192  gcc_like_host) without the parenthesised toolchain part.
193  """
194  return label.split('(')[0]
195
196
197def is_public_header(label):
198  """
199  Returns if this is a c++ header file that is part of the API.
200  Args:
201      label: Label to evaluate
202  """
203  return label.endswith('.h') and label.startswith('//include/perfetto/')
204
205
206@functools.total_ordering
207class Label(object):
208  """Represents a label in BUILD file terminology. This class wraps a string
209  label to allow for correct comparision of labels for sorting.
210
211  Args:
212      label: The string rerepsentation of the label.
213  """
214
215  def __init__(self, label):
216    self.label = label
217
218  def is_absolute(self):
219    return self.label.startswith('//')
220
221  def dirname(self):
222    return self.label.split(':')[0] if ':' in self.label else self.label
223
224  def basename(self):
225    return self.label.split(':')[1] if ':' in self.label else ''
226
227  def __eq__(self, other):
228    return self.label == other.label
229
230  def __lt__(self, other):
231    return (
232        self.is_absolute(),
233        self.dirname(),
234        self.basename()
235    ) < (
236        other.is_absolute(),
237        other.dirname(),
238        other.basename()
239    )
240
241  def __str__(self):
242    return self.label
243
244  def __hash__(self):
245    return hash(self.label)
246
247
248class Writer(object):
249  def __init__(self, output, width=79):
250    self.output = output
251    self.width = width
252
253  def comment(self, text):
254    for line in textwrap.wrap(text,
255                              self.width - 2,
256                              break_long_words=False,
257                              break_on_hyphens=False):
258      self.output.write('# {}\n'.format(line))
259
260  def newline(self):
261    self.output.write('\n')
262
263  def line(self, s, indent=0):
264    self.output.write('    ' * indent + s + '\n')
265
266  def variable(self, key, value, sort=True):
267    if value is None:
268      return
269    if isinstance(value, set) or isinstance(value, list):
270      if len(value) == 0:
271        return
272      self.line('{} = ['.format(key), indent=1)
273      for v in sorted(list(value)) if sort else value:
274        self.line('"{}",'.format(v), indent=2)
275      self.line('],', indent=1)
276    elif isinstance(value, basestring):
277      self.line('{} = "{}",'.format(key, value), indent=1)
278    else:
279      self.line('{} = {},'.format(key, value), indent=1)
280
281  def header(self):
282    self.output.write(header)
283
284
285class Target(object):
286  """In-memory representation of a BUILD target."""
287
288  def __init__(self, type, name, gn_name=None):
289    assert type in ('cc_binary', 'cc_library', 'cc_proto_library',
290                    'proto_library', 'filegroup', 'alias',
291                    'pbzero_cc_proto_library', 'genrule', )
292    self.type = type
293    self.name = name
294    self.srcs = set()
295    self.hdrs = set()
296    self.deps = set()
297    self.visibility = set()
298    self.gn_name = gn_name
299    self.is_pbzero = False
300    self.src_proto_library = None
301    self.outs = set()
302    self.cmd = None
303    self.tools = set()
304
305  def write(self, writer):
306    if self.gn_name:
307      writer.comment('GN target: {}'.format(self.gn_name))
308
309    writer.line('{}('.format(self.type))
310    writer.variable('name', self.name)
311    writer.variable('srcs', self.srcs)
312    writer.variable('hdrs', self.hdrs)
313
314    if self.type == 'proto_library' and not self.is_pbzero:
315      if self.srcs:
316        writer.variable('has_services', 1)
317      writer.variable('cc_api_version', 2)
318      if self.srcs:
319        writer.variable('cc_generic_services', 1)
320
321    writer.variable('src_proto_library', self.src_proto_library)
322
323    writer.variable('outs', self.outs)
324    writer.variable('cmd', self.cmd)
325    writer.variable('tools', self.tools)
326
327    # Keep visibility and deps last.
328    writer.variable('visibility', self.visibility)
329
330    if type != 'filegroup':
331      writer.variable('deps', self.deps)
332
333    writer.line(')')
334
335
336class Build(object):
337  """In-memory representation of a BUILD file."""
338
339  def __init__(self, public, header_lines=[]):
340    self.targets = {}
341    self.public = public
342    self.header_lines = header_lines
343
344  def add_target(self, target):
345    self.targets[target.name] = target
346
347  def write(self, writer):
348    writer.header()
349    writer.newline()
350    for line in self.header_lines:
351      writer.line(line)
352    if self.header_lines:
353      writer.newline()
354    if self.public:
355      writer.line(
356          'package(default_visibility = ["//visibility:public"])')
357    else:
358      writer.line(
359          'package(default_visibility = ["//third_party/perfetto:__subpackages__"])')
360    writer.newline()
361    writer.line('licenses(["notice"])  # Apache 2.0')
362    writer.newline()
363    writer.line('exports_files(["LICENSE"])')
364    writer.newline()
365
366    sorted_targets = sorted(
367        self.targets.itervalues(), key=lambda m: m.name)
368    for target in sorted_targets[:-1]:
369      target.write(writer)
370      writer.newline()
371
372    # BUILD files shouldn't have a trailing new line.
373    sorted_targets[-1].write(writer)
374
375
376class BuildGenerator(object):
377  def __init__(self, desc):
378    self.desc = desc
379    self.action_generated_files = set()
380
381    for target in self.desc.itervalues():
382      if target['type'] == 'action':
383        self.action_generated_files.update(target['outputs'])
384
385
386  def create_build_for_targets(self, targets):
387    """Generate a BUILD for a list of GN targets and aliases."""
388    self.build = Build(public=True)
389
390    proto_cc_import = 'load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")'
391    pbzero_cc_import = 'load("//third_party/perfetto/google:build_defs.bzl", "pbzero_cc_proto_library")'
392    self.proto_build = Build(public=False, header_lines=[
393                        proto_cc_import, pbzero_cc_import])
394
395    for target in targets:
396      self.create_target(target)
397
398    return (self.build, self.proto_build)
399
400
401  def resolve_dependencies(self, target_name):
402    """Return the set of direct dependent-on targets for a GN target.
403
404    Args:
405        desc: JSON GN description.
406        target_name: Name of target
407
408    Returns:
409        A set of transitive dependencies in the form of GN targets.
410    """
411
412    if label_without_toolchain(target_name) in builtin_deps:
413      return set()
414    target = self.desc[target_name]
415    resolved_deps = set()
416    for dep in target.get('deps', []):
417      resolved_deps.add(dep)
418    return resolved_deps
419
420
421  def apply_module_sources_to_target(self, target, module_desc):
422    """
423    Args:
424        target: Module to which dependencies should be added.
425        module_desc: JSON GN description of the module.
426        visibility: Whether the module is visible with respect to the target.
427    """
428    for src in module_desc['sources']:
429      label = Label(label_to_path(src))
430      if target.type == 'cc_library' and is_public_header(src):
431        target.hdrs.add(label)
432      else:
433        target.srcs.add(label)
434
435
436  def apply_module_dependency(self, target, dep_name):
437    """
438    Args:
439        build: BUILD instance which is being generated.
440        proto_build: BUILD instance which is being generated to hold protos.
441        desc: JSON GN description.
442        target: Module to which dependencies should be added.
443        dep_name: GN target of the dependency.
444    """
445    # If the dependency refers to a library which we can replace with an internal
446    # equivalent, stop recursing and patch the dependency in.
447    dep_name_no_toolchain = label_without_toolchain(dep_name)
448    if dep_name_no_toolchain in builtin_deps:
449      builtin_deps[dep_name_no_toolchain](target)
450      return
451
452    dep_desc = self.desc[dep_name]
453    if dep_desc['type'] == 'source_set':
454      for inner_name in self.resolve_dependencies(dep_name):
455        self.apply_module_dependency(target, inner_name)
456
457      # Any source set which has a source generated by an action doesn't need
458      # to be depended on as we will depend on the action directly.
459      if any(src in self.action_generated_files for src in dep_desc['sources']):
460        return
461
462      self.apply_module_sources_to_target(target, dep_desc)
463    elif dep_desc['type'] == 'action':
464      args = dep_desc['args']
465      if "gen_merged_sql_metrics" in dep_name:
466        dep_target = self.create_merged_sql_metrics_target(dep_name)
467        target.deps.add(Label("//third_party/perfetto:" + dep_target.name))
468
469        if target.type == 'cc_library' or target.type == 'cc_binary':
470          target.srcs.update(dep_target.outs)
471      elif args[0].endswith('/protoc'):
472        (proto_target, cc_target) = self.create_proto_target(dep_name)
473        if target.type == 'proto_library':
474          dep_target_name = proto_target.name
475        else:
476          dep_target_name = cc_target.name
477        target.deps.add(
478            Label("//third_party/perfetto/protos:" + dep_target_name))
479      else:
480        raise Error('Unsupported action in target %s: %s' % (dep_target_name,
481                                                            args))
482    elif dep_desc['type'] == 'static_library':
483      dep_target = self.create_target(dep_name)
484      target.deps.add(Label("//third_party/perfetto:" + dep_target.name))
485    elif dep_desc['type'] == 'group':
486      for inner_name in self.resolve_dependencies(dep_name):
487        self.apply_module_dependency(target, inner_name)
488    elif dep_desc['type'] == 'executable':
489      # Just create the dep target but don't add it as a dep because it's an
490      # executable.
491      self.create_target(dep_name)
492    else:
493      raise Error('Unknown target name %s with type: %s' %
494                  (dep_name, dep_desc['type']))
495
496  def create_merged_sql_metrics_target(self, gn_target_name):
497    target_desc = self.desc[gn_target_name]
498    gn_target_name_no_toolchain = label_without_toolchain(gn_target_name)
499    target = Target(
500      'genrule',
501      'gen_merged_sql_metrics',
502      gn_name=gn_target_name_no_toolchain,
503    )
504    target.outs.update(
505      Label(src[src.index('gen/') + len('gen/'):])
506      for src in target_desc.get('outputs', [])
507    )
508    target.cmd = '$(location gen_merged_sql_metrics_py) --cpp_out=$@ $(SRCS)'
509    target.tools.update([
510      'gen_merged_sql_metrics_py',
511    ])
512    target.srcs.update(
513      Label(label_to_path(src))
514      for src in target_desc.get('inputs', [])
515      if src not in self.action_generated_files
516    )
517    self.build.add_target(target)
518    return target
519
520  def create_proto_target(self, gn_target_name):
521    target_desc = self.desc[gn_target_name]
522    args = target_desc['args']
523
524    gn_target_name_no_toolchain = label_without_toolchain(gn_target_name)
525    stripped_path = gn_target_name_no_toolchain.replace("protos/perfetto/", "")
526    pretty_target_name = label_to_target_name_with_path(stripped_path)
527    pretty_target_name = pretty_target_name.replace("_lite_gen", "")
528    pretty_target_name = pretty_target_name.replace("_zero_gen", "_zero")
529
530    proto_target = Target(
531      'proto_library',
532      pretty_target_name,
533      gn_name=gn_target_name_no_toolchain
534    )
535    proto_target.is_pbzero = any("pbzero" in arg for arg in args)
536    proto_target.srcs.update([
537      Label(label_to_path(src).replace('protos/', ''))
538      for src in target_desc.get('sources', [])
539    ])
540    if not proto_target.is_pbzero:
541      proto_target.visibility.add("//visibility:public")
542    self.proto_build.add_target(proto_target)
543
544    for dep_name in self.resolve_dependencies(gn_target_name):
545      self.apply_module_dependency(proto_target, dep_name)
546
547    if proto_target.is_pbzero:
548      # Remove all the protozero srcs from the proto_library.
549      proto_target.srcs.difference_update(
550          [src for src in proto_target.srcs if not src.label.endswith('.proto')])
551
552      # Remove all the non-proto deps from the proto_library and add to the cc
553      # library.
554      cc_deps = [
555        dep for dep in proto_target.deps
556        if not dep.label.startswith('//third_party/perfetto/protos')
557      ]
558      proto_target.deps.difference_update(cc_deps)
559
560      cc_target_name = proto_target.name + "_cc_proto"
561      cc_target = Target('pbzero_cc_proto_library', cc_target_name,
562                         gn_name=gn_target_name_no_toolchain)
563
564      cc_target.deps.add(Label('//third_party/perfetto:libprotozero'))
565      cc_target.deps.update(cc_deps)
566
567      # Add the proto_library to the cc_target.
568      cc_target.src_proto_library = \
569          "//third_party/perfetto/protos:" + proto_target.name
570
571      self.proto_build.add_target(cc_target)
572    else:
573      cc_target_name = proto_target.name + "_cc_proto"
574      cc_target = Target('cc_proto_library',
575                        cc_target_name, gn_name=gn_target_name_no_toolchain)
576      cc_target.visibility.add("//visibility:public")
577      cc_target.deps.add(
578          Label("//third_party/perfetto/protos:" + proto_target.name))
579      self.proto_build.add_target(cc_target)
580
581    return (proto_target, cc_target)
582
583
584  def create_target(self, gn_target_name):
585    """Generate module(s) for a given GN target.
586
587    Given a GN target name, generate one or more corresponding modules into a
588    build file.
589
590    Args:
591        build: Build instance which is being generated.
592        desc: JSON GN description.
593        gn_target_name: GN target name for module generation.
594    """
595
596    target_desc = self.desc[gn_target_name]
597    if target_desc['type'] == 'action':
598      args = target_desc['args']
599      if args[0].endswith('/protoc'):
600        return self.create_proto_target(gn_target_name)
601      else:
602        raise Error('Unsupported action in target %s: %s' % (gn_target_name,
603                                                            args))
604    elif target_desc['type'] == 'executable':
605      target_type = 'cc_binary'
606    elif target_desc['type'] == 'static_library':
607      target_type = 'cc_library'
608    elif target_desc['type'] == 'source_set':
609      target_type = 'filegroup'
610    else:
611      raise Error('Unknown target type: %s' % target_desc['type'])
612
613    label_no_toolchain = label_without_toolchain(gn_target_name)
614    target_name_path = label_to_target_name_with_path(label_no_toolchain)
615    target_name = alias_targets.get(label_no_toolchain, target_name_path)
616    target = Target(target_type, target_name, gn_name=label_no_toolchain)
617    target.srcs.update(
618        Label(label_to_path(src))
619        for src in target_desc.get('sources', [])
620        if src not in self.action_generated_files
621    )
622
623    for dep_name in self.resolve_dependencies(gn_target_name):
624      self.apply_module_dependency(target, dep_name)
625
626    self.build.add_target(target)
627    return target
628
629def main():
630  parser = argparse.ArgumentParser(
631      description='Generate BUILD from a GN description.')
632  parser.add_argument(
633      '--desc',
634      help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
635  )
636  parser.add_argument(
637      '--repo-root',
638      help='Standalone Perfetto repository to generate a GN description',
639      default=repo_root(),
640  )
641  parser.add_argument(
642      '--extras',
643      help='Extra targets to include at the end of the BUILD file',
644      default=os.path.join(repo_root(), 'BUILD.extras'),
645  )
646  parser.add_argument(
647      '--output',
648      help='BUILD file to create',
649      default=os.path.join(repo_root(), 'BUILD'),
650  )
651  parser.add_argument(
652      '--output-proto',
653      help='Proto BUILD file to create',
654      default=os.path.join(repo_root(), 'protos', 'BUILD'),
655  )
656  parser.add_argument(
657      'targets',
658      nargs=argparse.REMAINDER,
659      help='Targets to include in the BUILD file (e.g., "//:perfetto_tests")')
660  args = parser.parse_args()
661
662  if args.desc:
663    with open(args.desc) as f:
664      desc = json.load(f)
665  else:
666    desc = create_build_description(args.repo_root)
667
668  build_generator = BuildGenerator(desc)
669  build, proto_build = build_generator.create_build_for_targets(
670      args.targets or default_targets)
671  with open(args.output, 'w') as f:
672    writer = Writer(f)
673    build.write(writer)
674    writer.newline()
675
676    with open(args.extras, 'r') as r:
677      for line in r:
678        writer.line(line.rstrip("\n\r"))
679
680  with open(args.output_proto, 'w') as f:
681    proto_build.write(Writer(f))
682
683  return 0
684
685
686if __name__ == '__main__':
687  sys.exit(main())
688