1#!/usr/bin/env python
2# Copyright 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""The frontend for the Mojo bindings system."""
7
8
9import argparse
10import cPickle
11import hashlib
12import importlib
13import json
14import os
15import pprint
16import re
17import struct
18import sys
19
20# Disable lint check for finding modules:
21# pylint: disable=F0401
22
23def _GetDirAbove(dirname):
24  """Returns the directory "above" this file containing |dirname| (which must
25  also be "above" this file)."""
26  path = os.path.abspath(__file__)
27  while True:
28    path, tail = os.path.split(path)
29    assert tail
30    if tail == dirname:
31      return path
32
33# Manually check for the command-line flag. (This isn't quite right, since it
34# ignores, e.g., "--", but it's close enough.)
35if "--use_bundled_pylibs" in sys.argv[1:]:
36  sys.path.insert(0, os.path.join(_GetDirAbove("mojo"), "third_party"))
37
38sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),
39                                "pylib"))
40
41from mojom.error import Error
42import mojom.fileutil as fileutil
43from mojom.generate import template_expander
44from mojom.generate import translate
45from mojom.generate.generator import AddComputedData, WriteFile
46from mojom.parse.conditional_features import RemoveDisabledDefinitions
47from mojom.parse.parser import Parse
48
49
50_BUILTIN_GENERATORS = {
51  "c++": "mojom_cpp_generator",
52  "javascript": "mojom_js_generator",
53  "java": "mojom_java_generator",
54}
55
56
57def LoadGenerators(generators_string):
58  if not generators_string:
59    return []  # No generators.
60
61  generators = {}
62  for generator_name in [s.strip() for s in generators_string.split(",")]:
63    language = generator_name.lower()
64    if language not in _BUILTIN_GENERATORS:
65      print "Unknown generator name %s" % generator_name
66      sys.exit(1)
67    generator_module = importlib.import_module(
68        "generators.%s" % _BUILTIN_GENERATORS[language])
69    generators[language] = generator_module
70  return generators
71
72
73def MakeImportStackMessage(imported_filename_stack):
74  """Make a (human-readable) message listing a chain of imports. (Returned
75  string begins with a newline (if nonempty) and does not end with one.)"""
76  return ''.join(
77      reversed(["\n  %s was imported by %s" % (a, b) for (a, b) in \
78                    zip(imported_filename_stack[1:], imported_filename_stack)]))
79
80
81class RelativePath(object):
82  """Represents a path relative to the source tree."""
83  def __init__(self, path, source_root):
84    self.path = path
85    self.source_root = source_root
86
87  def relative_path(self):
88    return os.path.relpath(os.path.abspath(self.path),
89                           os.path.abspath(self.source_root))
90
91
92def FindImportFile(rel_dir, file_name, search_rel_dirs):
93  """Finds |file_name| in either |rel_dir| or |search_rel_dirs|. Returns a
94  RelativePath with first file found, or an arbitrary non-existent file
95  otherwise."""
96  for rel_search_dir in [rel_dir] + search_rel_dirs:
97    path = os.path.join(rel_search_dir.path, file_name)
98    if os.path.isfile(path):
99      return RelativePath(path, rel_search_dir.source_root)
100  return RelativePath(os.path.join(rel_dir.path, file_name),
101                      rel_dir.source_root)
102
103
104def ScrambleMethodOrdinals(interfaces, salt):
105  already_generated = set()
106  for interface in interfaces:
107    i = 0
108    already_generated.clear()
109    for method in interface.methods:
110      while True:
111        i = i + 1
112        if i == 1000000:
113          raise Exception("Could not generate %d method ordinals for %s" %
114              (len(interface.methods), interface.mojom_name))
115        # Generate a scrambled method.ordinal value. The algorithm doesn't have
116        # to be very strong, cryptographically. It just needs to be non-trivial
117        # to guess the results without the secret salt, in order to make it
118        # harder for a compromised process to send fake Mojo messages.
119        sha256 = hashlib.sha256(salt)
120        sha256.update(interface.mojom_name)
121        sha256.update(str(i))
122        # Take the first 4 bytes as a little-endian uint32.
123        ordinal = struct.unpack('<L', sha256.digest()[:4])[0]
124        # Trim to 31 bits, so it always fits into a Java (signed) int.
125        ordinal = ordinal & 0x7fffffff
126        if ordinal in already_generated:
127          continue
128        already_generated.add(ordinal)
129        method.ordinal = ordinal
130        method.ordinal_comment = (
131            'The %s value is based on sha256(salt + "%s%d").' %
132            (ordinal, interface.mojom_name, i))
133        break
134
135
136def ReadFileContents(filename):
137  with open(filename, 'rb') as f:
138    return f.read()
139
140
141class MojomProcessor(object):
142  """Parses mojom files and creates ASTs for them.
143
144  Attributes:
145    _processed_files: {Dict[str, mojom.generate.module.Module]} Mapping from
146        relative mojom filename paths to the module AST for that mojom file.
147  """
148  def __init__(self, should_generate):
149    self._should_generate = should_generate
150    self._processed_files = {}
151    self._typemap = {}
152
153  def LoadTypemaps(self, typemaps):
154    # Support some very simple single-line comments in typemap JSON.
155    comment_expr = r"^\s*//.*$"
156    def no_comments(line):
157      return not re.match(comment_expr, line)
158    for filename in typemaps:
159      with open(filename) as f:
160        typemaps = json.loads("".join(filter(no_comments, f.readlines())))
161        for language, typemap in typemaps.iteritems():
162          language_map = self._typemap.get(language, {})
163          language_map.update(typemap)
164          self._typemap[language] = language_map
165
166  def _GenerateModule(self, args, remaining_args, generator_modules,
167                      rel_filename, imported_filename_stack):
168    # Return the already-generated module.
169    if rel_filename.path in self._processed_files:
170      return self._processed_files[rel_filename.path]
171
172    if rel_filename.path in imported_filename_stack:
173      print "%s: Error: Circular dependency" % rel_filename.path + \
174          MakeImportStackMessage(imported_filename_stack + [rel_filename.path])
175      sys.exit(1)
176
177    tree = _UnpickleAST(_FindPicklePath(rel_filename, args.gen_directories +
178                                        [args.output_dir]))
179    dirname = os.path.dirname(rel_filename.path)
180
181    # Process all our imports first and collect the module object for each.
182    # We use these to generate proper type info.
183    imports = {}
184    for parsed_imp in tree.import_list:
185      rel_import_file = FindImportFile(
186          RelativePath(dirname, rel_filename.source_root),
187          parsed_imp.import_filename, args.import_directories)
188      imports[parsed_imp.import_filename] = self._GenerateModule(
189          args, remaining_args, generator_modules, rel_import_file,
190          imported_filename_stack + [rel_filename.path])
191
192    # Set the module path as relative to the source root.
193    # Normalize to unix-style path here to keep the generators simpler.
194    module_path = rel_filename.relative_path().replace('\\', '/')
195
196    module = translate.OrderedModule(tree, module_path, imports)
197
198    if args.scrambled_message_id_salt_paths:
199      salt = ''.join(
200          map(ReadFileContents, args.scrambled_message_id_salt_paths))
201      ScrambleMethodOrdinals(module.interfaces, salt)
202
203    if self._should_generate(rel_filename.path):
204      AddComputedData(module)
205      for language, generator_module in generator_modules.iteritems():
206        generator = generator_module.Generator(
207            module, args.output_dir, typemap=self._typemap.get(language, {}),
208            variant=args.variant, bytecode_path=args.bytecode_path,
209            for_blink=args.for_blink,
210            use_once_callback=args.use_once_callback,
211            js_bindings_mode=args.js_bindings_mode,
212            export_attribute=args.export_attribute,
213            export_header=args.export_header,
214            generate_non_variant_code=args.generate_non_variant_code,
215            support_lazy_serialization=args.support_lazy_serialization,
216            disallow_native_types=args.disallow_native_types,
217            disallow_interfaces=args.disallow_interfaces,
218            generate_message_ids=args.generate_message_ids,
219            generate_fuzzing=args.generate_fuzzing)
220        filtered_args = []
221        if hasattr(generator_module, 'GENERATOR_PREFIX'):
222          prefix = '--' + generator_module.GENERATOR_PREFIX + '_'
223          filtered_args = [arg for arg in remaining_args
224                           if arg.startswith(prefix)]
225        generator.GenerateFiles(filtered_args)
226
227    # Save result.
228    self._processed_files[rel_filename.path] = module
229    return module
230
231
232def _Generate(args, remaining_args):
233  if args.variant == "none":
234    args.variant = None
235
236  for idx, import_dir in enumerate(args.import_directories):
237    tokens = import_dir.split(":")
238    if len(tokens) >= 2:
239      args.import_directories[idx] = RelativePath(tokens[0], tokens[1])
240    else:
241      args.import_directories[idx] = RelativePath(tokens[0], args.depth)
242  generator_modules = LoadGenerators(args.generators_string)
243
244  fileutil.EnsureDirectoryExists(args.output_dir)
245
246  processor = MojomProcessor(lambda filename: filename in args.filename)
247  processor.LoadTypemaps(set(args.typemaps))
248
249  if args.filelist:
250    with open(args.filelist) as f:
251      args.filename.extend(f.read().split())
252
253  for filename in args.filename:
254    processor._GenerateModule(args, remaining_args, generator_modules,
255                              RelativePath(filename, args.depth), [])
256
257  return 0
258
259
260def _FindPicklePath(rel_filename, search_dirs):
261  filename, _ = os.path.splitext(rel_filename.relative_path())
262  pickle_path = filename + '.p'
263  for search_dir in search_dirs:
264    path = os.path.join(search_dir, pickle_path)
265    if os.path.isfile(path):
266      return path
267  raise Exception("%s: Error: Could not find file in %r" % (pickle_path, search_dirs))
268
269
270def _GetPicklePath(rel_filename, output_dir):
271  filename, _ = os.path.splitext(rel_filename.relative_path())
272  pickle_path = filename + '.p'
273  return os.path.join(output_dir, pickle_path)
274
275
276def _PickleAST(ast, output_file):
277  full_dir = os.path.dirname(output_file)
278  fileutil.EnsureDirectoryExists(full_dir)
279
280  try:
281    WriteFile(cPickle.dumps(ast), output_file)
282  except (IOError, cPickle.PicklingError) as e:
283    print "%s: Error: %s" % (output_file, str(e))
284    sys.exit(1)
285
286def _UnpickleAST(input_file):
287    try:
288      with open(input_file, "rb") as f:
289        return cPickle.load(f)
290    except (IOError, cPickle.UnpicklingError) as e:
291      print "%s: Error: %s" % (input_file, str(e))
292      sys.exit(1)
293
294def _ParseFile(args, rel_filename):
295  try:
296    with open(rel_filename.path) as f:
297      source = f.read()
298  except IOError as e:
299    print "%s: Error: %s" % (rel_filename.path, e.strerror)
300    sys.exit(1)
301
302  try:
303    tree = Parse(source, rel_filename.path)
304    RemoveDisabledDefinitions(tree, args.enabled_features)
305  except Error as e:
306    print "%s: Error: %s" % (rel_filename.path, str(e))
307    sys.exit(1)
308  _PickleAST(tree, _GetPicklePath(rel_filename, args.output_dir))
309
310
311def _Parse(args, _):
312  fileutil.EnsureDirectoryExists(args.output_dir)
313
314  if args.filelist:
315    with open(args.filelist) as f:
316      args.filename.extend(f.read().split())
317
318  for filename in args.filename:
319    _ParseFile(args, RelativePath(filename, args.depth))
320  return 0
321
322
323def _Precompile(args, _):
324  generator_modules = LoadGenerators(",".join(_BUILTIN_GENERATORS.keys()))
325
326  template_expander.PrecompileTemplates(generator_modules, args.output_dir)
327  return 0
328
329def _VerifyImportDeps(args, __):
330  fileutil.EnsureDirectoryExists(args.gen_dir)
331
332  if args.filelist:
333    with open(args.filelist) as f:
334      args.filename.extend(f.read().split())
335
336  for filename in args.filename:
337    rel_path = RelativePath(filename, args.depth)
338    tree = _UnpickleAST(_GetPicklePath(rel_path, args.gen_dir))
339
340    mojom_imports = set(
341      parsed_imp.import_filename for parsed_imp in tree.import_list
342      )
343
344    # read the paths from the file
345    f_deps = open(args.deps_file, 'r')
346    deps_sources = set()
347    for deps_path in f_deps:
348      deps_path = deps_path.rstrip('\n')
349      f_sources = open(deps_path, 'r')
350
351      for source_file in f_sources:
352        source_dir = deps_path.split(args.gen_dir + "/", 1)[1]
353        full_source_path = os.path.dirname(source_dir) + "/" +  \
354          source_file
355        deps_sources.add(full_source_path.rstrip('\n'))
356
357    if (not deps_sources.issuperset(mojom_imports)):
358      print ">>> [%s] Missing dependencies for the following imports: %s" % ( \
359        args.filename[0], \
360        list(mojom_imports.difference(deps_sources)))
361      sys.exit(1)
362
363    source_filename, _ = os.path.splitext(rel_path.relative_path())
364    output_file = source_filename + '.v'
365    output_file_path = os.path.join(args.gen_dir, output_file)
366    WriteFile("", output_file_path)
367
368  return 0
369
370def main():
371  parser = argparse.ArgumentParser(
372      description="Generate bindings from mojom files.")
373  parser.add_argument("--use_bundled_pylibs", action="store_true",
374                      help="use Python modules bundled in the SDK")
375
376  subparsers = parser.add_subparsers()
377
378  parse_parser = subparsers.add_parser(
379      "parse", description="Parse mojom to AST and remove disabled definitions."
380                           " Pickle pruned AST into output_dir.")
381  parse_parser.add_argument("filename", nargs="*", help="mojom input file")
382  parse_parser.add_argument("--filelist", help="mojom input file list")
383  parse_parser.add_argument(
384      "-o",
385      "--output_dir",
386      dest="output_dir",
387      default=".",
388      help="output directory for generated files")
389  parse_parser.add_argument(
390      "-d", "--depth", dest="depth", default=".", help="depth from source root")
391  parse_parser.add_argument(
392      "--enable_feature",
393      dest = "enabled_features",
394      default=[],
395      action="append",
396      help="Controls which definitions guarded by an EnabledIf attribute "
397      "will be enabled. If an EnabledIf attribute does not specify a value "
398      "that matches one of the enabled features, it will be disabled.")
399  parse_parser.set_defaults(func=_Parse)
400
401  generate_parser = subparsers.add_parser(
402      "generate", description="Generate bindings from mojom files.")
403  generate_parser.add_argument("filename", nargs="*",
404                               help="mojom input file")
405  generate_parser.add_argument("--filelist", help="mojom input file list")
406  generate_parser.add_argument("-d", "--depth", dest="depth", default=".",
407                               help="depth from source root")
408  generate_parser.add_argument("-o", "--output_dir", dest="output_dir",
409                               default=".",
410                               help="output directory for generated files")
411  generate_parser.add_argument("-g", "--generators",
412                               dest="generators_string",
413                               metavar="GENERATORS",
414                               default="c++,javascript,java",
415                               help="comma-separated list of generators")
416  generate_parser.add_argument(
417      "--gen_dir", dest="gen_directories", action="append", metavar="directory",
418      default=[], help="add a directory to be searched for the syntax trees.")
419  generate_parser.add_argument(
420      "-I", dest="import_directories", action="append", metavar="directory",
421      default=[],
422      help="add a directory to be searched for import files. The depth from "
423           "source root can be specified for each import by appending it after "
424           "a colon")
425  generate_parser.add_argument("--typemap", action="append", metavar="TYPEMAP",
426                               default=[], dest="typemaps",
427                               help="apply TYPEMAP to generated output")
428  generate_parser.add_argument("--variant", dest="variant", default=None,
429                               help="output a named variant of the bindings")
430  generate_parser.add_argument(
431      "--bytecode_path", required=True, help=(
432          "the path from which to load template bytecode; to generate template "
433          "bytecode, run %s precompile BYTECODE_PATH" % os.path.basename(
434              sys.argv[0])))
435  generate_parser.add_argument("--for_blink", action="store_true",
436                               help="Use WTF types as generated types for mojo "
437                               "string/array/map.")
438  generate_parser.add_argument(
439      "--use_once_callback", action="store_true",
440      help="Use base::OnceCallback instead of base::RepeatingCallback.")
441  generate_parser.add_argument(
442      "--js_bindings_mode", choices=["new", "both", "old"], default="new",
443      help="This option only affects the JavaScript bindings. The value could "
444      "be: \"new\" - generate only the new-style JS bindings, which use the "
445      "new module loading approach and the core api exposed by Web IDL; "
446      "\"both\" - generate both the old- and new-style bindings; \"old\" - "
447      "generate only the old-style bindings.")
448  generate_parser.add_argument(
449      "--export_attribute", default="",
450      help="Optional attribute to specify on class declaration to export it "
451      "for the component build.")
452  generate_parser.add_argument(
453      "--export_header", default="",
454      help="Optional header to include in the generated headers to support the "
455      "component build.")
456  generate_parser.add_argument(
457      "--generate_non_variant_code", action="store_true",
458      help="Generate code that is shared by different variants.")
459  generate_parser.add_argument(
460      "--scrambled_message_id_salt_path",
461      dest="scrambled_message_id_salt_paths",
462      help="If non-empty, the path to a file whose contents should be used as"
463      "a salt for generating scrambled message IDs. If this switch is specified"
464      "more than once, the contents of all salt files are concatenated to form"
465      "the salt value.", default=[], action="append")
466  generate_parser.add_argument(
467      "--support_lazy_serialization",
468      help="If set, generated bindings will serialize lazily when possible.",
469      action="store_true")
470  generate_parser.add_argument(
471      "--disallow_native_types",
472      help="Disallows the [Native] attribute to be specified on structs or "
473      "enums within the mojom file.", action="store_true")
474  generate_parser.add_argument(
475      "--disallow_interfaces",
476      help="Disallows interface definitions within the mojom file. It is an "
477      "error to specify this flag when processing a mojom file which defines "
478      "any interface.", action="store_true")
479  generate_parser.add_argument(
480      "--generate_message_ids",
481      help="Generates only the message IDs header for C++ bindings. Note that "
482      "this flag only matters if --generate_non_variant_code is also "
483      "specified.", action="store_true")
484  generate_parser.add_argument(
485      "--generate_fuzzing",
486      action="store_true",
487      help="Generates additional bindings for fuzzing in JS.")
488  generate_parser.set_defaults(func=_Generate)
489
490  precompile_parser = subparsers.add_parser("precompile",
491      description="Precompile templates for the mojom bindings generator.")
492  precompile_parser.add_argument(
493      "-o", "--output_dir", dest="output_dir", default=".",
494      help="output directory for precompiled templates")
495  precompile_parser.set_defaults(func=_Precompile)
496
497  verify_parser = subparsers.add_parser("verify", description="Checks "
498      "the set of imports against the set of dependencies.")
499  verify_parser.add_argument("filename", nargs="*",
500      help="mojom input file")
501  verify_parser.add_argument("--filelist", help="mojom input file list")
502  verify_parser.add_argument("-f", "--file", dest="deps_file",
503      help="file containing paths to the sources files for "
504      "dependencies")
505  verify_parser.add_argument("-g", "--gen_dir",
506      dest="gen_dir",
507      help="directory with the syntax tree")
508  verify_parser.add_argument(
509      "-d", "--depth", dest="depth",
510      help="depth from source root")
511
512  verify_parser.set_defaults(func=_VerifyImportDeps)
513
514  args, remaining_args = parser.parse_known_args()
515  return args.func(args, remaining_args)
516
517
518if __name__ == "__main__":
519  sys.exit(main())
520