1#! /usr/bin/python
2#
3# Protocol Buffers - Google's data interchange format
4# Copyright 2015 Google Inc.  All rights reserved.
5# https://developers.google.com/protocol-buffers/
6#
7# Redistribution and use in source and binary forms, with or without
8# modification, are permitted provided that the following conditions are
9# met:
10#
11#     * Redistributions of source code must retain the above copyright
12# notice, this list of conditions and the following disclaimer.
13#     * Redistributions in binary form must reproduce the above
14# copyright notice, this list of conditions and the following disclaimer
15# in the documentation and/or other materials provided with the
16# distribution.
17#     * Neither the name of Google Inc. nor the names of its
18# contributors may be used to endorse or promote products derived from
19# this software without specific prior written permission.
20#
21# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
33"""PDDM - Poor Developers' Debug-able Macros
34
35A simple markup that can be added in comments of source so they can then be
36expanded out into code. Most of this could be done with CPP macros, but then
37developers can't really step through them in the debugger, this way they are
38expanded to the same code, but you can debug them.
39
40Any file can be processed, but the syntax is designed around a C based compiler.
41Processed lines start with "//%".  There are three types of sections you can
42create: Text (left alone), Macro Definitions, and Macro Expansions.  There is
43no order required between definitions and expansions, all definitions are read
44before any expansions are processed (thus, if desired, definitions can be put
45at the end of the file to keep them out of the way of the code).
46
47Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines
48afterwards that start with "//%" are included in the definition.  Multiple
49macros can be defined in one block by just using a another "//%PDDM-DEFINE"
50line to start the next macro.  Optionally, a macro can be ended with
51"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that
52trailing blank lines are included in the macro.  You can also end a definition
53with an expansion.
54
55Macro Expansions are started by single lines containing
56"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another
57expansions.  All lines in-between are replaced by the result of the expansion.
58The first line of the expansion is always a blank like just for readability.
59
60Expansion itself is pretty simple, one macro can invoke another macro, but
61you cannot nest the invoke of a macro in another macro (i.e. - can't do
62"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo()
63within its expansion.
64
65When macros are expanded, the arg references can also add "$O" suffix to the
66name (i.e. - "NAME$O") to specify an option to be applied. The options are:
67
68    $S - Replace each character in the value with a space.
69    $l - Lowercase the first letter of the value.
70    $L - Lowercase the whole value.
71    $u - Uppercase the first letter of the value.
72    $U - Uppercase the whole value.
73
74Within a macro you can use ## to cause things to get joined together after
75expansion (i.e. - "a##b" within a macro will become "ab").
76
77Example:
78
79    int foo(MyEnum x) {
80    switch (x) {
81    //%PDDM-EXPAND case(Enum_Left, 1)
82    //%PDDM-EXPAND case(Enum_Center, 2)
83    //%PDDM-EXPAND case(Enum_Right, 3)
84    //%PDDM-EXPAND-END
85    }
86
87    //%PDDM-DEFINE case(_A, _B)
88    //%  case _A:
89    //%    return _B;
90
91  A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END
92  can be used to avoid adding extra blank lines/returns (or make it clear when
93  it is desired).
94
95  One macro can invoke another by simply using its name NAME(ARGS). You cannot
96  nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported).
97
98  Within a macro you can use ## to cause things to get joined together after
99  processing (i.e. - "a##b" within a macro will become "ab").
100
101
102"""
103
104import optparse
105import os
106import re
107import sys
108
109
110# Regex for macro definition.
111_MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)')
112# Regex for macro's argument definition.
113_MACRO_ARG_NAME_RE = re.compile(r'^\w+$')
114
115# Line inserted after each EXPAND.
116_GENERATED_CODE_LINE = (
117  '// This block of code is generated, do not edit it directly.'
118)
119
120
121def _MacroRefRe(macro_names):
122  # Takes in a list of macro names and makes a regex that will match invokes
123  # of those macros.
124  return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' %
125                    '|'.join(macro_names))
126
127def _MacroArgRefRe(macro_arg_names):
128  # Takes in a list of macro arg names and makes a regex that will match
129  # uses of those args.
130  return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' %
131                    '|'.join(macro_arg_names))
132
133
134class PDDMError(Exception):
135  """Error thrown by pddm."""
136  pass
137
138
139class MacroCollection(object):
140  """Hold a set of macros and can resolve/expand them."""
141
142  def __init__(self, a_file=None):
143    """Initializes the collection.
144
145    Args:
146      a_file: The file like stream to parse.
147
148    Raises:
149      PDDMError if there are any issues.
150    """
151    self._macros = dict()
152    if a_file:
153      self.ParseInput(a_file)
154
155  class MacroDefinition(object):
156    """Holds a macro definition."""
157
158    def __init__(self, name, arg_names):
159      self._name = name
160      self._args = tuple(arg_names)
161      self._body = ''
162      self._needNewLine = False
163
164    def AppendLine(self, line):
165      if self._needNewLine:
166        self._body += '\n'
167      self._body += line
168      self._needNewLine = not line.endswith('\n')
169
170    @property
171    def name(self):
172      return self._name
173
174    @property
175    def args(self):
176      return self._args
177
178    @property
179    def body(self):
180      return self._body
181
182  def ParseInput(self, a_file):
183    """Consumes input extracting definitions.
184
185    Args:
186      a_file: The file like stream to parse.
187
188    Raises:
189      PDDMError if there are any issues.
190    """
191    input_lines = a_file.read().splitlines()
192    self.ParseLines(input_lines)
193
194  def ParseLines(self, input_lines):
195    """Parses list of lines.
196
197    Args:
198      input_lines: A list of strings of input to parse (no newlines on the
199                   strings).
200
201    Raises:
202      PDDMError if there are any issues.
203    """
204    current_macro = None
205    for line in input_lines:
206      if line.startswith('PDDM-'):
207        directive = line.split(' ', 1)[0]
208        if directive == 'PDDM-DEFINE':
209          name, args = self._ParseDefineLine(line)
210          if self._macros.get(name):
211            raise PDDMError('Attempt to redefine macro: "%s"' % line)
212          current_macro = self.MacroDefinition(name, args)
213          self._macros[name] = current_macro
214          continue
215        if directive == 'PDDM-DEFINE-END':
216          if not current_macro:
217            raise PDDMError('Got DEFINE-END directive without an active macro:'
218                            ' "%s"' % line)
219          current_macro = None
220          continue
221        raise PDDMError('Hit a line with an unknown directive: "%s"' % line)
222
223      if current_macro:
224        current_macro.AppendLine(line)
225        continue
226
227      # Allow blank lines between macro definitions.
228      if line.strip() == '':
229        continue
230
231      raise PDDMError('Hit a line that wasn\'t a directive and no open macro'
232                      ' definition: "%s"' % line)
233
234  def _ParseDefineLine(self, input_line):
235    assert input_line.startswith('PDDM-DEFINE')
236    line = input_line[12:].strip()
237    match = _MACRO_RE.match(line)
238    # Must match full line
239    if match is None or match.group(0) != line:
240      raise PDDMError('Failed to parse macro definition: "%s"' % input_line)
241    name = match.group('name')
242    args_str = match.group('args').strip()
243    args = []
244    if args_str:
245      for part in args_str.split(','):
246        arg = part.strip()
247        if arg == '':
248          raise PDDMError('Empty arg name in macro definition: "%s"'
249                          % input_line)
250        if not _MACRO_ARG_NAME_RE.match(arg):
251          raise PDDMError('Invalid arg name "%s" in macro definition: "%s"'
252                          % (arg, input_line))
253        if arg in args:
254          raise PDDMError('Arg name "%s" used more than once in macro'
255                          ' definition: "%s"' % (arg, input_line))
256        args.append(arg)
257    return (name, tuple(args))
258
259  def Expand(self, macro_ref_str):
260    """Expands the macro reference.
261
262    Args:
263      macro_ref_str: String of a macro reference (i.e. foo(a, b)).
264
265    Returns:
266      The text from the expansion.
267
268    Raises:
269      PDDMError if there are any issues.
270    """
271    match = _MACRO_RE.match(macro_ref_str)
272    if match is None or match.group(0) != macro_ref_str:
273      raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str)
274    if match.group('name') not in self._macros:
275      raise PDDMError('No macro named "%s".' % match.group('name'))
276    return self._Expand(match, [], macro_ref_str)
277
278  def _FormatStack(self, macro_ref_stack):
279    result = ''
280    for _, macro_ref in reversed(macro_ref_stack):
281      result += '\n...while expanding "%s".' % macro_ref
282    return result
283
284  def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None):
285    if macro_ref_str is None:
286      macro_ref_str = macro_ref_match.group('macro_ref')
287    name = macro_ref_match.group('name')
288    for prev_name, prev_macro_ref in macro_stack:
289      if name == prev_name:
290        raise PDDMError('Found macro recusion, invoking "%s":%s' %
291                        (macro_ref_str, self._FormatStack(macro_stack)))
292    macro = self._macros[name]
293    args_str = macro_ref_match.group('args').strip()
294    args = []
295    if args_str or len(macro.args):
296      args = [x.strip() for x in args_str.split(',')]
297    if len(args) != len(macro.args):
298      raise PDDMError('Expected %d args, got: "%s".%s' %
299                      (len(macro.args), macro_ref_str,
300                       self._FormatStack(macro_stack)))
301    # Replace args usages.
302    result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack)
303    # Expand any macro invokes.
304    new_macro_stack = macro_stack + [(name, macro_ref_str)]
305    while True:
306      eval_result = self._EvalMacrosRefs(result, new_macro_stack)
307      # Consume all ## directives to glue things together.
308      eval_result = eval_result.replace('##', '')
309      if eval_result == result:
310        break
311      result = eval_result
312    return result
313
314  def _ReplaceArgValues(self,
315                        macro, arg_values, macro_ref_to_report, macro_stack):
316    if len(arg_values) == 0:
317      # Nothing to do
318      return macro.body
319    assert len(arg_values) == len(macro.args)
320    args = dict(zip(macro.args, arg_values))
321    def _lookupArg(match):
322      val = args[match.group('name')]
323      opt = match.group('option')
324      if opt:
325        if opt == 'S': # Spaces for the length
326          return ' ' * len(val)
327        elif opt == 'l': # Lowercase first character
328          if val:
329            return val[0].lower() + val[1:]
330          else:
331            return val
332        elif opt == 'L': # All Lowercase
333          return val.lower()
334        elif opt == 'u': # Uppercase first character
335          if val:
336            return val[0].upper() + val[1:]
337          else:
338            return val
339        elif opt == 'U': # All Uppercase
340          return val.upper()
341        else:
342          raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s'
343                          % (match.group('name'), match.group('option'),
344                             macro_ref_to_report,
345                             self._FormatStack(macro_stack)))
346      return val
347    # Let the regex do the work!
348    macro_arg_ref_re = _MacroArgRefRe(macro.args)
349    return macro_arg_ref_re.sub(_lookupArg, macro.body)
350
351  def _EvalMacrosRefs(self, text, macro_stack):
352    macro_ref_re = _MacroRefRe(self._macros.keys())
353    def _resolveMacro(match):
354      return self._Expand(match, macro_stack)
355    return macro_ref_re.sub(_resolveMacro, text)
356
357
358class SourceFile(object):
359  """Represents a source file with PDDM directives in it."""
360
361  def __init__(self, a_file, import_resolver=None):
362    """Initializes the file reading in the file.
363
364    Args:
365      a_file: The file to read in.
366      import_resolver: a function that given a path will return a stream for
367        the contents.
368
369    Raises:
370      PDDMError if there are any issues.
371    """
372    self._sections = []
373    self._original_content = a_file.read()
374    self._import_resolver = import_resolver
375    self._processed_content = None
376
377  class SectionBase(object):
378
379    def __init__(self, first_line_num):
380      self._lines = []
381      self._first_line_num = first_line_num
382
383    def TryAppend(self, line, line_num):
384      """Try appending a line.
385
386      Args:
387        line: The line to append.
388        line_num: The number of the line.
389
390      Returns:
391        A tuple of (SUCCESS, CAN_ADD_MORE).  If SUCCESS if False, the line
392        wasn't append.  If SUCCESS is True, then CAN_ADD_MORE is True/False to
393        indicate if more lines can be added after this one.
394      """
395      assert False, "sublcass should have overridden"
396      return (False, False)
397
398    def HitEOF(self):
399      """Called when the EOF was reached for for a given section."""
400      pass
401
402    def BindMacroCollection(self, macro_collection):
403      """Binds the chunk to a macro collection.
404
405      Args:
406        macro_collection: The collection to bind too.
407      """
408      pass
409
410    def Append(self, line):
411      self._lines.append(line)
412
413    @property
414    def lines(self):
415      return self._lines
416
417    @property
418    def num_lines_captured(self):
419      return len(self._lines)
420
421    @property
422    def first_line_num(self):
423      return self._first_line_num
424
425    @property
426    def first_line(self):
427      if not self._lines:
428        return ''
429      return self._lines[0]
430
431    @property
432    def text(self):
433      return '\n'.join(self.lines) + '\n'
434
435  class TextSection(SectionBase):
436    """Text section that is echoed out as is."""
437
438    def TryAppend(self, line, line_num):
439      if line.startswith('//%PDDM'):
440        return (False, False)
441      self.Append(line)
442      return (True, True)
443
444  class ExpansionSection(SectionBase):
445    """Section that is the result of an macro expansion."""
446
447    def __init__(self, first_line_num):
448      SourceFile.SectionBase.__init__(self, first_line_num)
449      self._macro_collection = None
450
451    def TryAppend(self, line, line_num):
452      if line.startswith('//%PDDM'):
453        directive = line.split(' ', 1)[0]
454        if directive == '//%PDDM-EXPAND':
455          self.Append(line)
456          return (True, True)
457        if directive == '//%PDDM-EXPAND-END':
458          assert self.num_lines_captured > 0
459          return (True, False)
460        raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' %
461                        (directive, line_num, self.first_line))
462      # Eat other lines.
463      return (True, True)
464
465    def HitEOF(self):
466      raise PDDMError('Hit the end of the file while in "%s".' %
467                      self.first_line)
468
469    def BindMacroCollection(self, macro_collection):
470      self._macro_collection = macro_collection
471
472    @property
473    def lines(self):
474      captured_lines = SourceFile.SectionBase.lines.fget(self)
475      directive_len = len('//%PDDM-EXPAND')
476      result = []
477      for line in captured_lines:
478        result.append(line)
479        if self._macro_collection:
480          # Always add a blank line, seems to read better. (If need be, add an
481          # option to the EXPAND to indicate if this should be done.)
482          result.extend([_GENERATED_CODE_LINE, ''])
483          macro = line[directive_len:].strip()
484          try:
485            expand_result = self._macro_collection.Expand(macro)
486            # Since expansions are line oriented, strip trailing whitespace
487            # from the lines.
488            lines = [x.rstrip() for x in expand_result.split('\n')]
489            result.append('\n'.join(lines))
490          except PDDMError as e:
491            raise PDDMError('%s\n...while expanding "%s" from the section'
492                            ' that started:\n   Line %d: %s' %
493                            (e.message, macro,
494                             self.first_line_num, self.first_line))
495
496      # Add the ending marker.
497      if len(captured_lines) == 1:
498        result.append('//%%PDDM-EXPAND-END %s' %
499                       captured_lines[0][directive_len:].strip())
500      else:
501        result.append('//%%PDDM-EXPAND-END (%s expansions)' % len(captured_lines))
502
503      return result
504
505  class DefinitionSection(SectionBase):
506    """Section containing macro definitions"""
507
508    def TryAppend(self, line, line_num):
509      if not line.startswith('//%'):
510        return (False, False)
511      if line.startswith('//%PDDM'):
512        directive = line.split(' ', 1)[0]
513        if directive == "//%PDDM-EXPAND":
514          return False, False
515        if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'):
516          raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' %
517                          (directive, line_num, self.first_line))
518      self.Append(line)
519      return (True, True)
520
521    def BindMacroCollection(self, macro_collection):
522      if macro_collection:
523        try:
524          # Parse the lines after stripping the prefix.
525          macro_collection.ParseLines([x[3:] for x in self.lines])
526        except PDDMError as e:
527          raise PDDMError('%s\n...while parsing section that started:\n'
528                          '  Line %d: %s' %
529                          (e.message, self.first_line_num, self.first_line))
530
531  class ImportDefinesSection(SectionBase):
532    """Section containing an import of PDDM-DEFINES from an external file."""
533
534    def __init__(self, first_line_num, import_resolver):
535      SourceFile.SectionBase.__init__(self, first_line_num)
536      self._import_resolver = import_resolver
537
538    def TryAppend(self, line, line_num):
539      if not line.startswith('//%PDDM-IMPORT-DEFINES '):
540        return (False, False)
541      assert self.num_lines_captured == 0
542      self.Append(line)
543      return (True, False)
544
545    def BindMacroCollection(self, macro_colletion):
546      if not macro_colletion:
547        return
548      if self._import_resolver is None:
549        raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):'
550                        ' "%s".' % (self.first_line_num, self.first_line))
551      import_name = self.first_line.split(' ', 1)[1].strip()
552      imported_file = self._import_resolver(import_name)
553      if imported_file is None:
554        raise PDDMError('Resolver failed to find "%s" (line %d):'
555                        ' "%s".' %
556                        (import_name, self.first_line_num, self.first_line))
557      try:
558        imported_src_file = SourceFile(imported_file, self._import_resolver)
559        imported_src_file._ParseFile()
560        for section in imported_src_file._sections:
561          section.BindMacroCollection(macro_colletion)
562      except PDDMError as e:
563        raise PDDMError('%s\n...while importing defines:\n'
564                        '  Line %d: %s' %
565                        (e.message, self.first_line_num, self.first_line))
566
567  def _ParseFile(self):
568    self._sections = []
569    lines = self._original_content.splitlines()
570    cur_section = None
571    for line_num, line in enumerate(lines, 1):
572      if not cur_section:
573        cur_section = self._MakeSection(line, line_num)
574      was_added, accept_more = cur_section.TryAppend(line, line_num)
575      if not was_added:
576        cur_section = self._MakeSection(line, line_num)
577        was_added, accept_more = cur_section.TryAppend(line, line_num)
578        assert was_added
579      if not accept_more:
580        cur_section = None
581
582    if cur_section:
583      cur_section.HitEOF()
584
585  def _MakeSection(self, line, line_num):
586    if not line.startswith('//%PDDM'):
587      section = self.TextSection(line_num)
588    else:
589      directive = line.split(' ', 1)[0]
590      if directive == '//%PDDM-EXPAND':
591        section = self.ExpansionSection(line_num)
592      elif directive == '//%PDDM-DEFINE':
593        section = self.DefinitionSection(line_num)
594      elif directive == '//%PDDM-IMPORT-DEFINES':
595        section = self.ImportDefinesSection(line_num, self._import_resolver)
596      else:
597        raise PDDMError('Unexpected line %d: "%s".' % (line_num, line))
598    self._sections.append(section)
599    return section
600
601  def ProcessContent(self, strip_expansion=False):
602    """Processes the file contents."""
603    self._ParseFile()
604    if strip_expansion:
605      # Without a collection the expansions become blank, removing them.
606      collection = None
607    else:
608      collection = MacroCollection()
609    for section in self._sections:
610      section.BindMacroCollection(collection)
611    result = ''
612    for section in self._sections:
613      result += section.text
614    self._processed_content = result
615
616  @property
617  def original_content(self):
618    return self._original_content
619
620  @property
621  def processed_content(self):
622    return self._processed_content
623
624
625def main(args):
626  usage = '%prog [OPTIONS] PATH ...'
627  description = (
628      'Processes PDDM directives in the given paths and write them back out.'
629  )
630  parser = optparse.OptionParser(usage=usage, description=description)
631  parser.add_option('--dry-run',
632                    default=False, action='store_true',
633                    help='Don\'t write back to the file(s), just report if the'
634                    ' contents needs an update and exit with a value of 1.')
635  parser.add_option('--verbose',
636                    default=False, action='store_true',
637                    help='Reports is a file is already current.')
638  parser.add_option('--collapse',
639                    default=False, action='store_true',
640                    help='Removes all the generated code.')
641  opts, extra_args = parser.parse_args(args)
642
643  if not extra_args:
644    parser.error('Need atleast one file to process')
645
646  result = 0
647  for a_path in extra_args:
648    if not os.path.exists(a_path):
649      sys.stderr.write('ERROR: File not found: %s\n' % a_path)
650      return 100
651
652    def _ImportResolver(name):
653      # resolve based on the file being read.
654      a_dir = os.path.dirname(a_path)
655      import_path = os.path.join(a_dir, name)
656      if not os.path.exists(import_path):
657        return None
658      return open(import_path, 'r')
659
660    with open(a_path, 'r') as f:
661      src_file = SourceFile(f, _ImportResolver)
662
663    try:
664      src_file.ProcessContent(strip_expansion=opts.collapse)
665    except PDDMError as e:
666      sys.stderr.write('ERROR: %s\n...While processing "%s"\n' %
667                       (e.message, a_path))
668      return 101
669
670    if src_file.processed_content != src_file.original_content:
671      if not opts.dry_run:
672        print 'Updating for "%s".' % a_path
673        with open(a_path, 'w') as f:
674          f.write(src_file.processed_content)
675      else:
676        # Special result to indicate things need updating.
677        print 'Update needed for "%s".' % a_path
678        result = 1
679    elif opts.verbose:
680      print 'No update for "%s".' % a_path
681
682  return result
683
684
685if __name__ == '__main__':
686  sys.exit(main(sys.argv[1:]))
687