1#!/usr/bin/env python3
2
3"""Ninja File Parser.
4"""
5
6from __future__ import print_function
7
8import argparse
9import collections
10import os
11import re
12import struct
13import sys
14
15try:
16    import cPickle as pickle  # Python 2
17except ImportError:
18    import pickle  # Python 3
19
20try:
21    from cStringIO import StringIO  # Python 2
22except ImportError:
23    from io import StringIO  # Python 3
24
25try:
26    from sys import intern
27except ImportError:
28    pass  # In Python 2, intern() is a built-in function.
29
30if sys.version_info < (3,):
31    # Wrap built-in open() function to ignore encoding in Python 2.
32    _builtin_open = open
33    def open(path, mode, encoding=None):
34        return _builtin_open(path, mode)
35
36    # Replace built-in zip() function with itertools.izip
37    from itertools import izip as zip
38
39
40class EvalEnv(dict):
41    __slots__ = ('parent')
42
43
44    def __init__(self, *args, **kwargs):
45        super(EvalEnv, self).__init__(*args, **kwargs)
46        self.parent = None
47
48
49    def get_recursive(self, key, default=None):
50        try:
51            return self[key]
52        except KeyError:
53            if self.parent:
54                return self.parent.get_recursive(key, default)
55            return default
56
57
58class BuildEvalEnv(EvalEnv):
59    __slots__ = ('_build_env', '_rule_env')
60
61
62    def __init__(self, build_env, rule_env):
63        self._build_env = build_env
64        self._rule_env = rule_env
65
66
67    def get_recursive(self, key, default=None):
68        try:
69            return self._build_env[key]
70        except KeyError:
71            pass
72
73        if self._rule_env:
74            try:
75                return self._rule_env[key]
76            except KeyError:
77                pass
78
79        if self._build_env.parent:
80            return self._build_env.parent.get_recursive(key, default)
81        return default
82
83
84class EvalError(ValueError):
85    """Exceptions for ``EvalString`` evalution errors."""
86    pass
87
88
89class EvalCircularError(EvalError):
90    """Exception for circular substitution in ``EvalString``."""
91
92
93    def __init__(self, expanded_vars):
94        super(EvalCircularError, self).__init__(
95                'circular evaluation: ' + ' -> '.join(expanded_vars))
96
97
98class EvalString(tuple):
99    """Strings with variables to be substituted."""
100
101
102    def __bool__(self):
103        """Check whether this is an empty string."""
104        return len(self) > 1
105
106
107    def __nonzero__(self):
108        """Check whether this is an empty string (Python2)."""
109        return self.__bool__()
110
111
112    def create_iters(self):
113        """Create descriptors and segments iterators."""
114        curr_iter = iter(self)
115        descs = next(curr_iter)
116        return zip(descs, curr_iter)
117
118
119def _eval_string(s, env, expanded_vars, result_buf):
120    """Evaluate each segments in ``EvalString`` and write result to the
121    given StringIO buffer.
122
123    Args:
124        env: A ``dict`` that maps a name to ``EvalString`` object.
125        expanded_vars: A ``list`` that keeps the variable under evaluation.
126        result_buf: Output buffer.
127    """
128    if type(s) is str:
129        result_buf.write(s)
130        return
131
132    for desc, seg in s.create_iters():
133        if desc == 't':
134            # Append raw text
135            result_buf.write(seg)
136        else:
137            # Substitute variables
138            varname = seg
139            if varname in expanded_vars:
140                raise EvalCircularError(expanded_vars + [varname])
141            expanded_vars.append(varname)
142            try:
143                next_es = env.get_recursive(varname)
144                if next_es:
145                    _eval_string(next_es, env, expanded_vars, result_buf)
146            finally:
147                expanded_vars.pop()
148
149
150def eval_string(s, env):
151    """Evaluate a ``str`` or ``EvalString`` in an environment.
152
153    Args:
154        env: A ``dict`` that maps a name to an ``EvalString`` object.
155
156    Returns:
157        str: The result of evaluation.
158
159    Raises:
160        EvalNameError: Unknown variable name occurs.
161        EvalCircularError: Circular variable substitution occurs.
162    """
163    expanded_vars = []
164    result_buf = StringIO()
165    _eval_string(s, env, expanded_vars, result_buf)
166    return result_buf.getvalue()
167
168
169def eval_path_strings(strs, env):
170    """Evalute a list of ``EvalString`` in an environment and normalize paths.
171
172    Args:
173        strs: A list of ``EvalString`` which should be treated as paths.
174        env: A ``dict`` that maps a name to an ``EvalString`` object.
175
176    Returns:
177        The list of evaluated strings.
178    """
179    return [intern(os.path.normpath(eval_string(s, env))) for s in strs]
180
181
182class EvalStringBuilder(object):
183    def __init__(self):
184        self._segs = ['']
185
186
187    def append_raw(self, text):
188        descs = self._segs[0]
189        if descs and descs[-1] == 't':
190            self._segs[-1] += text
191        else:
192            self._segs[0] += 't'
193            self._segs.append(text)
194        return self
195
196
197    def append_var(self, varname):
198        self._segs[0] += 'v'
199        self._segs.append(varname)
200        return self
201
202
203    def getvalue(self):
204        return EvalString(intern(seg) for seg in self._segs)
205
206
207class Build(object):
208    __slots__ = ('explicit_outs', 'implicit_outs', 'rule', 'explicit_ins',
209                 'implicit_ins', 'prerequisites', 'bindings',
210                 'depfile_implicit_ins')
211
212
213class Rule(object):
214    __slots__ = ('name', 'bindings')
215
216
217class Pool(object):
218    __slots__ = ('name', 'bindings')
219
220
221class Default(object):
222    __slots__ = ('outs')
223
224
225Token = collections.namedtuple('Token', 'kind line column value')
226
227
228class TK(object):
229    """Token ID enumerations."""
230
231    # Trivial tokens
232    EOF = 0
233    COMMENT = 1
234    NEWLINE = 2
235    SPACE = 3
236    ESC_NEWLINE = 4
237    IDENT = 5
238    PIPE2 = 6
239    PIPE = 7
240    COLON = 8
241    ASSIGN = 9
242
243    # Non-trivial tokens
244    PATH = 10
245    STRING = 11
246
247
248class TokenMatcher(object):
249    def __init__(self, patterns):
250        self._matcher = re.compile('|'.join('(' + p + ')' for k, p in patterns))
251        self._kinds = [k for k, p in patterns]
252
253
254    def match(self, buf, pos):
255        match = self._matcher.match(buf, pos)
256        if not match:
257            return None
258        return (self._kinds[match.lastindex - 1], match.start(), match.end())
259
260
261class ParseError(ValueError):
262    def __init__(self, path, line, column, reason=None):
263        self.path = path
264        self.line = line
265        self.column = column
266        self.reason = reason
267
268
269    def __repr__(self):
270        s = 'ParseError: {}:{}:{}'.format(self.path, self.line, self.column)
271        if self.reason:
272            s += ': ' + self.reason
273        return s
274
275
276class Lexer(object):
277    def __init__(self, lines_iterable, path='<stdin>', encoding='utf-8'):
278        self.encoding = encoding
279        self.path = path
280
281        self._line_iter = iter(lines_iterable)
282        self._line_buf = None
283        self._line = 0
284        self._line_pos = 0
285        self._line_end = 0
286
287        self._line_start = True
288
289        self._next_token = None
290        self._next_pos = None
291
292
293    def raise_error(self, reason=None):
294        raise ParseError(self.path, self._line, self._line_pos + 1, reason)
295
296
297    def _read_next_line(self):
298        try:
299            self._line_buf = next(self._line_iter)
300            self._line_pos = 0
301            self._line_end = len(self._line_buf)
302            self._line += 1
303            return True
304        except StopIteration:
305            self._line_buf = None
306            return False
307
308
309    def _ensure_line(self):
310        if self._line_buf and self._line_pos < self._line_end:
311            return True
312        return self._read_next_line()
313
314    _COMMENT_MATCHER = re.compile(r'[ \t]*(?:#[^\n]*)?(?=\n)')
315
316
317    def _ensure_non_comment_line(self):
318        if not self._ensure_line():
319            return False
320        # Match comments or spaces
321        match = self._COMMENT_MATCHER.match(self._line_buf)
322        if not match:
323            return True
324        # Move the cursor to the newline character
325        self._line_pos = match.end()
326        return True
327
328    _SPACE_MATCHER = re.compile(r'[ \t]+')
329
330
331    def _skip_space(self):
332        match = self._SPACE_MATCHER.match(self._line_buf, self._line_pos)
333        if match:
334            self._line_pos = match.end()
335
336    _SIMPLE_TOKEN_MATCHER = TokenMatcher([
337        (TK.COMMENT, r'#[^\n]*'),
338        (TK.NEWLINE, r'[\r\n]'),
339        (TK.SPACE, r'[ \t]+'),
340        (TK.ESC_NEWLINE, r'\$[\r\n]'),
341        (TK.IDENT, r'[\w_.-]+'),
342        (TK.PIPE2, r'\|\|'),
343        (TK.PIPE, r'\|'),
344        (TK.COLON, r':'),
345        (TK.ASSIGN, r'='),
346    ])
347
348
349    def peek(self):
350        if self._next_token is not None:
351            return self._next_token
352        while True:
353            if not self._ensure_non_comment_line():
354                return Token(TK.EOF, self._line, self._line_pos + 1, '')
355
356            match = self._SIMPLE_TOKEN_MATCHER.match(
357                    self._line_buf, self._line_pos)
358            if not match:
359                return None
360            kind, start, end = match
361
362            # Skip comments and spaces
363            if ((kind == TK.SPACE and not self._line_start) or
364                (kind == TK.ESC_NEWLINE) or
365                (kind == TK.COMMENT)):
366                self._line_pos = end
367                continue
368
369            # Save the peaked token
370            token = Token(kind, self._line, self._line_pos + 1,
371                          self._line_buf[start:end])
372            self._next_token = token
373            self._next_pos = end
374            return token
375
376
377    def lex(self):
378        token = self.peek()
379        if not token:
380            self.raise_error()
381        self._line_start = token.kind == TK.NEWLINE
382        self._line_pos = self._next_pos
383        self._next_token = None
384        self._next_pos = None
385        return token
386
387
388    def lex_match(self, match_set):
389        token = self.lex()
390        if token.kind not in match_set:
391            self.raise_error()
392        return token
393
394
395    class STR_TK(object):
396        END = 0
397        CHARS = 1
398        ESC_CHAR = 2
399        ESC_NEWLINE = 3
400        VAR = 4
401        CURVE_VAR = 5
402
403
404    _PATH_TOKEN_MATCHER = TokenMatcher([
405        (STR_TK.END, r'[ \t\n|:]'),
406        (STR_TK.CHARS, r'[^ \t\n|:$]+'),
407        (STR_TK.ESC_CHAR, r'\$[^\n{\w_-]'),
408        (STR_TK.ESC_NEWLINE, r'\$\n[ \t]*'),
409        (STR_TK.VAR, r'\$[\w_-]+'),
410        (STR_TK.CURVE_VAR, r'\$\{[\w_.-]+\}'),
411    ])
412
413
414    _STR_TOKEN_MATCHER = TokenMatcher([
415        (STR_TK.END, r'\n+'),
416        (STR_TK.CHARS, r'[^\n$]+'),
417        (STR_TK.ESC_CHAR, r'\$[^\n{\w_-]'),
418        (STR_TK.ESC_NEWLINE, r'\$\n[ \t]*'),
419        (STR_TK.VAR, r'\$[\w_-]+'),
420        (STR_TK.CURVE_VAR, r'\$\{[\w_.-]+\}'),
421    ])
422
423
424    def _lex_string_or_path(self, matcher, result_kind):
425        self._ensure_line()
426        self._skip_space()
427
428        start_line = self._line
429        start_column = self._line_pos + 1
430
431        builder = EvalStringBuilder()
432
433        while True:
434            if not self._ensure_line():
435                break
436
437            match = matcher.match(self._line_buf, self._line_pos)
438            if not match:
439                self.raise_error('unknown character sequence')
440
441            kind, start, end = match
442            if kind == self.STR_TK.END:
443                break
444
445            self._line_pos = end
446
447            if kind == self.STR_TK.CHARS:
448                builder.append_raw(self._line_buf[start:end])
449            elif kind == self.STR_TK.ESC_CHAR:
450                ch = self._line_buf[start + 1]
451                if ch in ' \t:$':
452                    builder.append_raw(ch)
453                else:
454                    self.raise_error('bad escape sequence')
455            elif kind == self.STR_TK.ESC_NEWLINE:
456                if not self._read_next_line():
457                    break
458                self._skip_space()
459            elif kind == self.STR_TK.VAR:
460                builder.append_var(self._line_buf[start + 1 : end])
461            else:
462                assert kind == self.STR_TK.CURVE_VAR
463                builder.append_var(self._line_buf[start + 2 : end - 1])
464
465        self._next_token = None
466        return Token(result_kind, start_line, start_column, builder.getvalue())
467
468
469    def lex_path(self):
470        return self._lex_string_or_path(self._PATH_TOKEN_MATCHER, TK.PATH)
471
472
473    def lex_string(self):
474        return self._lex_string_or_path(self._STR_TOKEN_MATCHER, TK.STRING)
475
476
477Manifest = collections.namedtuple('Manifest', 'builds rules pools defaults')
478
479
480class Parser(object):
481    """Ninja Manifest Parser
482
483    This parser parses ninja-build manifest files, such as::
484
485        cflags = -Wall
486
487        pool cc_pool
488          depth = 1
489
490        rule cc
491          command = gcc -c -o $out $in $cflags $extra_cflags
492          pool = cc_pool
493
494        build test.o : cc test.c
495          extra_cflags = -Werror
496
497        default test.o
498
499    Example:
500        >>> manifest = Parser().parse('build.ninja', 'utf-8')
501        >>> print(manifest.builds)
502
503    """
504
505
506    def __init__(self, base_dir=None):
507        if base_dir is None:
508            self._base_dir = os.getcwd()
509        else:
510            self._base_dir = base_dir
511
512        # File context
513        self._context = []
514        self._lexer = None
515        self._env = None
516
517        # Intermediate results
518        self._builds = []
519        self._rules = []
520        self._pools = []
521        self._defaults = []
522
523        self._rules_dict = {}
524
525
526    def _push_context(self, lexer, env):
527        """Push a parsing file context.
528
529        Args:
530            lexer: Lexer for the associated file.
531            env: Environment for global variable bindings.
532        """
533
534        self._context.append((self._lexer, self._env))
535        self._lexer = lexer
536        self._env = env
537
538
539    def _pop_context(self):
540        """Push a parsing file context."""
541
542        current_context = (self._lexer, self._env)
543        self._lexer, self._env = self._context.pop()
544        return current_context
545
546
547    def parse(self, path, encoding, depfile=None):
548        """Parse a ninja-build manifest file.
549
550        Args:
551            path (str): Input file path to be parsed.
552            encoding (str): Input file encoding.
553
554        Returns:
555            Manifest: Parsed manifest for the given ninja-build manifest file.
556        """
557
558        self._parse_internal(path, encoding, EvalEnv())
559        if depfile:
560            self.parse_dep_file(depfile, encoding)
561        return Manifest(self._builds, self._rules, self._pools, self._defaults)
562
563
564    def _parse_internal(self, path, encoding, env):
565        path = os.path.join(self._base_dir, path)
566        with open(path, 'r', encoding=encoding) as fp:
567            self._push_context(Lexer(fp, path, encoding), env)
568            try:
569                self._parse_all_top_level_stmts()
570            finally:
571                self._pop_context()
572
573
574    def _parse_all_top_level_stmts(self):
575        """Parse all top-level statements in a file."""
576        while self._parse_top_level_stmt():
577            pass
578
579
580    def _parse_top_level_stmt(self):
581        """Parse a top level statement."""
582
583        token = self._lexer.peek()
584        if not token:
585            # An unexpected non-trivial token occurs.  Raise an error.
586            self._lexer.raise_error()
587
588        if token.kind == TK.EOF:
589            return False
590        elif token.kind == TK.NEWLINE:
591            self._lexer.lex()
592        elif token.kind == TK.IDENT:
593            ident = token.value
594            if ident == 'rule':
595                self._parse_rule_stmt()
596            elif ident == 'build':
597                self._parse_build_stmt()
598            elif ident == 'default':
599                self._parse_default_stmt()
600            elif ident == 'pool':
601                self._parse_pool_stmt()
602            elif ident in {'subninja', 'include'}:
603                self._parse_include_stmt()
604            else:
605                self._parse_global_binding_stmt()
606        else:
607            # An unexpected trivial token occurs.  Raise an error.
608            self._lexer.raise_error()
609        return True
610
611
612    def _parse_path_list(self, end_set):
613        """Parse a list of paths."""
614
615        result = []
616        while True:
617            token = self._lexer.peek()
618            if token:
619                if token.kind in end_set:
620                    break
621                elif token.kind != TK.IDENT:
622                    self._lexer.raise_error()
623
624            token = self._lexer.lex_path()
625            result.append(token.value)
626        return result
627
628
629    def _parse_binding_stmt(self):
630        """Parse a variable binding statement.
631
632        Example:
633            IDENT = STRING
634        """
635        key = self._lexer.lex_match({TK.IDENT}).value
636        self._lexer.lex_match({TK.ASSIGN})
637        token = self._lexer.lex_string()
638        value = token.value
639        self._lexer.lex_match({TK.NEWLINE, TK.EOF})
640        return (key, value)
641
642
643    def _parse_global_binding_stmt(self):
644        """Parse a global variable binding statement.
645
646        Example:
647            IDENT = STRING
648        """
649
650        key, value = self._parse_binding_stmt()
651        value = eval_string(value, self._env)
652        self._env[key] = value
653
654
655    def _parse_local_binding_block(self):
656        """Parse several local variable bindings.
657
658        Example:
659            SPACE IDENT1 = STRING1
660            SPACE IDENT2 = STRING2
661        """
662        result = EvalEnv()
663        while True:
664            token = self._lexer.peek()
665            if not token or token.kind != TK.SPACE:
666                break
667            self._lexer.lex()
668            key, value = self._parse_binding_stmt()
669            result[key] = value
670        return result
671
672
673    def _parse_build_stmt(self):
674        """Parse `build` statement.
675
676        Example:
677            build PATH1 PATH2 | PATH3 PATH4 : IDENT PATH5 PATH6 | $
678                  PATH7 PATH8 || PATH9 PATH10
679            SPACE IDENT1 = STRING1
680            SPACE IDENT2 = STRING2
681        """
682
683        token = self._lexer.lex_match({TK.IDENT})
684        assert token.value == 'build'
685
686        build = Build()
687
688        # Parse explicit outs
689        explicit_outs = self._parse_path_list({TK.PIPE, TK.COLON})
690
691        # Parse implicit outs
692        token = self._lexer.peek()
693        if token.kind == TK.PIPE:
694            self._lexer.lex()
695            implicit_outs = self._parse_path_list({TK.COLON})
696        else:
697            implicit_outs = tuple()
698
699        self._lexer.lex_match({TK.COLON})
700
701        # Parse rule name for this build statement
702        build.rule = self._lexer.lex_match({TK.IDENT}).value
703        try:
704            rule_env = self._rules_dict[build.rule].bindings
705        except KeyError:
706            if build.rule != 'phony':
707                self._lexer.raise_error('undeclared rule name')
708            rule_env = self._env
709
710        # Parse explicit ins
711        explicit_ins = self._parse_path_list(
712                {TK.PIPE, TK.PIPE2, TK.NEWLINE, TK.EOF})
713
714        # Parse implicit ins
715        token = self._lexer.peek()
716        if token.kind == TK.PIPE:
717            self._lexer.lex()
718            implicit_ins = self._parse_path_list({TK.PIPE2, TK.NEWLINE, TK.EOF})
719        else:
720            implicit_ins = tuple()
721
722        # Parse order-only prerequisites
723        token = self._lexer.peek()
724        if token.kind == TK.PIPE2:
725            self._lexer.lex()
726            prerequisites = self._parse_path_list({TK.NEWLINE, TK.EOF})
727        else:
728            prerequisites = tuple()
729
730        self._lexer.lex_match({TK.NEWLINE, TK.EOF})
731
732        # Parse local bindings
733        bindings = self._parse_local_binding_block()
734        bindings.parent = self._env
735        if bindings:
736            build.bindings = bindings
737        else:
738            # Don't keep the empty ``dict`` object if there are no bindings
739            build.bindings = None
740
741        # Evaluate all paths
742        env = BuildEvalEnv(bindings, rule_env)
743
744        build.explicit_outs = eval_path_strings(explicit_outs, env)
745        build.implicit_outs = eval_path_strings(implicit_outs, env)
746        build.explicit_ins = eval_path_strings(explicit_ins, env)
747        build.implicit_ins = eval_path_strings(implicit_ins, env)
748        build.prerequisites = eval_path_strings(prerequisites, env)
749        build.depfile_implicit_ins = tuple()
750
751        self._builds.append(build)
752
753
754    def _parse_rule_stmt(self):
755        """Parse a `rule` statement.
756
757        Example:
758            rule IDENT
759            SPACE IDENT1 = STRING1
760            SPACE IDENT2 = STRING2
761        """
762
763        token = self._lexer.lex_match({TK.IDENT})
764        assert token.value == 'rule'
765
766        rule = Rule()
767        rule.name = self._lexer.lex_match({TK.IDENT}).value
768        self._lexer.lex_match({TK.NEWLINE, TK.EOF})
769        rule.bindings = self._parse_local_binding_block()
770
771        self._rules.append(rule)
772        self._rules_dict[rule.name] = rule
773
774
775    def _parse_default_stmt(self):
776        """Parse a `default` statement.
777
778        Example:
779            default PATH1 PATH2 PATH3
780        """
781
782        token = self._lexer.lex_match({TK.IDENT})
783        assert token.value == 'default'
784
785        default = Default()
786        outs = self._parse_path_list({TK.NEWLINE, TK.EOF})
787        default.outs = eval_path_strings(outs, self._env)
788
789        self._lexer.lex_match({TK.NEWLINE, TK.EOF})
790
791        self._defaults.append(default)
792
793
794    def _parse_pool_stmt(self):
795        """Parse a `pool` statement.
796
797        Example:
798            pool IDENT
799            SPACE IDENT1 = STRING1
800            SPACE IDENT2 = STRING2
801        """
802        token = self._lexer.lex_match({TK.IDENT})
803        assert token.value == 'pool'
804
805        pool = Pool()
806
807        token = self._lexer.lex()
808        assert token.kind == TK.IDENT
809        pool.name = token.value
810
811        self._lexer.lex_match({TK.NEWLINE, TK.EOF})
812
813        pool.bindings = self._parse_local_binding_block()
814
815        self._pools.append(pool)
816
817
818    def _parse_include_stmt(self):
819        """Parse an `include` or `subninja` statement.
820
821        Example:
822            include PATH
823            subninja PATH
824        """
825
826        token = self._lexer.lex_match({TK.IDENT})
827        assert token.value in {'include', 'subninja'}
828        wrap_env = token.value == 'subninja'
829
830        token = self._lexer.lex_path()
831        path = eval_string(token.value, self._env)  # XXX: Check lookup order
832        self._lexer.lex_match({TK.NEWLINE, TK.EOF})
833
834        if wrap_env:
835            env = EvalEnv()
836            env.parent = self._env
837        else:
838            env = self._env
839        self._parse_internal(path, self._lexer.encoding, env)
840
841
842    def parse_dep_file(self, path, encoding):
843        depfile = DepFileParser().parse(path, encoding)
844        for build in self._builds:
845            depfile_implicit_ins = set()
846            for explicit_out in build.explicit_outs:
847                deps = depfile.get(explicit_out)
848                if deps:
849                    depfile_implicit_ins.update(deps.implicit_ins)
850            build.depfile_implicit_ins = tuple(sorted(depfile_implicit_ins))
851
852
853class DepFileError(ValueError):
854    pass
855
856
857class DepFileRecord(object):
858    __slots__ = ('id', 'explicit_out', 'mtime', 'implicit_ins')
859
860
861    def __init__(self, id, explicit_out, mtime, implicit_ins):
862        self.id = id
863        self.explicit_out = explicit_out
864        self.mtime = mtime
865        self.implicit_ins = implicit_ins
866
867
868class DepFileParser(object):
869    """Ninja deps log parser which parses ``.ninja_deps`` file.
870    """
871
872
873    def __init__(self):
874        self._deps = []
875        self._paths = []
876        self._path_deps = {}
877
878
879    def parse(self, path, encoding):
880        with open(path, 'rb') as fp:
881            return self._parse(fp, encoding)
882
883
884    @staticmethod
885    def _unpack_uint32(buf):
886        return struct.unpack('<I', buf)[0]
887
888
889    @staticmethod
890    def _unpack_uint32_iter(buf):
891        for p in struct.iter_unpack('<I', buf):
892            yield p[0]
893
894
895    if sys.version_info < (3,):
896        @staticmethod
897        def _extract_path(s, encoding):
898            pos = len(s)
899            count = 3
900            while count > 0 and pos > 0 and s[pos - 1] == b'\0':
901                pos -= 1
902                count -= 1
903            return intern(s[0:pos])
904    else:
905        @staticmethod
906        def _extract_path(s, encoding):
907            pos = len(s)
908            count = 3
909            while count > 0 and pos > 0 and s[pos - 1] == 0:
910                pos -= 1
911                count -= 1
912            return intern(s[0:pos].decode(encoding))
913
914
915    def _get_path(self, index):
916        try:
917            return self._paths[index]
918        except IndexError:
919            raise DepFileError('path index overflow')
920
921
922    def _parse(self, fp, encoding):
923        # Check the magic word
924        if fp.readline() != b'# ninjadeps\n':
925            raise DepFileError('bad magic word')
926
927        # Check the file format version
928        version = self._unpack_uint32(fp.read(4))
929        if version != 3:
930            raise DepFileError('unsupported deps log version: ' + str(version))
931
932        # Read the records
933        MAX_RECORD_SIZE = (1 << 19) - 1
934        while True:
935            buf = fp.read(4)
936            if not buf:
937                break
938
939            record_size = self._unpack_uint32(buf)
940            is_dep = bool(record_size >> 31)
941            record_size &= (1 << 31) - 1
942
943            if record_size > MAX_RECORD_SIZE:
944                raise DepFileError('record size overflow')
945
946            if is_dep:
947                if record_size % 4 != 0 or record_size < 8:
948                    raise DepFileError('corrupted deps record')
949
950                buf = fp.read(record_size)
951
952                dep_iter = self._unpack_uint32_iter(buf)
953
954                idx = len(self._deps)
955                explicit_out = self._get_path(next(dep_iter))
956                mtime = next(dep_iter)
957                implicit_ins = [self._get_path(p) for p in dep_iter]
958
959                deps = DepFileRecord(idx, explicit_out, mtime, implicit_ins)
960
961                old_deps = self._path_deps.get(explicit_out)
962                if not old_deps:
963                    self._deps.append(deps)
964                    self._path_deps[explicit_out] = deps
965                elif old_deps.mtime > deps.mtime:
966                    self._deps.append(None)
967                else:
968                    self._deps[old_deps.id] = None
969                    self._deps.append(deps)
970                    self._path_deps[explicit_out] = deps
971            else:
972                if record_size < 4:
973                    raise DepFileError('corrupted path record')
974                buf = fp.read(record_size - 4)
975                path = self._extract_path(buf, encoding)
976                buf = fp.read(4)
977                checksum = 0xffffffff ^ self._unpack_uint32(buf)
978                if len(self._paths) != checksum:
979                    raise DepFileError('bad path record checksum')
980                self._paths.append(path)
981
982        return self._path_deps
983
984
985def _parse_args():
986    """Parse command line options."""
987
988    parser = argparse.ArgumentParser()
989    subparsers = parser.add_subparsers(dest='command')
990
991    def _register_input_file_args(parser):
992        parser.add_argument('input_file', help='input ninja file')
993        parser.add_argument('--ninja-deps', help='.ninja_deps file')
994        parser.add_argument('--cwd', help='working directory for ninja')
995        parser.add_argument('--encoding', default='utf-8',
996                            help='ninja file encoding')
997
998    # dump sub-command
999    parser_dump = subparsers.add_parser('dump', help='dump dependency graph')
1000    _register_input_file_args(parser_dump)
1001    parser_dump.add_argument('-o', '--output', help='output file')
1002
1003    # pickle sub-command
1004    parser_pickle = subparsers.add_parser(
1005            'pickle', help='serialize dependency graph with pickle')
1006    _register_input_file_args(parser_pickle)
1007    parser_pickle.add_argument('-o', '--output', required=True,
1008                               help='output file')
1009
1010    # Parse arguments and check sub-command
1011    args = parser.parse_args()
1012    if args.command is None:
1013        parser.print_help()
1014        sys.exit(1)
1015
1016    return args
1017
1018
1019def load_manifest_from_args(args):
1020    """Load the input manifest specified by command line options."""
1021
1022    input_file = args.input_file
1023
1024    # If the input file name ends with `.pickle`, load it with pickle.load().
1025    if input_file.endswith('.pickle'):
1026        with open(input_file, 'rb') as pickle_file:
1027            return pickle.load(pickle_file)
1028
1029    # Parse the ninja file
1030    return Parser(args.cwd).parse(args.input_file, args.encoding,
1031                                  args.ninja_deps)
1032
1033
1034def dump_manifest(manifest, file):
1035    """Dump a manifest to a text file."""
1036
1037    for rule in manifest.rules:
1038        print('rule', rule.name, file=file)
1039
1040    for build in manifest.builds:
1041        print('build', file=file)
1042        for path in build.explicit_outs:
1043            print('  explicit_out:', path, file=file)
1044        for path in build.implicit_outs:
1045            print('  implicit_out:', path, file=file)
1046        for path in build.explicit_ins:
1047            print('  explicit_in:', path, file=file)
1048        for path in build.implicit_ins:
1049            print('  implicit_in:', path, file=file)
1050        for path in build.prerequisites:
1051            print('  prerequisites:', path, file=file)
1052        for path in build.depfile_implicit_ins:
1053            print('  depfile_implicit_in:', path, file=file)
1054
1055    for pool in manifest.pools:
1056        print('pool', pool.name, file=file)
1057
1058    for default in manifest.defaults:
1059        print('default', file=file)
1060        for path in default.outs:
1061            print('  out:', path, file=file)
1062
1063
1064def command_dump_main(args):
1065    """Main function for the dump sub-command"""
1066    if args.output is None:
1067        dump_manifest(load_manifest_from_args(args), sys.stdout)
1068    else:
1069        with open(args.output, 'w') as output_file:
1070            dump_manifest(load_manifest_from_args(args), output_file)
1071
1072
1073def command_pickle_main(args):
1074    """Main function for the pickle sub-command"""
1075    with open(args.output, 'wb') as output_file:
1076        pickle.dump(load_manifest_from_args(args), output_file)
1077
1078
1079def main():
1080    """Main function for the executable"""
1081    args = _parse_args()
1082    if args.command == 'dump':
1083        command_dump_main(args)
1084    elif args.command == 'pickle':
1085        command_pickle_main(args)
1086    else:
1087        raise KeyError('unknown command ' + args.command)
1088
1089
1090if __name__ == '__main__':
1091    import ninja
1092    ninja.main()
1093