105     def __init__(self, tu=None, group=None, int_data=None, ptr_data=None,  argument
107         clang.cindex.Token.__init__(self)
108         self._id = None
109         self._tu = tu
110         self._group = group
111         self._cursor = cursor
115             self.int_data = int_data
116         self.ptr_data = ptr_data
119     def id(self):  argument
121         if self._id is None:
122             return self.spelling
124             return self._id
127     def id(self, new_id):  argument
129         self._id = new_id
132     def cursor(self):  argument
133         if self._cursor is None:
134             self._cursor = clang.cindex.Token.cursor
135         return self._cursor
138     def cursor(self, new_cursor):  argument
139         self._cursor = new_cursor
141     def __repr__(self):  argument
142         if self.id == 'defined':
143             return self.id
144         elif self.kind == TokenKind.IDENTIFIER:
145             return "(ident %s)" % self.id
147         return self.id
149     def __str__(self):  argument
150         return self.id
167 def SourceRange__contains__(self, other):  argument
171     if other.file is None and self.start.file is None:
173     elif (self.start.file.name != other.file.name or
174           other.file.name != self.end.file.name):
178     if self.start.line < other.line < self.end.line:
181     elif self.start.line == other.line == self.end.line:
182         if self.start.column <= other.column <= self.end.column:
184     elif self.start.line == other.line:
186         if self.start.column <= other.column:
188     elif other.line == self.end.line:
190         if other.column <= self.end.column:
218     def __init__(self):  argument
220         self._indexer = clang.cindex.Index.create()
221         self._tu = None
222         self._index = 0
223         self.tokens = None
225     def _getTokensWithCursors(self):  argument
239         conf.lib.clang_tokenize(self._tu, self._tu.cursor.extent,
253         conf.lib.clang_annotateTokens(self._tu, tokens_memory, count,
259         token_group = TokenGroup(self._tu, tokens_memory, tokens_count)
263             token = Token(self._tu, token_group,
273     def parseString(self, lines):  argument
276         self._tu = self._indexer.parse(file_, self.clang_flags,
278                                        options=self.options)
279         self.tokens = self._getTokensWithCursors()
281     def parseFile(self, file_):  argument
283         self._tu = self._indexer.parse(file_, self.clang_flags,
284                                        options=self.options)
285         self.tokens = self._getTokensWithCursors()
287     def nextToken(self):  argument
289         if self._index < len(self.tokens):
290             t = self.tokens[self._index]
291             self._index += 1
300     def __init__(self, line):  argument
301         CppTokenizer.__init__(self)
302         self.parseString(line)
308     def __init__(self, file_):  argument
309         CppTokenizer.__init__(self)
310         self.parseFile(file_)
318     def get_tokens(self, token_string, line_col=False):  argument
332     def test_hash(self):  argument
333         self.assertEqual(self.get_tokens("#an/example  && (01923_xy)"),
337     def test_parens(self):  argument
338         self.assertEqual(self.get_tokens("FOO(BAR) && defined(BAZ)"),
342     def test_comment(self):  argument
343         self.assertEqual(self.get_tokens("/*\n#\n*/"), [])
345     def test_line_cross(self):  argument
346         self.assertEqual(self.get_tokens("first\nsecond"), ["first", "second"])
348     def test_line_cross_line_col(self):  argument
349         self.assertEqual(self.get_tokens("first second\n  third", True),
352     def test_comment_line_col(self):  argument
353         self.assertEqual(self.get_tokens("boo /* what the\nhell */", True),
356     def test_escapes(self):  argument
357         self.assertEqual(self.get_tokens("an \\\n example", True),
395     def __init__(self, tokens):  argument
397         self.tokens = tokens
398         self._num_tokens = len(tokens)
399         self._index = 0
403         self.expr = self.parseExpression(0)
405             print("CppExpr: got " + repr(self.expr))
406         if self._index != self._num_tokens:
407             self.throw(BadExpectedToken, "crap at end of input (%d != %d): %s"
408                        % (self._index, self._num_tokens, repr(tokens)))
410     def throw(self, exception, msg):  argument
411         if self._index < self._num_tokens:
412             tok = self.tokens[self._index]
418     def expectId(self, id):  argument
420         token = self.tokens[self._index]
421         if self._index >= self._num_tokens or token.id != id:
422             self.throw(BadExpectedToken,
425         self._index += 1
427     def is_decimal(self):  argument
428         token = self.tokens[self._index].id
433             self._index += 1
438     def is_octal(self):  argument
439         token = self.tokens[self._index].id
446             self._index += 1
451     def is_hexadecimal(self):  argument
452         token = self.tokens[self._index].id
459             self._index += 1
464     def is_integer(self):  argument
465         if self.tokens[self._index].kind != TokenKind.LITERAL:
468         c = self.is_hexadecimal()
472         c = self.is_octal()
476         c = self.is_decimal()
482     def is_number(self):  argument
483         t = self.tokens[self._index]
484         if t.id == tokMINUS and self._index + 1 < self._num_tokens:
485             self._index += 1
486             c = self.is_integer()
490         if t.id == tokPLUS and self._index + 1 < self._num_tokens:
491             self._index += 1
492             c = self.is_integer()
496         return self.is_integer()
498     def is_defined(self):  argument
499         t = self.tokens[self._index]
504         self._index += 1
506         if (self._index < self._num_tokens and
507             self.tokens[self._index].id == tokLPAREN):
509             self._index += 1
511         if self._index >= self._num_tokens:
512             self.throw(BadExpectedToken,
516         t = self.tokens[self._index]
518             self.throw(BadExpectedToken,
521         self._index += 1
523             self.expectId(tokRPAREN)
527     def is_call_or_ident(self):  argument
528         if self._index >= self._num_tokens:
531         t = self.tokens[self._index]
537         self._index += 1
538         if (self._index >= self._num_tokens or
539             self.tokens[self._index].id != tokLPAREN):
544         self._index += 1
545         j = self._index
546         while self._index < self._num_tokens:
547             id = self.tokens[self._index].id
551                 k = self._index
552                 param = self.tokens[j:k]
556                 j = self._index + 1
559             self._index += 1
561         if self._index >= self._num_tokens:
564         self._index += 1
573     def parseExpression(self, minPrecedence):  argument
574         if self._index >= self._num_tokens:
577         node = self.parsePrimary()
578         while (self.token() and self.isBinary(self.token()) and
579                self.precedence(self.token()) >= minPrecedence):
580             op = self.token()
581             self.nextToken()
582             rhs = self.parseExpression(self.precedence(op) + 1)
587     def parsePrimary(self):  argument
588         op = self.token()
589         if self.isUnary(op):
590             self.nextToken()
591             return (op.id, self.parseExpression(self.precedence(op)))
595             self.nextToken()
596             primary = self.parseExpression(0)
597             self.expectId(tokRPAREN)
599             self.nextToken()
600             primary = self.parseExpression(0)
601             self.expectId(":")
603             primary = self.is_number()
607             primary = self.is_defined()
609             primary = self.is_call_or_ident()
611             self.throw(BadExpectedToken,
613                            self.tokens[self._index].id))
616     def isBinary(self, token):  argument
617         return token.id in self.binaries
619     def isUnary(self, token):  argument
620         return token.id in self.unaries
622     def precedence(self, token):  argument
623         return self.precedences.get(token.id)
625     def token(self):  argument
626         if self._index >= self._num_tokens:
628         return self.tokens[self._index]
630     def nextToken(self):  argument
631         self._index += 1
632         if self._index >= self._num_tokens:
634         return self.tokens[self._index]
636     def dump_node(self, e):  argument
661             line += " %s)" % self.dump_node(e[1])
663             line += " %s %s)" % (self.dump_node(e[1]), self.dump_node(e[2]))
669     def __repr__(self):  argument
670         return self.dump_node(self.expr)
672     def source_node(self, e):  argument
689             arg_src = self.source_node(arg)
700             arg1_src = self.source_node(arg)
701             arg2_src = self.source_node(arg2)
710     def __str__(self):  argument
711         return self.source_node(self.expr)
720     def toInt(self):  argument
721         return self.int_node(self.expr)
723     def optimize_node(self, e, macros=None):  argument
753                 return self.optimize_node(expanded, macros)
758             v = self.optimize_node(v, macros)
768             l = self.optimize_node(l, macros)
769             r = self.optimize_node(r, macros)
770             li = self.int_node(l)
771             ri = self.int_node(r)
786             l = self.optimize_node(l, macros)
787             r = self.optimize_node(r, macros)
788             li = self.int_node(l)
789             ri = self.int_node(r)
805     def optimize(self, macros=None):  argument
808         self.expr = self.optimize_node(self.expr, macros)
813     def get_expr(self, expr):  argument
816     def test_cpp_expr(self):  argument
817         self.assertEqual(self.get_expr("0"), "(int 0)")
818         self.assertEqual(self.get_expr("1"), "(int 1)")
819         self.assertEqual(self.get_expr("-5"), "(int -5)")
820         self.assertEqual(self.get_expr("+1"), "(int 1)")
821         self.assertEqual(self.get_expr("0U"), "(int 0)")
822         self.assertEqual(self.get_expr("015"), "(oct 015)")
823         self.assertEqual(self.get_expr("015l"), "(oct 015)")
824         self.assertEqual(self.get_expr("0x3e"), "(hex 0x3e)")
825         self.assertEqual(self.get_expr("(0)"), "(int 0)")
826         self.assertEqual(self.get_expr("1 && 1"), "(&& (int 1) (int 1))")
827         self.assertEqual(self.get_expr("1 && 0"), "(&& (int 1) (int 0))")
828         self.assertEqual(self.get_expr("EXAMPLE"), "(ident EXAMPLE)")
829         self.assertEqual(self.get_expr("EXAMPLE - 3"),
831         self.assertEqual(self.get_expr("defined(EXAMPLE)"),
833         self.assertEqual(self.get_expr("defined ( EXAMPLE ) "),
835         self.assertEqual(self.get_expr("!defined(EXAMPLE)"),
837         self.assertEqual(self.get_expr("defined(ABC) || defined(BINGO)"),
839         self.assertEqual(self.get_expr("FOO(BAR,5)"), "(call FOO [BAR,5])")
840         self.assertEqual(self.get_expr("A == 1 || defined(B)"),
843     def get_expr_optimize(self, expr, macros=None):  argument
850     def test_cpp_expr_optimize(self):  argument
851         self.assertEqual(self.get_expr_optimize("0"), "(int 0)")
852         self.assertEqual(self.get_expr_optimize("1"), "(int 1)")
853         self.assertEqual(self.get_expr_optimize("1 && 1"), "(int 1)")
854         self.assertEqual(self.get_expr_optimize("1 && +1"), "(int 1)")
855         self.assertEqual(self.get_expr_optimize("0x1 && 01"), "(oct 01)")
856         self.assertEqual(self.get_expr_optimize("1 && 0"), "(int 0)")
857         self.assertEqual(self.get_expr_optimize("0 && 1"), "(int 0)")
858         self.assertEqual(self.get_expr_optimize("0 && 0"), "(int 0)")
859         self.assertEqual(self.get_expr_optimize("1 || 1"), "(int 1)")
860         self.assertEqual(self.get_expr_optimize("1 || 0"), "(int 1)")
861         self.assertEqual(self.get_expr_optimize("0 || 1"), "(int 1)")
862         self.assertEqual(self.get_expr_optimize("0 || 0"), "(int 0)")
863         self.assertEqual(self.get_expr_optimize("A"), "(ident A)")
864         self.assertEqual(self.get_expr_optimize("A", {"A": 1}), "(int 1)")
865         self.assertEqual(self.get_expr_optimize("A || B", {"A": 1}), "(int 1)")
866         self.assertEqual(self.get_expr_optimize("A || B", {"B": 1}), "(int 1)")
867         self.assertEqual(self.get_expr_optimize("A && B", {"A": 1}), "(ident B)")
868         self.assertEqual(self.get_expr_optimize("A && B", {"B": 1}), "(ident A)")
869         self.assertEqual(self.get_expr_optimize("A && B"), "(&& (ident A) (ident B))")
870         self.assertEqual(self.get_expr_optimize("EXAMPLE"), "(ident EXAMPLE)")
871         self.assertEqual(self.get_expr_optimize("EXAMPLE - 3"), "(- (ident EXAMPLE) (int 3))")
872         self.assertEqual(self.get_expr_optimize("defined(EXAMPLE)"), "(defined EXAMPLE)")
873         self.assertEqual(self.get_expr_optimize("defined(EXAMPLE)",
876         self.assertEqual(self.get_expr_optimize("defined(EXAMPLE)",
879         self.assertEqual(self.get_expr_optimize("!defined(EXAMPLE)"), "(! (defined EXAMPLE))")
880         self.assertEqual(self.get_expr_optimize("!defined(EXAMPLE)",
883         self.assertEqual(self.get_expr_optimize("!defined(EXAMPLE)",
886         self.assertEqual(self.get_expr_optimize("defined(A) || defined(B)"),
888         self.assertEqual(self.get_expr_optimize("defined(A) || defined(B)",
891         self.assertEqual(self.get_expr_optimize("defined(A) || defined(B)",
894         self.assertEqual(self.get_expr_optimize("defined(A) || defined(B)",
897         self.assertEqual(self.get_expr_optimize("defined(A) || defined(B)",
901         self.assertEqual(self.get_expr_optimize("defined(A) && defined(B)"),
903         self.assertEqual(self.get_expr_optimize("defined(A) && defined(B)",
906         self.assertEqual(self.get_expr_optimize("defined(A) && defined(B)",
909         self.assertEqual(self.get_expr_optimize("defined(A) && defined(B)",
912         self.assertEqual(self.get_expr_optimize("defined(A) && defined(B)",
915         self.assertEqual(self.get_expr_optimize("A == 1 || defined(B)"),
917         self.assertEqual(self.get_expr_optimize(
922     def get_expr_string(self, expr):  argument
925     def test_cpp_expr_string(self):  argument
926         self.assertEqual(self.get_expr_string("0"), "0")
927         self.assertEqual(self.get_expr_string("1"), "1")
928         self.assertEqual(self.get_expr_string("1 && 1"), "1 && 1")
929         self.assertEqual(self.get_expr_string("1 && 0"), "1 && 0")
930         self.assertEqual(self.get_expr_string("0 && 1"), "0 && 1")
931         self.assertEqual(self.get_expr_string("0 && 0"), "0 && 0")
932         self.assertEqual(self.get_expr_string("1 || 1"), "1 || 1")
933         self.assertEqual(self.get_expr_string("1 || 0"), "1 || 0")
934         self.assertEqual(self.get_expr_string("0 || 1"), "0 || 1")
935         self.assertEqual(self.get_expr_string("0 || 0"), "0 || 0")
936         self.assertEqual(self.get_expr_string("EXAMPLE"), "EXAMPLE")
937         self.assertEqual(self.get_expr_string("EXAMPLE - 3"), "EXAMPLE - 3")
938         self.assertEqual(self.get_expr_string("defined(EXAMPLE)"), "defined(EXAMPLE)")
939         self.assertEqual(self.get_expr_string("defined EXAMPLE"), "defined(EXAMPLE)")
940         self.assertEqual(self.get_expr_string("A == 1 || defined(B)"), "A == 1 || defined(B)")
964     def __init__(self, tokens, directive=None, lineno=0, identifier=None):  argument
985         self.tokens = tokens
986         self.directive = directive
987         self.define_id = identifier
989             self.lineno = lineno
991             self.lineno = self.tokens[0].location.line
993         if self.isIf():
994             self.expr = CppExpr(self.tokens)
996     def isDirective(self):  argument
998         return self.directive is not None
1000     def isConditional(self):  argument
1002         return self.directive in ["if", "ifdef", "ifndef", "else", "elif",
1005     def isDefine(self):  argument
1007         if self.directive != "define":
1009         return self.define_id
1011     def isIf(self):  argument
1013         return self.directive in ["if", "ifdef", "ifndef", "elif"]
1015     def isEndif(self):  argument
1017         return self.directive == "endif"
1019     def isInclude(self):  argument
1026         if self.directive != "include":
1028         return ''.join([str(x) for x in self.tokens])
1086     def write(self, out, indent):  argument
1093         if self.directive is None and not self.tokens:
1096         if self.directive:
1097             out.write(str(self) + '\n')
1099             lines, indent = self.format_blocks(self.tokens, indent)
1105     def __repr__(self):  argument
1107         if self.directive:
1108             result = "#%s " % self.directive
1109             if self.isIf():
1110                 result += repr(self.expr)
1112                 for tok in self.tokens:
1116             for tok in self.tokens:
1121     def __str__(self):  argument
1123         if self.directive:
1125             if self.directive == "if":
1127                 e = self.expr.expr
1134                     result = "#if " + str(self.expr)
1137             elif self.isDefine():
1138                 result = "#%s %s" % (self.directive, self.define_id)
1139                 if self.tokens:
1141                 expr = strip_space(' '.join([tok.id for tok in self.tokens]))
1148             elif self.directive == "error":
1149                 result = "#error %s" % ' '.join([tok.id for tok in self.tokens])
1152                 result = "#%s" % self.directive
1153                 if self.tokens:
1155                 result += ''.join([tok.id for tok in self.tokens])
1157             lines, _ = self.format_blocks(self.tokens)
1169     def __init__(self, blocks):  argument
1170         self.blocks = blocks
1172     def __len__(self):  argument
1173         return len(self.blocks)
1175     def __getitem__(self, n):  argument
1176         return self.blocks[n]
1178     def __repr__(self):  argument
1179         return repr(self.blocks)
1181     def __str__(self):  argument
1182         result = '\n'.join([str(b) for b in self.blocks])
1185     def dump(self):  argument
1188         for i, b in enumerate(self.blocks):
1193     def optimizeIf01(self):  argument
1195         self.blocks = optimize_if01(self.blocks)
1197     def optimizeMacros(self, macros):  argument
1199         for b in self.blocks:
1203     def removeStructs(self, structs):  argument
1207         num_blocks = len(self.blocks)
1209             b = self.blocks[block_num]
1279                                 if not self.blocks[block_num].directive:
1285 …                       self.blocks = self.blocks[0:start_block] + self.blocks[block_num:num_blocks]
1286                             num_blocks = len(self.blocks)
1287                             b = self.blocks[start_block]
1297             self.blocks.insert(2, Block(replacement.tokens, directive='include'))
1299     def optimizeAll(self, macros):  argument
1300         self.optimizeMacros(macros)
1301         self.optimizeIf01()
1304     def findIncludes(self):  argument
1307         for b in self.blocks:
1313     def write(self, out):  argument
1315         for b in self.blocks:
1318     def removeVarsAndFuncs(self, keep):  argument
1351         for block in self.blocks:
1458         self.blocks = blocksToKeep
1460     def replaceTokens(self, replacements):  argument
1462         for b in self.blocks:
1520     def __init__(self, tokzer=None):  argument
1525         self._tokzer = tokzer
1526         self._parsed = False
1529     def parsed(self):  argument
1530         return self._parsed
1537     def getBlocks(self, tokzer=None):  argument
1575             tokzer = self._tokzer
1590                                         self._short_extent(cursor.extent)))
1671         self._parsed = True
1675     def parse(self, tokzer):  argument
1676         return self.getBlocks(tokzer)
1678     def parseFile(self, path):  argument
1679         return self.getBlocks(CppFileTokenizer(path))
1685     def get_blocks(self, lines):  argument
1689     def test_hash(self):  argument
1690         self.assertEqual(self.get_blocks(["#error hello"]), ["#error hello"])
1692     def test_empty_line(self):  argument
1693         self.assertEqual(self.get_blocks(["foo", "", "bar"]), ["foo bar"])
1695     def test_hash_with_space(self):  argument
1701     def test_with_comment(self):  argument
1702         self.assertEqual(self.get_blocks(["foo",
1840     def parse(self, text, macros=None):  argument
1847     def test_if1(self):  argument
1856         self.assertEqual(self.parse(text), expected)
1858     def test_if0(self):  argument
1866         self.assertEqual(self.parse(text), expected)
1868     def test_if1_else(self):  argument
1879         self.assertEqual(self.parse(text), expected)
1881     def test_if0_else(self):  argument
1892         self.assertEqual(self.parse(text), expected)
1894     def test_if_elif1(self):  argument
1909         self.assertEqual(self.parse(text), expected)
1911     def test_if_elif1_macro(self):  argument
1926         self.assertEqual(self.parse(text, {"WILL_BE_ONE": "1"}), expected)
1929     def test_if_elif1_else(self):  argument
1946         self.assertEqual(self.parse(text), expected)
1948     def test_if_elif1_else_macro(self):  argument
1965         self.assertEqual(self.parse(text, {"WILL_BE_ONE": "1"}), expected)
1968     def test_if_elif1_else_macro(self):  argument
1985         self.assertEqual(self.parse(text, {"WILL_BE_ONE": "1"}), expected)
1987     def test_macro_set_to_undefined_single(self):  argument
1995         self.assertEqual(self.parse(text, macros), expected)
1997     def test_macro_set_to_undefined_if(self):  argument
2009         self.assertEqual(self.parse(text, macros), expected)
2011     def test_endif_comment_removed(self):  argument
2022         self.assertEqual(self.parse(text), expected)
2024     def test_multilevel_if0(self):  argument
2033         self.assertEqual(self.parse(text), expected)
2036     def parse(self, text, structs):  argument
2043     def test_remove_struct_from_start(self):  argument
2060         self.assertEqual(self.parse(text, {"remove": True}), expected)
2062     def test_remove_struct_from_end(self):  argument
2079         self.assertEqual(self.parse(text, {"remove": True}), expected)
2081     def test_remove_minimal_struct(self):  argument
2087         self.assertEqual(self.parse(text, {"remove": True}), expected)
2089     def test_remove_struct_with_struct_fields(self):  argument
2107         self.assertEqual(self.parse(text, {"remove": True}), expected)
2109     def test_remove_consecutive_structs(self):  argument
2139         self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
2141     def test_remove_multiple_structs(self):  argument
2172         self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
2174     def test_remove_struct_with_inline_structs(self):  argument
2197         self.assertEqual(self.parse(text, {"remove": True}), expected)
2199     def test_remove_struct_across_blocks(self):  argument
2222         self.assertEqual(self.parse(text, {"remove": True}), expected)
2224     def test_remove_struct_across_blocks_multiple_structs(self):  argument
2249         self.assertEqual(self.parse(text, {"remove1": True, "remove2": True}), expected)
2251     def test_remove_multiple_struct_and_add_includes(self):  argument
2266         self.assertEqual(self.parse(text, {"remove1": False, "remove2": False}), expected)
2272     def parse(self, text, keep=None):  argument
2286     def test_function_removed(self):  argument
2293         self.assertEqual(self.parse(text), expected)
2295     def test_function_removed_with_struct(self):  argument
2302         self.assertEqual(self.parse(text), expected)
2304     def test_function_kept(self):  argument
2314         self.assertEqual(self.parse(text, set(["function"])), expected)
2316     def test_var_removed(self):  argument
2319         self.assertEqual(self.parse(text), expected)
2321     def test_var_kept(self):  argument
2324         self.assertEqual(self.parse(text, set(["variable"])), expected)
2326     def test_keep_function_typedef(self):  argument
2329         self.assertEqual(self.parse(text), expected)
2331     def test_struct_keep_attribute(self):  argument
2344         self.assertEqual(self.parse(text), expected)
2346     def test_function_keep_attribute_structs(self):  argument
2355         self.assertEqual(self.parse(text, set(["function"])), expected)
2357     def test_struct_after_struct(self):  argument
2377         self.assertEqual(self.parse(text), expected)
2379     def test_other_not_removed(self):  argument
2423         self.assertEqual(self.parse(text), expected)
2425     def test_semicolon_after_function(self):  argument
2440         self.assertEqual(self.parse(text), expected)
2442     def test_define_in_middle_keep(self):  argument
2459         self.assertEqual(self.parse(text), expected)
2461     def test_define_in_middle_remove(self):  argument
2476         self.assertEqual(self.parse(text), expected)
2478     def test_define_in_middle_force_keep(self):  argument
2499         self.assertEqual(self.parse(text, set(["function"])), expected)
2501     def test_define_before_remove(self):  argument
2520         self.assertEqual(self.parse(text), expected)
2522     def test_extern_C(self):  argument
2545         self.assertEqual(self.parse(text), expected)
2547     def test_macro_definition_removed(self):  argument
2563         self.assertEqual(self.parse(text), expected)
2565     def test_verify_timeval_itemerval(self):  argument
2584         self.assertEqual(self.parse(text), expected)
2586     def test_token_replacement(self):  argument
2597         self.assertEqual(self.parse(text), expected)