1#!/usr/bin/env python 2# 3# Copyright 2008 The Closure Linter Authors. All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS-IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Classes to represent tokens and positions within them.""" 18 19__author__ = ('robbyw@google.com (Robert Walker)', 20 'ajp@google.com (Andy Perelson)') 21 22 23class TokenType(object): 24 """Token types common to all languages.""" 25 NORMAL = 'normal' 26 WHITESPACE = 'whitespace' 27 BLANK_LINE = 'blank line' 28 29 30class Token(object): 31 """Token class for intelligent text splitting. 32 33 The token class represents a string of characters and an identifying type. 34 35 Attributes: 36 type: The type of token. 37 string: The characters the token comprises. 38 length: The length of the token. 39 line: The text of the line the token is found in. 40 line_number: The number of the line the token is found in. 41 values: Dictionary of values returned from the tokens regex match. 42 previous: The token before this one. 43 next: The token after this one. 44 start_index: The character index in the line where this token starts. 45 attached_object: Object containing more information about this token. 46 metadata: Object containing metadata about this token. Must be added by 47 a separate metadata pass. 48 """ 49 50 def __init__(self, string, token_type, line, line_number, values=None, 51 orig_line_number=None): 52 """Creates a new Token object. 53 54 Args: 55 string: The string of input the token contains. 56 token_type: The type of token. 57 line: The text of the line this token is in. 58 line_number: The line number of the token. 59 values: A dict of named values within the token. For instance, a 60 function declaration may have a value called 'name' which captures the 61 name of the function. 62 orig_line_number: The line number of the original file this token comes 63 from. This should be only set during the tokenization process. For newly 64 created error fix tokens after that, it should be None. 65 """ 66 self.type = token_type 67 self.string = string 68 self.length = len(string) 69 self.line = line 70 self.line_number = line_number 71 self.orig_line_number = orig_line_number 72 self.values = values 73 self.is_deleted = False 74 75 # These parts can only be computed when the file is fully tokenized 76 self.previous = None 77 self.next = None 78 self.start_index = None 79 80 # This part is set in statetracker.py 81 # TODO(robbyw): Wrap this in to metadata 82 self.attached_object = None 83 84 # This part is set in *metadatapass.py 85 self.metadata = None 86 87 def IsFirstInLine(self): 88 """Tests if this token is the first token in its line. 89 90 Returns: 91 Whether the token is the first token in its line. 92 """ 93 return not self.previous or self.previous.line_number != self.line_number 94 95 def IsLastInLine(self): 96 """Tests if this token is the last token in its line. 97 98 Returns: 99 Whether the token is the last token in its line. 100 """ 101 return not self.next or self.next.line_number != self.line_number 102 103 def IsType(self, token_type): 104 """Tests if this token is of the given type. 105 106 Args: 107 token_type: The type to test for. 108 109 Returns: 110 True if the type of this token matches the type passed in. 111 """ 112 return self.type == token_type 113 114 def IsAnyType(self, *token_types): 115 """Tests if this token is any of the given types. 116 117 Args: 118 token_types: The types to check. Also accepts a single array. 119 120 Returns: 121 True if the type of this token is any of the types passed in. 122 """ 123 if not isinstance(token_types[0], basestring): 124 return self.type in token_types[0] 125 else: 126 return self.type in token_types 127 128 def __repr__(self): 129 return '<Token: %s, "%s", %r, %d, %r>' % (self.type, self.string, 130 self.values, self.line_number, 131 self.metadata) 132 133 def __iter__(self): 134 """Returns a token iterator.""" 135 node = self 136 while node: 137 yield node 138 node = node.next 139 140 def __reversed__(self): 141 """Returns a reverse-direction token iterator.""" 142 node = self 143 while node: 144 yield node 145 node = node.previous 146