1#!/usr/bin/env python
2#
3# Copyright 2008 The Closure Linter Authors. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS-IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Classes to represent tokens and positions within them."""
18
19__author__ = ('robbyw@google.com (Robert Walker)',
20              'ajp@google.com (Andy Perelson)')
21
22
23class TokenType(object):
24  """Token types common to all languages."""
25  NORMAL = 'normal'
26  WHITESPACE = 'whitespace'
27  BLANK_LINE = 'blank line'
28
29
30class Token(object):
31  """Token class for intelligent text splitting.
32
33  The token class represents a string of characters and an identifying type.
34
35  Attributes:
36    type: The type of token.
37    string: The characters the token comprises.
38    length: The length of the token.
39    line: The text of the line the token is found in.
40    line_number: The number of the line the token is found in.
41    values: Dictionary of values returned from the tokens regex match.
42    previous: The token before this one.
43    next: The token after this one.
44    start_index: The character index in the line where this token starts.
45    attached_object: Object containing more information about this token.
46    metadata: Object containing metadata about this token.  Must be added by
47        a separate metadata pass.
48  """
49
50  def __init__(self, string, token_type, line, line_number, values=None,
51               orig_line_number=None):
52    """Creates a new Token object.
53
54    Args:
55      string: The string of input the token contains.
56      token_type: The type of token.
57      line: The text of the line this token is in.
58      line_number: The line number of the token.
59      values: A dict of named values within the token.  For instance, a
60        function declaration may have a value called 'name' which captures the
61        name of the function.
62      orig_line_number: The line number of the original file this token comes
63        from. This should be only set during the tokenization process. For newly
64        created error fix tokens after that, it should be None.
65    """
66    self.type = token_type
67    self.string = string
68    self.length = len(string)
69    self.line = line
70    self.line_number = line_number
71    self.orig_line_number = orig_line_number
72    self.values = values
73    self.is_deleted = False
74
75    # These parts can only be computed when the file is fully tokenized
76    self.previous = None
77    self.next = None
78    self.start_index = None
79
80    # This part is set in statetracker.py
81    # TODO(robbyw): Wrap this in to metadata
82    self.attached_object = None
83
84    # This part is set in *metadatapass.py
85    self.metadata = None
86
87  def IsFirstInLine(self):
88    """Tests if this token is the first token in its line.
89
90    Returns:
91      Whether the token is the first token in its line.
92    """
93    return not self.previous or self.previous.line_number != self.line_number
94
95  def IsLastInLine(self):
96    """Tests if this token is the last token in its line.
97
98    Returns:
99      Whether the token is the last token in its line.
100    """
101    return not self.next or self.next.line_number != self.line_number
102
103  def IsType(self, token_type):
104    """Tests if this token is of the given type.
105
106    Args:
107      token_type: The type to test for.
108
109    Returns:
110      True if the type of this token matches the type passed in.
111    """
112    return self.type == token_type
113
114  def IsAnyType(self, *token_types):
115    """Tests if this token is any of the given types.
116
117    Args:
118      token_types: The types to check.  Also accepts a single array.
119
120    Returns:
121      True if the type of this token is any of the types passed in.
122    """
123    if not isinstance(token_types[0], basestring):
124      return self.type in token_types[0]
125    else:
126      return self.type in token_types
127
128  def __repr__(self):
129    return '<Token: %s, "%s", %r, %d, %r>' % (self.type, self.string,
130                                              self.values, self.line_number,
131                                              self.metadata)
132
133  def __iter__(self):
134    """Returns a token iterator."""
135    node = self
136    while node:
137      yield node
138      node = node.next
139
140  def __reversed__(self):
141    """Returns a reverse-direction token iterator."""
142    node = self
143    while node:
144      yield node
145      node = node.previous
146