Lines Matching refs:TokenInfo
94 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): class
498 yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
523 yield TokenInfo(STRING, contstr + line[:end],
528 yield TokenInfo(ERRORTOKEN, contstr + line,
557 yield TokenInfo(COMMENT, comment_token,
561 yield TokenInfo(NL, line[pos:],
567 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
575 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
593 yield TokenInfo(NUMBER, token, spos, epos, line)
596 yield TokenInfo(NL, token, spos, epos, line)
598 yield TokenInfo(NEWLINE, token, spos, epos, line)
602 yield TokenInfo(COMMENT, token, spos, epos, line)
610 yield TokenInfo(STRING, token, spos, (lnum, pos), line)
645 yield TokenInfo(STRING, token, spos, epos, line)
648 yield TokenInfo(NAME, token, spos, epos, line)
656 yield TokenInfo(OP, token, spos, epos, line)
658 yield TokenInfo(ERRORTOKEN, line[pos],
664 yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
666 yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
667 yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')