Lines Matching refs:lexer
42 static void mTokens (pANTLR3_LEXER lexer);
43 static void setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input);
44 static void pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input);
45 static void popCharStream (pANTLR3_LEXER lexer);
47 static void emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token);
48 static pANTLR3_COMMON_TOKEN emit (pANTLR3_LEXER lexer);
49 static ANTLR3_BOOLEAN matchs (pANTLR3_LEXER lexer, ANTLR3_UCHAR * string);
50 static ANTLR3_BOOLEAN matchc (pANTLR3_LEXER lexer, ANTLR3_UCHAR c);
51 static ANTLR3_BOOLEAN matchRange (pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high);
52 static void matchAny (pANTLR3_LEXER lexer);
53 static void recover (pANTLR3_LEXER lexer);
54 static ANTLR3_UINT32 getLine (pANTLR3_LEXER lexer);
55 static ANTLR3_MARKER getCharIndex (pANTLR3_LEXER lexer);
56 static ANTLR3_UINT32 getCharPositionInLine (pANTLR3_LEXER lexer);
57 static pANTLR3_STRING getText (pANTLR3_LEXER lexer);
68 static void freeLexer (pANTLR3_LEXER lexer);
74 pANTLR3_LEXER lexer; in antlr3LexerNew() local
79 lexer = (pANTLR3_LEXER) ANTLR3_MALLOC(sizeof(ANTLR3_LEXER)); in antlr3LexerNew()
81 if (lexer == NULL) in antlr3LexerNew()
88 lexer->rec = antlr3BaseRecognizerNew(ANTLR3_TYPE_LEXER, sizeHint, state); in antlr3LexerNew()
90 if (lexer->rec == NULL) in antlr3LexerNew()
92 lexer->free(lexer); in antlr3LexerNew()
95 lexer->rec->super = lexer; in antlr3LexerNew()
97 lexer->rec->displayRecognitionError = displayRecognitionError; in antlr3LexerNew()
98 lexer->rec->reportError = reportError; in antlr3LexerNew()
99 lexer->rec->reset = reset; in antlr3LexerNew()
100 lexer->rec->getCurrentInputSymbol = getCurrentInputSymbol; in antlr3LexerNew()
101 lexer->rec->getMissingSymbol = getMissingSymbol; in antlr3LexerNew()
105 if (lexer->rec->state->tokSource == NULL) in antlr3LexerNew()
107 …lexer->rec->state->tokSource = (pANTLR3_TOKEN_SOURCE)ANTLR3_CALLOC(1, sizeof(ANTLR3_TOKEN_SOURCE)); in antlr3LexerNew()
109 if (lexer->rec->state->tokSource == NULL) in antlr3LexerNew()
111 lexer->rec->free(lexer->rec); in antlr3LexerNew()
112 lexer->free(lexer); in antlr3LexerNew()
116 lexer->rec->state->tokSource->super = lexer; in antlr3LexerNew()
121 lexer->rec->state->tokSource->nextToken = nextToken; in antlr3LexerNew()
122 lexer->rec->state->tokSource->strFactory = NULL; in antlr3LexerNew()
124 lexer->rec->state->tokFactory = NULL; in antlr3LexerNew()
129 lexer->setCharStream = setCharStream; in antlr3LexerNew()
130 lexer->mTokens = (void (*)(void *))(mTokens); in antlr3LexerNew()
131 lexer->setCharStream = setCharStream; in antlr3LexerNew()
132 lexer->pushCharStream = pushCharStream; in antlr3LexerNew()
133 lexer->popCharStream = popCharStream; in antlr3LexerNew()
134 lexer->emit = emit; in antlr3LexerNew()
135 lexer->emitNew = emitNew; in antlr3LexerNew()
136 lexer->matchs = matchs; in antlr3LexerNew()
137 lexer->matchc = matchc; in antlr3LexerNew()
138 lexer->matchRange = matchRange; in antlr3LexerNew()
139 lexer->matchAny = matchAny; in antlr3LexerNew()
140 lexer->recover = recover; in antlr3LexerNew()
141 lexer->getLine = getLine; in antlr3LexerNew()
142 lexer->getCharIndex = getCharIndex; in antlr3LexerNew()
143 lexer->getCharPositionInLine = getCharPositionInLine; in antlr3LexerNew()
144 lexer->getText = getText; in antlr3LexerNew()
145 lexer->free = freeLexer; in antlr3LexerNew()
149 specialT = &(lexer->rec->state->tokSource->eofToken); in antlr3LexerNew()
162 specialT = &(lexer->rec->state->tokSource->skipToken); in antlr3LexerNew()
171 return lexer; in antlr3LexerNew()
177 pANTLR3_LEXER lexer; in reset() local
179 lexer = rec->super; in reset()
181 lexer->rec->state->token = NULL; in reset()
182 lexer->rec->state->type = ANTLR3_TOKEN_INVALID; in reset()
183 lexer->rec->state->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; in reset()
184 lexer->rec->state->tokenStartCharIndex = -1; in reset()
185 lexer->rec->state->tokenStartCharPositionInLine = -1; in reset()
186 lexer->rec->state->tokenStartLine = -1; in reset()
188 lexer->rec->state->text = NULL; in reset()
193 if (lexer->rec->state->tokFactory != NULL) in reset()
195 lexer->rec->state->tokFactory->reset(lexer->rec->state->tokFactory); in reset()
219 pANTLR3_LEXER lexer; in nextTokenStr() local
224 lexer = (pANTLR3_LEXER)(toksource->super); in nextTokenStr()
225 state = lexer->rec->state; in nextTokenStr()
226 input = lexer->input; in nextTokenStr()
263 teof->setStartIndex (teof, lexer->getCharIndex(lexer)); in nextTokenStr()
264 teof->setStopIndex (teof, lexer->getCharIndex(lexer)); in nextTokenStr()
265 teof->setLine (teof, lexer->getLine(lexer)); in nextTokenStr()
276 lexer->mTokens(lexer->ctx); in nextTokenStr()
283 lexer->rec->reportError(lexer->rec); in nextTokenStr()
284 lexer->recover(lexer); in nextTokenStr()
292 emit(lexer); in nextTokenStr()
353 pANTLR3_LEXER lexer; in nextToken() local
355 lexer = (pANTLR3_LEXER)(toksource->super); in nextToken()
357 …if (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->str… in nextToken()
363 lexer->popCharStream(lexer); in nextToken()
386 pANTLR3_LEXER lexer; in antlr3LexerNewStream() local
390 lexer = antlr3LexerNew(sizeHint, state); in antlr3LexerNewStream()
392 if (lexer != NULL) in antlr3LexerNewStream()
396 setCharStream(lexer, input); in antlr3LexerNewStream()
399 return lexer; in antlr3LexerNewStream()
402 static void mTokens (pANTLR3_LEXER lexer) in mTokens() argument
404 if (lexer) // Fool compiler, avoid pragmas in mTokens()
429 pANTLR3_LEXER lexer; in displayRecognitionError() local
433 lexer = (pANTLR3_LEXER)(recognizer->super); in displayRecognitionError()
434 ex = lexer->rec->state->exception; in displayRecognitionError()
457 …width = ANTLR3_UINT32_CAST(( (pANTLR3_UINT8)(lexer->input->data) + (lexer->input->size(lexer->inpu… in displayRecognitionError()
475 (ANTLR3_UINT32)(lexer->rec->state->tokenStartLine), in displayRecognitionError()
476 (ANTLR3_UINT32)(lexer->rec->state->tokenStartCharPositionInLine) in displayRecognitionError()
478 … ANTLR3_UINT32_CAST(((pANTLR3_UINT8)(lexer->input->data)+(lexer->input->size(lexer->input))) - (pA… in displayRecognitionError()
482 … "looks like this:\n\t\t%.*s\n", width > 20 ? 20 : width ,(pANTLR3_UINT8)(lexer->rec->state->token… in displayRecognitionError()
492 static void setCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input) in setCharStream() argument
496 lexer->input = input; in setCharStream()
503 if (lexer->rec->state->tokFactory == NULL) in setCharStream()
505 lexer->rec->state->tokFactory = antlr3TokenFactoryNew(input); in setCharStream()
515 lexer->rec->state->tokFactory->setInputStream(lexer->rec->state->tokFactory, input); in setCharStream()
521 if (lexer->rec->state->tokSource->strFactory == NULL) in setCharStream()
523 lexer->rec->state->tokSource->strFactory = input->strFactory; in setCharStream()
528 if (lexer->rec->state->tokSource->eofToken.strFactory == NULL) in setCharStream()
530 lexer->rec->state->tokSource->eofToken.strFactory = input->strFactory; in setCharStream()
536 lexer->rec->exConstruct = antlr3RecognitionExceptionNew; in setCharStream()
540 lexer->rec->state->token = NULL; in setCharStream()
541 lexer->rec->state->text = NULL; in setCharStream()
542 lexer->rec->state->tokenStartCharIndex = -1; in setCharStream()
546 lexer->rec->state->tokSource->fileName = input->fileName; in setCharStream()
564 pushCharStream (pANTLR3_LEXER lexer, pANTLR3_INPUT_STREAM input) in pushCharStream() argument
568 if (lexer->rec->state->streams == NULL) in pushCharStream()
573 lexer->rec->state->streams = antlr3StackNew(0); in pushCharStream()
575 if (lexer->rec->state->streams == NULL) in pushCharStream()
587 lexer->input->istream->mark(lexer->input->istream); in pushCharStream()
588 lexer->rec->state->streams->push(lexer->rec->state->streams, lexer->input, NULL); in pushCharStream()
592 lexer->setCharStream(lexer, input); in pushCharStream()
610 popCharStream (pANTLR3_LEXER lexer) in popCharStream() argument
617 …if (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->stre… in popCharStream()
625 input = (pANTLR3_INPUT_STREAM)(lexer->rec->state->streams->top); in popCharStream()
626 lexer->rec->state->streams->pop(lexer->rec->state->streams); in popCharStream()
630 lexer->setCharStream(lexer, input); in popCharStream()
631 lexer->input->istream->rewindLast(lexer->input->istream); in popCharStream()
636 static void emitNew (pANTLR3_LEXER lexer, pANTLR3_COMMON_TOKEN token) in emitNew() argument
638 lexer->rec->state->token = token; /* Voila! */ in emitNew()
642 emit (pANTLR3_LEXER lexer) in emit() argument
651 token = lexer->rec->state->tokFactory->newToken(lexer->rec->state->tokFactory); in emit()
657 token->type = lexer->rec->state->type; in emit()
658 token->channel = lexer->rec->state->channel; in emit()
659 token->start = lexer->rec->state->tokenStartCharIndex; in emit()
660 token->stop = lexer->getCharIndex(lexer) - 1; in emit()
661 token->line = lexer->rec->state->tokenStartLine; in emit()
662 token->charPosition = lexer->rec->state->tokenStartCharPositionInLine; in emit()
664 if (lexer->rec->state->text != NULL) in emit()
667 token->tokText.text = lexer->rec->state->text; in emit()
673 token->lineStart = lexer->input->currentLine; in emit()
674 token->user1 = lexer->rec->state->user1; in emit()
675 token->user2 = lexer->rec->state->user2; in emit()
676 token->user3 = lexer->rec->state->user3; in emit()
677 token->custom = lexer->rec->state->custom; in emit()
679 lexer->rec->state->token = token; in emit()
688 freeLexer (pANTLR3_LEXER lexer) in freeLexer() argument
694 if (lexer->rec->state != NULL) in freeLexer()
696 if (lexer->rec->state->streams != NULL) in freeLexer()
698 lexer->rec->state->streams->free(lexer->rec->state->streams); in freeLexer()
700 if (lexer->rec->state->tokFactory != NULL) in freeLexer()
702 lexer->rec->state->tokFactory->close(lexer->rec->state->tokFactory); in freeLexer()
703 lexer->rec->state->tokFactory = NULL; in freeLexer()
705 if (lexer->rec->state->tokSource != NULL) in freeLexer()
707 ANTLR3_FREE(lexer->rec->state->tokSource); in freeLexer()
708 lexer->rec->state->tokSource = NULL; in freeLexer()
711 if (lexer->rec != NULL) in freeLexer()
713 lexer->rec->free(lexer->rec); in freeLexer()
714 lexer->rec = NULL; in freeLexer()
716 ANTLR3_FREE(lexer); in freeLexer()
727 matchs(pANTLR3_LEXER lexer, ANTLR3_UCHAR * string) in matchs() argument
731 if (lexer->input->istream->_LA(lexer->input->istream, 1) != (*string)) in matchs()
733 if (lexer->rec->state->backtracking > 0) in matchs()
735 lexer->rec->state->failed = ANTLR3_TRUE; in matchs()
739 lexer->rec->exConstruct(lexer->rec); in matchs()
740 lexer->rec->state->failed = ANTLR3_TRUE; in matchs()
744 lexer->recover(lexer); in matchs()
750 lexer->input->istream->consume(lexer->input->istream); in matchs()
755 lexer->rec->state->failed = ANTLR3_FALSE; in matchs()
770 matchc(pANTLR3_LEXER lexer, ANTLR3_UCHAR c) in matchc() argument
772 if (lexer->input->istream->_LA(lexer->input->istream, 1) == c) in matchc()
776 lexer->input->istream->consume(lexer->input->istream); in matchc()
780 lexer->rec->state->failed = ANTLR3_FALSE; in matchc()
787 if (lexer->rec->state->backtracking > 0) in matchc()
789 lexer->rec->state->failed = ANTLR3_TRUE; in matchc()
793 lexer->rec->exConstruct(lexer->rec); in matchc()
797 lexer->recover(lexer); in matchc()
810 matchRange(pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high) in matchRange() argument
816 c = lexer->input->istream->_LA(lexer->input->istream, 1); in matchRange()
821 lexer->input->istream->consume(lexer->input->istream); in matchRange()
825 lexer->rec->state->failed = ANTLR3_FALSE; in matchRange()
833 if (lexer->rec->state->backtracking > 0) in matchRange()
835 lexer->rec->state->failed = ANTLR3_TRUE; in matchRange()
839 lexer->rec->exConstruct(lexer->rec); in matchRange()
843 lexer->recover(lexer); in matchRange()
849 matchAny (pANTLR3_LEXER lexer) in matchAny() argument
851 lexer->input->istream->consume(lexer->input->istream); in matchAny()
855 recover (pANTLR3_LEXER lexer) in recover() argument
857 lexer->input->istream->consume(lexer->input->istream); in recover()
861 getLine (pANTLR3_LEXER lexer) in getLine() argument
863 return lexer->input->getLine(lexer->input); in getLine()
867 getCharPositionInLine (pANTLR3_LEXER lexer) in getCharPositionInLine() argument
869 return lexer->input->charPositionInLine; in getCharPositionInLine()
872 static ANTLR3_MARKER getCharIndex (pANTLR3_LEXER lexer) in getCharIndex() argument
874 return lexer->input->istream->index(lexer->input->istream); in getCharIndex()
878 getText (pANTLR3_LEXER lexer) in getText() argument
880 if (lexer->rec->state->text) in getText()
882 return lexer->rec->state->text; in getText()
885 return lexer->input->substr( in getText()
886 lexer->input, in getText()
887 lexer->rec->state->tokenStartCharIndex, in getText()
888 lexer->getCharIndex(lexer) - lexer->input->charByteSize in getText()