Sources/Antlr3.Runtime/Lexer.cs

/*
 * [The "BSD licence"]
 * Copyright (c) 2005-2008 Terence Parr
 * All rights reserved.
 *
 * Conversion to C#:
 * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

namespace Antlr.Runtime
{
    using ConditionalAttribute = System.Diagnostics.ConditionalAttribute;

    /** <summary>
     *  A lexer is recognizer that draws input symbols from a character stream.
     *  lexer grammars result in a subclass of this object. A Lexer object
     *  uses simplified match() and error recovery mechanisms in the interest
     *  of speed.
     *  </summary>
     */
    public abstract class Lexer : BaseRecognizer, ITokenSource
    {
        /** <summary>Where is the lexer drawing characters from?</summary> */
        protected ICharStream input;

        public Lexer()
        {
        }

        public Lexer( ICharStream input )
        {
            this.input = input;
        }

        public Lexer( ICharStream input, RecognizerSharedState state )
            : base(state)
        {
            this.input = input;
        }

        #region Properties
        public string Text
        {
            /** <summary>Return the text matched so far for the current token or any text override.</summary> */
            get
            {
                if ( state.text != null )
                {
                    return state.text;
                }
                return input.Substring( state.tokenStartCharIndex, CharIndex - state.tokenStartCharIndex );
            }
            /** <summary>Set the complete text of this token; it wipes any previous changes to the text.</summary> */
            set
            {
                state.text = value;
            }
        }
        public int Line
        {
            get
            {
                return input.Line;
            }
            set
            {
                input.Line = value;
            }
        }
        public int CharPositionInLine
        {
            get
            {
                return input.CharPositionInLine;
            }
            set
            {
                input.CharPositionInLine = value;
            }
        }
        #endregion

        public override void Reset()
        {
            base.Reset(); // reset all recognizer state variables
            // wack Lexer state variables
            if ( input != null )
            {
                input.Seek( 0 ); // rewind the input
            }
            if ( state == null )
            {
                return; // no shared state work to do
            }
            state.token = null;
            state.type = TokenTypes.Invalid;
            state.channel = TokenChannels.Default;
            state.tokenStartCharIndex = -1;
            state.tokenStartCharPositionInLine = -1;
            state.tokenStartLine = -1;
            state.text = null;
        }

        /** <summary>Return a token from this source; i.e., match a token on the char stream.</summary> */
        public virtual IToken NextToken()
        {
            for ( ; ; )
            {
                state.token = null;
                state.channel = TokenChannels.Default;
                state.tokenStartCharIndex = input.Index;
                state.tokenStartCharPositionInLine = input.CharPositionInLine;
                state.tokenStartLine = input.Line;
                state.text = null;
                if ( input.LA( 1 ) == CharStreamConstants.EndOfFile )
                {
                    return GetEndOfFileToken();
                }
                try
                {
                    ParseNextToken();
                    if ( state.token == null )
                    {
                        Emit();
                    }
                    else if ( state.token == Tokens.Skip )
                    {
                        continue;
                    }
                    return state.token;
                }
                catch (MismatchedRangeException mre)
                {
                    ReportError(mre);
                    // MatchRange() routine has already called recover()
                }
                catch (MismatchedTokenException mte)
                {
                    ReportError(mte);
                    // Match() routine has already called recover()
                }
                catch ( RecognitionException re )
                {
                    ReportError( re );
                    Recover( re ); // throw out current char and try again
                }
            }
        }

        /** Returns the EOF token (default), if you need
         *  to return a custom token instead override this method.
         */
        public virtual IToken GetEndOfFileToken()
        {
            IToken eof = new CommonToken((ICharStream)input, CharStreamConstants.EndOfFile, TokenChannels.Default, input.Index, input.Index);
            eof.Line = Line;
            eof.CharPositionInLine = CharPositionInLine;
            return eof;
        }

        /** <summary>
         *  Instruct the lexer to skip creating a token for current lexer rule
         *  and look for another token.  nextToken() knows to keep looking when
         *  a lexer rule finishes with token set to SKIP_TOKEN.  Recall that
         *  if token==null at end of any token rule, it creates one for you
         *  and emits it.
         *  </summary>
         */
        public virtual void Skip()
        {
            state.token = Tokens.Skip;
        }

        /** <summary>This is the lexer entry point that sets instance var 'token'</summary> */
        public abstract void mTokens();

        public virtual ICharStream CharStream
        {
            get
            {
                return input;
            }
            /** <summary>Set the char stream and reset the lexer</summary> */
            set
            {
                input = null;
                Reset();
                input = value;
            }
        }

        public override string SourceName
        {
            get
            {
                return input.SourceName;
            }
        }

        /** <summary>
         *  Currently does not support multiple emits per nextToken invocation
         *  for efficiency reasons.  Subclass and override this method and
         *  nextToken (to push tokens into a list and pull from that list rather
         *  than a single variable as this implementation does).
         *  </summary>
         */
        public virtual void Emit( IToken token )
        {
            state.token = token;
        }

        /** <summary>
         *  The standard method called to automatically emit a token at the
         *  outermost lexical rule.  The token object should point into the
         *  char buffer start..stop.  If there is a text override in 'text',
         *  use that to set the token's text.  Override this method to emit
         *  custom Token objects.
         *  </summary>
         *
         *  <remarks>
         *  If you are building trees, then you should also override
         *  Parser or TreeParser.getMissingSymbol().
         *  </remarks>
         */
        public virtual IToken Emit()
        {
            IToken t = new CommonToken( input, state.type, state.channel, state.tokenStartCharIndex, CharIndex - 1 );
            t.Line = state.tokenStartLine;
            t.Text = state.text;
            t.CharPositionInLine = state.tokenStartCharPositionInLine;
            Emit( t );
            return t;
        }

        public virtual void Match( string s )
        {
            int i = 0;
            while ( i < s.Length )
            {
                if ( input.LA( 1 ) != s[i] )
                {
                    if ( state.backtracking > 0 )
                    {
                        state.failed = true;
                        return;
                    }
                    MismatchedTokenException mte = new MismatchedTokenException(s[i], input, TokenNames);
                    Recover( mte );
                    throw mte;
                }
                i++;
                input.Consume();
                state.failed = false;
            }
        }

        public virtual void MatchAny()
        {
            input.Consume();
        }

        public virtual void Match( int c )
        {
            if ( input.LA( 1 ) != c )
            {
                if ( state.backtracking > 0 )
                {
                    state.failed = true;
                    return;
                }
                MismatchedTokenException mte = new MismatchedTokenException(c, input, TokenNames);
                Recover( mte );  // don't really recover; just consume in lexer
                throw mte;
            }
            input.Consume();
            state.failed = false;
        }

        public virtual void MatchRange( int a, int b )
        {
            if ( input.LA( 1 ) < a || input.LA( 1 ) > b )
            {
                if ( state.backtracking > 0 )
                {
                    state.failed = true;
                    return;
                }
                MismatchedRangeException mre = new MismatchedRangeException(a, b, input);
                Recover( mre );
                throw mre;
            }
            input.Consume();
            state.failed = false;
        }

        /** <summary>What is the index of the current character of lookahead?</summary> */
        public virtual int CharIndex
        {
            get
            {
                return input.Index;
            }
        }

        public override void ReportError( RecognitionException e )
        {
            /** TODO: not thought about recovery in lexer yet.
             *
            // if we've already reported an error and have not matched a token
            // yet successfully, don't report any errors.
            if ( errorRecovery ) {
                //System.err.print("[SPURIOUS] ");
                return;
            }
            errorRecovery = true;
             */

            DisplayRecognitionError( this.TokenNames, e );
        }

        public override string GetErrorMessage( RecognitionException e, string[] tokenNames )
        {
            string msg = null;
            if ( e is MismatchedTokenException )
            {
                MismatchedTokenException mte = (MismatchedTokenException)e;
                msg = "mismatched character " + GetCharErrorDisplay( e.Character ) + " expecting " + GetCharErrorDisplay( mte.Expecting );
            }
            else if ( e is NoViableAltException )
            {
                NoViableAltException nvae = (NoViableAltException)e;
                // for development, can add "decision=<<"+nvae.grammarDecisionDescription+">>"
                // and "(decision="+nvae.decisionNumber+") and
                // "state "+nvae.stateNumber
                msg = "no viable alternative at character " + GetCharErrorDisplay( e.Character );
            }
            else if ( e is EarlyExitException )
            {
                EarlyExitException eee = (EarlyExitException)e;
                // for development, can add "(decision="+eee.decisionNumber+")"
                msg = "required (...)+ loop did not match anything at character " + GetCharErrorDisplay( e.Character );
            }
            else if ( e is MismatchedNotSetException )
            {
                MismatchedNotSetException mse = (MismatchedNotSetException)e;
                msg = "mismatched character " + GetCharErrorDisplay( e.Character ) + " expecting set " + mse.Expecting;
            }
            else if ( e is MismatchedSetException )
            {
                MismatchedSetException mse = (MismatchedSetException)e;
                msg = "mismatched character " + GetCharErrorDisplay( e.Character ) + " expecting set " + mse.Expecting;
            }
            else if ( e is MismatchedRangeException )
            {
                MismatchedRangeException mre = (MismatchedRangeException)e;
                msg = "mismatched character " + GetCharErrorDisplay( e.Character ) + " expecting set " +
                      GetCharErrorDisplay( mre.A ) + ".." + GetCharErrorDisplay( mre.B );
            }
            else
            {
                msg = base.GetErrorMessage( e, tokenNames );
            }
            return msg;
        }

        public virtual string GetCharErrorDisplay( int c )
        {
            string s = ( (char)c ).ToString();
            switch ( c )
            {
            case TokenTypes.EndOfFile:
                s = "<EOF>";
                break;
            case '\n':
                s = "\\n";
                break;
            case '\t':
                s = "\\t";
                break;
            case '\r':
                s = "\\r";
                break;
            }
            return "'" + s + "'";
        }

        /** <summary>
         *  Lexers can normally match any char in it's vocabulary after matching
         *  a token, so do the easy thing and just kill a character and hope
         *  it all works out.  You can instead use the rule invocation stack
         *  to do sophisticated error recovery if you are in a fragment rule.
         *  </summary>
         */
        public virtual void Recover( RecognitionException re )
        {
            //System.out.println("consuming char "+(char)input.LA(1)+" during recovery");
            //re.printStackTrace();
            input.Consume();
        }

        [Conditional("ANTLR_TRACE")]
        public virtual void TraceIn( string ruleName, int ruleIndex )
        {
            string inputSymbol = ( (char)input.LT( 1 ) ) + " line=" + Line + ":" + CharPositionInLine;
            base.TraceIn( ruleName, ruleIndex, inputSymbol );
        }

        [Conditional("ANTLR_TRACE")]
        public virtual void TraceOut( string ruleName, int ruleIndex )
        {
            string inputSymbol = ( (char)input.LT( 1 ) ) + " line=" + Line + ":" + CharPositionInLine;
            base.TraceOut( ruleName, ruleIndex, inputSymbol );
        }

        protected virtual void ParseNextToken()
        {
            mTokens();
        }
    }
}