Antlr3.Runtime/Antlr.Runtime/BufferedTokenStream.cs

/*
 * [The "BSD licence"]
 * Copyright (c) 2005-2008 Terence Parr
 * All rights reserved.
 *
 * Conversion to C#:
 * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

namespace Antlr.Runtime {
    using System.Collections.Generic;
    using CLSCompliant = System.CLSCompliantAttribute;
    using IndexOutOfRangeException = System.IndexOutOfRangeException;
    using StringBuilder = System.Text.StringBuilder;

    /** Buffer all input tokens but do on-demand fetching of new tokens from
     *  lexer. Useful when the parser or lexer has to set context/mode info before
     *  proper lexing of future tokens. The ST template parser needs this,
     *  for example, because it has to constantly flip back and forth between
     *  inside/output templates. E.g., <names:{hi, <it>}> has to parse names
     *  as part of an expression but "hi, <it>" as a nested template.
     *
     *  You can't use this stream if you pass whitespace or other off-channel
     *  tokens to the parser. The stream can't ignore off-channel tokens.
     *  (UnbufferedTokenStream is the same way.)
     *
     *  This is not a subclass of UnbufferedTokenStream because I don't want
     *  to confuse small moving window of tokens it uses for the full buffer.
     */
    [System.Serializable]
    public class BufferedTokenStream : ITokenStream, ITokenStreamInformation {
        private ITokenSource _tokenSource;

        /** Record every single token pulled from the source so we can reproduce
         *  chunks of it later.  The buffer in LookaheadStream overlaps sometimes
         *  as its moving window moves through the input.  This list captures
         *  everything so we can access complete input text.
         */
        [CLSCompliant(false)]
        protected List<IToken> _tokens = new List<IToken>(100);

        /** Track the last mark() call result value for use in rewind(). */
        private int _lastMarker;

        /** The index into the tokens list of the current token (next token
         *  to consume).  tokens[p] should be LT(1).  p=-1 indicates need
         *  to initialize with first token.  The ctor doesn't get a token.
         *  First call to LT(1) or whatever gets the first token and sets p=0;
         */
        [CLSCompliant(false)]
        protected int _p = -1;

        public BufferedTokenStream() {
        }

        public BufferedTokenStream(ITokenSource tokenSource) {
            this._tokenSource = tokenSource;
        }

        public virtual ITokenSource TokenSource {
            get {
                return _tokenSource;
            }
            set {
                this._tokenSource = value;
                _tokens.Clear();
                _p = -1;
            }
        }

        public virtual int Index {
            get {
                return _p;
            }
        }

        /// <summary>
        /// How deep have we gone?
        /// </summary>
        public virtual int Range {
            get;
            protected set;
        }

        public virtual int Count {
            get {
                return _tokens.Count;
            }
        }

        public virtual string SourceName {
            get {
                return _tokenSource.SourceName;
            }
        }

        public virtual IToken LastToken {
            get {
                return LB(1);
            }
        }

        public virtual IToken LastRealToken {
            get {
                int i = 0;
                IToken token;
                do {
                    i++;
                    token = LB(i);
                } while (token != null && token.Line <= 0);

                return token;
            }
        }

        public virtual int MaxLookBehind {
            get {
                return int.MaxValue;
            }
        }

        public virtual int Mark() {
            if (_p == -1)
                Setup();
            _lastMarker = Index;
            return _lastMarker;
        }

        public virtual void Release(int marker) {
            // no resources to release
        }

        public virtual void Rewind(int marker) {
            Seek(marker);
        }

        public virtual void Rewind() {
            Seek(_lastMarker);
        }

        public virtual void Reset() {
            _p = 0;
            _lastMarker = 0;
        }

        public virtual void Seek(int index) {
            _p = index;
        }

        /** Move the input pointer to the next incoming token.  The stream
         *  must become active with LT(1) available.  consume() simply
         *  moves the input pointer so that LT(1) points at the next
         *  input symbol. Consume at least one token.
         *
         *  Walk past any token not on the channel the parser is listening to.
         */
        public virtual void Consume() {
            if (_p == -1)
                Setup();
            _p++;
            Sync(_p);
        }

        /** Make sure index i in tokens has a token. */
        protected virtual void Sync(int i) {
            int n = i - _tokens.Count + 1; // how many more elements we need?
            if (n > 0)
                Fetch(n);
        }

        /** add n elements to buffer */
        protected virtual void Fetch(int n) {
            for (int i = 0; i < n; i++) {
                IToken t = TokenSource.NextToken();
                t.TokenIndex = _tokens.Count;
                _tokens.Add(t);
                if (t.Type == CharStreamConstants.EndOfFile)
                    break;
            }
        }

        public virtual IToken Get(int i) {
            if (i < 0 || i >= _tokens.Count) {
                throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1));
            }
            return _tokens[i];
        }

#if false // why is this different from GetTokens(start, count) ?
        /// <summary>
        /// Get all tokens from start..(start+count-1) inclusively
        /// </summary>
        public virtual List<IToken> Get(int start, int count)
        {
            if (start < 0)
                throw new ArgumentOutOfRangeException("start");
            if (count < 0)
                throw new ArgumentOutOfRangeException("count");
            if (start + count >= _tokens.Count)
                throw new ArgumentException();

            if (_p == -1)
                Setup();

            List<IToken> subset = new List<IToken>(count);
            for (int i = 0; i < count; i++)
            {
                IToken token = _tokens[i];
                if (token.Type == TokenTypes.EndOfFile)
                    break;

                subset.Add(token);
            }

            return subset;
        }
#endif

        public virtual int LA(int i) {
            return LT(i).Type;
        }

        protected virtual IToken LB(int k) {
            if ((_p - k) < 0)
                return null;

            return _tokens[_p - k];
        }

        public virtual IToken LT(int k) {
            if (_p == -1)
                Setup();
            if (k == 0)
                return null;
            if (k < 0)
                return LB(-k);

            int i = _p + k - 1;
            Sync(i);
            if (i >= _tokens.Count) {
                // EOF must be last token
                return _tokens[_tokens.Count - 1];
            }

            if (i > Range)
                Range = i;

            return _tokens[_p + k - 1];
        }

        protected virtual void Setup() {
            Sync(0);
            _p = 0;
        }

        public virtual List<IToken> GetTokens() {
            return _tokens;
        }

        public virtual List<IToken> GetTokens(int start, int stop) {
            return GetTokens(start, stop, default(BitSet));
        }

        /** Given a start and stop index, return a List of all tokens in
         *  the token type BitSet.  Return null if no tokens were found.  This
         *  method looks at both on and off channel tokens.
         */
        public virtual List<IToken> GetTokens(int start, int stop, BitSet types) {
            if (_p == -1)
                Setup();
            if (stop >= _tokens.Count)
                stop = _tokens.Count - 1;
            if (start < 0)
                start = 0;
            if (start > stop)
                return null;

            // list = tokens[start:stop]:{Token t, t.getType() in types}
            List<IToken> filteredTokens = new List<IToken>();
            for (int i = start; i <= stop; i++) {
                IToken t = _tokens[i];
                if (types == null || types.Member(t.Type)) {
                    filteredTokens.Add(t);
                }
            }
            if (filteredTokens.Count == 0) {
                filteredTokens = null;
            }
            return filteredTokens;
        }

        public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) {
            return GetTokens(start, stop, new BitSet(types));
        }

        public virtual List<IToken> GetTokens(int start, int stop, int ttype) {
            return GetTokens(start, stop, BitSet.Of(ttype));
        }

        public override string ToString() {
            if (_p == -1)
                Setup();

            Fill();
            return ToString(0, _tokens.Count - 1);
        }

        public virtual string ToString(int start, int stop) {
            if (start < 0 || stop < 0)
                return null;
            if (_p == -1)
                Setup();
            if (stop >= _tokens.Count)
                stop = _tokens.Count - 1;

            StringBuilder buf = new StringBuilder();
            for (int i = start; i <= stop; i++) {
                IToken t = _tokens[i];
                if (t.Type == CharStreamConstants.EndOfFile)
                    break;
                buf.Append(t.Text);
            }

            return buf.ToString();
        }

        public virtual string ToString(IToken start, IToken stop) {
            if (start != null && stop != null) {
                return ToString(start.TokenIndex, stop.TokenIndex);
            }
            return null;
        }

        public virtual void Fill() {
            if (_p == -1)
                Setup();

            if (_tokens[_p].Type == CharStreamConstants.EndOfFile)
                return;

            int i = _p + 1;
            Sync(i);
            while (_tokens[i].Type != CharStreamConstants.EndOfFile) {
                i++;
                Sync(i);
            }
        }
    }
}