Antlr3.Runtime/Antlr.Runtime/LegacyCommonTokenStream.cs

/*
 * NOTE TO JL: The original contains references to 3.5-only stuff!
 *
 * [The "BSD licence"]
 * Copyright (c) 2005-2008 Terence Parr
 * All rights reserved.
 *
 * Conversion to C#:
 * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

namespace Antlr.Runtime {
    using System.Collections.Generic;

    using ArgumentOutOfRangeException = System.ArgumentOutOfRangeException;
    using InvalidOperationException = System.InvalidOperationException;
    using StringBuilder = System.Text.StringBuilder;

    /** <summary>
     *  The most common stream of tokens is one where every token is buffered up
     *  and tokens are prefiltered for a certain channel (the parser will only
     *  see these tokens and cannot change the filter channel number during the
     *  parse).
     *  </summary>
     *
     *  <remarks>TODO: how to access the full token stream?  How to track all tokens matched per rule?</remarks>
     */
    [System.Serializable]
    public class LegacyCommonTokenStream : ITokenStream {
        [System.NonSerialized]
        ITokenSource _tokenSource;

        /** <summary>
         *  Record every single token pulled from the source so we can reproduce
         *  chunks of it later.
         *  </summary>
         */
        protected List<IToken> tokens;

        /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */
        protected IDictionary<int, int> channelOverrideMap;

        /** <summary>Set<tokentype>; discard any tokens with this type</summary> */
        protected Dictionary<int, int> discardSet;

        /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */
        protected int channel = TokenChannels.Default;

        /** <summary>By default, track all incoming tokens</summary> */
        protected bool discardOffChannelTokens = false;

        /** <summary>Track the last mark() call result value for use in rewind().</summary> */
        protected int lastMarker;

        /** <summary>
         *  The index into the tokens list of the current token (next token
         *  to consume).  p==-1 indicates that the tokens list is empty
         *  </summary>
         */
        protected int p = -1;

        public LegacyCommonTokenStream() {
            tokens = new List<IToken>(500);
        }

        public LegacyCommonTokenStream(ITokenSource tokenSource)
            : this() {
            this._tokenSource = tokenSource;
        }

        public LegacyCommonTokenStream(ITokenSource tokenSource, int channel)
            : this(tokenSource) {
            this.channel = channel;
        }

        public virtual int Index {
            get {
                return p;
            }
        }

        /// <summary>
        /// How deep have we gone?
        /// </summary>
        public virtual int Range {
            get;
            protected set;
        }

        /** <summary>Reset this token stream by setting its token source.</summary> */
        public virtual void SetTokenSource(ITokenSource tokenSource) {
            this._tokenSource = tokenSource;
            tokens.Clear();
            p = -1;
            channel = TokenChannels.Default;
        }

        /** <summary>
         *  Load all tokens from the token source and put in tokens.
         *  This is done upon first LT request because you might want to
         *  set some token type / channel overrides before filling buffer.
         *  </summary>
         */
        public virtual void FillBuffer() {
            // fast return if the buffer is already full
            if (p != -1)
                return;

            int index = 0;
            IToken t = _tokenSource.NextToken();
            while (t != null && t.Type != CharStreamConstants.EndOfFile) {
                bool discard = false;
                // is there a channel override for token type?
                int channelI;
                if (channelOverrideMap != null && channelOverrideMap.TryGetValue(t.Type, out channelI))
                    t.Channel = channelI;

                //if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) )
                //{
                //    object channelI = channelOverrideMap.get( t.getType() );
                //    if ( channelI != null )
                //    {
                //        t.setChannel( (int)channelI );
                //    }
                //}
                if (discardSet != null &&
                     discardSet.ContainsKey(t.Type)) {
                    discard = true;
                } else if (discardOffChannelTokens && t.Channel != this.channel) {
                    discard = true;
                }
                if (!discard) {
                    t.TokenIndex = index;
                    tokens.Add(t);
                    index++;
                }
                t = _tokenSource.NextToken();
            }
            // leave p pointing at first token on channel
            p = 0;
            p = SkipOffTokenChannels(p);
        }

        /** <summary>
         *  Move the input pointer to the next incoming token.  The stream
         *  must become active with LT(1) available.  consume() simply
         *  moves the input pointer so that LT(1) points at the next
         *  input symbol. Consume at least one token.
         *  </summary>
         *
         *  <remarks>
         *  Walk past any token not on the channel the parser is listening to.
         *  </remarks>
         */
        public virtual void Consume() {
            if (p < tokens.Count) {
                p++;
                p = SkipOffTokenChannels(p); // leave p on valid token
            }
        }

        /** <summary>Given a starting index, return the index of the first on-channel token.</summary> */
        protected virtual int SkipOffTokenChannels(int i) {
            int n = tokens.Count;
            while (i < n && ((IToken)tokens[i]).Channel != channel) {
                i++;
            }
            return i;
        }

        protected virtual int SkipOffTokenChannelsReverse(int i) {
            while (i >= 0 && ((IToken)tokens[i]).Channel != channel) {
                i--;
            }
            return i;
        }

        /** <summary>
         *  A simple filter mechanism whereby you can tell this token stream
         *  to force all tokens of type ttype to be on channel.  For example,
         *  when interpreting, we cannot exec actions so we need to tell
         *  the stream to force all WS and NEWLINE to be a different, ignored
         *  channel.
         *  </summary>
         */
        public virtual void SetTokenTypeChannel(int ttype, int channel) {
            if (channelOverrideMap == null) {
                channelOverrideMap = new Dictionary<int, int>();
            }
            channelOverrideMap[ttype] = channel;
        }

        public virtual void DiscardTokenType(int ttype) {
            if (discardSet == null) {
                discardSet = new Dictionary<int, int>();
            }
            discardSet.Add(ttype, ttype);
        }

        public virtual void SetDiscardOffChannelTokens(bool discardOffChannelTokens) {
            this.discardOffChannelTokens = discardOffChannelTokens;
        }

        public virtual IList<IToken> GetTokens() {
            if (p == -1) {
                FillBuffer();
            }
            return tokens;
        }

        public virtual IList<IToken> GetTokens(int start, int stop) {
            return GetTokens(start, stop, (BitSet)null);
        }

        /** <summary>
         *  Given a start and stop index, return a List of all tokens in
         *  the token type BitSet.  Return null if no tokens were found.  This
         *  method looks at both on and off channel tokens.
         *  </summary>
         */
        public virtual IList<IToken> GetTokens(int start, int stop, BitSet types) {
            if (p == -1) {
                FillBuffer();
            }
            if (stop >= tokens.Count) {
                stop = tokens.Count - 1;
            }
            if (start < 0) {
                start = 0;
            }
            if (start > stop) {
                return null;
            }

            // list = tokens[start:stop]:{Token t, t.getType() in types}
            IList<IToken> filteredTokens = new List<IToken>();
            for (int i = start; i <= stop; i++) {
                IToken t = tokens[i];
                if (types == null || types.Member(t.Type)) {
                    filteredTokens.Add(t);
                }
            }
            if (filteredTokens.Count == 0) {
                filteredTokens = null;
            }
            return filteredTokens;
        }

        public virtual IList<IToken> GetTokens(int start, int stop, IList<int> types) {
            return GetTokens(start, stop, new BitSet(types));
        }

        public virtual IList<IToken> GetTokens(int start, int stop, int ttype) {
            return GetTokens(start, stop, BitSet.Of(ttype));
        }

        /** <summary>
         *  Get the ith token from the current position 1..n where k=1 is the
         *  first symbol of lookahead.
         *  </summary>
         */
        public virtual IToken LT(int k) {
            if (p == -1) {
                FillBuffer();
            }
            if (k == 0) {
                return null;
            }
            if (k < 0) {
                return LB(-k);
            }
            //System.out.print("LT(p="+p+","+k+")=");
            if ((p + k - 1) >= tokens.Count) {
                return tokens[tokens.Count - 1];
            }
            //System.out.println(tokens.get(p+k-1));
            int i = p;
            int n = 1;
            // find k good tokens
            while (n < k) {
                // skip off-channel tokens
                i = SkipOffTokenChannels(i + 1); // leave p on valid token
                n++;
            }
            if (i >= tokens.Count) {
                return tokens[tokens.Count - 1];
            }

            if (i > Range)
                Range = i;

            return (IToken)tokens[i];
        }

        /** <summary>Look backwards k tokens on-channel tokens</summary> */
        protected virtual IToken LB(int k) {
            //System.out.print("LB(p="+p+","+k+") ");
            if (p == -1) {
                FillBuffer();
            }
            if (k == 0) {
                return null;
            }
            if ((p - k) < 0) {
                return null;
            }

            int i = p;
            int n = 1;
            // find k good tokens looking backwards
            while (n <= k) {
                // skip off-channel tokens
                i = SkipOffTokenChannelsReverse(i - 1); // leave p on valid token
                n++;
            }
            if (i < 0) {
                return null;
            }
            return (IToken)tokens[i];
        }

        /** <summary>
         *  Return absolute token i; ignore which channel the tokens are on;
         *  that is, count all tokens not just on-channel tokens.
         *  </summary>
         */
        public virtual IToken Get(int i) {
            return (IToken)tokens[i];
        }

#if false
        /** Get all tokens from start..stop inclusively */
        public virtual List<IToken> Get(int start, int count)
        {
            if (start < 0)
                throw new ArgumentOutOfRangeException("start");
            if (count < 0)
                throw new ArgumentOutOfRangeException("count");

            if (p == -1)
                FillBuffer();

            return new List<IToken>(tokens.Skip(start).Take(count));
        }
#endif

        public virtual int LA(int i) {
            return LT(i).Type;
        }

        public virtual int Mark() {
            if (p == -1) {
                FillBuffer();
            }
            lastMarker = Index;
            return lastMarker;
        }

        public virtual void Release(int marker) {
            // no resources to release
        }

        public virtual int Count {
            get {
                return tokens.Count;
            }
        }

        public virtual void Rewind(int marker) {
            Seek(marker);
        }

        public virtual void Rewind() {
            Seek(lastMarker);
        }

        public virtual void Reset() {
            p = 0;
            lastMarker = 0;
        }

        public virtual void Seek(int index) {
            p = index;
        }

        public virtual ITokenSource TokenSource {
            get {
                return _tokenSource;
            }
        }

        public virtual string SourceName {
            get {
                return TokenSource.SourceName;
            }
        }

        public override string ToString() {
            if (p == -1) {
                throw new InvalidOperationException("Buffer is not yet filled.");
            }
            return ToString(0, tokens.Count - 1);
        }

        public virtual string ToString(int start, int stop) {
            if (start < 0 || stop < 0) {
                return null;
            }
            if (p == -1) {
                throw new InvalidOperationException("Buffer is not yet filled.");
            }
            if (stop >= tokens.Count) {
                stop = tokens.Count - 1;
            }
            StringBuilder buf = new StringBuilder();
            for (int i = start; i <= stop; i++) {
                IToken t = tokens[i];
                buf.Append(t.Text);
            }
            return buf.ToString();
        }

        public virtual string ToString(IToken start, IToken stop) {
            if (start != null && stop != null) {
                return ToString(start.TokenIndex, stop.TokenIndex);
            }
            return null;
        }
    }
}