1 /*
2  * [The "BSD licence"]
3  * Copyright (c) 2005-2008 Terence Parr
4  * All rights reserved.
5  *
6  * Conversion to C#:
7  * Copyright (c) 2009 Sam Harwell
8  * All rights reserved.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. The name of the author may not be used to endorse or promote products
19  *    derived from this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
22  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
23  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
24  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
26  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
30  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 namespace Antlr.Runtime
34 {
35     using Antlr.Runtime.Misc;
36     using CLSCompliant = System.CLSCompliantAttribute;
37     using NotSupportedException = System.NotSupportedException;
38     using IndexOutOfRangeException = System.IndexOutOfRangeException;
39 
40     /** A token stream that pulls tokens from the code source on-demand and
41      *  without tracking a complete buffer of the tokens. This stream buffers
42      *  the minimum number of tokens possible.  It's the same as
43      *  OnDemandTokenStream except that OnDemandTokenStream buffers all tokens.
44      *
45      *  You can't use this stream if you pass whitespace or other off-channel
46      *  tokens to the parser. The stream can't ignore off-channel tokens.
47      *
48      *  You can only look backwards 1 token: LT(-1).
49      *
50      *  Use this when you need to read from a socket or other infinite stream.
51      *
52      *  @see BufferedTokenStream
53      *  @see CommonTokenStream
54      */
55     public class UnbufferedTokenStream : LookaheadStream<IToken>, ITokenStream, ITokenStreamInformation
56     {
57         [CLSCompliant(false)]
58         protected ITokenSource tokenSource;
59         protected int tokenIndex; // simple counter to set token index in tokens
60 
61         /** Skip tokens on any channel but this one; this is how we skip whitespace... */
62         protected int channel = TokenChannels.Default;
63 
64         private readonly ListStack<IToken> _realTokens = new ListStack<IToken>() { null };
65 
UnbufferedTokenStream(ITokenSource tokenSource)66         public UnbufferedTokenStream(ITokenSource tokenSource)
67         {
68             this.tokenSource = tokenSource;
69         }
70 
71         public ITokenSource TokenSource
72         {
73             get
74             {
75                 return this.tokenSource;
76             }
77         }
78 
79         public string SourceName
80         {
81             get
82             {
83                 return TokenSource.SourceName;
84             }
85         }
86 
87         #region ITokenStreamInformation Members
88 
89         public IToken LastToken
90         {
91             get
92             {
93                 return LB(1);
94             }
95         }
96 
97         public IToken LastRealToken
98         {
99             get
100             {
101                 return _realTokens.Peek();
102             }
103         }
104 
105         public int MaxLookBehind
106         {
107             get
108             {
109                 return 1;
110             }
111         }
112 
Mark()113         public override int Mark()
114         {
115             _realTokens.Push(_realTokens.Peek());
116             return base.Mark();
117         }
118 
Release(int marker)119         public override void Release(int marker)
120         {
121             base.Release(marker);
122             _realTokens.Pop();
123         }
124 
Clear()125         public override void Clear()
126         {
127             _realTokens.Clear();
128             _realTokens.Push(null);
129         }
130 
Consume()131         public override void Consume()
132         {
133             base.Consume();
134             if (PreviousElement != null && PreviousElement.Line > 0)
135                 _realTokens[_realTokens.Count - 1] = PreviousElement;
136         }
137 
138         #endregion
139 
NextElement()140         public override IToken NextElement()
141         {
142             IToken t = this.tokenSource.NextToken();
143             t.TokenIndex = this.tokenIndex++;
144             return t;
145         }
146 
IsEndOfFile(IToken o)147         public override bool IsEndOfFile(IToken o)
148         {
149             return o.Type == CharStreamConstants.EndOfFile;
150         }
151 
Get(int i)152         public IToken Get(int i)
153         {
154             throw new NotSupportedException("Absolute token indexes are meaningless in an unbuffered stream");
155         }
156 
LA(int i)157         public int LA(int i)
158         {
159             return LT(i).Type;
160         }
161 
ToString(int start, int stop)162         public string ToString(int start, int stop)
163         {
164             return "n/a";
165         }
166 
ToString(IToken start, IToken stop)167         public string ToString(IToken start, IToken stop)
168         {
169             return "n/a";
170         }
171     }
172 }
173