Antlr3.Runtime/Antlr.Runtime/TokenRewriteStream.cs

/*
 * [The "BSD licence"]
 * Copyright (c) 2005-2008 Terence Parr
 * All rights reserved.
 *
 * Conversion to C#:
 * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

namespace Antlr.Runtime {
    using System.Collections.Generic;

    using ArgumentException = System.ArgumentException;
    using DebuggerDisplay = System.Diagnostics.DebuggerDisplayAttribute;
    using Exception = System.Exception;
    using StringBuilder = System.Text.StringBuilder;
    using Type = System.Type;

    /** Useful for dumping out the input stream after doing some
     *  augmentation or other manipulations.
     *
     *  You can insert stuff, replace, and delete chunks.  Note that the
     *  operations are done lazily--only if you convert the buffer to a
     *  String.  This is very efficient because you are not moving data around
     *  all the time.  As the buffer of tokens is converted to strings, the
     *  toString() method(s) check to see if there is an operation at the
     *  current index.  If so, the operation is done and then normal String
     *  rendering continues on the buffer.  This is like having multiple Turing
     *  machine instruction streams (programs) operating on a single input tape. :)
     *
     *  Since the operations are done lazily at toString-time, operations do not
     *  screw up the token index values.  That is, an insert operation at token
     *  index i does not change the index values for tokens i+1..n-1.
     *
     *  Because operations never actually alter the buffer, you may always get
     *  the original token stream back without undoing anything.  Since
     *  the instructions are queued up, you can easily simulate transactions and
     *  roll back any changes if there is an error just by removing instructions.
     *  For example,
     *
     *   CharStream input = new ANTLRFileStream("input");
     *   TLexer lex = new TLexer(input);
     *   TokenRewriteStream tokens = new TokenRewriteStream(lex);
     *   T parser = new T(tokens);
     *   parser.startRule();
     *
     * 	 Then in the rules, you can execute
     *      Token t,u;
     *      ...
     *      input.insertAfter(t, "text to put after t");}
     * 		input.insertAfter(u, "text after u");}
     * 		System.out.println(tokens.toString());
     *
     *  Actually, you have to cast the 'input' to a TokenRewriteStream. :(
     *
     *  You can also have multiple "instruction streams" and get multiple
     *  rewrites from a single pass over the input.  Just name the instruction
     *  streams and use that name again when printing the buffer.  This could be
     *  useful for generating a C file and also its header file--all from the
     *  same buffer:
     *
     *      tokens.insertAfter("pass1", t, "text to put after t");}
     * 		tokens.insertAfter("pass2", u, "text after u");}
     * 		System.out.println(tokens.toString("pass1"));
     * 		System.out.println(tokens.toString("pass2"));
     *
     *  If you don't use named rewrite streams, a "default" stream is used as
     *  the first example shows.
     */
    [System.Serializable]
    [DebuggerDisplay("TODO: TokenRewriteStream debugger display")]
    public class TokenRewriteStream : CommonTokenStream {
        public const string DEFAULT_PROGRAM_NAME = "default";
        public const int PROGRAM_INIT_SIZE = 100;
        public const int MIN_TOKEN_INDEX = 0;

        // Define the rewrite operation hierarchy

        protected class RewriteOperation {
            /** <summary>What index into rewrites List are we?</summary> */
            public int instructionIndex;
            /** <summary>Token buffer index.</summary> */
            public int index;
            public object text;
            // outer
            protected TokenRewriteStream stream;

            protected RewriteOperation(TokenRewriteStream stream, int index, object text) {
                this.index = index;
                this.text = text;
                this.stream = stream;
            }
            /** <summary>
             *  Execute the rewrite operation by possibly adding to the buffer.
             *  Return the index of the next token to operate on.
             *  </summary>
             */
            public virtual int Execute(StringBuilder buf) {
                return index;
            }
            public override string ToString() {
                string opName = this.GetType().Name;
                int index = opName.IndexOf('$');
                opName = opName.Substring(index + 1);
                return "<" + opName + "@" + this.index + ":\"" + text + "\">";
            }
        }

        class InsertBeforeOp : RewriteOperation {
            public InsertBeforeOp(TokenRewriteStream stream, int index, object text) :
                base(stream, index, text) {
            }

            public override int Execute(StringBuilder buf) {
                buf.Append(text);
                if (stream._tokens[index].Type != CharStreamConstants.EndOfFile)
                    buf.Append(stream._tokens[index].Text);
                return index + 1;
            }
        }

        /** <summary>
         *  I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp
         *  instructions.
         *  </summary>
         */
        class ReplaceOp : RewriteOperation {
            public int lastIndex;
            public ReplaceOp(TokenRewriteStream stream, int from, int to, object text)
                : base(stream, from, text) {
                lastIndex = to;
            }
            public override int Execute(StringBuilder buf) {
                if (text != null) {
                    buf.Append(text);
                }
                return lastIndex + 1;
            }
            public override string ToString() {
                return "<ReplaceOp@" + index + ".." + lastIndex + ":\"" + text + "\">";
            }
        }

        class DeleteOp : ReplaceOp {
            public DeleteOp(TokenRewriteStream stream, int from, int to) :
                base(stream, from, to, null) {
            }
            public override string ToString() {
                return "<DeleteOp@" + index + ".." + lastIndex + ">";
            }
        }

        /** <summary>
         *  You may have multiple, named streams of rewrite operations.
         *  I'm calling these things "programs."
         *  Maps String (name) -> rewrite (List)
         *  </summary>
         */
        protected IDictionary<string, IList<RewriteOperation>> programs = null;

        /** <summary>Map String (program name) -> Integer index</summary> */
        protected IDictionary<string, int> lastRewriteTokenIndexes = null;

        public TokenRewriteStream() {
            Init();
        }

        protected void Init() {
            programs = new Dictionary<string, IList<RewriteOperation>>();
            programs[DEFAULT_PROGRAM_NAME] = new List<RewriteOperation>(PROGRAM_INIT_SIZE);
            lastRewriteTokenIndexes = new Dictionary<string, int>();
        }

        public TokenRewriteStream(ITokenSource tokenSource)
            : base(tokenSource) {
            Init();
        }

        public TokenRewriteStream(ITokenSource tokenSource, int channel)
            : base(tokenSource, channel) {
            Init();
        }

        public virtual void Rollback(int instructionIndex) {
            Rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
        }

        /** <summary>
         *  Rollback the instruction stream for a program so that
         *  the indicated instruction (via instructionIndex) is no
         *  longer in the stream.  UNTESTED!
         *  </summary>
         */
        public virtual void Rollback(string programName, int instructionIndex) {
            IList<RewriteOperation> @is;
            if (programs.TryGetValue(programName, out @is) && @is != null) {
                List<RewriteOperation> sublist = new List<RewriteOperation>();
                for (int i = MIN_TOKEN_INDEX; i <= instructionIndex; i++)
                    sublist.Add(@is[i]);

                programs[programName] = sublist;
            }
        }

        public virtual void DeleteProgram() {
            DeleteProgram(DEFAULT_PROGRAM_NAME);
        }

        /** <summary>Reset the program so that no instructions exist</summary> */
        public virtual void DeleteProgram(string programName) {
            Rollback(programName, MIN_TOKEN_INDEX);
        }

        public virtual void InsertAfter(IToken t, object text) {
            InsertAfter(DEFAULT_PROGRAM_NAME, t, text);
        }

        public virtual void InsertAfter(int index, object text) {
            InsertAfter(DEFAULT_PROGRAM_NAME, index, text);
        }

        public virtual void InsertAfter(string programName, IToken t, object text) {
            InsertAfter(programName, t.TokenIndex, text);
        }

        public virtual void InsertAfter(string programName, int index, object text) {
            // to insert after, just insert before next index (even if past end)
            InsertBefore(programName, index + 1, text);
            //addToSortedRewriteList(programName, new InsertAfterOp(index,text));
        }

        public virtual void InsertBefore(IToken t, object text) {
            InsertBefore(DEFAULT_PROGRAM_NAME, t, text);
        }

        public virtual void InsertBefore(int index, object text) {
            InsertBefore(DEFAULT_PROGRAM_NAME, index, text);
        }

        public virtual void InsertBefore(string programName, IToken t, object text) {
            InsertBefore(programName, t.TokenIndex, text);
        }

        public virtual void InsertBefore(string programName, int index, object text) {
            //addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
            RewriteOperation op = new InsertBeforeOp(this, index, text);
            IList<RewriteOperation> rewrites = GetProgram(programName);
            op.instructionIndex = rewrites.Count;
            rewrites.Add(op);
        }

        public virtual void Replace(int index, object text) {
            Replace(DEFAULT_PROGRAM_NAME, index, index, text);
        }

        public virtual void Replace(int from, int to, object text) {
            Replace(DEFAULT_PROGRAM_NAME, from, to, text);
        }

        public virtual void Replace(IToken indexT, object text) {
            Replace(DEFAULT_PROGRAM_NAME, indexT, indexT, text);
        }

        public virtual void Replace(IToken from, IToken to, object text) {
            Replace(DEFAULT_PROGRAM_NAME, from, to, text);
        }

        public virtual void Replace(string programName, int from, int to, object text) {
            if (from > to || from < 0 || to < 0 || to >= _tokens.Count) {
                throw new ArgumentException("replace: range invalid: " + from + ".." + to + "(size=" + _tokens.Count + ")");
            }
            RewriteOperation op = new ReplaceOp(this, from, to, text);
            IList<RewriteOperation> rewrites = GetProgram(programName);
            op.instructionIndex = rewrites.Count;
            rewrites.Add(op);
        }

        public virtual void Replace(string programName, IToken from, IToken to, object text) {
            Replace(programName,
                    from.TokenIndex,
                    to.TokenIndex,
                    text);
        }

        public virtual void Delete(int index) {
            Delete(DEFAULT_PROGRAM_NAME, index, index);
        }

        public virtual void Delete(int from, int to) {
            Delete(DEFAULT_PROGRAM_NAME, from, to);
        }

        public virtual void Delete(IToken indexT) {
            Delete(DEFAULT_PROGRAM_NAME, indexT, indexT);
        }

        public virtual void Delete(IToken from, IToken to) {
            Delete(DEFAULT_PROGRAM_NAME, from, to);
        }

        public virtual void Delete(string programName, int from, int to) {
            Replace(programName, from, to, null);
        }

        public virtual void Delete(string programName, IToken from, IToken to) {
            Replace(programName, from, to, null);
        }

        public virtual int GetLastRewriteTokenIndex() {
            return GetLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
        }

        protected virtual int GetLastRewriteTokenIndex(string programName) {
            int value;
            if (lastRewriteTokenIndexes.TryGetValue(programName, out value))
                return value;

            return -1;
        }

        protected virtual void SetLastRewriteTokenIndex(string programName, int i) {
            lastRewriteTokenIndexes[programName] = i;
        }

        protected virtual IList<RewriteOperation> GetProgram(string name) {
            IList<RewriteOperation> @is;
            if (!programs.TryGetValue(name, out @is) || @is == null) {
                @is = InitializeProgram(name);
            }
            return @is;
        }

        private IList<RewriteOperation> InitializeProgram(string name) {
            IList<RewriteOperation> @is = new List<RewriteOperation>(PROGRAM_INIT_SIZE);
            programs[name] = @is;
            return @is;
        }

        public virtual string ToOriginalString() {
            Fill();
            return ToOriginalString(MIN_TOKEN_INDEX, Count - 1);
        }

        public virtual string ToOriginalString(int start, int end) {
            StringBuilder buf = new StringBuilder();
            for (int i = start; i >= MIN_TOKEN_INDEX && i <= end && i < _tokens.Count; i++) {
                if (Get(i).Type != CharStreamConstants.EndOfFile)
                    buf.Append(Get(i).Text);
            }
            return buf.ToString();
        }

        public override string ToString() {
            Fill();
            return ToString(MIN_TOKEN_INDEX, Count - 1);
        }

        public virtual string ToString(string programName) {
            Fill();
            return ToString(programName, MIN_TOKEN_INDEX, Count - 1);
        }

        public override string ToString(int start, int end) {
            return ToString(DEFAULT_PROGRAM_NAME, start, end);
        }

        public virtual string ToString(string programName, int start, int end) {
            IList<RewriteOperation> rewrites;
            if (!programs.TryGetValue(programName, out rewrites))
                rewrites = null;

            // ensure start/end are in range
            if (end > _tokens.Count - 1)
                end = _tokens.Count - 1;
            if (start < 0)
                start = 0;

            if (rewrites == null || rewrites.Count == 0) {
                return ToOriginalString(start, end); // no instructions to execute
            }
            StringBuilder buf = new StringBuilder();

            // First, optimize instruction stream
            IDictionary<int, RewriteOperation> indexToOp = ReduceToSingleOperationPerIndex(rewrites);

            // Walk buffer, executing instructions and emitting tokens
            int i = start;
            while (i <= end && i < _tokens.Count) {
                RewriteOperation op;
                bool exists = indexToOp.TryGetValue(i, out op);

                if (exists) {
                    // remove so any left have index size-1
                    indexToOp.Remove(i);
                }

                if (!exists || op == null) {
                    IToken t = _tokens[i];
                    // no operation at that index, just dump token
                    if (t.Type != CharStreamConstants.EndOfFile)
                        buf.Append(t.Text);
                    i++; // move to next token
                } else {
                    i = op.Execute(buf); // execute operation and skip
                }
            }

            // include stuff after end if it's last index in buffer
            // So, if they did an insertAfter(lastValidIndex, "foo"), include
            // foo if end==lastValidIndex.
            if (end == _tokens.Count - 1) {
                // Scan any remaining operations after last token
                // should be included (they will be inserts).
                foreach (RewriteOperation op in indexToOp.Values) {
                    if (op.index >= _tokens.Count - 1)
                        buf.Append(op.text);
                }
            }
            return buf.ToString();
        }

        /** We need to combine operations and report invalid operations (like
         *  overlapping replaces that are not completed nested).  Inserts to
         *  same index need to be combined etc...   Here are the cases:
         *
         *  I.i.u I.j.v								leave alone, nonoverlapping
         *  I.i.u I.i.v								combine: Iivu
         *
         *  R.i-j.u R.x-y.v	| i-j in x-y			delete first R
         *  R.i-j.u R.i-j.v							delete first R
         *  R.i-j.u R.x-y.v	| x-y in i-j			ERROR
         *  R.i-j.u R.x-y.v	| boundaries overlap	ERROR
         *
         *  I.i.u R.x-y.v | i in x-y				delete I
         *  I.i.u R.x-y.v | i not in x-y			leave alone, nonoverlapping
         *  R.x-y.v I.i.u | i in x-y				ERROR
         *  R.x-y.v I.x.u 							R.x-y.uv (combine, delete I)
         *  R.x-y.v I.i.u | i not in x-y			leave alone, nonoverlapping
         *
         *  I.i.u = insert u before op @ index i
         *  R.x-y.u = replace x-y indexed tokens with u
         *
         *  First we need to examine replaces.  For any replace op:
         *
         * 		1. wipe out any insertions before op within that range.
         *		2. Drop any replace op before that is contained completely within
         *         that range.
         *		3. Throw exception upon boundary overlap with any previous replace.
         *
         *  Then we can deal with inserts:
         *
         * 		1. for any inserts to same index, combine even if not adjacent.
         * 		2. for any prior replace with same left boundary, combine this
         *         insert with replace and delete this replace.
         * 		3. throw exception if index in same range as previous replace
         *
         *  Don't actually delete; make op null in list. Easier to walk list.
         *  Later we can throw as we add to index -> op map.
         *
         *  Note that I.2 R.2-2 will wipe out I.2 even though, technically, the
         *  inserted stuff would be before the replace range.  But, if you
         *  add tokens in front of a method body '{' and then delete the method
         *  body, I think the stuff before the '{' you added should disappear too.
         *
         *  Return a map from token index to operation.
         */
        protected virtual IDictionary<int, RewriteOperation> ReduceToSingleOperationPerIndex(IList<RewriteOperation> rewrites) {
            //System.out.println("rewrites="+rewrites);

            // WALK REPLACES
            for (int i = 0; i < rewrites.Count; i++) {
                RewriteOperation op = rewrites[i];
                if (op == null)
                    continue;
                if (!(op is ReplaceOp))
                    continue;
                ReplaceOp rop = (ReplaceOp)rewrites[i];
                // Wipe prior inserts within range
                var inserts = GetKindOfOps(rewrites, typeof(InsertBeforeOp), i);
                for (int j = 0; j < inserts.Count; j++) {
                    InsertBeforeOp iop = (InsertBeforeOp)inserts[j];
                    if (iop.index >= rop.index && iop.index <= rop.lastIndex) {
                        // delete insert as it's a no-op.
                        rewrites[iop.instructionIndex] = null;
                    }
                }
                // Drop any prior replaces contained within
                var prevReplaces = GetKindOfOps(rewrites, typeof(ReplaceOp), i);
                for (int j = 0; j < prevReplaces.Count; j++) {
                    ReplaceOp prevRop = (ReplaceOp)prevReplaces[j];
                    if (prevRop.index >= rop.index && prevRop.lastIndex <= rop.lastIndex) {
                        // delete replace as it's a no-op.
                        rewrites[prevRop.instructionIndex] = null;
                        continue;
                    }
                    // throw exception unless disjoint or identical
                    bool disjoint =
                        prevRop.lastIndex < rop.index || prevRop.index > rop.lastIndex;
                    bool same =
                        prevRop.index == rop.index && prevRop.lastIndex == rop.lastIndex;
                    if (!disjoint && !same) {
                        throw new ArgumentException("replace op boundaries of " + rop +
                                                           " overlap with previous " + prevRop);
                    }
                }
            }

            // WALK INSERTS
            for (int i = 0; i < rewrites.Count; i++) {
                RewriteOperation op = (RewriteOperation)rewrites[i];
                if (op == null)
                    continue;
                if (!(op is InsertBeforeOp))
                    continue;
                InsertBeforeOp iop = (InsertBeforeOp)rewrites[i];
                // combine current insert with prior if any at same index
                var prevInserts = GetKindOfOps(rewrites, typeof(InsertBeforeOp), i);
                for (int j = 0; j < prevInserts.Count; j++) {
                    InsertBeforeOp prevIop = (InsertBeforeOp)prevInserts[j];
                    if (prevIop.index == iop.index) { // combine objects
                        // convert to strings...we're in process of toString'ing
                        // whole token buffer so no lazy eval issue with any templates
                        iop.text = CatOpText(iop.text, prevIop.text);
                        // delete redundant prior insert
                        rewrites[prevIop.instructionIndex] = null;
                    }
                }
                // look for replaces where iop.index is in range; error
                var prevReplaces = GetKindOfOps(rewrites, typeof(ReplaceOp), i);
                for (int j = 0; j < prevReplaces.Count; j++) {
                    ReplaceOp rop = (ReplaceOp)prevReplaces[j];
                    if (iop.index == rop.index) {
                        rop.text = CatOpText(iop.text, rop.text);
                        rewrites[i] = null;  // delete current insert
                        continue;
                    }
                    if (iop.index >= rop.index && iop.index <= rop.lastIndex) {
                        throw new ArgumentException("insert op " + iop +
                                                           " within boundaries of previous " + rop);
                    }
                }
            }
            // System.out.println("rewrites after="+rewrites);
            IDictionary<int, RewriteOperation> m = new Dictionary<int, RewriteOperation>();
            for (int i = 0; i < rewrites.Count; i++) {
                RewriteOperation op = (RewriteOperation)rewrites[i];
                if (op == null)
                    continue; // ignore deleted ops

                RewriteOperation existing;
                if (m.TryGetValue(op.index, out existing) && existing != null) {
                    throw new Exception("should only be one op per index");
                }
                m[op.index] = op;
            }
            //System.out.println("index to op: "+m);
            return m;
        }

        protected virtual string CatOpText(object a, object b) {
            return string.Concat(a, b);
        }
        protected virtual IList<RewriteOperation> GetKindOfOps(IList<RewriteOperation> rewrites, Type kind) {
            return GetKindOfOps(rewrites, kind, rewrites.Count);
        }

        /** <summary>Get all operations before an index of a particular kind</summary> */
        protected virtual IList<RewriteOperation> GetKindOfOps(IList<RewriteOperation> rewrites, Type kind, int before) {
            IList<RewriteOperation> ops = new List<RewriteOperation>();
            for (int i = 0; i < before && i < rewrites.Count; i++) {
                RewriteOperation op = rewrites[i];
                if (op == null)
                    continue; // ignore deleted
                if (op.GetType() == kind)
                    ops.Add(op);
            }
            return ops;
        }

        public virtual string ToDebugString() {
            return ToDebugString(MIN_TOKEN_INDEX, Count - 1);
        }

        public virtual string ToDebugString(int start, int end) {
            StringBuilder buf = new StringBuilder();
            for (int i = start; i >= MIN_TOKEN_INDEX && i <= end && i < _tokens.Count; i++) {
                buf.Append(Get(i));
            }
            return buf.ToString();
        }
    }
}