C/src/antlr3tokenstream.c

/// \file
/// Default implementation of CommonTokenStream
///

// [The "BSD licence"]
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
// http://www.temporal-wave.com
// http://www.linkedin.com/in/jimidle
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include    <antlr3tokenstream.h>

#ifdef	ANTLR3_WINDOWS
#pragma warning( disable : 4100 )
#endif

// COMMON_TOKEN_STREAM API
//
static void		setTokenTypeChannel     (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
static void		discardTokenType        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
static void		discardOffChannel       (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
static pANTLR3_VECTOR	getTokens	        (pANTLR3_COMMON_TOKEN_STREAM cts);
static pANTLR3_LIST	getTokenRange	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
static pANTLR3_LIST	getTokensSet	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
static pANTLR3_LIST	getTokensList	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
static pANTLR3_LIST	getTokensType	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
static void             reset                   (pANTLR3_COMMON_TOKEN_STREAM cts);

// TOKEN_STREAM API
//
static pANTLR3_COMMON_TOKEN tokLT	        (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
static pANTLR3_COMMON_TOKEN dbgTokLT		(pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
static pANTLR3_COMMON_TOKEN get			(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
static pANTLR3_TOKEN_SOURCE getTokenSource	(pANTLR3_TOKEN_STREAM ts);
static void		    setTokenSource	(pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
static pANTLR3_STRING	    toString		(pANTLR3_TOKEN_STREAM ts);
static pANTLR3_STRING	    toStringSS		(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
static pANTLR3_STRING	    toStringTT		(pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
static void		    setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);

// INT STREAM API
//
static void		    consume			(pANTLR3_INT_STREAM is);
static void		    dbgConsume			(pANTLR3_INT_STREAM is);
static ANTLR3_UINT32	    _LA				(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
static ANTLR3_UINT32	    dbgLA			(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
static ANTLR3_MARKER	    mark			(pANTLR3_INT_STREAM is);
static ANTLR3_MARKER	    dbgMark			(pANTLR3_INT_STREAM is);
static void		    release			(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
static ANTLR3_UINT32	    size			(pANTLR3_INT_STREAM is);
static ANTLR3_MARKER	    tindex			(pANTLR3_INT_STREAM is);
static void		    rewindStream		(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
static void		    dbgRewindStream		(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
static void		    rewindLast			(pANTLR3_INT_STREAM is);
static void		    dbgRewindLast		(pANTLR3_INT_STREAM is);
static void		    seek			(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
static void		    dbgSeek			(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
static pANTLR3_STRING	    getSourceName		(pANTLR3_INT_STREAM is);
static void		    antlr3TokenStreamFree	(pANTLR3_TOKEN_STREAM	    stream);
static void		    antlr3CTSFree		(pANTLR3_COMMON_TOKEN_STREAM    stream);

// Helpers
//
static void		    fillBuffer			(pANTLR3_COMMON_TOKEN_STREAM tokenStream);
static ANTLR3_UINT32	    skipOffTokenChannels	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
static ANTLR3_UINT32	    skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
static pANTLR3_COMMON_TOKEN LB				(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);

ANTLR3_API pANTLR3_TOKEN_STREAM
antlr3TokenStreamNew()
{
    pANTLR3_TOKEN_STREAM stream;

    // Memory for the interface structure
    //
    stream  = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));

    if	(stream == NULL)
    {
		return	NULL;
    }

    // Install basic API
    //
    stream->free    =  antlr3TokenStreamFree;


    return stream;
}

static void
antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
{
    ANTLR3_FREE(stream);
}

static void
antlr3CTSFree	    (pANTLR3_COMMON_TOKEN_STREAM stream)
{
	// We only free up our subordinate interfaces if they belong
	// to us, otherwise we let whoever owns them deal with them.
	//
	if	(stream->tstream->super == stream)
	{
		if	(stream->tstream->istream->super == stream->tstream)
		{
			stream->tstream->istream->free(stream->tstream->istream);
			stream->tstream->istream = NULL;
		}
		stream->tstream->free(stream->tstream);
	}

	// Now we free our own resources
	//
	if	(stream->tokens != NULL)
	{
		stream->tokens->free(stream->tokens);
		stream->tokens	= NULL;
	}
	if	(stream->discardSet != NULL)
	{
		stream->discardSet->free(stream->discardSet);
		stream->discardSet  = NULL;
	}
	if	(stream->channelOverrides != NULL)
	{
		stream->channelOverrides->free(stream->channelOverrides);
		stream->channelOverrides = NULL;
	}

	// Free our memory now
	//
	ANTLR3_FREE(stream);
}

// Reset a token stream so it can be used again and can reuse it's
// resources.
//
static void
reset   (pANTLR3_COMMON_TOKEN_STREAM cts)
{

    // Free any resources that ar most like specifc to the
    // run we just did.
    //
    if	(cts->discardSet != NULL)
    {
        cts->discardSet->free(cts->discardSet);
        cts->discardSet  = NULL;
    }
    if	(cts->channelOverrides != NULL)
    {
        cts->channelOverrides->free(cts->channelOverrides);
        cts->channelOverrides = NULL;
    }

    // Now, if there were any existing tokens in the stream,
    // then we just reset the vector count so that it starts
    // again. We must traverse the entries unfortunately as
    // there may be free pointers for custom token types and
    // so on. However that is just a quick NULL check on the
    // vector entries.
    //
    if	(cts->tokens != NULL)
    {
        cts->tokens->clear(cts->tokens);
    }
    else
    {
        /* Install the token tracking tables
         */
        cts->tokens  = antlr3VectorNew(0);
    }

    // Reset to defaults
    //
    cts->discardOffChannel  = ANTLR3_FALSE;
    cts->channel            = ANTLR3_TOKEN_DEFAULT_CHANNEL;
    cts->p	            = -1;
}

ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
{
    pANTLR3_COMMON_TOKEN_STREAM	stream;

	// Create a standard token stream
	//
	stream = antlr3CommonTokenStreamSourceNew(hint, source);

	// Install the debugger object
	//
	stream->tstream->debugger = debugger;

	// Override standard token stream methods with debugging versions
	//
	stream->tstream->initialStreamState	= ANTLR3_FALSE;

	stream->tstream->_LT				= dbgTokLT;

	stream->tstream->istream->consume		= dbgConsume;
	stream->tstream->istream->_LA			= dbgLA;
	stream->tstream->istream->mark			= dbgMark;
	stream->tstream->istream->rewind		= dbgRewindStream;
	stream->tstream->istream->rewindLast	= dbgRewindLast;
	stream->tstream->istream->seek			= dbgSeek;

	return stream;
}

ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
{
    pANTLR3_COMMON_TOKEN_STREAM	stream;

    stream = antlr3CommonTokenStreamNew(hint);

    stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;

    stream->channelOverrides	= NULL;
    stream->discardSet		= NULL;
    stream->discardOffChannel	= ANTLR3_FALSE;

    stream->tstream->setTokenSource(stream->tstream, source);

    stream->free		=  antlr3CTSFree;
    return  stream;
}

ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
{
    pANTLR3_COMMON_TOKEN_STREAM stream;

    /* Memory for the interface structure
     */
    stream  = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));

    if	(stream == NULL)
    {
	return	NULL;
    }

    /* Create space for the token stream interface
     */
    stream->tstream	    = antlr3TokenStreamNew();
    stream->tstream->super  =  stream;

    /* Create space for the INT_STREAM interfacce
     */
    stream->tstream->istream		    =  antlr3IntStreamNew();
    stream->tstream->istream->super	    =  (stream->tstream);
    stream->tstream->istream->type	    = ANTLR3_TOKENSTREAM;

    /* Install the token tracking tables
     */
    stream->tokens  = antlr3VectorNew(0);

    /* Defaults
     */
    stream->p	    = -1;

    /* Install the common token stream API
     */
    stream->setTokenTypeChannel	    = setTokenTypeChannel;
    stream->discardTokenType	    = discardTokenType;
    stream->discardOffChannelToks   = discardOffChannel;
    stream->getTokens		    = getTokens;
    stream->getTokenRange	    = getTokenRange;
    stream->getTokensSet	    = getTokensSet;
    stream->getTokensList	    = getTokensList;
    stream->getTokensType	    = getTokensType;
    stream->reset                   = reset;

    /* Install the token stream API
     */
    stream->tstream->_LT			=  tokLT;
    stream->tstream->get			=  get;
    stream->tstream->getTokenSource	        =  getTokenSource;
    stream->tstream->setTokenSource	        =  setTokenSource;
    stream->tstream->toString		        =  toString;
    stream->tstream->toStringSS		        =  toStringSS;
    stream->tstream->toStringTT		        =  toStringTT;
    stream->tstream->setDebugListener           =  setDebugListener;

    /* Install INT_STREAM interface
     */
    stream->tstream->istream->_LA	=  _LA;
    stream->tstream->istream->mark	=  mark;
    stream->tstream->istream->release	=  release;
    stream->tstream->istream->size	=  size;
    stream->tstream->istream->index	=  tindex;
    stream->tstream->istream->rewind	=  rewindStream;
    stream->tstream->istream->rewindLast=  rewindLast;
    stream->tstream->istream->seek	=  seek;
    stream->tstream->istream->consume	=  consume;
    stream->tstream->istream->getSourceName = getSourceName;

    return  stream;
}

// Install a debug listener adn switch to debug mode methods
//
static void
setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
{
		// Install the debugger object
	//
	ts->debugger = debugger;

	// Override standard token stream methods with debugging versions
	//
	ts->initialStreamState	= ANTLR3_FALSE;

	ts->_LT				= dbgTokLT;

	ts->istream->consume		= dbgConsume;
	ts->istream->_LA			= dbgLA;
	ts->istream->mark			= dbgMark;
	ts->istream->rewind			= dbgRewindStream;
	ts->istream->rewindLast		= dbgRewindLast;
	ts->istream->seek			= dbgSeek;
}

/** Get the ith token from the current position 1..n where k=1 is the
*  first symbol of lookahead.
*/
static pANTLR3_COMMON_TOKEN
tokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
{
	ANTLR3_INT32    i;
	ANTLR3_INT32    n;
	pANTLR3_COMMON_TOKEN_STREAM cts;

	cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;

        if	(k < 0)
	{
		return LB(cts, -k);
	}

	if	(cts->p == -1)
	{
		fillBuffer(cts);
	}

        // Here we used to check for k == 0 and return 0, but this seems
        // a superfluous check to me. LT(k=0) is therefore just undefined
        // and we won't waste the clock cycles on the check
        //

	if	((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
	{
		pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);

		teof->setStartIndex (teof, ts->istream->index	    (ts->istream));
		teof->setStopIndex  (teof, ts->istream->index	    (ts->istream));
		return  teof;
	}

	i	= cts->p;
	n	= 1;

	/* Need to find k good tokens, skipping ones that are off channel
	*/
	while   ( n < k)
	{
		/* Skip off-channel tokens */
		i = skipOffTokenChannels(cts, i+1); /* leave p on valid token    */
		n++;
	}
	if	( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
	{
		pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);

		teof->setStartIndex (teof, ts->istream->index(ts->istream));
		teof->setStopIndex  (teof, ts->istream->index(ts->istream));
		return  teof;
	}

	// Here the token must be in the input vector. Rather then incur
	// function call penalty, we just return the pointer directly
	// from the vector
	//
	return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
	//return  (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
}

/// Debug only method to flag consumption of initial off-channel
/// tokens in the input stream
///
static void
consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
{
	ANTLR3_MARKER	first;
	ANTLR3_INT32	i;
	pANTLR3_TOKEN_STREAM	ts;

	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
	first	= is->index(is);

	for	(i=0; i<first; i++)
	{
		ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
	}

	ts->initialStreamState = ANTLR3_FALSE;

}

/// As per the normal tokLT but sends information to the debugger
///
static pANTLR3_COMMON_TOKEN
dbgTokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
{
	if	(ts->initialStreamState == ANTLR3_TRUE)
	{
		consumeInitialHiddenTokens(ts->istream);
	}
	return tokLT(ts, k);
}

#ifdef	ANTLR3_WINDOWS
	/* When fully optimized VC7 complains about non reachable code.
	 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
	 */
#pragma warning( disable : 4702 )
#endif

static pANTLR3_COMMON_TOKEN
LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
{
    ANTLR3_INT32 i;
    ANTLR3_INT32 n;

    if (cts->p == -1)
    {
        fillBuffer(cts);
    }
    if (k == 0)
    {
        return NULL;
    }
    if ((cts->p - k) < 0)
    {
        return NULL;
    }

    i = cts->p;
    n = 1;

    /* Need to find k good tokens, going backwards, skipping ones that are off channel
     */
    while (n <= (ANTLR3_INT32) k)
    {
        /* Skip off-channel tokens
         */

        i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token    */
        n++;
    }
    if (i < 0)
    {
        return NULL;
    }
	// Here the token must be in the input vector. Rather then incut
	// function call penalty, we jsut return the pointer directly
	// from the vector
	//
	return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
}

static pANTLR3_COMMON_TOKEN
get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
{
    pANTLR3_COMMON_TOKEN_STREAM cts;

    cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;

    return  (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i));  /* Token index is zero based but vectors are 1 based */
}

static pANTLR3_TOKEN_SOURCE
getTokenSource	(pANTLR3_TOKEN_STREAM ts)
{
    return  ts->tokenSource;
}

static void
setTokenSource	(   pANTLR3_TOKEN_STREAM ts,
		    pANTLR3_TOKEN_SOURCE tokenSource)
{
    ts->tokenSource	= tokenSource;
}

static pANTLR3_STRING
toString    (pANTLR3_TOKEN_STREAM ts)
{
    pANTLR3_COMMON_TOKEN_STREAM cts;

    cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;

    if	(cts->p == -1)
    {
	fillBuffer(cts);
    }

    return  ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
}

static pANTLR3_STRING
toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
{
    pANTLR3_STRING string;
    pANTLR3_TOKEN_SOURCE tsource;
    pANTLR3_COMMON_TOKEN tok;
    ANTLR3_UINT32 i;
    pANTLR3_COMMON_TOKEN_STREAM cts;

    cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;

    if (cts->p == -1)
    {
        fillBuffer(cts);
    }
    if (stop >= ts->istream->size(ts->istream))
    {
        stop = ts->istream->size(ts->istream) - 1;
    }

    /* Who is giving us these tokens?
     */
    tsource = ts->getTokenSource(ts);

    if (tsource != NULL && cts->tokens != NULL)
    {
        /* Finally, let's get a string
         */
        string = tsource->strFactory->newRaw(tsource->strFactory);

        for (i = start; i <= stop; i++)
        {
            tok = ts->get(ts, i);
            if (tok != NULL)
            {
                string->appendS(string, tok->getText(tok));
            }
        }

        return string;
    }
    return NULL;

}

static pANTLR3_STRING
toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
{
	if	(start != NULL && stop != NULL)
	{
		return	ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
	}
	else
	{
		return	NULL;
	}
}

/** Move the input pointer to the next incoming token.  The stream
 *  must become active with LT(1) available.  consume() simply
 *  moves the input pointer so that LT(1) points at the next
 *  input symbol. Consume at least one token.
 *
 *  Walk past any token not on the channel the parser is listening to.
 */
static void
consume	(pANTLR3_INT_STREAM is)
{
	pANTLR3_COMMON_TOKEN_STREAM cts;
	pANTLR3_TOKEN_STREAM	ts;

	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
	cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;

        if	((ANTLR3_UINT32)cts->p < cts->tokens->count)
	{
		cts->p++;
		cts->p	= skipOffTokenChannels(cts, cts->p);
	}
}


/// As per ordinary consume but notifies the debugger about hidden
/// tokens and so on.
///
static void
dbgConsume	(pANTLR3_INT_STREAM is)
{
	pANTLR3_TOKEN_STREAM	ts;
	ANTLR3_MARKER			a;
	ANTLR3_MARKER			b;
	pANTLR3_COMMON_TOKEN	t;

	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;

	if	(ts->initialStreamState == ANTLR3_TRUE)
	{
		consumeInitialHiddenTokens(is);
	}

	a = is->index(is);		// Where are we right now?
	t = ts->_LT(ts, 1);		// Current token from stream

	consume(is);			// Standard consumer

	b = is->index(is);		// Where are we after consuming 1 on channel token?

	ts->debugger->consumeToken(ts->debugger, t);	// Tell the debugger that we consumed the first token

	if	(b>a+1)
	{
		// The standard consume caused the index to advance by more than 1,
		// which can only happen if it skipped some off-channel tokens.
		// we need to tell the debugger about those tokens.
		//
		ANTLR3_MARKER	i;

		for	(i = a+1; i<b; i++)
		{
			ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
		}

	}
}

/** A simple filter mechanism whereby you can tell this token stream
 *  to force all tokens of type ttype to be on channel.  For example,
 *  when interpreting, we cannot execute actions so we need to tell
 *  the stream to force all WS and NEWLINE to be a different, ignored,
 *  channel.
 */
static void
setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
{
    if	(tokenStream->channelOverrides == NULL)
    {
	tokenStream->channelOverrides	= antlr3ListNew(10);
    }

    /* We add one to the channel so we can distinguish NULL as being no entry in the
     * table for a particular token type.
     */
    tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
}

static void
discardTokenType    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
{
    if	(tokenStream->discardSet == NULL)
    {
	tokenStream->discardSet	= antlr3ListNew(31);
    }

    /* We add one to the channel so we can distinguish NULL as being no entry in the
     * table for a particular token type. We could use bitsets for this I suppose too.
     */
    tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
}

static void
discardOffChannel   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
{
    tokenStream->discardOffChannel  = discard;
}

static pANTLR3_VECTOR
getTokens   (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
{
    if	(tokenStream->p == -1)
    {
	fillBuffer(tokenStream);
    }

    return  tokenStream->tokens;
}

static pANTLR3_LIST
getTokenRange	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
{
    return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
}
/** Given a start and stop index, return a List of all tokens in
 *  the token type BitSet.  Return null if no tokens were found.  This
 *  method looks at both on and off channel tokens.
 */
static pANTLR3_LIST
getTokensSet	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
{
    pANTLR3_LIST	    filteredList;
    ANTLR3_UINT32	    i;
    ANTLR3_UINT32	    n;
    pANTLR3_COMMON_TOKEN    tok;

    if	(tokenStream->p == -1)
    {
	fillBuffer(tokenStream);
    }
    if	(stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
    {
	stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
    }
    if	(start > stop)
    {
	return NULL;
    }

    /* We have the range set, now we need to iterate through the
     * installed tokens and create a new list with just the ones we want
     * in it. We are just moving pointers about really.
     */
    filteredList    = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));

    for	(i = start, n = 0; i<= stop; i++)
    {
	tok = tokenStream->tstream->get(tokenStream->tstream, i);

	if  (	   types == NULL
		|| types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
	    )
	{
	    filteredList->put(filteredList, n++, (void *)tok, NULL);
	}
    }

    /* Did we get any then?
     */
    if	(filteredList->size(filteredList) == 0)
    {
	filteredList->free(filteredList);
	filteredList	= NULL;
    }

    return  filteredList;
}

static pANTLR3_LIST
getTokensList	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
{
    pANTLR3_BITSET  bitSet;
    pANTLR3_LIST    newlist;

    bitSet  = antlr3BitsetList(list->table);

    newlist    = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);

    bitSet->free(bitSet);

    return  newlist;

}

static pANTLR3_LIST
getTokensType	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
{
    pANTLR3_BITSET  bitSet;
    pANTLR3_LIST    newlist;

    bitSet  = antlr3BitsetOf(type, -1);
    newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);

    bitSet->free(bitSet);

    return  newlist;
}

static ANTLR3_UINT32
_LA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
{
	pANTLR3_TOKEN_STREAM    ts;
	pANTLR3_COMMON_TOKEN    tok;

	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;

	tok	    =  ts->_LT(ts, i);

	if	(tok != NULL)
	{
		return	tok->getType(tok);
	}
	else
	{
		return	ANTLR3_TOKEN_INVALID;
	}
}

/// As per _LA() but for debug mode.
///
static ANTLR3_UINT32
dbgLA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
{
    pANTLR3_TOKEN_STREAM    ts;

    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;

	if	(ts->initialStreamState == ANTLR3_TRUE)
	{
		consumeInitialHiddenTokens(is);
	}
	ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
	return	_LA(is, i);
}

static ANTLR3_MARKER
mark	(pANTLR3_INT_STREAM is)
{
    is->lastMarker = is->index(is);
    return  is->lastMarker;
}

/// As per mark() but with a call to tell the debugger we are doing this
///
static ANTLR3_MARKER
dbgMark	(pANTLR3_INT_STREAM is)
{
    pANTLR3_TOKEN_STREAM    ts;

    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;

	is->lastMarker = is->index(is);
	ts->debugger->mark(ts->debugger, is->lastMarker);

    return  is->lastMarker;
}

static void
release	(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
{
    return;
}

static ANTLR3_UINT32
size	(pANTLR3_INT_STREAM is)
{
    pANTLR3_COMMON_TOKEN_STREAM cts;
    pANTLR3_TOKEN_STREAM	ts;

    if (is->cachedSize > 0)
    {
	return  is->cachedSize;
    }
    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
    cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;

    is->cachedSize =  cts->tokens->count;
    return  is->cachedSize;
}

static ANTLR3_MARKER
tindex	(pANTLR3_INT_STREAM is)
{
    pANTLR3_COMMON_TOKEN_STREAM cts;
    pANTLR3_TOKEN_STREAM	ts;

    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
    cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;

    return  cts->p;
}

static void
dbgRewindLast	(pANTLR3_INT_STREAM is)
{
	pANTLR3_TOKEN_STREAM	ts;

    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;

	ts->debugger->rewindLast(ts->debugger);

    is->rewind(is, is->lastMarker);
}
static void
rewindLast	(pANTLR3_INT_STREAM is)
{
    is->rewind(is, is->lastMarker);
}
static void
rewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
{
    is->seek(is, (ANTLR3_UINT32)(marker));
}
static void
dbgRewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
{
    pANTLR3_TOKEN_STREAM	ts;

    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;

	ts->debugger->rewind(ts->debugger, marker);

    is->seek(is, (ANTLR3_UINT32)(marker));
}

static void
seek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
{
    pANTLR3_COMMON_TOKEN_STREAM cts;
    pANTLR3_TOKEN_STREAM	ts;

    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
    cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;

    cts->p  = (ANTLR3_UINT32)index;
}
static void
dbgSeek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
{
	// TODO: Implement seek in debugger when Ter adds it to Java
	//
	seek(is, index);
}
ANTLR3_API void
fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
{
    fillBuffer(tokenStream);
}
static void
fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
    ANTLR3_UINT32 index;
    pANTLR3_COMMON_TOKEN tok;
    ANTLR3_BOOLEAN discard;
    void * channelI;

    /* Start at index 0 of course
     */
    index = 0;

    /* Pick out the next token from the token source
     * Remember we just get a pointer (reference if you like) here
     * and so if we store it anywhere, we don't set any pointers to auto free it.
     */
    tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);

    while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
    {
        discard = ANTLR3_FALSE; /* Assume we are not discarding	*/

        /* I employ a bit of a trick, or perhaps hack here. Rather than
         * store a pointer to a structure in the override map and discard set
         * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
         * we can distinguish "not being there" from "being channel or type 0"
         */

        if (tokenStream->discardSet != NULL
            && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
        {
            discard = ANTLR3_TRUE;
        }
        else if (   tokenStream->discardOffChannel == ANTLR3_TRUE
                 && tok->getChannel(tok) != tokenStream->channel
                 )
        {
            discard = ANTLR3_TRUE;
        }
        else if (tokenStream->channelOverrides != NULL)
        {
            /* See if this type is in the override map
             */
            channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);

            if (channelI != NULL)
            {
                /* Override found
                 */
                tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
            }
        }

        /* If not discarding it, add it to the list at the current index
         */
        if (discard == ANTLR3_FALSE)
        {
            /* Add it, indicating that we will delete it and the table should not
             */
            tok->setTokenIndex(tok, index);
            tokenStream->p++;
            tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
            index++;
        }

        tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
    }

    /* Cache the size so we don't keep doing indirect method calls. We do this as
     * early as possible so that anything after this may utilize the cached value.
     */
    tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;

    /* Set the consume pointer to the first token that is on our channel
     */
    tokenStream->p = 0;
    tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);

}

/// Given a starting index, return the index of the first on-channel
///  token.
///
static ANTLR3_UINT32
skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
    ANTLR3_INT32 n;
    pANTLR3_COMMON_TOKEN tok;

    n = tokenStream->tstream->istream->cachedSize;

    while (i < n)
    {
        tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;

        if (tok->channel!= tokenStream->channel)
        {
            i++;
        }
        else
        {
            return i;
        }
    }
    return i;
}

static ANTLR3_UINT32
skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
{
    pANTLR3_COMMON_TOKEN tok;

    while (x >= 0)
    {
        tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;

        if ((tok->channel != tokenStream->channel))
        {
            x--;
        }
        else
        {
            return x;
        }
    }
    return x;
}

/// Return a string that represents the name assoicated with the input source
///
/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
///
/// /returns
/// /implements ANTLR3_INT_STREAM_struct::getSourceName()
///
static pANTLR3_STRING
getSourceName				(pANTLR3_INT_STREAM is)
{
	// Slightly convoluted as we must trace back to the lexer's input source
	// via the token source. The streamName that is here is not initialized
	// because this is a token stream, not a file or string stream, which are the
	// only things that have a context for a source name.
	//
	return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
}