xref: /external/antlr/antlr-3.4/runtime/C/src/antlr3commontoken.c

/**
 * Contains the default implementation of the common token used within
 * java. Custom tokens should create this structure and then append to it using the
 * custom pointer to install their own structure and API.
 */

// [The "BSD licence"]
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
// http://www.temporal-wave.com
// http://www.linkedin.com/in/jimidle
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include    <antlr3.h>

/* Token API
 */
static  pANTLR3_STRING	getText					(pANTLR3_COMMON_TOKEN token);
static  void			setText					(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
static  void			setText8				(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
static	ANTLR3_UINT32   getType					(pANTLR3_COMMON_TOKEN token);
static  void			setType					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
static  ANTLR3_UINT32   getLine					(pANTLR3_COMMON_TOKEN token);
static  void			setLine					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token);
static  void			setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
static  ANTLR3_UINT32   getChannel				(pANTLR3_COMMON_TOKEN token);
static  void			setChannel				(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
static  ANTLR3_MARKER   getTokenIndex			(pANTLR3_COMMON_TOKEN token);
static  void			setTokenIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
static  ANTLR3_MARKER   getStartIndex			(pANTLR3_COMMON_TOKEN token);
static  void			setStartIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
static  ANTLR3_MARKER   getStopIndex			(pANTLR3_COMMON_TOKEN token);
static  void			setStopIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
static  pANTLR3_STRING  toString				(pANTLR3_COMMON_TOKEN token);

/* Factory API
 */
static	void			factoryClose	(pANTLR3_TOKEN_FACTORY factory);
static	pANTLR3_COMMON_TOKEN	newToken	(void);
static  void			setInputStream	(pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
static	void                    factoryReset    (pANTLR3_TOKEN_FACTORY factory);

/* Internal management functions
 */
static	void			newPool		(pANTLR3_TOKEN_FACTORY factory);
static	pANTLR3_COMMON_TOKEN    newPoolToken	(pANTLR3_TOKEN_FACTORY factory);


ANTLR3_API pANTLR3_COMMON_TOKEN
antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
{
	pANTLR3_COMMON_TOKEN    token;

	// Create a raw token with the interface installed
	//
	token   = newToken();

	if	(token != NULL)
	{
		token->setType(token, ttype);
	}

	// All good
	//
	return  token;
}

ANTLR3_API pANTLR3_TOKEN_FACTORY
antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
{
    pANTLR3_TOKEN_FACTORY   factory;

    /* allocate memory
     */
    factory	= (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));

    if	(factory == NULL)
    {
	return	NULL;
    }

    /* Install factory API
     */
    factory->newToken	    = newPoolToken;
    factory->close	    = factoryClose;
    factory->setInputStream = setInputStream;
    factory->reset          = factoryReset;

    /* Allocate the initial pool
     */
    factory->thisPool	= -1;
    factory->pools      = NULL;
    factory->maxPool    = -1;
    newPool(factory);

    /* Factory space is good, we now want to initialize our cheating token
     * which one it is initialized is the model for all tokens we manufacture
     */
    antlr3SetTokenAPI(&factory->unTruc);

    /* Set some initial variables for future copying
     */
    factory->unTruc.factoryMade	= ANTLR3_TRUE;

    // Input stream
    //
    setInputStream(factory, input);

    return  factory;

}

static void
setInputStream	(pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
{
    factory->input          =  input;
    factory->unTruc.input   =  input;
	if	(input != NULL)
	{
		factory->unTruc.strFactory	= input->strFactory;
	}
	else
	{
		factory->unTruc.strFactory = NULL;
    }
}

static void
newPool(pANTLR3_TOKEN_FACTORY factory)
{
    /* Increment factory count
     */
    factory->thisPool++;

    // If we were reusing this token factory then we may already have a pool
    // allocated. If we exceeded the max avaible then we must allocate a new
    // one.
    if  (factory->thisPool > factory->maxPool)
    {
        /* Ensure we have enough pointers allocated
         */
        factory->pools = (pANTLR3_COMMON_TOKEN *)
		         ANTLR3_REALLOC(	(void *)factory->pools,	    /* Current pools pointer (starts at NULL)	*/
					    (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *))	/* Memory for new pool pointers */
					    );

        /* Allocate a new pool for the factory
         */
        factory->pools[factory->thisPool]	=
			        (pANTLR3_COMMON_TOKEN)
				    ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));

        // We now have a new pool and can track it as the maximum we have created so far
        //
        factory->maxPool = factory->thisPool;
    }

    /* Reset the counters
     */
    factory->nextToken	= 0;

    /* Done
     */
    return;
}

static pANTLR3_COMMON_TOKEN
newPoolToken(pANTLR3_TOKEN_FACTORY factory)
{
    pANTLR3_COMMON_TOKEN token;

    /* See if we need a new token pool before allocating a new
     * one
     */
    if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
    {
        /* We ran out of tokens in the current pool, so we need a new pool
         */
        newPool(factory);
    }

    /* Assuming everything went well (we are trying for performance here so doing minimal
     * error checking. Then we can work out what the pointer is to the next token.
     */
    token = factory->pools[factory->thisPool] + factory->nextToken;
    factory->nextToken++;

    /* We have our token pointer now, so we can initialize it to the predefined model.
     * We only need do this though if the token is not already initialized, we just check
     * an api function pointer for this as they are allocated via calloc.
     */
    if  (token->setStartIndex == NULL)
    {
        antlr3SetTokenAPI(token);

        // It is factory made, and we need to copy the string factory pointer
        //
        token->factoryMade  = ANTLR3_TRUE;
        token->strFactory   = factory->input == NULL ? NULL : factory->input->strFactory;
        token->input        = factory->input;
    }

    /* And we are done
     */
    return token;
}

static	void
factoryReset	    (pANTLR3_TOKEN_FACTORY factory)
{
    // Just start again with pool #0 when we are
    // called.
    //
    factory->thisPool   = -1;
    newPool(factory);
}

static	void
factoryClose	    (pANTLR3_TOKEN_FACTORY factory)
{
    pANTLR3_COMMON_TOKEN    pool;
    ANTLR3_INT32	    poolCount;
    ANTLR3_UINT32	    limit;
    ANTLR3_UINT32	    token;
    pANTLR3_COMMON_TOKEN    check;

    /* We iterate the token pools one at a time
     */
    for	(poolCount = 0; poolCount <= factory->thisPool; poolCount++)
    {
	/* Pointer to current pool
	 */
	pool	= factory->pools[poolCount];

	/* Work out how many tokens we need to check in this pool.
	 */
	limit	= (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);

	/* Marginal condition, we might be at the start of a brand new pool
	 * where the nextToken is 0 and nothing has been allocated.
	 */
	if  (limit > 0)
	{
	    /* We have some tokens allocated from this pool
	     */
	    for (token = 0; token < limit; token++)
	    {
		/* Next one in the chain
		 */
		check	= pool + token;

		/* If the programmer made this a custom token, then
		 * see if we need to call their free routine.
		 */
		if  (check->custom != NULL && check->freeCustom != NULL)
		{
		    check->freeCustom(check->custom);
		    check->custom = NULL;
		}
	    }
	}

	/* We can now free this pool allocation
	 */
	ANTLR3_FREE(factory->pools[poolCount]);
	factory->pools[poolCount] = NULL;
    }

    /* All the pools are deallocated we can free the pointers to the pools
     * now.
     */
    ANTLR3_FREE(factory->pools);

    /* Finally, we can free the space for the factory itself
     */
    ANTLR3_FREE(factory);
}


static	pANTLR3_COMMON_TOKEN
newToken(void)
{
    pANTLR3_COMMON_TOKEN    token;

    /* Allocate memory for this
     */
    token   = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN)));

    if	(token == NULL)
    {
	return	NULL;
    }

    // Install the API
    //
    antlr3SetTokenAPI(token);
    token->factoryMade = ANTLR3_FALSE;

    return  token;
}

ANTLR3_API void
antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
{
    token->getText		    = getText;
    token->setText		    = setText;
    token->setText8		    = setText8;
    token->getType		    = getType;
    token->setType		    = setType;
    token->getLine		    = getLine;
    token->setLine		    = setLine;
    token->setLine		    = setLine;
    token->getCharPositionInLine    = getCharPositionInLine;
    token->setCharPositionInLine    = setCharPositionInLine;
    token->getChannel		    = getChannel;
    token->setChannel		    = setChannel;
    token->getTokenIndex	    = getTokenIndex;
    token->setTokenIndex	    = setTokenIndex;
    token->getStartIndex	    = getStartIndex;
    token->setStartIndex	    = setStartIndex;
    token->getStopIndex		    = getStopIndex;
    token->setStopIndex		    = setStopIndex;
    token->toString		    = toString;

    return;
}

static  pANTLR3_STRING  getText			(pANTLR3_COMMON_TOKEN token)
{
	switch (token->textState)
	{
		case ANTLR3_TEXT_STRING:

			// Someone already created a string for this token, so we just
			// use it.
			//
			return	token->tokText.text;
			break;

		case ANTLR3_TEXT_CHARP:

			// We had a straight text pointer installed, now we
			// must convert it to a string. Note we have to do this here
			// or otherwise setText8() will just install the same char*
			//
			if	(token->strFactory != NULL)
			{
				token->tokText.text	= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
				token->textState	= ANTLR3_TEXT_STRING;
				return token->tokText.text;
			}
			else
			{
				// We cannot do anything here
				//
				return NULL;
			}
			break;

		default:

			// EOF is a special case
			//
			if (token->type == ANTLR3_TOKEN_EOF)
			{
				token->tokText.text				= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
				token->textState				= ANTLR3_TEXT_STRING;
				token->tokText.text->factory	= token->strFactory;
				return token->tokText.text;
			}


			// We had nothing installed in the token, create a new string
			// from the input stream
			//

			if	(token->input != NULL)
			{

				return	token->input->substr(	token->input,
												token->getStartIndex(token),
 												token->getStopIndex(token)
											);
			}

			// Nothing to return, there is no input stream
			//
			return NULL;
			break;
	}
}
static  void		setText8		(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
{
	// No text to set, so ignore
	//
	if	(text == NULL) return;

	switch	(token->textState)
	{
		case	ANTLR3_TEXT_NONE:
		case	ANTLR3_TEXT_CHARP:	// Caller must free before setting again, if it needs to be freed

			// Nothing in there yet, or just a char *, so just set the
			// text as a pointer
			//
			token->textState		= ANTLR3_TEXT_CHARP;
			token->tokText.chars	= (pANTLR3_UCHAR)text;
			break;

		default:

			// It was already a pANTLR3_STRING, so just override it
			//
			token->tokText.text->set8(token->tokText.text, (const char *)text);
			break;
	}

	// We are done
	//
	return;
}

/** \brief Install the supplied text string as teh text for the token.
 * The method assumes that the existing text (if any) was created by a factory
 * and so does not attempt to release any memory it is using.Text not created
 * by a string fctory (not advised) should be released prior to this call.
 */
static  void		setText			(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
{
	// Merely replaces and existing pre-defined text with the supplied
	// string
	//
	token->textState	= ANTLR3_TEXT_STRING;
	token->tokText.text	= text;

	/* We are done
	*/
	return;
}

static	ANTLR3_UINT32   getType			(pANTLR3_COMMON_TOKEN token)
{
    return  token->type;
}

static  void		setType			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
{
    token->type = type;
}

static  ANTLR3_UINT32   getLine			(pANTLR3_COMMON_TOKEN token)
{
    return  token->line;
}

static  void		setLine			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
{
    token->line = line;
}

static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token)
{
    return  token->charPosition;
}

static  void		setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
{
    token->charPosition = pos;
}

static  ANTLR3_UINT32   getChannel		(pANTLR3_COMMON_TOKEN token)
{
    return  token->channel;
}

static  void		setChannel		(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
{
    token->channel  = channel;
}

static  ANTLR3_MARKER   getTokenIndex		(pANTLR3_COMMON_TOKEN token)
{
    return  token->index;
}

static  void		setTokenIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
{
    token->index    = index;
}

static  ANTLR3_MARKER   getStartIndex		(pANTLR3_COMMON_TOKEN token)
{
	return  token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
}

static  void		setStartIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
{
    token->start    = start;
}

static  ANTLR3_MARKER   getStopIndex		(pANTLR3_COMMON_TOKEN token)
{
    return  token->stop;
}

static  void		setStopIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
{
    token->stop	= stop;
}

static  pANTLR3_STRING    toString		(pANTLR3_COMMON_TOKEN token)
{
    pANTLR3_STRING  text;
    pANTLR3_STRING  outtext;

    text    =	token->getText(token);

    if	(text == NULL)
    {
		return NULL;
    }

	if	(text->factory == NULL)
	{
		return text;		// This usally means it is the EOF token
	}

    /* A new empty string to assemble all the stuff in
     */
    outtext = text->factory->newRaw(text->factory);

    /* Now we use our handy dandy string utility to assemble the
     * the reporting string
     * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
     */
    outtext->append8(outtext, "[Index: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
    outtext->append8(outtext, " (Start: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getStartIndex(token));
    outtext->append8(outtext, "-Stop: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getStopIndex(token));
    outtext->append8(outtext, ") ='");
    outtext->appendS(outtext, text);
    outtext->append8(outtext, "', type<");
    outtext->addi   (outtext, token->type);
    outtext->append8(outtext, "> ");

    if	(token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
    {
		outtext->append8(outtext, "(channel = ");
		outtext->addi	(outtext, (ANTLR3_INT32)token->getChannel(token));
		outtext->append8(outtext, ") ");
    }

    outtext->append8(outtext, "Line: ");
    outtext->addi   (outtext, (ANTLR3_INT32)token->getLine(token));
    outtext->append8(outtext, " LinePos:");
    outtext->addi   (outtext, token->getCharPositionInLine(token));
    outtext->addc   (outtext, ']');

    return  outtext;
}