2  * Contains the default implementation of the common token used within
40 static  pANTLR3_STRING	getText					(pANTLR3_COMMON_TOKEN token);
41 static  void			setText					(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
42 static  void			setText8				(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
43 static	ANTLR3_UINT32   getType					(pANTLR3_COMMON_TOKEN token);
44 static  void			setType					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
45 static  ANTLR3_UINT32   getLine					(pANTLR3_COMMON_TOKEN token);
46 static  void			setLine					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
47 static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token);
48 static  void			setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
49 static  ANTLR3_UINT32   getChannel				(pANTLR3_COMMON_TOKEN token);
50 static  void			setChannel				(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
51 static  ANTLR3_MARKER   getTokenIndex			(pANTLR3_COMMON_TOKEN token);
52 static  void			setTokenIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
53 static  ANTLR3_MARKER   getStartIndex			(pANTLR3_COMMON_TOKEN token);
54 static  void			setStartIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
55 static  ANTLR3_MARKER   getStopIndex			(pANTLR3_COMMON_TOKEN token);
56 static  void			setStopIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
57 static  pANTLR3_STRING  toString				(pANTLR3_COMMON_TOKEN token);
75 	pANTLR3_COMMON_TOKEN    token;
77 	// Create a raw token with the interface installed
79 	token   = newToken();
81 	if	(token != NULL)
83 		token->setType(token, ttype);
88 	return  token;
119     /* Factory space is good, we now want to initialize our cheating token
158     // If we were reusing this token factory then we may already have a pool
193     pANTLR3_COMMON_TOKEN token;
195     /* See if we need a new token pool before allocating a new
206      * error checking. Then we can work out what the pointer is to the next token.
208     token = factory->pools[factory->thisPool] + factory->nextToken;
211     /* We have our token pointer now, so we can initialize it to the predefined model.
212      * We only need do this though if the token is not already initialized, we just check
215     if  (token->setStartIndex == NULL)
217         antlr3SetTokenAPI(token);
221         token->factoryMade  = ANTLR3_TRUE;
222         token->strFactory   = factory->input == NULL ? NULL : factory->input->strFactory;
223         token->input        = factory->input;
228     return token;
247     ANTLR3_UINT32	    token;
250     /* We iterate the token pools one at a time
269 	    for (token = 0; token < limit; token++)
273 		check	= pool + token;
275 		/* If the programmer made this a custom token, then
306     pANTLR3_COMMON_TOKEN    token;
310     token   = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
312     if	(token == NULL)
319     antlr3SetTokenAPI(token);
320     token->factoryMade = ANTLR3_FALSE;
322     return  token;
326 antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
328     token->getText		    = getText;
329     token->setText		    = setText;
330     token->setText8		    = setText8;
331     token->getType		    = getType;
332     token->setType		    = setType;
333     token->getLine		    = getLine;
334     token->setLine		    = setLine;
335     token->setLine		    = setLine;
336     token->getCharPositionInLine    = getCharPositionInLine;
337     token->setCharPositionInLine    = setCharPositionInLine;
338     token->getChannel		    = getChannel;
339     token->setChannel		    = setChannel;
340     token->getTokenIndex	    = getTokenIndex;
341     token->setTokenIndex	    = setTokenIndex;
342     token->getStartIndex	    = getStartIndex;
343     token->setStartIndex	    = setStartIndex;
344     token->getStopIndex		    = getStopIndex;
345     token->setStopIndex		    = setStopIndex;
346     token->toString		    = toString;
351 static  pANTLR3_STRING  getText			(pANTLR3_COMMON_TOKEN token)
353 	switch (token->textState)
357 			// Someone already created a string for this token, so we just
360 			return	token->tokText.text;
369 			if	(token->strFactory != NULL)
371 				token->tokText.text	= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
372 				token->textState	= ANTLR3_TEXT_STRING;
373 				return token->tokText.text;
387 			if (token->type == ANTLR3_TOKEN_EOF)
389 				token->tokText.text				= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
390 				token->textState				= ANTLR3_TEXT_STRING;
391 				token->tokText.text->factory	= token->strFactory;
392 				return token->tokText.text;
396 			// We had nothing installed in the token, create a new string
400 			if	(token->input != NULL)
403 				return	token->input->substr(	token->input, 
404 												token->getStartIndex(token), 
405  												token->getStopIndex(token)
415 static  void		setText8		(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
421 	switch	(token->textState)
429 			token->textState		= ANTLR3_TEXT_CHARP;
430 			token->tokText.chars	= (pANTLR3_UCHAR)text;
437 			token->tokText.text->set8(token->tokText.text, (const char *)text);
446 /** \brief Install the supplied text string as teh text for the token.
451 static  void		setText			(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
456 	token->textState	= ANTLR3_TEXT_STRING;
457 	token->tokText.text	= text;
464 static	ANTLR3_UINT32   getType			(pANTLR3_COMMON_TOKEN token)
466     return  token->type;
469 static  void		setType			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
471     token->type = type;
474 static  ANTLR3_UINT32   getLine			(pANTLR3_COMMON_TOKEN token)
476     return  token->line;
479 static  void		setLine			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
481     token->line = line;
484 static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token)
486     return  token->charPosition;
489 static  void		setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
491     token->charPosition = pos;
494 static  ANTLR3_UINT32   getChannel		(pANTLR3_COMMON_TOKEN token)
496     return  token->channel;
499 static  void		setChannel		(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
501     token->channel  = channel;
504 static  ANTLR3_MARKER   getTokenIndex		(pANTLR3_COMMON_TOKEN token)
506     return  token->index;
509 static  void		setTokenIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
511     token->index    = index;
514 static  ANTLR3_MARKER   getStartIndex		(pANTLR3_COMMON_TOKEN token)
516 	return  token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
519 static  void		setStartIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
521     token->start    = start;
524 static  ANTLR3_MARKER   getStopIndex		(pANTLR3_COMMON_TOKEN token)
526     return  token->stop;
529 static  void		setStopIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
531     token->stop	= stop;
534 static  pANTLR3_STRING    toString		(pANTLR3_COMMON_TOKEN token)
539     text    =	token->getText(token);
548 		return text;		// This usally means it is the EOF token
560     outtext->addi   (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
562     outtext->addi   (outtext, (ANTLR3_INT32)token->getStartIndex(token));
564     outtext->addi   (outtext, (ANTLR3_INT32)token->getStopIndex(token));
568     outtext->addi   (outtext, token->type);
571     if	(token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
574 		outtext->addi	(outtext, (ANTLR3_INT32)token->getChannel(token));
579     outtext->addi   (outtext, (ANTLR3_INT32)token->getLine(token));
581     outtext->addi   (outtext, token->getCharPositionInLine(token));

Indexes created Fri Mar 13 02:32:08 CET 2015