tokens.c revision ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddb 
1/****************************************************************************\
2Copyright (c) 2002, NVIDIA Corporation.
3
4NVIDIA Corporation("NVIDIA") supplies this software to you in
5consideration of your agreement to the following terms, and your use,
6installation, modification or redistribution of this NVIDIA software
7constitutes acceptance of these terms.  If you do not agree with these
8terms, please do not use, install, modify or redistribute this NVIDIA
9software.
10
11In consideration of your agreement to abide by the following terms, and
12subject to these terms, NVIDIA grants you a personal, non-exclusive
13license, under NVIDIA's copyrights in this original NVIDIA software (the
14"NVIDIA Software"), to use, reproduce, modify and redistribute the
15NVIDIA Software, with or without modifications, in source and/or binary
16forms; provided that if you redistribute the NVIDIA Software, you must
17retain the copyright notice of NVIDIA, this notice and the following
18text and disclaimers in all such redistributions of the NVIDIA Software.
19Neither the name, trademarks, service marks nor logos of NVIDIA
20Corporation may be used to endorse or promote products derived from the
21NVIDIA Software without specific prior written permission from NVIDIA.
22Except as expressly stated in this notice, no other rights or licenses
23express or implied, are granted by NVIDIA herein, including but not
24limited to any patent rights that may be infringed by your derivative
25works or by other works in which the NVIDIA Software may be
26incorporated. No hardware is licensed hereunder.
27
28THE NVIDIA SOFTWARE IS BEING PROVIDED ON AN "AS IS" BASIS, WITHOUT
29WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED,
30INCLUDING WITHOUT LIMITATION, WARRANTIES OR CONDITIONS OF TITLE,
31NON-INFRINGEMENT, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
32ITS USE AND OPERATION EITHER ALONE OR IN COMBINATION WITH OTHER
33PRODUCTS.
34
35IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT,
36INCIDENTAL, EXEMPLARY, CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
37TO, LOST PROFITS; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
38USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) OR ARISING IN ANY WAY
39OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION OF THE
40NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
41TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
42NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43\****************************************************************************/
44//
45// tokens.c
46//
47
48#include <stdlib.h>
49#include <stdio.h>
50#include <string.h>
51#include <ctype.h>
52
53#include "compiler/debug.h"
54#include "compiler/preprocessor/slglobals.h"
55#include "compiler/util.h"
56
57///////////////////////////////////////////////////////////////////////////////////////////////
58//////////////////////// Preprocessor and Token Recorder and Playback: ////////////////////////
59///////////////////////////////////////////////////////////////////////////////////////////////
60
61/*
62 * idstr()
63 * Copy a string to a malloc'ed block and convert it into something suitable
64 * for an ID
65 *
66 */
67
68static char *idstr(const char *fstr, MemoryPool *pool)
69{
70    size_t len;
71    char *str, *t;
72    const char *f;
73
74    len = strlen(fstr);
75    if (!pool)
76        str = (char *) malloc(len + 1);
77    else
78        str = (char *) mem_Alloc(pool, len + 1);
79
80    for (f=fstr, t=str; *f; f++) {
81        if (isalnum(*f)) *t++ = *f;
82        else if (*f == '.' || *f == '/') *t++ = '_';
83    }
84    *t = 0;
85    return str;
86} // idstr
87
88
89/*
90 * lNewBlock()
91 *
92 */
93
94static TokenBlock *lNewBlock(TokenStream *fTok, MemoryPool *pool)
95{
96    TokenBlock *lBlock;
97
98    if (!pool)
99        lBlock = (TokenBlock *) malloc(sizeof(TokenBlock) + 256);
100    else
101        lBlock = (TokenBlock *) mem_Alloc(pool, sizeof(TokenBlock) + 256);
102    lBlock->count = 0;
103    lBlock->current = 0;
104    lBlock->data = (unsigned char *) lBlock + sizeof(TokenBlock);
105    lBlock->max = 256;
106    lBlock->next = NULL;
107    if (fTok->head) {
108        fTok->current->next = lBlock;
109    } else {
110        fTok->head = lBlock;
111    }
112    fTok->current = lBlock;
113    return lBlock;
114} // lNewBlock
115
116/*
117 * lAddByte()
118 *
119 */
120
121static void lAddByte(TokenStream *fTok, unsigned char fVal)
122{
123    TokenBlock *lBlock;
124    lBlock = fTok->current;
125    if (lBlock->count >= lBlock->max)
126        lBlock = lNewBlock(fTok, 0);
127    lBlock->data[lBlock->count++] = fVal;
128} // lAddByte
129
130
131
132/*
133 * lReadByte() - Get the next byte from a stream.
134 *
135 */
136
137static int lReadByte(TokenStream *pTok)
138{
139    TokenBlock *lBlock;
140    int lval = -1;
141
142    lBlock = pTok->current;
143    if (lBlock) {
144        if (lBlock->current >= lBlock->count) {
145            lBlock = lBlock->next;
146            if (lBlock)
147                lBlock->current = 0;
148            pTok->current = lBlock;
149        }
150        if (lBlock)
151            lval = lBlock->data[lBlock->current++];
152    }
153    return lval;
154} // lReadByte
155
156/////////////////////////////////////// Global Functions://////////////////////////////////////
157
158/*
159 * NewTokenStream()
160 *
161 */
162
163TokenStream *NewTokenStream(const char *name, MemoryPool *pool)
164{
165    TokenStream *pTok;
166
167    if (!pool)
168        pTok = (TokenStream *) malloc(sizeof(TokenStream));
169    else
170        pTok = (TokenStream*)mem_Alloc(pool, sizeof(TokenStream));
171    pTok->next = NULL;
172    pTok->name = idstr(name, pool);
173    pTok->head = NULL;
174    pTok->current = NULL;
175    lNewBlock(pTok, pool);
176    return pTok;
177} // NewTokenStream
178
179/*
180 * DeleteTokenStream()
181 *
182 */
183
184void DeleteTokenStream(TokenStream *pTok)
185{
186    TokenBlock *pBlock, *nBlock;
187
188    if (pTok) {
189        pBlock = pTok->head;
190        while (pBlock) {
191            nBlock = pBlock->next;
192            free(pBlock);
193            pBlock = nBlock;
194        }
195        if (pTok->name)
196            free(pTok->name);
197        free(pTok);
198    }
199} // DeleteTokenStream
200
201/*
202 * RecordToken() - Add a token to the end of a list for later playback or printout.
203 *
204 */
205
206void RecordToken(TokenStream *pTok, int token, yystypepp * yylvalpp)
207{
208    const char *s;
209    char *str=NULL;
210
211    if (token > 256)
212        lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
213    else
214        lAddByte(pTok, (unsigned char)(token & 0x7f));
215    switch (token) {
216    case CPP_IDENTIFIER:
217    case CPP_TYPEIDENTIFIER:
218    case CPP_STRCONSTANT:
219        s = GetAtomString(atable, yylvalpp->sc_ident);
220        while (*s)
221            lAddByte(pTok, (unsigned char) *s++);
222        lAddByte(pTok, 0);
223        break;
224    case CPP_FLOATCONSTANT:
225    case CPP_INTCONSTANT:
226         str=yylvalpp->symbol_name;
227         while (*str){
228            lAddByte(pTok, (unsigned char) *str++);
229         }
230         lAddByte(pTok, 0);
231         break;
232    case '(':
233        lAddByte(pTok, (unsigned char)(yylvalpp->sc_int ? 1 : 0));
234    default:
235        break;
236    }
237} // RecordToken
238
239/*
240 * RewindTokenStream() - Reset a token stream in preperation for reading.
241 *
242 */
243
244void RewindTokenStream(TokenStream *pTok)
245{
246    if (pTok->head) {
247        pTok->current = pTok->head;
248        pTok->current->current = 0;
249    }
250} // RewindTokenStream
251
252/*
253 * ReadToken() - Read the next token from a stream.
254 *
255 */
256
257int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
258{
259    char symbol_name[MAX_SYMBOL_NAME_LEN + 1];
260    char string_val[MAX_STRING_LEN + 1];
261    int ltoken, len;
262    char ch;
263
264    ltoken = lReadByte(pTok);
265    if (ltoken >= 0) {
266        if (ltoken > 127)
267            ltoken += 128;
268        switch (ltoken) {
269        case CPP_IDENTIFIER:
270        case CPP_TYPEIDENTIFIER:
271            len = 0;
272            ch = lReadByte(pTok);
273            while ((ch >= 'a' && ch <= 'z') ||
274                     (ch >= 'A' && ch <= 'Z') ||
275                     (ch >= '0' && ch <= '9') ||
276                     ch == '_')
277            {
278                if (len < MAX_SYMBOL_NAME_LEN) {
279                    symbol_name[len++] = ch;
280                    ch = lReadByte(pTok);
281                }
282            }
283            symbol_name[len] = '\0';
284            assert(ch == '\0');
285            yylvalpp->sc_ident = LookUpAddString(atable, symbol_name);
286            return CPP_IDENTIFIER;
287            break;
288        case CPP_STRCONSTANT:
289            len = 0;
290            while ((ch = lReadByte(pTok)) != 0)
291                if (len < MAX_STRING_LEN)
292                    string_val[len++] = ch;
293            string_val[len] = '\0';
294            yylvalpp->sc_ident = LookUpAddString(atable, string_val);
295            break;
296        case CPP_FLOATCONSTANT:
297            len = 0;
298            ch = lReadByte(pTok);
299            while ((ch >= '0' && ch <= '9')||(ch=='e'||ch=='E'||ch=='.')||(ch=='+'||ch=='-'))
300            {
301                if (len < MAX_SYMBOL_NAME_LEN) {
302                    symbol_name[len++] = ch;
303                    ch = lReadByte(pTok);
304                }
305            }
306            symbol_name[len] = '\0';
307            assert(ch == '\0');
308            strcpy(yylvalpp->symbol_name,symbol_name);
309            yylvalpp->sc_fval=(float)atof_dot(yylvalpp->symbol_name);
310            break;
311        case CPP_INTCONSTANT:
312            len = 0;
313            ch = lReadByte(pTok);
314            while ((ch >= '0' && ch <= '9'))
315            {
316                if (len < MAX_SYMBOL_NAME_LEN) {
317                    symbol_name[len++] = ch;
318                    ch = lReadByte(pTok);
319                }
320            }
321            symbol_name[len] = '\0';
322            assert(ch == '\0');
323            strcpy(yylvalpp->symbol_name,symbol_name);
324            yylvalpp->sc_int=atoi(yylvalpp->symbol_name);
325            break;
326        case '(':
327            yylvalpp->sc_int = lReadByte(pTok);
328            break;
329        }
330        return ltoken;
331    }
332    return EOF_SY;
333} // ReadToken
334
335typedef struct TokenInputSrc {
336    InputSrc            base;
337    TokenStream         *tokens;
338    int                 (*final)(CPPStruct *);
339} TokenInputSrc;
340
341static int scan_token(TokenInputSrc *in, yystypepp * yylvalpp)
342{
343    int token = ReadToken(in->tokens, yylvalpp);
344    int (*final)(CPPStruct *);
345    cpp->tokenLoc->file = cpp->currentInput->name;
346    cpp->tokenLoc->line = cpp->currentInput->line;
347    if (token == '\n') {
348        in->base.line++;
349        return token;
350    }
351    if (token > 0) return token;
352    cpp->currentInput = in->base.prev;
353    final = in->final;
354    free(in);
355    if (final && !final(cpp)) return -1;
356    return cpp->currentInput->scan(cpp->currentInput, yylvalpp);
357}
358
359int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(CPPStruct *))
360{
361    TokenInputSrc *in = malloc(sizeof(TokenInputSrc));
362    memset(in, 0, sizeof(TokenInputSrc));
363    in->base.name = name;
364    in->base.prev = cpp->currentInput;
365    in->base.scan = (int (*)(InputSrc *, yystypepp *))scan_token;
366    in->base.line = 1;
367    in->tokens = ts;
368    in->final = final;
369    RewindTokenStream(ts);
370    cpp->currentInput = &in->base;
371    return 1;
372}
373
374typedef struct UngotToken {
375    InputSrc    base;
376    int         token;
377    yystypepp     lval;
378} UngotToken;
379
380static int reget_token(UngotToken *t, yystypepp * yylvalpp)
381{
382    int token = t->token;
383    *yylvalpp = t->lval;
384    cpp->currentInput = t->base.prev;
385    free(t);
386    return token;
387}
388
389void UngetToken(int token, yystypepp * yylvalpp) {
390    UngotToken *t = malloc(sizeof(UngotToken));
391    memset(t, 0, sizeof(UngotToken));
392    t->token = token;
393    t->lval = *yylvalpp;
394    t->base.scan = (void *)reget_token;
395    t->base.prev = cpp->currentInput;
396    t->base.name = cpp->currentInput->name;
397    t->base.line = cpp->currentInput->line;
398    cpp->currentInput = &t->base;
399}
400
401
402void DumpTokenStream(FILE *fp, TokenStream *s, yystypepp * yylvalpp) {
403    int token;
404    char str[100];
405
406    if (fp == 0) fp = stdout;
407    RewindTokenStream(s);
408    while ((token = ReadToken(s, yylvalpp)) > 0) {
409        switch (token) {
410        case CPP_IDENTIFIER:
411        case CPP_TYPEIDENTIFIER:
412            sprintf(str, "%s ", GetAtomString(atable, yylvalpp->sc_ident));
413            break;
414        case CPP_STRCONSTANT:
415            sprintf(str, "\"%s\"", GetAtomString(atable, yylvalpp->sc_ident));
416            break;
417        case CPP_FLOATCONSTANT:
418            //printf("%g9.6 ", yylvalpp->sc_fval);
419            break;
420        case CPP_INTCONSTANT:
421            //printf("%d ", yylvalpp->sc_int);
422            break;
423        default:
424            if (token >= 127)
425                sprintf(str, "%s ", GetAtomString(atable, token));
426            else
427                sprintf(str, "%c", token);
428            break;
429        }
430        CPPDebugLogMsg(str);
431    }
432}
433
434///////////////////////////////////////////////////////////////////////////////////////////////
435/////////////////////////////////////// End of tokens.c ///////////////////////////////////////
436///////////////////////////////////////////////////////////////////////////////////////////////
437