Lines Matching refs:token
231 * In unames.dat, it is marked as token[';']==-1 only if the
234 * So, it will be token[';']==-1 if we store U1.0 names/ISO comments/aliases
242 uint16_t token, tokenCount=*tokens++, bufferPos=0;
249 * if the semicolon byte value is a character, not a token number
263 * the semicolon byte value is a token number, therefore
271 /* write each letter directly, and write a token word per token */
285 token=tokens[c];
286 if(token==(uint16_t)(-2)) {
287 /* this is a lead byte for a double-byte token */
288 token=tokens[c<<8|*name++];
291 if(token==(uint16_t)(-1)) {
308 /* write token word */
309 uint8_t *tokenString=tokenStrings+token;
335 uint16_t token, tokenCount=*tokens++;
343 * if the semicolon byte value is a character, not a token number
357 * the semicolon byte value is a token number, therefore
365 /* compare each letter directly, and compare a token word per token */
381 token=tokens[c];
382 if(token==(uint16_t)(-2)) {
383 /* this is a lead byte for a double-byte token */
384 token=tokens[c<<8|*name++];
387 if(token==(uint16_t)(-1)) {
406 /* write token word */
407 uint8_t *tokenString=tokenStrings+token;
1302 uint16_t c, token;
1310 token=tokens[c];
1311 if(token==(uint16_t)(-2)) {
1312 /* this is a lead byte for a double-byte token */
1314 token=tokens[c];
1316 if(token==(uint16_t)(-1)) {
1321 /* count token word */
1323 /* use cached token length */
1326 tokenLength=calcStringSetLength(set, (const char *)tokenStrings+token);
1330 tokenLength=calcStringSetLength(set, (const char *)tokenStrings+token);
1754 * The token table contains non-negative entries for token bytes,
1926 /* read the first 512 tokens and make the token maps */
1971 * swap the token strings but not a possible padding byte after
1977 udata_printError(ds, "uchar_swapNames(token strings) failed\n");
2022 /* token lead byte: swap the trail byte, too */