1#! /usr/bin/env python 2 3"""Token constants (from "token.h").""" 4 5# Taken from Python (r53757) and modified to include some tokens 6# originally monkeypatched in by pgen2.tokenize 7 8#--start constants-- 9ENDMARKER = 0 10NAME = 1 11NUMBER = 2 12STRING = 3 13NEWLINE = 4 14INDENT = 5 15DEDENT = 6 16LPAR = 7 17RPAR = 8 18LSQB = 9 19RSQB = 10 20COLON = 11 21COMMA = 12 22SEMI = 13 23PLUS = 14 24MINUS = 15 25STAR = 16 26SLASH = 17 27VBAR = 18 28AMPER = 19 29LESS = 20 30GREATER = 21 31EQUAL = 22 32DOT = 23 33PERCENT = 24 34BACKQUOTE = 25 35LBRACE = 26 36RBRACE = 27 37EQEQUAL = 28 38NOTEQUAL = 29 39LESSEQUAL = 30 40GREATEREQUAL = 31 41TILDE = 32 42CIRCUMFLEX = 33 43LEFTSHIFT = 34 44RIGHTSHIFT = 35 45DOUBLESTAR = 36 46PLUSEQUAL = 37 47MINEQUAL = 38 48STAREQUAL = 39 49SLASHEQUAL = 40 50PERCENTEQUAL = 41 51AMPEREQUAL = 42 52VBAREQUAL = 43 53CIRCUMFLEXEQUAL = 44 54LEFTSHIFTEQUAL = 45 55RIGHTSHIFTEQUAL = 46 56DOUBLESTAREQUAL = 47 57DOUBLESLASH = 48 58DOUBLESLASHEQUAL = 49 59AT = 50 60OP = 51 61COMMENT = 52 62NL = 53 63RARROW = 54 64ERRORTOKEN = 55 65N_TOKENS = 56 66NT_OFFSET = 256 67#--end constants-- 68 69tok_name = {} 70for _name, _value in globals().items(): 71 if type(_value) is type(0): 72 tok_name[_value] = _name 73 74 75def ISTERMINAL(x): 76 return x < NT_OFFSET 77 78def ISNONTERMINAL(x): 79 return x >= NT_OFFSET 80 81def ISEOF(x): 82 return x == ENDMARKER 83