Lines Matching refs:tokens
18 # Return a list of tokens (strings).
21 tokens = []
36 tokens.append(s[:i])
39 tokens.append(c)
47 tokens.append(s[:i])
49 return tokens
53 # Return a list of tokens (strings).
56 tokens = []
60 tokens = tokens + tokenize_string(line)
61 return tokens
70 # Input is a list of tokens as returned by tokenize_*().
71 # Return a pair (expr, tokens)
73 # and tokens contains the remaining tokens.
76 def parse_expr(tokens):
77 if (not tokens) or tokens[0] != '(':
79 tokens = tokens[1:]
82 if not tokens:
84 if tokens[0] == ')':
85 return expr, tokens[1:]
86 elif tokens[0] == '(':
87 subexpr, tokens = parse_expr(tokens)
90 expr.append(tokens[0])
91 tokens = tokens[1:]
98 tokens = tokenize_file(fp)
100 while tokens:
101 expr, tokens = parse_expr(tokens)
116 # plus an empty token list (because there are no tokens left.