1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport antlr3 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport testbase 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport unittest 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass t011lexer(testbase.ANTLRTest): 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setUp(self): 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.compileGrammar() 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def lexerClass(self, base): 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver class TLexer(base): 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def emitErrorMessage(self, msg): 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # report errors to /dev/null 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def reportError(self, re): 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # no error recovery yet, just crash! 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise re 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return TLexer 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def testValid(self): 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3.StringStream('foobar _Ab98 \n A12sdf') 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = self.getLexer(stream) 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.IDENTIFIER 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 0, token.start 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 5, token.stop 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == 'foobar', token.text 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.WS 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 6, token.start 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 6, token.stop 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == ' ', token.text 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.IDENTIFIER 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 7, token.start 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 11, token.stop 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == '_Ab98', token.text 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.WS 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 12, token.start 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 14, token.stop 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == ' \n ', token.text 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.IDENTIFIER 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 15, token.start 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 20, token.stop 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == 'A12sdf', token.text 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.EOF 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def testMalformedInput(self): 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3.StringStream('a-b') 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = self.getLexer(stream) 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.nextToken() 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise AssertionError, token 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except antlr3.NoViableAltException, exc: 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert exc.unexpectedType == '-', repr(exc.unexpectedType) 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert exc.charPositionInLine == 1, repr(exc.charPositionInLine) 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert exc.line == 1, repr(exc.line) 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverif __name__ == '__main__': 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver unittest.main() 79