1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport antlr3 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport testbase 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport unittest 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass t004lexer(testbase.ANTLRTest): 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setUp(self): 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.compileGrammar() 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def lexerClass(self, base): 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver class TLexer(base): 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def emitErrorMessage(self, msg): 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # report errors to /dev/null 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def reportError(self, re): 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # no error recovery yet, just crash! 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise re 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return TLexer 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def testValid(self): 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3.StringStream('ffofoofooo') 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = self.getLexer(stream) 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.FOO 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 0, token.start 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 0, token.stop 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == 'f', token.text 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.FOO 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 1, token.start 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 2, token.stop 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == 'fo', token.text 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.FOO 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 3, token.start 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 5, token.stop 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == 'foo', token.text 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.FOO 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 6, token.start 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 9, token.stop 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == 'fooo', token.text 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.EOF 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def testMalformedInput(self): 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3.StringStream('2') 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = self.getLexer(stream) 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fail() 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except antlr3.MismatchedTokenException, exc: 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.failUnlessEqual(exc.expecting, 'f') 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.failUnlessEqual(exc.unexpectedType, '2') 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverif __name__ == '__main__': 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver unittest.main() 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 71