1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport antlr3 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport testbase 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport unittest 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass t009lexer(testbase.ANTLRTest): 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setUp(self): 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.compileGrammar() 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def lexerClass(self, base): 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver class TLexer(base): 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def emitErrorMessage(self, msg): 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # report errors to /dev/null 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def reportError(self, re): 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # no error recovery yet, just crash! 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise re 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return TLexer 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def testValid(self): 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3.StringStream('085') 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = self.getLexer(stream) 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.DIGIT 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 0, token.start 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 0, token.stop 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == '0', token.text 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.DIGIT 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 1, token.start 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 1, token.stop 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == '8', token.text 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.DIGIT 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.start == 2, token.start 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.stop == 2, token.stop 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.text == '5', token.text 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert token.type == self.lexerModule.EOF 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def testMalformedInput(self): 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream = antlr3.StringStream('2a') 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = self.getLexer(stream) 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.nextToken() 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.nextToken() 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise AssertionError, token 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except antlr3.MismatchedSetException, exc: 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # TODO: This should provide more useful information 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert exc.expecting is None 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert exc.unexpectedType == 'a', repr(exc.unexpectedType) 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert exc.charPositionInLine == 1, repr(exc.charPositionInLine) 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert exc.line == 1, repr(exc.line) 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverif __name__ == '__main__': 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver unittest.main() 68