1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#!/usr/bin/ruby 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# encoding: utf-8 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverrequire 'antlr3/test/functional' 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest001 < ANTLR3::Test::Functional 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar Zero; 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ZERO: '0'; 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example %(lexing '0') do 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Zero::Lexer.new( '0' ) 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'ZERO' 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example %(iterating over tokens) do 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Zero::Lexer.new( '0' ) 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token_types = lexer.map { |token| token.name } 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token_types.should == %w(ZERO) 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "mismatched token" do 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Zero::Lexer.new( '1' ) 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver }.should raise_error( ANTLR3::Error::MismatchedToken ) do |e| 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver e.expecting.should == '0' 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver e.unexpected_type.should == '1' 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest002 < ANTLR3::Test::Functional 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar Binary; 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ZERO: '0'; 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ONE: '1'; 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing '01'" do 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Binary::Lexer.new( '01' ) 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'ZERO' 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'ONE' 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "no matching token rule" do 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Binary::Lexer.new( '2' ) 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver b = lambda { token = lexer.next_token } 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver b.should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == '2' 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest003 < ANTLR3::Test::Functional 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar BinaryFooze; 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ZERO: '0'; 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ONE: '1'; 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FOOZE: 'fooze'; 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing '0fooze1'" do 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = BinaryFooze::Lexer.new( '0fooze1' ) 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'ZERO' 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOOZE' 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'ONE' 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "no token match" do 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = BinaryFooze::Lexer.new( '2' ) 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == '2' 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest004 < ANTLR3::Test::Functional 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar FooStar; 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FOO: 'f' 'o'*; 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing 'ffofoofooo'" do 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = FooStar::Lexer.new( 'ffofoofooo' ) 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 0 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'f' 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fo' 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 1 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 2 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 3 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 5 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'foo' 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 6 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 9 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fooo' 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "mismatched token" do 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = FooStar::Lexer.new( '2' ) 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.expecting.should == 'f' 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == '2' 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest005 < ANTLR3::Test::Functional 179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar FooPlus; 181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FOO: 'f' 'o'+; 188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing 'fofoofooo'" do 191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = FooPlus::Lexer.new( 'fofoofooo' ) 192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 1 197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fo' 198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'foo' 202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 2 203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 4 204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 205324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 5 208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 8 209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fooo' 210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "mismatched token" do 216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = FooPlus::Lexer.new( '2' ) 217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.expecting.should == 'f' 221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == '2' 222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "early exit" do 226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = FooPlus::Lexer.new( 'f' ) 227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { token = lexer.next_token }. 229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::EarlyExit ) { |exc| 230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == ANTLR3::Constants::EOF 231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 234324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest006 < ANTLR3::Test::Functional 237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar FoaStar; 239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FOO: 'f' ('o' | 'a')*; 246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing 'fofaaooa'" do 249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = FoaStar::Lexer.new( 'fofaaooa' ) 250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 253324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 1 255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fo' 256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'faaooa' 260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 2 261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 7 262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "mismatched token" do 268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = FoaStar::Lexer.new( 'fofoaooaoa2' ) 269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.next_token 271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.next_token 272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.expecting.should == 'f' 275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == '2' 276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.column.should == 10 277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.line.should == 1 278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest007 < ANTLR3::Test::Functional 283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar Foab; 285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FOO: 'f' ('o' | 'a' 'b'+)*; 292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing 'fofababbooabb'" do 295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Foab::Lexer.new( 'fofababbooabb' ) 296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 1 301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fo' 302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 304324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 305324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 2 306324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 12 307324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fababbooabb' 308324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 309324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 310324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 311324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 312324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 313324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "early exit" do 314324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Foab::Lexer.new( 'foaboao' ) 315324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 316324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 317324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::EarlyExit ) do |exc| 318324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == 'o' 319324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.column.should == 6 320324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.line.should == 1 321324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 322324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 323324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 324324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 325324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest008 < ANTLR3::Test::Functional 326324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 327324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar Fa; 328324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 329324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 330324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 331324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 332324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 333324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 334324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FOO: 'f' 'a'?; 335324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 336324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 337324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing 'ffaf'" do 338324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Fa::Lexer.new( 'ffaf' ) 339324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 340324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 341324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 342324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 343324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 0 344324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'f' 345324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 346324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 347324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 348324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 1 349324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 2 350324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'fa' 351324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 352324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 353324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'FOO' 354324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 3 355324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 3 356324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'f' 357324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 358324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 359324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 360324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 361324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 362324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "mismatched token" do 363324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Fa::Lexer.new( 'fafb' ) 364324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 365324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.next_token 366324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.next_token 367324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 368324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 369324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == 'b' 370324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.column.should == 3 371324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.line.should == 1 372324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 373324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 374324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 375324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 376324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 377324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest009 < ANTLR3::Test::Functional 378324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 379324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar Digit; 380324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 381324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 382324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 383324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 384324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 385324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 386324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver DIGIT: '0' .. '9'; 387324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 388324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 389324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing '085'" do 390324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Digit::Lexer.new( '085' ) 391324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 392324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 393324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'DIGIT' 394324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 395324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 0 396324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == '0' 397324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 398324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 399324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'DIGIT' 400324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 1 401324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 1 402324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == '8' 403324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 404324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 405324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'DIGIT' 406324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 2 407324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 2 408324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == '5' 409324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 410324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 411324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 412324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 413324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 414324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "mismatched range" do 415324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = Digit::Lexer.new( '2a' ) 416324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 417324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.next_token 418324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 419324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::MismatchedRange ) do |exc| 420324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.min.should == '0' 421324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.max.should == '9' 422324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == 'a' 423324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.column.should == 1 424324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.line.should == 1 425324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 426324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 427324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 428324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 429324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest010 < ANTLR3::Test::Functional 430324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 431324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar IDsAndSpaces; 432324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options { 433324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver language = Ruby; 434324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 435324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 436324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 437324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 438324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IDENTIFIER: ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*; 439324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver WS: (' ' | '\n')+; 440324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 441324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 442324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing 'foobar _Ab98 \n A12sdf'" do 443324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = IDsAndSpaces::Lexer.new( "foobar _Ab98 \n A12sdf" ) 444324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 445324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 446324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'IDENTIFIER' 447324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 448324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 5 449324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'foobar' 450324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 451324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 452324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'WS' 453324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 6 454324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 6 455324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == ' ' 456324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 457324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 458324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'IDENTIFIER' 459324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 7 460324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 11 461324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == '_Ab98' 462324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 463324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 464324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'WS' 465324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 12 466324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 14 467324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == " \n " 468324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 469324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 470324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'IDENTIFIER' 471324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 15 472324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 20 473324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'A12sdf' 474324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 475324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 476324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 477324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 478324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 479324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "contains characters without a matching token rule" do 480324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = IDsAndSpaces::Lexer.new( 'a-b' ) 481324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 482324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.next_token 483324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 484324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 485324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == '-' 486324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.column.should == 1 487324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.line.should == 1 488324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 489324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 490324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 491324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 492324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass LexerTest011 < ANTLR3::Test::Functional 493324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inline_grammar( <<-'END' ) 494324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer grammar IDsWithAction; 495324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver options {language = Ruby;} 496324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 497324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @members { include ANTLR3::Test::RaiseErrors } 498324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 499324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IDENTIFIER: 500324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ('a'..'z'|'A'..'Z'|'_') 501324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ('a'..'z' 502324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver |'A'..'Z' 503324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver |'0'..'9' 504324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver |'_' { \$action_var = '_' } 505324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver )* 506324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ; 507324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 508324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver WS: (' ' | '\n')+; 509324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver END 510324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 511324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "lexing 'foobar _Ab98 \n A12sdf'" do 512324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = IDsWithAction::Lexer.new( "foobar _Ab98 \n A12sdf" ) 513324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 514324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 515324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'IDENTIFIER' 516324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 0 517324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 5 518324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'foobar' 519324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 520324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 521324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'WS' 522324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 6 523324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 6 524324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == ' ' 525324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 526324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 527324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'IDENTIFIER' 528324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 7 529324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 11 530324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == '_Ab98' 531324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 532324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 533324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'WS' 534324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 12 535324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 14 536324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == " \n " 537324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 538324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 539324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == 'IDENTIFIER' 540324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.start.should == 15 541324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.stop.should == 20 542324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.text.should == 'A12sdf' 543324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 544324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = lexer.next_token 545324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token.name.should == '<EOF>' 546324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 547324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 548324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver example "contains characters without a matching token" do 549324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer = IDsWithAction::Lexer.new( 'a-b' ) 550324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 551324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer.next_token 552324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver proc { lexer.next_token }. 553324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 554324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.unexpected_type.should == '-' 555324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.column.should == 1 556324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver exc.line.should == 1 557324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 558324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end 559324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverend 560