basic.rb revision 324c4644fee44b9898524c09511bd33c3f12e2df
186c27eebc0a4aaeb3de1ee615e8c31c6ab60af14Chris Lattner#!/usr/bin/ruby 286c27eebc0a4aaeb3de1ee615e8c31c6ab60af14Chris Lattner# encoding: utf-8 3ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukmanrequire 'antlr3/test/functional' 486c27eebc0a4aaeb3de1ee615e8c31c6ab60af14Chris Lattner 5b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukmanclass LexerTest001 < ANTLR3::Test::Functional 6e71bd5aead677d8b073ed44f6fbb181e78900a12Bill Wendling inline_grammar( <<-'END' ) 7e71bd5aead677d8b073ed44f6fbb181e78900a12Bill Wendling lexer grammar Zero; 8e71bd5aead677d8b073ed44f6fbb181e78900a12Bill Wendling options { 9e71bd5aead677d8b073ed44f6fbb181e78900a12Bill Wendling language = Ruby; 10b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman } 11b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 12b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman @members { include ANTLR3::Test::RaiseErrors } 13b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 14b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman ZERO: '0'; 15b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman END 1671a1872451e9348fa5b3e3295d584d56b0ddf1c4Owen Anderson 1771a1872451e9348fa5b3e3295d584d56b0ddf1c4Owen Anderson example %(lexing '0') do 1889fc9428f3ad2a3cb1716d68310b14644ab9e506Owen Anderson lexer = Zero::Lexer.new( '0' ) 1989fc9428f3ad2a3cb1716d68310b14644ab9e506Owen Anderson 2071a1872451e9348fa5b3e3295d584d56b0ddf1c4Owen Anderson token = lexer.next_token 21f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner token.name.should == 'ZERO' 22f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner 23f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner token = lexer.next_token 24f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman token.name.should == '<EOF>' 25f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman end 26e5ce8479244a293aa46d94d1aa49d1769f35ba7bNate Begeman 27e5ce8479244a293aa46d94d1aa49d1769f35ba7bNate Begeman example %(iterating over tokens) do 28f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman lexer = Zero::Lexer.new( '0' ) 294e6b17a41bc97da1e7addd6b184e6901a0f638a0Chris Lattner 304e6b17a41bc97da1e7addd6b184e6901a0f638a0Chris Lattner token_types = lexer.map { |token| token.name } 314e6b17a41bc97da1e7addd6b184e6901a0f638a0Chris Lattner token_types.should == %w(ZERO) 322f5d5937eccfaa17c01ab5136bfde20f2f6d767cDaniel Berlin end 334e6b17a41bc97da1e7addd6b184e6901a0f638a0Chris Lattner 340b061363e310f96b21fe24f7abc5d230b7936236Neil Booth example "mismatched token" do 350b061363e310f96b21fe24f7abc5d230b7936236Neil Booth lexer = Zero::Lexer.new( '1' ) 36ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman 370b061363e310f96b21fe24f7abc5d230b7936236Neil Booth proc { 38b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token = lexer.next_token 392d7721457a7587285c46f1543185cef93164457cMisha Brukman }.should raise_error( ANTLR3::Error::MismatchedToken ) do |e| 40b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman e.expecting.should == '0' 41f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman e.unexpected_type.should == '1' 42f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman end 43b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman end 44b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukmanend 452d7721457a7587285c46f1543185cef93164457cMisha Brukman 460fc20c82bcf82a3a802bedf480b8e89048df2371Misha Brukmanclass LexerTest002 < ANTLR3::Test::Functional 47b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman inline_grammar( <<-'END' ) 48bb7a62435264f8f00ae502fcf985a3956d914cd4Chandler Carruth lexer grammar Binary; 49bb7a62435264f8f00ae502fcf985a3956d914cd4Chandler Carruth options { 50bb7a62435264f8f00ae502fcf985a3956d914cd4Chandler Carruth language = Ruby; 51bb7a62435264f8f00ae502fcf985a3956d914cd4Chandler Carruth } 52b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 532d7721457a7587285c46f1543185cef93164457cMisha Brukman @members { include ANTLR3::Test::RaiseErrors } 54b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 55b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman ZERO: '0'; 564c286c373168269827e104749c75066b26588d58Chris Lattner ONE: '1'; 574c286c373168269827e104749c75066b26588d58Chris Lattner END 58b3ff6b75390803166301d850754670713193f8b8Evan Cheng 59fc32dcbdda713cd1f3e27aeaa5a0722df1896c66Evan Cheng example "lexing '01'" do 605d286d8d6f49d9911f5a40ac91f723cd20874e53Evan Cheng lexer = Binary::Lexer.new( '01' ) 61fc32dcbdda713cd1f3e27aeaa5a0722df1896c66Evan Cheng 62fc32dcbdda713cd1f3e27aeaa5a0722df1896c66Evan Cheng token = lexer.next_token 634c286c373168269827e104749c75066b26588d58Chris Lattner token.name.should == 'ZERO' 64bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen 65bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen token = lexer.next_token 66bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen token.name.should == 'ONE' 67bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen 68bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen token = lexer.next_token 69bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen token.name.should == '<EOF>' 70bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen end 71bf22bc66b9969fb6c119569e89c79179dcd557c0Dan Villiom Podlaski Christiansen 72f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner example "no matching token rule" do 730939d7745298ae84ef4e0adcbc872125c0faf544Jeff Cohen lexer = Binary::Lexer.new( '2' ) 740939d7745298ae84ef4e0adcbc872125c0faf544Jeff Cohen 75f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner b = lambda { token = lexer.next_token } 76f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner b.should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 772d7721457a7587285c46f1543185cef93164457cMisha Brukman exc.unexpected_type.should == '2' 782d7721457a7587285c46f1543185cef93164457cMisha Brukman end 790a4a48948782b5593fbd1aaea6aadf85f3615463John Criswell end 802d7721457a7587285c46f1543185cef93164457cMisha Brukman 81b3e849cf151b4dde8b5a4931daa0bca188322ecaStefanus Du Toitend 82b3e849cf151b4dde8b5a4931daa0bca188322ecaStefanus Du Toit 83b3e849cf151b4dde8b5a4931daa0bca188322ecaStefanus Du Toitclass LexerTest003 < ANTLR3::Test::Functional 84b3e849cf151b4dde8b5a4931daa0bca188322ecaStefanus Du Toit inline_grammar( <<-'END' ) 85a3fdc024669beedc938360c7fee3349fe7a4a48fRafael Espindola lexer grammar BinaryFooze; 86a3fdc024669beedc938360c7fee3349fe7a4a48fRafael Espindola options { 87a3fdc024669beedc938360c7fee3349fe7a4a48fRafael Espindola language = Ruby; 88a3fdc024669beedc938360c7fee3349fe7a4a48fRafael Espindola } 894ce08a0a47d0708455195237ac30b626dcdf969aChris Lattner 90ab5948f2427a1b7f3f92e338fb70f682d81c0fb6Alkis Evlogimenos @members { include ANTLR3::Test::RaiseErrors } 914ce08a0a47d0708455195237ac30b626dcdf969aChris Lattner 924ce08a0a47d0708455195237ac30b626dcdf969aChris Lattner ZERO: '0'; 9305e8de739b386571f1cfc754726d8d79583e5050Ryan Flynn ONE: '1'; 9405e8de739b386571f1cfc754726d8d79583e5050Ryan Flynn FOOZE: 'fooze'; 9505e8de739b386571f1cfc754726d8d79583e5050Ryan Flynn END 9605e8de739b386571f1cfc754726d8d79583e5050Ryan Flynn 97b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman example "lexing '0fooze1'" do 98b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman lexer = BinaryFooze::Lexer.new( '0fooze1' ) 99b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 1004e514c1f0c5794d24eff32b9a7ecce935110113eChris Lattner token = lexer.next_token 101ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman token.name.should == 'ZERO' 1022d7721457a7587285c46f1543185cef93164457cMisha Brukman 103b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token = lexer.next_token 10445cdf30d0aedbc437cd1029695de8f66f3136facNicolas Geoffray token.name.should == 'FOOZE' 10545cdf30d0aedbc437cd1029695de8f66f3136facNicolas Geoffray 10645cdf30d0aedbc437cd1029695de8f66f3136facNicolas Geoffray token = lexer.next_token 10745cdf30d0aedbc437cd1029695de8f66f3136facNicolas Geoffray token.name.should == 'ONE' 10845cdf30d0aedbc437cd1029695de8f66f3136facNicolas Geoffray 109f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman token = lexer.next_token 110f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman token.name.should == '<EOF>' 111f200eee6ac761044d24de0c89ee3224d589fd72bMisha Brukman end 1124e514c1f0c5794d24eff32b9a7ecce935110113eChris Lattner 1134e514c1f0c5794d24eff32b9a7ecce935110113eChris Lattner example "no token match" do 1144e514c1f0c5794d24eff32b9a7ecce935110113eChris Lattner lexer = BinaryFooze::Lexer.new( '2' ) 1154e514c1f0c5794d24eff32b9a7ecce935110113eChris Lattner 116f5e941b9f690e54597af6b33df582ea2c5b72e4aAnton Korobeynikov proc { lexer.next_token }. 117f5e941b9f690e54597af6b33df582ea2c5b72e4aAnton Korobeynikov should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 118f5e941b9f690e54597af6b33df582ea2c5b72e4aAnton Korobeynikov exc.unexpected_type.should == '2' 119f5e941b9f690e54597af6b33df582ea2c5b72e4aAnton Korobeynikov end 120addccfd0060473f0ef3b731a08d4b384e8b9f8f9Dan Gohman end 121cf891f4dd40eac746e861e7a700f56555b7e8349Dan Gohmanend 122addccfd0060473f0ef3b731a08d4b384e8b9f8f9Dan Gohman 123addccfd0060473f0ef3b731a08d4b384e8b9f8f9Dan Gohman 1244de77f8afcf5920234c86bd8fba3b164b1a17a72David Goodwinclass LexerTest004 < ANTLR3::Test::Functional 1254de77f8afcf5920234c86bd8fba3b164b1a17a72David Goodwin inline_grammar( <<-'END' ) 1264de77f8afcf5920234c86bd8fba3b164b1a17a72David Goodwin lexer grammar FooStar; 1274de77f8afcf5920234c86bd8fba3b164b1a17a72David Goodwin options { 12855236d2885df05031810cf77a48f25c4f9eee40cDavid Greene language = Ruby; 12955236d2885df05031810cf77a48f25c4f9eee40cDavid Greene } 13055236d2885df05031810cf77a48f25c4f9eee40cDavid Greene 13155236d2885df05031810cf77a48f25c4f9eee40cDavid Greene @members { include ANTLR3::Test::RaiseErrors } 13255236d2885df05031810cf77a48f25c4f9eee40cDavid Greene 133efe65369a74871c3140a540a6c95ce5d1f080954Gabor Greif FOO: 'f' 'o'*; 134efe65369a74871c3140a540a6c95ce5d1f080954Gabor Greif END 135efe65369a74871c3140a540a6c95ce5d1f080954Gabor Greif 136efe65369a74871c3140a540a6c95ce5d1f080954Gabor Greif example "lexing 'ffofoofooo'" do 137bd3c63c70ed5e146e6d057cb842d4c2eec4f1cf8Jim Grosbach lexer = FooStar::Lexer.new( 'ffofoofooo' ) 138bd3c63c70ed5e146e6d057cb842d4c2eec4f1cf8Jim Grosbach 139bd3c63c70ed5e146e6d057cb842d4c2eec4f1cf8Jim Grosbach token = lexer.next_token 140bd3c63c70ed5e146e6d057cb842d4c2eec4f1cf8Jim Grosbach token.name.should == 'FOO' 141bd3c63c70ed5e146e6d057cb842d4c2eec4f1cf8Jim Grosbach token.start.should == 0 142ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman token.stop.should == 0 143ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman token.text.should == 'f' 144ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman 145ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman token = lexer.next_token 146baa2fa5823fe4598d158794711d88af21976c182Gordon Henriksen token.name.should == 'FOO' 147baa2fa5823fe4598d158794711d88af21976c182Gordon Henriksen token.text.should == 'fo' 148fcd78f88f346c951d7fde3c5abc99679afb11868Gordon Henriksen token.start.should == 1 149baa2fa5823fe4598d158794711d88af21976c182Gordon Henriksen token.stop.should == 2 150fcd78f88f346c951d7fde3c5abc99679afb11868Gordon Henriksen 151baa2fa5823fe4598d158794711d88af21976c182Gordon Henriksen token = lexer.next_token 15230b0f2f7c3730204c081b5a1b48b4447e7ed7a13Raul Herbster token.name.should == 'FOO' 15330b0f2f7c3730204c081b5a1b48b4447e7ed7a13Raul Herbster token.start.should == 3 15430b0f2f7c3730204c081b5a1b48b4447e7ed7a13Raul Herbster token.stop.should == 5 15530b0f2f7c3730204c081b5a1b48b4447e7ed7a13Raul Herbster token.text.should == 'foo' 156f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner 157309ca69064f027702499605fc98f76ea8d34fd37Misha Brukman token = lexer.next_token 158ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman token.name.should == 'FOO' 159f69dcb15c3b75a267c6e35d914a83d6c3445a192Chris Lattner token.start.should == 6 160df4283c8f9670edc50ab5440d655c290cf0f6d49Tanya Lattner token.stop.should == 9 161df4283c8f9670edc50ab5440d655c290cf0f6d49Tanya Lattner token.text.should == 'fooo' 162df4283c8f9670edc50ab5440d655c290cf0f6d49Tanya Lattner 163df4283c8f9670edc50ab5440d655c290cf0f6d49Tanya Lattner token = lexer.next_token 1642fcdf64ddd12cb91101df2e86c26838e988e605bDale Johannesen token.name.should == '<EOF>' 1652fcdf64ddd12cb91101df2e86c26838e988e605bDale Johannesen end 16682434387dd203e86c1fbc8beb8deeb868ed15d1bDale Johannesen 167f8ad9552215503f7573b5049567ed59c37cc9636Dale Johannesen example "mismatched token" do 168f8ad9552215503f7573b5049567ed59c37cc9636Dale Johannesen lexer = FooStar::Lexer.new( '2' ) 169005ac653f2c8e04fcf95afcecc58ff06b7a3b5adDale Johannesen 170005ac653f2c8e04fcf95afcecc58ff06b7a3b5adDale Johannesen proc { lexer.next_token }. 1712fcdf64ddd12cb91101df2e86c26838e988e605bDale Johannesen should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 172e4261471bec1485cf9dbb3fa9a727d3ff8d3e0ebBill Wendling exc.expecting.should == 'f' 173e4261471bec1485cf9dbb3fa9a727d3ff8d3e0ebBill Wendling exc.unexpected_type.should == '2' 174e4261471bec1485cf9dbb3fa9a727d3ff8d3e0ebBill Wendling end 175e4261471bec1485cf9dbb3fa9a727d3ff8d3e0ebBill Wendling end 176633a016493e6ed2d3039e0633b72f36bc0cee8f7Chris Lattnerend 177633a016493e6ed2d3039e0633b72f36bc0cee8f7Chris Lattner 178633a016493e6ed2d3039e0633b72f36bc0cee8f7Chris Lattnerclass LexerTest005 < ANTLR3::Test::Functional 179633a016493e6ed2d3039e0633b72f36bc0cee8f7Chris Lattner inline_grammar( <<-'END' ) 180b195d9d365e5f960c93ddfdf10638d5d147d9f6fReid Spencer lexer grammar FooPlus; 181b195d9d365e5f960c93ddfdf10638d5d147d9f6fReid Spencer options { 182b195d9d365e5f960c93ddfdf10638d5d147d9f6fReid Spencer language = Ruby; 183b195d9d365e5f960c93ddfdf10638d5d147d9f6fReid Spencer } 1848f2105bcb952eeb712f0cac237233bc092d2119fAnton Korobeynikov 1858f2105bcb952eeb712f0cac237233bc092d2119fAnton Korobeynikov @members { include ANTLR3::Test::RaiseErrors } 1864d396b1a47621f336fe92fb80b5238284bbe89e7Anton Korobeynikov 1874d396b1a47621f336fe92fb80b5238284bbe89e7Anton Korobeynikov FOO: 'f' 'o'+; 1884d396b1a47621f336fe92fb80b5238284bbe89e7Anton Korobeynikov END 1898f2105bcb952eeb712f0cac237233bc092d2119fAnton Korobeynikov 1903acd9181df5d8f76f75a384266e2a78b9a6c4f45Chris Lattner example "lexing 'fofoofooo'" do 1913acd9181df5d8f76f75a384266e2a78b9a6c4f45Chris Lattner lexer = FooPlus::Lexer.new( 'fofoofooo' ) 1923acd9181df5d8f76f75a384266e2a78b9a6c4f45Chris Lattner 1933acd9181df5d8f76f75a384266e2a78b9a6c4f45Chris Lattner token = lexer.next_token 1945aeac0507065a29d145eedb1b4c6ed91ab3ccc8eBenjamin Kramer token.name.should == 'FOO' 1955aeac0507065a29d145eedb1b4c6ed91ab3ccc8eBenjamin Kramer token.start.should == 0 1965aeac0507065a29d145eedb1b4c6ed91ab3ccc8eBenjamin Kramer token.stop.should == 1 1975aeac0507065a29d145eedb1b4c6ed91ab3ccc8eBenjamin Kramer token.text.should == 'fo' 1982330e4d4c4f8008d17f5a38ac0d7b04e139d4131Christopher Lamb 1992330e4d4c4f8008d17f5a38ac0d7b04e139d4131Christopher Lamb token = lexer.next_token 200a1eb155e521e06d2b3e33699a4cf87ae5d012087Christopher Lamb token.name.should == 'FOO' 201a1eb155e521e06d2b3e33699a4cf87ae5d012087Christopher Lamb token.text.should == 'foo' 20252cf47e331f53903c1ce28a31a060d3d3f2c1a93Christopher Lamb token.start.should == 2 2032330e4d4c4f8008d17f5a38ac0d7b04e139d4131Christopher Lamb token.stop.should == 4 2044c286c373168269827e104749c75066b26588d58Chris Lattner 2054c286c373168269827e104749c75066b26588d58Chris Lattner token = lexer.next_token 2064c286c373168269827e104749c75066b26588d58Chris Lattner token.name.should == 'FOO' 207abf41f2e0a71274011220ec3db6d1299afd1b2c1Jim Laskey token.start.should == 5 208ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman token.stop.should == 8 209abf41f2e0a71274011220ec3db6d1299afd1b2c1Jim Laskey token.text.should == 'fooo' 2104c286c373168269827e104749c75066b26588d58Chris Lattner 211b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token = lexer.next_token 212b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token.name.should == '<EOF>' 213b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman end 214b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 215b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman example "mismatched token" do 216706a9cab44d9e08eaeef8f6418b81d7814013b51Tanya Lattner lexer = FooPlus::Lexer.new( '2' ) 217f2dce8d29b58a86fe5a9221e47247bcbe0d97118Tanya Lattner 218f2dce8d29b58a86fe5a9221e47247bcbe0d97118Tanya Lattner proc { lexer.next_token }. 219f2dce8d29b58a86fe5a9221e47247bcbe0d97118Tanya Lattner should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 220f2dce8d29b58a86fe5a9221e47247bcbe0d97118Tanya Lattner exc.expecting.should == 'f' 221df4283c8f9670edc50ab5440d655c290cf0f6d49Tanya Lattner exc.unexpected_type.should == '2' 222f2dce8d29b58a86fe5a9221e47247bcbe0d97118Tanya Lattner end 223e9b0a02579b9a5153fe8fe5c029870e2e93e48a1Misha Brukman end 224e9b0a02579b9a5153fe8fe5c029870e2e93e48a1Misha Brukman 225e9b0a02579b9a5153fe8fe5c029870e2e93e48a1Misha Brukman example "early exit" do 226e9b0a02579b9a5153fe8fe5c029870e2e93e48a1Misha Brukman lexer = FooPlus::Lexer.new( 'f' ) 22763223c93d83f9f3bb1f2369c84639696268a2858Andrew Lenharth 228e9b0a02579b9a5153fe8fe5c029870e2e93e48a1Misha Brukman proc { token = lexer.next_token }. 22997198c9b1b983b15156585098d13d5f4898ab994Nick Lewycky should raise_error( ANTLR3::Error::EarlyExit ) { |exc| 23097198c9b1b983b15156585098d13d5f4898ab994Nick Lewycky exc.unexpected_type.should == ANTLR3::Constants::EOF 23197198c9b1b983b15156585098d13d5f4898ab994Nick Lewycky } 23297198c9b1b983b15156585098d13d5f4898ab994Nick Lewycky end 233cb0a660142618410f107e5ae6d775e67efd9be18Bill Wendling 234cb0a660142618410f107e5ae6d775e67efd9be18Bill Wendlingend 235cb0a660142618410f107e5ae6d775e67efd9be18Bill Wendling 236cb0a660142618410f107e5ae6d775e67efd9be18Bill Wendlingclass LexerTest006 < ANTLR3::Test::Functional 237cb0a660142618410f107e5ae6d775e67efd9be18Bill Wendling inline_grammar( <<-'END' ) 238735e0acba5695d26826bb4aa0beda497a60e7810Duraid Madina lexer grammar FoaStar; 239735e0acba5695d26826bb4aa0beda497a60e7810Duraid Madina options { 240735e0acba5695d26826bb4aa0beda497a60e7810Duraid Madina language = Ruby; 2412672a8e4077a537d91d3c829d146e33104b253c1Duraid Madina } 242735e0acba5695d26826bb4aa0beda497a60e7810Duraid Madina 2439626470f109752ddd3b12469aae30df8d121a056Chris Lattner @members { include ANTLR3::Test::RaiseErrors } 2449626470f109752ddd3b12469aae30df8d121a056Chris Lattner 2459626470f109752ddd3b12469aae30df8d121a056Chris Lattner FOO: 'f' ('o' | 'a')*; 2469626470f109752ddd3b12469aae30df8d121a056Chris Lattner END 247b8356b3ce2d1913d466ac4dc1812d7e2520527ddChris Lattner 248b8356b3ce2d1913d466ac4dc1812d7e2520527ddChris Lattner example "lexing 'fofaaooa'" do 249323ed35e4fda5247799a8ec384a2d1f39b4c461dMisha Brukman lexer = FoaStar::Lexer.new( 'fofaaooa' ) 250b8356b3ce2d1913d466ac4dc1812d7e2520527ddChris Lattner 25108f8b33e2c758143bca088408fc4a3db21c451c2Bill Wendling token = lexer.next_token 25208f8b33e2c758143bca088408fc4a3db21c451c2Bill Wendling token.name.should == 'FOO' 25308f8b33e2c758143bca088408fc4a3db21c451c2Bill Wendling token.start.should == 0 25408f8b33e2c758143bca088408fc4a3db21c451c2Bill Wendling token.stop.should == 1 255ca6829aebb04ea1098723ed7a817438fef8be339Duncan Sands token.text.should == 'fo' 256ca6829aebb04ea1098723ed7a817438fef8be339Duncan Sands 257ca6829aebb04ea1098723ed7a817438fef8be339Duncan Sands token = lexer.next_token 258ca6829aebb04ea1098723ed7a817438fef8be339Duncan Sands token.name.should == 'FOO' 259ca6829aebb04ea1098723ed7a817438fef8be339Duncan Sands token.text.should == 'faaooa' 260ca6829aebb04ea1098723ed7a817438fef8be339Duncan Sands token.start.should == 2 261ca6829aebb04ea1098723ed7a817438fef8be339Duncan Sands token.stop.should == 7 262309ca69064f027702499605fc98f76ea8d34fd37Misha Brukman 263309ca69064f027702499605fc98f76ea8d34fd37Misha Brukman token = lexer.next_token 264ce07e99dd6fafc51805c21d53286ae5765d1cffcMisha Brukman token.name.should == '<EOF>' 265309ca69064f027702499605fc98f76ea8d34fd37Misha Brukman end 266268e35d99fd1de6dec9055f52691424188de161cJakob Stoklund Olesen 267268e35d99fd1de6dec9055f52691424188de161cJakob Stoklund Olesen example "mismatched token" do 268268e35d99fd1de6dec9055f52691424188de161cJakob Stoklund Olesen lexer = FoaStar::Lexer.new( 'fofoaooaoa2' ) 269268e35d99fd1de6dec9055f52691424188de161cJakob Stoklund Olesen 270268e35d99fd1de6dec9055f52691424188de161cJakob Stoklund Olesen lexer.next_token 2713fa75ff2f229cf4d50a5898472cdb8377e29b509Richard Osborne lexer.next_token 2723fa75ff2f229cf4d50a5898472cdb8377e29b509Richard Osborne proc { lexer.next_token }. 2733fa75ff2f229cf4d50a5898472cdb8377e29b509Richard Osborne should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 2743fa75ff2f229cf4d50a5898472cdb8377e29b509Richard Osborne exc.expecting.should == 'f' 2751f7a9bc982aaad336963dce582c0ad82d5076274Devang Patel exc.unexpected_type.should == '2' 2761f7a9bc982aaad336963dce582c0ad82d5076274Devang Patel exc.column.should == 10 2771f7a9bc982aaad336963dce582c0ad82d5076274Devang Patel exc.line.should == 1 2781f7a9bc982aaad336963dce582c0ad82d5076274Devang Patel end 2795d02a0cf9b4aec8c1659c2d729ed18ea4dd8ee77Devang Patel end 2801f7a9bc982aaad336963dce582c0ad82d5076274Devang Patelend 281774350a60694fee45403d87b5edfeff27ac54d58Sandeep Patel 282774350a60694fee45403d87b5edfeff27ac54d58Sandeep Patelclass LexerTest007 < ANTLR3::Test::Functional 283774350a60694fee45403d87b5edfeff27ac54d58Sandeep Patel inline_grammar( <<-'END' ) 284774350a60694fee45403d87b5edfeff27ac54d58Sandeep Patel lexer grammar Foab; 28524fdd27683960157eaed38a18d95958873f74959Bill Wendling options { 28624fdd27683960157eaed38a18d95958873f74959Bill Wendling language = Ruby; 28724fdd27683960157eaed38a18d95958873f74959Bill Wendling } 28824fdd27683960157eaed38a18d95958873f74959Bill Wendling 28924fdd27683960157eaed38a18d95958873f74959Bill Wendling @members { include ANTLR3::Test::RaiseErrors } 290b7206e74fc9cae6533593853a9763a4b6257f5a2Misha Brukman 2911100eea8d2d948306efb70159eb89073a686d234Vladimir Prus FOO: 'f' ('o' | 'a' 'b'+)*; 292b7206e74fc9cae6533593853a9763a4b6257f5a2Misha Brukman END 2939cd94d72b0cc5014c09775f2050cdd868baaa3e3Misha Brukman 294b7206e74fc9cae6533593853a9763a4b6257f5a2Misha Brukman example "lexing 'fofababbooabb'" do 295af2d3793c2c16ad4069d18b1ec7b3fa16fdd2d97Anton Korobeynikov lexer = Foab::Lexer.new( 'fofababbooabb' ) 296af2d3793c2c16ad4069d18b1ec7b3fa16fdd2d97Anton Korobeynikov 297af2d3793c2c16ad4069d18b1ec7b3fa16fdd2d97Anton Korobeynikov token = lexer.next_token 298af2d3793c2c16ad4069d18b1ec7b3fa16fdd2d97Anton Korobeynikov token.name.should == 'FOO' 299ef476fdce2d194b5b335c15cd8a545ceeeb633b1Duncan Sands token.start.should == 0 300ef476fdce2d194b5b335c15cd8a545ceeeb633b1Duncan Sands token.stop.should == 1 301586eccb8cb281c0caf133bbd82a7c95741c231cbDuncan Sands token.text.should == 'fo' 302ef476fdce2d194b5b335c15cd8a545ceeeb633b1Duncan Sands 303b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token = lexer.next_token 304b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token.name.should == 'FOO' 305b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token.start.should == 2 306b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman token.stop.should == 12 30768c8411ef591818d89c6a81098a328fcd75be72aArnold Schwaighofer token.text.should == 'fababbooabb' 30868c8411ef591818d89c6a81098a328fcd75be72aArnold Schwaighofer 30968c8411ef591818d89c6a81098a328fcd75be72aArnold Schwaighofer token = lexer.next_token 31068c8411ef591818d89c6a81098a328fcd75be72aArnold Schwaighofer token.name.should == '<EOF>' 3111a1ee832c91647e0fa2dbb7e4ac92cd21be32182Shantonu Sen end 3121a1ee832c91647e0fa2dbb7e4ac92cd21be32182Shantonu Sen 3131a1ee832c91647e0fa2dbb7e4ac92cd21be32182Shantonu Sen example "early exit" do 3141a1ee832c91647e0fa2dbb7e4ac92cd21be32182Shantonu Sen lexer = Foab::Lexer.new( 'foaboao' ) 315b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 316b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman proc { lexer.next_token }. 3172d7721457a7587285c46f1543185cef93164457cMisha Brukman should raise_error( ANTLR3::Error::EarlyExit ) do |exc| 318b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman exc.unexpected_type.should == 'o' 31924fdd27683960157eaed38a18d95958873f74959Bill Wendling exc.column.should == 6 32024fdd27683960157eaed38a18d95958873f74959Bill Wendling exc.line.should == 1 32124fdd27683960157eaed38a18d95958873f74959Bill Wendling end 32224fdd27683960157eaed38a18d95958873f74959Bill Wendling end 32375920ee4608c556711df852cda647b5a13075f03Brian Gaekeend 3243336aa056f050770374ac9c92bd23a435dd5d7ccReid Spencer 3253336aa056f050770374ac9c92bd23a435dd5d7ccReid Spencerclass LexerTest008 < ANTLR3::Test::Functional 3265c18c8134e04b12aa26ecfad3444770c7ebb91f1Reid Spencer inline_grammar( <<-'END' ) 32775920ee4608c556711df852cda647b5a13075f03Brian Gaeke lexer grammar Fa; 32863c283e784e8a2588377c928b8897c6bb2e21da9Torok Edwin options { 3295bfeded6e5980b3902e355c49f366808e2109351Torok Edwin language = Ruby; 3305bfeded6e5980b3902e355c49f366808e2109351Torok Edwin } 3315bfeded6e5980b3902e355c49f366808e2109351Torok Edwin 3323363e2786c55097e145abf7235c1ae944ebe0bb6Chris Lattner @members { include ANTLR3::Test::RaiseErrors } 3333363e2786c55097e145abf7235c1ae944ebe0bb6Chris Lattner 3343363e2786c55097e145abf7235c1ae944ebe0bb6Chris Lattner FOO: 'f' 'a'?; 3353363e2786c55097e145abf7235c1ae944ebe0bb6Chris Lattner END 3362d6b4bfc844ba6a57a08a27b21b0fbea75b3c9cdLauro Ramos Venancio 3372d6b4bfc844ba6a57a08a27b21b0fbea75b3c9cdLauro Ramos Venancio example "lexing 'ffaf'" do 3382d6b4bfc844ba6a57a08a27b21b0fbea75b3c9cdLauro Ramos Venancio lexer = Fa::Lexer.new( 'ffaf' ) 3392d6b4bfc844ba6a57a08a27b21b0fbea75b3c9cdLauro Ramos Venancio 3402d6b4bfc844ba6a57a08a27b21b0fbea75b3c9cdLauro Ramos Venancio token = lexer.next_token 341bd594a14a07e24f717b7f8d90679cb5b6bc98212Xerxes Ranby token.name.should == 'FOO' 342bd594a14a07e24f717b7f8d90679cb5b6bc98212Xerxes Ranby token.start.should == 0 343bd594a14a07e24f717b7f8d90679cb5b6bc98212Xerxes Ranby token.stop.should == 0 344bd594a14a07e24f717b7f8d90679cb5b6bc98212Xerxes Ranby token.text.should == 'f' 345b296160ce9693149d6f8d8b8f2596a05d4a8330dMisha Brukman 34624fdd27683960157eaed38a18d95958873f74959Bill Wendling token = lexer.next_token 347e4261471bec1485cf9dbb3fa9a727d3ff8d3e0ebBill Wendling token.name.should == 'FOO' 34827a2dbbdf5cc74a12a9a849bd890da79af5aacd1Bob Wilson token.start.should == 1 34927a2dbbdf5cc74a12a9a849bd890da79af5aacd1Bob Wilson token.stop.should == 2 35027a2dbbdf5cc74a12a9a849bd890da79af5aacd1Bob Wilson token.text.should == 'fa' 35127a2dbbdf5cc74a12a9a849bd890da79af5aacd1Bob Wilson 352 token = lexer.next_token 353 token.name.should == 'FOO' 354 token.start.should == 3 355 token.stop.should == 3 356 token.text.should == 'f' 357 358 token = lexer.next_token 359 token.name.should == '<EOF>' 360 end 361 362 example "mismatched token" do 363 lexer = Fa::Lexer.new( 'fafb' ) 364 365 lexer.next_token 366 lexer.next_token 367 proc { lexer.next_token }. 368 should raise_error( ANTLR3::Error::MismatchedToken ) do |exc| 369 exc.unexpected_type.should == 'b' 370 exc.column.should == 3 371 exc.line.should == 1 372 end 373 end 374end 375 376 377class LexerTest009 < ANTLR3::Test::Functional 378 inline_grammar( <<-'END' ) 379 lexer grammar Digit; 380 options { 381 language = Ruby; 382 } 383 384 @members { include ANTLR3::Test::RaiseErrors } 385 386 DIGIT: '0' .. '9'; 387 END 388 389 example "lexing '085'" do 390 lexer = Digit::Lexer.new( '085' ) 391 392 token = lexer.next_token 393 token.name.should == 'DIGIT' 394 token.start.should == 0 395 token.stop.should == 0 396 token.text.should == '0' 397 398 token = lexer.next_token 399 token.name.should == 'DIGIT' 400 token.start.should == 1 401 token.stop.should == 1 402 token.text.should == '8' 403 404 token = lexer.next_token 405 token.name.should == 'DIGIT' 406 token.start.should == 2 407 token.stop.should == 2 408 token.text.should == '5' 409 410 token = lexer.next_token 411 token.name.should == '<EOF>' 412 end 413 414 example "mismatched range" do 415 lexer = Digit::Lexer.new( '2a' ) 416 417 lexer.next_token 418 proc { lexer.next_token }. 419 should raise_error( ANTLR3::Error::MismatchedRange ) do |exc| 420 exc.min.should == '0' 421 exc.max.should == '9' 422 exc.unexpected_type.should == 'a' 423 exc.column.should == 1 424 exc.line.should == 1 425 end 426 end 427end 428 429class LexerTest010 < ANTLR3::Test::Functional 430 inline_grammar( <<-'END' ) 431 lexer grammar IDsAndSpaces; 432 options { 433 language = Ruby; 434 } 435 436 @members { include ANTLR3::Test::RaiseErrors } 437 438 IDENTIFIER: ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*; 439 WS: (' ' | '\n')+; 440 END 441 442 example "lexing 'foobar _Ab98 \n A12sdf'" do 443 lexer = IDsAndSpaces::Lexer.new( "foobar _Ab98 \n A12sdf" ) 444 445 token = lexer.next_token 446 token.name.should == 'IDENTIFIER' 447 token.start.should == 0 448 token.stop.should == 5 449 token.text.should == 'foobar' 450 451 token = lexer.next_token 452 token.name.should == 'WS' 453 token.start.should == 6 454 token.stop.should == 6 455 token.text.should == ' ' 456 457 token = lexer.next_token 458 token.name.should == 'IDENTIFIER' 459 token.start.should == 7 460 token.stop.should == 11 461 token.text.should == '_Ab98' 462 463 token = lexer.next_token 464 token.name.should == 'WS' 465 token.start.should == 12 466 token.stop.should == 14 467 token.text.should == " \n " 468 469 token = lexer.next_token 470 token.name.should == 'IDENTIFIER' 471 token.start.should == 15 472 token.stop.should == 20 473 token.text.should == 'A12sdf' 474 475 token = lexer.next_token 476 token.name.should == '<EOF>' 477 end 478 479 example "contains characters without a matching token rule" do 480 lexer = IDsAndSpaces::Lexer.new( 'a-b' ) 481 482 lexer.next_token 483 proc { lexer.next_token }. 484 should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 485 exc.unexpected_type.should == '-' 486 exc.column.should == 1 487 exc.line.should == 1 488 end 489 end 490end 491 492class LexerTest011 < ANTLR3::Test::Functional 493 inline_grammar( <<-'END' ) 494 lexer grammar IDsWithAction; 495 options {language = Ruby;} 496 497 @members { include ANTLR3::Test::RaiseErrors } 498 499 IDENTIFIER: 500 ('a'..'z'|'A'..'Z'|'_') 501 ('a'..'z' 502 |'A'..'Z' 503 |'0'..'9' 504 |'_' { \$action_var = '_' } 505 )* 506 ; 507 508 WS: (' ' | '\n')+; 509 END 510 511 example "lexing 'foobar _Ab98 \n A12sdf'" do 512 lexer = IDsWithAction::Lexer.new( "foobar _Ab98 \n A12sdf" ) 513 514 token = lexer.next_token 515 token.name.should == 'IDENTIFIER' 516 token.start.should == 0 517 token.stop.should == 5 518 token.text.should == 'foobar' 519 520 token = lexer.next_token 521 token.name.should == 'WS' 522 token.start.should == 6 523 token.stop.should == 6 524 token.text.should == ' ' 525 526 token = lexer.next_token 527 token.name.should == 'IDENTIFIER' 528 token.start.should == 7 529 token.stop.should == 11 530 token.text.should == '_Ab98' 531 532 token = lexer.next_token 533 token.name.should == 'WS' 534 token.start.should == 12 535 token.stop.should == 14 536 token.text.should == " \n " 537 538 token = lexer.next_token 539 token.name.should == 'IDENTIFIER' 540 token.start.should == 15 541 token.stop.should == 20 542 token.text.should == 'A12sdf' 543 544 token = lexer.next_token 545 token.name.should == '<EOF>' 546 end 547 548 example "contains characters without a matching token" do 549 lexer = IDsWithAction::Lexer.new( 'a-b' ) 550 551 lexer.next_token 552 proc { lexer.next_token }. 553 should raise_error( ANTLR3::Error::NoViableAlternative ) do |exc| 554 exc.unexpected_type.should == '-' 555 exc.column.should == 1 556 exc.line.should == 1 557 end 558 end 559end 560