1# 2006 September 9 2# 3# The author disclaims copyright to this source code. In place of 4# a legal notice, here is a blessing: 5# 6# May you do good and not evil. 7# May you find forgiveness for yourself and forgive others. 8# May you share freely, never taking more than you give. 9# 10#************************************************************************* 11# This file implements regression tests for SQLite library. The 12# focus of this script is testing the FTS3 module. 13# 14# $Id: fts3expr.test,v 1.9 2009/07/28 16:44:26 danielk1977 Exp $ 15# 16 17set testdir [file dirname $argv0] 18source $testdir/tester.tcl 19 20# If SQLITE_ENABLE_FTS3 is defined, omit this file. 21ifcapable !fts3 { 22 finish_test 23 return 24} 25 26set sqlite_fts3_enable_parentheses 1 27 28proc test_fts3expr {expr} { 29 db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')} 30} 31do_test fts3expr-1.0 { 32 test_fts3expr "abcd" 33} {PHRASE 3 0 abcd} 34do_test fts3expr-1.1 { 35 test_fts3expr " tag " 36} {PHRASE 3 0 tag} 37 38do_test fts3expr-1.2 { 39 test_fts3expr "ab AND cd" 40} {AND {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 41do_test fts3expr-1.2.1 { 42 test_fts3expr "ab cd" 43} {AND {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 44do_test fts3expr-1.3 { 45 test_fts3expr "ab OR cd" 46} {OR {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 47do_test fts3expr-1.4 { 48 test_fts3expr "ab NOT cd" 49} {NOT {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 50do_test fts3expr-1.5 { 51 test_fts3expr "ab NEAR cd" 52} {NEAR/10 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 53do_test fts3expr-1.6.1 { 54 test_fts3expr "ab NEAR/5 cd" 55} {NEAR/5 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 56do_test fts3expr-1.6.2 { 57 test_fts3expr "ab NEAR/87654321 cd" 58} {NEAR/87654321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 59do_test fts3expr-1.6.3 { 60 test_fts3expr "ab NEAR/7654321 cd" 61} {NEAR/7654321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 62do_test fts3expr-1.6.4 { 63 test_fts3expr "ab NEAR/654321 cd" 64} {NEAR/654321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 65do_test fts3expr-1.6.5 { 66 test_fts3expr "ab NEAR/54321 cd" 67} {NEAR/54321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 68do_test fts3expr-1.6.6 { 69 test_fts3expr "ab NEAR/4321 cd" 70} {NEAR/4321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 71do_test fts3expr-1.6.7 { 72 test_fts3expr "ab NEAR/321 cd" 73} {NEAR/321 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 74do_test fts3expr-1.6.8 { 75 test_fts3expr "ab NEAR/21 cd" 76} {NEAR/21 {PHRASE 3 0 ab} {PHRASE 3 0 cd}} 77 78do_test fts3expr-1.7 { 79 test_fts3expr {"one two three"} 80} {PHRASE 3 0 one two three} 81do_test fts3expr-1.8.1 { 82 test_fts3expr {zero "one two three" four} 83} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}} 84do_test fts3expr-1.8.2 { 85 test_fts3expr {zero AND "one two three" four} 86} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}} 87do_test fts3expr-1.8.3 { 88 test_fts3expr {zero "one two three" AND four} 89} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}} 90do_test fts3expr-1.8.4 { 91 test_fts3expr {zero AND "one two three" AND four} 92} {AND {AND {PHRASE 3 0 zero} {PHRASE 3 0 one two three}} {PHRASE 3 0 four}} 93do_test fts3expr-1.9.1 { 94 test_fts3expr {"one* two three"} 95} {PHRASE 3 0 one+ two three} 96do_test fts3expr-1.9.2 { 97 test_fts3expr {"one two* three"} 98} {PHRASE 3 0 one two+ three} 99do_test fts3expr-1.9.3 { 100 test_fts3expr {"one* two* three"} 101} {PHRASE 3 0 one+ two+ three} 102do_test fts3expr-1.9.4 { 103 test_fts3expr {"one two three*"} 104} {PHRASE 3 0 one two three+} 105do_test fts3expr-1.9.5 { 106 test_fts3expr {"one* two three*"} 107} {PHRASE 3 0 one+ two three+} 108do_test fts3expr-1.9.6 { 109 test_fts3expr {"one two* three*"} 110} {PHRASE 3 0 one two+ three+} 111do_test fts3expr-1.9.7 { 112 test_fts3expr {"one* two* three*"} 113} {PHRASE 3 0 one+ two+ three+} 114 115do_test fts3expr-1.10 { 116 test_fts3expr {one* two} 117} {AND {PHRASE 3 0 one+} {PHRASE 3 0 two}} 118do_test fts3expr-1.11 { 119 test_fts3expr {one two*} 120} {AND {PHRASE 3 0 one} {PHRASE 3 0 two+}} 121 122do_test fts3expr-1.14 { 123 test_fts3expr {a:one two} 124} {AND {PHRASE 0 0 one} {PHRASE 3 0 two}} 125do_test fts3expr-1.15.1 { 126 test_fts3expr {one b:two} 127} {AND {PHRASE 3 0 one} {PHRASE 1 0 two}} 128do_test fts3expr-1.15.2 { 129 test_fts3expr {one B:two} 130} {AND {PHRASE 3 0 one} {PHRASE 1 0 two}} 131 132do_test fts3expr-1.16 { 133 test_fts3expr {one AND two AND three AND four AND five} 134} [list AND \ 135 [list AND \ 136 [list AND \ 137 [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 138 {PHRASE 3 0 three} \ 139 ] \ 140 {PHRASE 3 0 four} \ 141 ] \ 142 {PHRASE 3 0 five} \ 143 ] 144do_test fts3expr-1.17 { 145 test_fts3expr {(one AND two) AND ((three AND four) AND five)} 146} [list AND \ 147 [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 148 [list AND \ 149 [list AND {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 150 {PHRASE 3 0 five} \ 151 ] \ 152 ] 153do_test fts3expr-1.18 { 154 test_fts3expr {(one AND two) OR ((three AND four) AND five)} 155} [list OR \ 156 [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 157 [list AND \ 158 [list AND {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 159 {PHRASE 3 0 five} \ 160 ] \ 161 ] 162do_test fts3expr-1.19 { 163 test_fts3expr {(one AND two) AND ((three AND four) OR five)} 164} [list AND \ 165 [list AND {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 166 [list OR \ 167 [list AND {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 168 {PHRASE 3 0 five} \ 169 ] \ 170 ] 171do_test fts3expr-1.20 { 172 test_fts3expr {(one OR two) AND ((three OR four) AND five)} 173} [list AND \ 174 [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 175 [list AND \ 176 [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 177 {PHRASE 3 0 five} \ 178 ] \ 179 ] 180do_test fts3expr-1.21 { 181 test_fts3expr {(one OR two) AND ((three NOT four) AND five)} 182} [list AND \ 183 [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 184 [list AND \ 185 [list NOT {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 186 {PHRASE 3 0 five} \ 187 ] \ 188 ] 189do_test fts3expr-1.22 { 190 test_fts3expr {(one OR two) NOT ((three OR four) AND five)} 191} [list NOT \ 192 [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 193 [list AND \ 194 [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 195 {PHRASE 3 0 five} \ 196 ] \ 197 ] 198do_test fts3expr-1.23 { 199 test_fts3expr {(((((one OR two))))) NOT (((((three OR four))) AND five))} 200} [list NOT \ 201 [list OR {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 202 [list AND \ 203 [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 204 {PHRASE 3 0 five} \ 205 ] \ 206 ] 207do_test fts3expr-1.24 { 208 test_fts3expr {one NEAR two} 209} [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}] 210do_test fts3expr-1.25 { 211 test_fts3expr {(one NEAR two)} 212} [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}] 213do_test fts3expr-1.26 { 214 test_fts3expr {((((((one NEAR two))))))} 215} [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}] 216do_test fts3expr-1.27 { 217 test_fts3expr {(one NEAR two) OR ((three OR four) AND five)} 218} [list OR \ 219 [list NEAR/10 {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 220 [list AND \ 221 [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 222 {PHRASE 3 0 five} \ 223 ] \ 224 ] 225do_test fts3expr-1.28 { 226 test_fts3expr {(one NEAR/321 two) OR ((three OR four) AND five)} 227} [list OR \ 228 [list NEAR/321 {PHRASE 3 0 one} {PHRASE 3 0 two}] \ 229 [list AND \ 230 [list OR {PHRASE 3 0 three} {PHRASE 3 0 four}] \ 231 {PHRASE 3 0 five} \ 232 ] \ 233 ] 234 235proc strip_phrase_data {L} { 236 if {[lindex $L 0] eq "PHRASE"} { 237 return [lrange $L 3 end] 238 } 239 return [list \ 240 [lindex $L 0] \ 241 [strip_phrase_data [lindex $L 1]] \ 242 [strip_phrase_data [lindex $L 2]] \ 243 ] 244} 245proc test_fts3expr2 {expr} { 246 strip_phrase_data [ 247 db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')} 248 ] 249} 250do_test fts3expr-2.1 { 251 test_fts3expr2 "ab OR cd AND ef" 252} {OR ab {AND cd ef}} 253do_test fts3expr-2.2 { 254 test_fts3expr2 "cd AND ef OR ab" 255} {OR {AND cd ef} ab} 256do_test fts3expr-2.3 { 257 test_fts3expr2 "ab AND cd AND ef OR gh" 258} {OR {AND {AND ab cd} ef} gh} 259do_test fts3expr-2.4 { 260 test_fts3expr2 "ab AND cd OR ef AND gh" 261} {OR {AND ab cd} {AND ef gh}} 262do_test fts3expr-2.5 { 263 test_fts3expr2 "ab cd" 264} {AND ab cd} 265 266do_test fts3expr-3.1 { 267 test_fts3expr2 "(ab OR cd) AND ef" 268} {AND {OR ab cd} ef} 269do_test fts3expr-3.2 { 270 test_fts3expr2 "ef AND (ab OR cd)" 271} {AND ef {OR ab cd}} 272do_test fts3expr-3.3 { 273 test_fts3expr2 "(ab OR cd)" 274} {OR ab cd} 275do_test fts3expr-3.4 { 276 test_fts3expr2 "(((ab OR cd)))" 277} {OR ab cd} 278 279do_test fts3expr-3.5 { 280 test_fts3expr2 "one AND (two NEAR three)" 281} {AND one {NEAR/10 two three}} 282do_test fts3expr-3.6 { 283 test_fts3expr2 "one (two NEAR three)" 284} {AND one {NEAR/10 two three}} 285do_test fts3expr-3.7 { 286 test_fts3expr2 "(two NEAR three) one" 287} {AND {NEAR/10 two three} one} 288do_test fts3expr-3.8 { 289 test_fts3expr2 "(two NEAR three) AND one" 290} {AND {NEAR/10 two three} one} 291do_test fts3expr-3.9 { 292 test_fts3expr2 "(two NEAR three) (four five)" 293} {AND {NEAR/10 two three} {AND four five}} 294do_test fts3expr-3.10 { 295 test_fts3expr2 "(two NEAR three) AND (four five)" 296} {AND {NEAR/10 two three} {AND four five}} 297do_test fts3expr-3.11 { 298 test_fts3expr2 "(two NEAR three) (four NEAR five)" 299} {AND {NEAR/10 two three} {NEAR/10 four five}} 300do_test fts3expr-3.12 { 301 test_fts3expr2 "(two NEAR three) OR (four NEAR five)" 302} {OR {NEAR/10 two three} {NEAR/10 four five}} 303 304do_test fts3expr-3.13 { 305 test_fts3expr2 "(two NEAR/1a three)" 306} {AND {AND {AND two near} 1a} three} 307 308do_test fts3expr-3.14 { 309 test_fts3expr2 "(two NEAR// three)" 310} {AND {AND two near} three} 311do_test fts3expr-3.15 { 312 test_fts3expr2 "(two NEAR/: three)" 313} {AND {AND two near} three} 314 315do_test fts3expr-3.16 { 316 test_fts3expr2 "(two NEAR three)OR(four NEAR five)" 317} {OR {NEAR/10 two three} {NEAR/10 four five}} 318do_test fts3expr-3.17 { 319 test_fts3expr2 "(two NEAR three)OR\"four five\"" 320} {OR {NEAR/10 two three} {four five}} 321do_test fts3expr-3.18 { 322 test_fts3expr2 "one \u0080wo" 323} "AND one \u0080wo" 324 325 326 327#------------------------------------------------------------------------ 328# The following tests, fts3expr-4.*, test the parsers response to syntax 329# errors in query expressions. This is done using a real fts3 table and 330# MATCH clauses, not the parser test interface. 331# 332do_test fts3expr-4.1 { 333 execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b, c) } 334} {} 335 336# Mismatched parenthesis: 337do_test fts3expr-4.2.1 { 338 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example AND (hello OR world))' } 339} {1 {malformed MATCH expression: [example AND (hello OR world))]}} 340do_test fts3expr-4.2.2 { 341 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example AND (hello OR world' } 342} {1 {malformed MATCH expression: [example AND (hello OR world]}} 343do_test fts3expr-4.2.3 { 344 catchsql { SELECT * FROM t1 WHERE t1 MATCH '(hello' } 345} {1 {malformed MATCH expression: [(hello]}} 346do_test fts3expr-4.2.4 { 347 catchsql { SELECT * FROM t1 WHERE t1 MATCH '(' } 348} {1 {malformed MATCH expression: [(]}} 349do_test fts3expr-4.2.5 { 350 catchsql { SELECT * FROM t1 WHERE t1 MATCH ')' } 351} {1 {malformed MATCH expression: [)]}} 352 353do_test fts3expr-4.2.6 { 354 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example (hello world' } 355} {1 {malformed MATCH expression: [example (hello world]}} 356 357# Unterminated quotation marks: 358do_test fts3expr-4.3.1 { 359 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example OR "hello world' } 360} {1 {malformed MATCH expression: [example OR "hello world]}} 361do_test fts3expr-4.3.2 { 362 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example OR hello world"' } 363} {1 {malformed MATCH expression: [example OR hello world"]}} 364 365# Binary operators without the required operands. 366do_test fts3expr-4.4.1 { 367 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'OR hello world' } 368} {1 {malformed MATCH expression: [OR hello world]}} 369do_test fts3expr-4.4.2 { 370 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'hello world OR' } 371} {1 {malformed MATCH expression: [hello world OR]}} 372do_test fts3expr-4.4.3 { 373 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one (hello world OR) two' } 374} {1 {malformed MATCH expression: [one (hello world OR) two]}} 375do_test fts3expr-4.4.4 { 376 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one (OR hello world) two' } 377} {1 {malformed MATCH expression: [one (OR hello world) two]}} 378 379# NEAR operators with something other than phrases as arguments. 380do_test fts3expr-4.5.1 { 381 catchsql { SELECT * FROM t1 WHERE t1 MATCH '(hello OR world) NEAR one' } 382} {1 {malformed MATCH expression: [(hello OR world) NEAR one]}} 383do_test fts3expr-4.5.2 { 384 catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one NEAR (hello OR world)' } 385} {1 {malformed MATCH expression: [one NEAR (hello OR world)]}} 386 387#------------------------------------------------------------------------ 388# The following OOM tests are designed to cover cases in fts3_expr.c. 389# 390source $testdir/malloc_common.tcl 391do_malloc_test fts3expr-malloc-1 -sqlbody { 392 SELECT fts3_exprtest('simple', 'a b c "d e f"', 'a', 'b', 'c') 393} 394do_malloc_test fts3expr-malloc-2 -tclprep { 395 set sqlite_fts3_enable_parentheses 0 396} -sqlbody { 397 SELECT fts3_exprtest('simple', 'a -b', 'a', 'b', 'c') 398} -cleanup { 399 set sqlite_fts3_enable_parentheses 1 400} 401 402#------------------------------------------------------------------------ 403# The following tests are not very important. They cover error handling 404# cases in the test code, which makes test coverage easier to measure. 405# 406do_test fts3expr-5.1 { 407 catchsql { SELECT fts3_exprtest('simple', 'a b') } 408} {1 {Usage: fts3_exprtest(tokenizer, expr, col1, ...}} 409do_test fts3expr-5.2 { 410 catchsql { SELECT fts3_exprtest('doesnotexist', 'a b', 'c') } 411} {1 {No such tokenizer module}} 412do_test fts3expr-5.3 { 413 catchsql { SELECT fts3_exprtest('simple', 'a b OR', 'c') } 414} {1 {Error parsing expression}} 415 416#------------------------------------------------------------------------ 417# The next set of tests verifies that things actually work as they are 418# supposed to when using the new syntax. 419# 420do_test fts3expr-6.1 { 421 execsql { 422 CREATE VIRTUAL TABLE t1 USING fts3(a); 423 } 424 for {set ii 1} {$ii < 32} {incr ii} { 425 set v [list] 426 if {$ii & 1} { lappend v one } 427 if {$ii & 2} { lappend v two } 428 if {$ii & 4} { lappend v three } 429 if {$ii & 8} { lappend v four } 430 if {$ii & 16} { lappend v five } 431 execsql { INSERT INTO t1 VALUES($v) } 432 } 433 434 execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'five four one' ORDER BY rowid} 435} {25 27 29 31} 436 437foreach {id expr res} { 438 439 2 "five four NOT one" {24 26 28 30} 440 441 3 "five AND four OR one" 442 {1 3 5 7 9 11 13 15 17 19 21 23 24 25 26 27 28 29 30 31} 443 444 4 "five AND (four OR one)" {17 19 21 23 24 25 26 27 28 29 30 31} 445 446 5 "five NOT (four OR one)" {16 18 20 22} 447 448 6 "(five NOT (four OR one)) OR (five AND (four OR one))" 449 {16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31} 450 451 7 "(five OR one) AND two AND three" {7 15 22 23 30 31} 452 453 8 "five OR one AND two AND three" 454 {7 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31} 455 456 9 "five OR one two three" 457 {7 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31} 458 459 10 "five OR \"one two three\"" 460 {7 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31} 461 462 11 "one two OR four five NOT three" {3 7 11 15 19 23 24 25 26 27 31} 463 464 12 "(one two OR four five) NOT three" {3 11 19 24 25 26 27} 465 466 13 "((((((one two OR four five)))))) NOT three" {3 11 19 24 25 26 27} 467 468} { 469 do_test fts3expr-6.1.$id { 470 execsql { SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid } 471 } $res 472} 473 474set sqlite_fts3_enable_parentheses 0 475foreach {id expr res} { 476 1 "one -two three" {5 13 21 29} 477 2 "-two one three" {5 13 21 29} 478 3 "one three -two" {5 13 21 29} 479 4 "-one -two three" {4 12 20 28} 480 5 "three -one -two" {4 12 20 28} 481 6 "-one three -two" {4 12 20 28} 482} { 483 do_test fts3expr-6.2.$id { 484 execsql { SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid } 485 } $res 486} 487set sqlite_fts3_enable_parentheses 1 488 489do_test fts3expr-7.1 { 490 execsql { 491 CREATE VIRTUAL TABLE test USING fts3 (keyword); 492 INSERT INTO test VALUES ('abc'); 493 SELECT * FROM test WHERE keyword MATCH '""'; 494 } 495} {} 496 497 498set sqlite_fts3_enable_parentheses 0 499finish_test 500