1# 2007 June 21
2#
3# The author disclaims copyright to this source code.  In place of
4# a legal notice, here is a blessing:
5#
6#    May you do good and not evil.
7#    May you find forgiveness for yourself and forgive others.
8#    May you share freely, never taking more than you give.
9#
10#*************************************************************************
11# This file implements regression tests for SQLite library. The focus 
12# of this script is testing the pluggable tokeniser feature of the 
13# FTS3 module.
14#
15# $Id: fts3atoken.test,v 1.1 2007/08/20 17:38:42 shess Exp $
16#
17
18set testdir [file dirname $argv0]
19source $testdir/tester.tcl
20
21# If SQLITE_ENABLE_FTS3 is defined, omit this file.
22ifcapable !fts3 {
23  finish_test
24  return
25}
26
27proc escape_string {str} {
28  set out ""
29  foreach char [split $str ""] {
30    scan $char %c i
31    if {$i<=127} {
32      append out $char
33    } else {
34      append out [format {\x%.4x} $i]
35    }
36  }
37  set out
38}
39
40#--------------------------------------------------------------------------
41# Test cases fts3token-1.* are the warm-body test for the SQL scalar
42# function fts3_tokenizer(). The procedure is as follows:
43#
44#   1: Verify that there is no such fts3 tokenizer as 'blah'.
45#
46#   2: Query for the built-in tokenizer 'simple'. Insert a copy of the
47#      retrieved value as tokenizer 'blah'.
48#
49#   3: Test that the value returned for tokenizer 'blah' is now the
50#      same as that retrieved for 'simple'.
51#
52#   4: Test that it is now possible to create an fts3 table using 
53#      tokenizer 'blah' (it was not possible in step 1).
54#
55#   5: Test that the table created to use tokenizer 'blah' is usable.
56#
57do_test fts3token-1.1 {
58  catchsql {
59    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
60  }
61} {1 {unknown tokenizer: blah}}
62do_test fts3token-1.2 {
63  execsql {
64    SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
65  }
66} {0}
67do_test fts3token-1.3 {
68  execsql {
69    SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
70  }
71} {1}
72do_test fts3token-1.4 {
73  catchsql {
74    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
75  }
76} {0 {}}
77do_test fts3token-1.5 {
78  execsql {
79    INSERT INTO t1(content) VALUES('There was movement at the station');
80    INSERT INTO t1(content) VALUES('For the word has passed around');
81    INSERT INTO t1(content) VALUES('That the colt from ol regret had got away');
82    SELECT content FROM t1 WHERE content MATCH 'movement'
83  }
84} {{There was movement at the station}}
85
86#--------------------------------------------------------------------------
87# Test cases fts3token-2.* test error cases in the scalar function based
88# API for getting and setting tokenizers.
89#
90do_test fts3token-2.1 {
91  catchsql {
92    SELECT fts3_tokenizer('nosuchtokenizer');
93  }
94} {1 {unknown tokenizer: nosuchtokenizer}}
95
96#--------------------------------------------------------------------------
97# Test cases fts3token-3.* test the three built-in tokenizers with a
98# simple input string via the built-in test function. This is as much
99# to test the test function as the tokenizer implementations.
100#
101do_test fts3token-3.1 {
102  execsql {
103    SELECT fts3_tokenizer_test('simple', 'I don''t see how');
104  }
105} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
106do_test fts3token-3.2 {
107  execsql {
108    SELECT fts3_tokenizer_test('porter', 'I don''t see how');
109  }
110} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
111ifcapable icu {
112  do_test fts3token-3.3 {
113    execsql {
114      SELECT fts3_tokenizer_test('icu', 'I don''t see how');
115    }
116  } {{0 i I 1 don't don't 2 see see 3 how how}}
117}
118
119#--------------------------------------------------------------------------
120# Test cases fts3token-4.* test the ICU tokenizer. In practice, this
121# tokenizer only has two modes - "thai" and "everybody else". Some other
122# Asian languages (Lao, Khmer etc.) require the same special treatment as 
123# Thai, but ICU doesn't support them yet.
124#
125ifcapable icu {
126
127  proc do_icu_test {name locale input output} {
128    set ::out [db eval { SELECT fts3_tokenizer_test('icu', $locale, $input) }]
129    do_test $name {
130      lindex $::out 0
131    } $output
132  }
133  
134  do_icu_test fts3token-4.1 en_US  {}   {}
135  do_icu_test fts3token-4.2 en_US {Test cases fts3} [list \
136    0 test Test 1 cases cases 2 fts3 fts3
137  ]
138
139  # The following test shows that ICU is smart enough to recognise
140  # Thai chararacters, even when the locale is set to English/United 
141  # States.
142  #
143  set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a"
144  set output    "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 "
145  append output "1 \u0e19\u0e30 \u0e19\u0e30 "
146  append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a"
147
148  do_icu_test fts3token-4.3 th_TH  $input $output
149  do_icu_test fts3token-4.4 en_US  $input $output
150
151  # ICU handles an unknown locale by falling back to the default.
152  # So this is not an error.
153  do_icu_test fts3token-4.5 MiddleOfTheOcean  $input $output
154
155  set    longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire"
156  append longtoken "AReallocInTheIcuTokenizerCode"
157
158  set    input "short tokens then "
159  append input $longtoken
160  set    output "0 short short "
161  append output "1 tokens tokens "
162  append output "2 then then "
163  append output "3 [string tolower $longtoken] $longtoken"
164
165  do_icu_test fts3token-4.6 MiddleOfTheOcean  $input $output
166  do_icu_test fts3token-4.7 th_TH  $input $output
167  do_icu_test fts3token-4.8 en_US  $input $output
168}
169
170do_test fts3token-internal {
171  execsql { SELECT fts3_tokenizer_internal_test() }
172} {ok}
173
174finish_test
175