15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2007 June 22 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The author disclaims copyright to this source code. In place of 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a legal notice, here is a blessing: 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you do good and not evil. 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you find forgiveness for yourself and forgive others. 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you share freely, never taking more than you give. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)****************************************************************************** 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This is part of an SQLite module implementing full-text search. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This particular file implements the generic tokenizer interface. 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The code in this file is only compiled if: 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** * The FTS3 module is being built as an extension 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** (in which case SQLITE_CORE is not defined), or 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** * The FTS3 module is being built into the core of 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "sqlite3ext.h" 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef SQLITE_CORE 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SQLITE_EXTENSION_INIT1 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fts3Int.h" 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <assert.h> 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string.h> 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Implementation of the SQL scalar function for accessing the underlying 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** hash table. This function may be called as follows: 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SELECT <function-name>(<key-name>); 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SELECT <function-name>(<key-name>, <pointer>); 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** where <function-name> is the name passed as the second argument 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If the <pointer> argument is specified, it must be a blob value 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** containing a pointer to be stored as the hash data corresponding 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** to the string <key-name>. If <pointer> is not specified, then 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the string <key-name> must already exist in the has table. Otherwise, 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** an error is returned. 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Whether or not the <pointer> argument is specified, the value returned 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** is a blob containing the pointer stored as the hash data corresponding 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** to string <key-name> (after the hash-table is updated, if applicable). 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void scalarFunc( 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_context *context, 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int argc, 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **argv 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Hash *pHash; 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *pPtr = 0; 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const unsigned char *zName; 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nName; 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( argc==1 || argc==2 ); 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pHash = (Fts3Hash *)sqlite3_user_data(context); 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zName = sqlite3_value_text(argv[0]); 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nName = sqlite3_value_bytes(argv[0])+1; 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( argc==2 ){ 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *pOld; 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n = sqlite3_value_bytes(argv[1]); 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( n!=sizeof(pPtr) ){ 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_error(context, "argument type mismatch", -1); 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pPtr = *(void **)sqlite3_value_blob(argv[1]); 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pOld==pPtr ){ 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_error(context, "out of memory", -1); 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pPtr = sqlite3Fts3HashFind(pHash, zName, nName); 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pPtr ){ 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_error(context, zErr, -1); 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(zErr); 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3IsIdChar(char c){ 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) static const char isFtsIdChar[] = { 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }; 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (c&0x80 || isFtsIdChar[(int)(c)]); 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *z1; 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *z2 = 0; 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Find the start of the next token. */ 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z1 = zStr; 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( z2==0 ){ 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char c = *z1; 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) switch( c ){ 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) case '\0': return 0; /* No more tokens here */ 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) case '\'': 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) case '"': 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) case '`': { 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z2 = z1; 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( *++z2 && (*z2!=c || *++z2==c) ); 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) case '[': 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z2 = &z1[1]; 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( *z2 && z2[0]!=']' ) z2++; 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( *z2 ) z2++; 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) default: 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( sqlite3Fts3IsIdChar(*z1) ){ 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z2 = &z1[1]; 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( sqlite3Fts3IsIdChar(*z2) ) z2++; 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z1++; 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pn = (int)(z2-z1); 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return z1; 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3InitTokenizer( 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Hash *pHash, /* Tokenizer hash table */ 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zArg, /* Tokenizer name */ 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char **pzErr /* OUT: Set to malloced error message */ 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *z = (char *)zArg; 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n; 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zCopy; 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zEnd; /* Pointer to nul-term of zCopy */ 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_module *m; 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zCopy = sqlite3_mprintf("%s", zArg); 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !zCopy ) return SQLITE_NOMEM; 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zEnd = &zCopy[strlen(zCopy)]; 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z = (char *)sqlite3Fts3NextToken(zCopy, &n); 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z[n] = '\0'; 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3Dequote(z); 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !m ){ 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_ERROR; 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char const **aArg = 0; 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iArg = 0; 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z = &z[n+1]; 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){ 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nNew = sizeof(char *)*(iArg+1); 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew); 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !aNew ){ 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(zCopy); 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free((void *)aArg); 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aArg = aNew; 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aArg[iArg++] = z; 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z[n] = '\0'; 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3Dequote(z); 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) z = &z[n+1]; 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = m->xCreate(iArg, aArg, ppTok); 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc!=SQLITE_OK || *ppTok ); 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pzErr = sqlite3_mprintf("unknown tokenizer"); 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (*ppTok)->pModule = m; 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free((void *)aArg); 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(zCopy); 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef SQLITE_TEST 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <tcl.h> 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string.h> 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Implementation of a special SQL scalar function for testing tokenizers 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** designed to be used in concert with the Tcl testing framework. This 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** function must be called with two arguments: 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SELECT <function-name>(<key-name>, <input-string>); 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SELECT <function-name>(<key-name>, <pointer>); 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** where <function-name> is the name passed as the second argument 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The return value is a string that may be interpreted as a Tcl 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** list. For each token in the <input-string>, three elements are 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** added to the returned list. The first is the token position, the 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** second is the token text (folded, stemmed, etc.) and the third is the 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** substring of <input-string> associated with the token. For example, 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** using the built-in "simple" tokenizer: 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SELECT fts_tokenizer_test('simple', 'I don't see how'); 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** will return the string: 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** "{0 i I 1 dont don't 2 see see 3 how how}" 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void testFunc( 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_context *context, 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int argc, 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **argv 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Hash *pHash; 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_module *p; 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer *pTokenizer = 0; 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_cursor *pCsr = 0; 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zErr = 0; 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zName; 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nName; 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zInput; 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nInput; 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zArg = 0; 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zToken; 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nToken; 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iStart; 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iEnd; 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iPos; 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tcl_Obj *pRet; 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( argc==2 || argc==3 ); 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nName = sqlite3_value_bytes(argv[0]); 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zName = (const char *)sqlite3_value_text(argv[0]); 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nInput = sqlite3_value_bytes(argv[argc-1]); 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zInput = (const char *)sqlite3_value_text(argv[argc-1]); 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( argc==3 ){ 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zArg = (const char *)sqlite3_value_text(argv[1]); 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pHash = (Fts3Hash *)sqlite3_user_data(context); 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !p ){ 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_error(context, zErr, -1); 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(zErr); 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pRet = Tcl_NewObj(); 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tcl_IncrRefCount(pRet); 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){ 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zErr = "error in xCreate()"; 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) goto finish; 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTokenizer->pModule = p; 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){ 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zErr = "error in xOpen()"; 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) goto finish; 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->pTokenizer = pTokenizer; 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zToken = &zInput[iStart]; 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nToken = iEnd-iStart; 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK!=p->xClose(pCsr) ){ 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zErr = "error in xClose()"; 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) goto finish; 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zErr = "error in xDestroy()"; 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) goto finish; 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)finish: 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( zErr ){ 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_error(context, zErr, -1); 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tcl_DecrRefCount(pRet); 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int registerTokenizer( 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3 *db, 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zName, 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const sqlite3_tokenizer_module *p 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pStmt); 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return sqlite3_finalize(pStmt); 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int queryTokenizer( 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3 *db, 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zName, 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const sqlite3_tokenizer_module **pp 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char zSql[] = "SELECT fts3_tokenizer(?)"; 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pp = 0; 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_ROW==sqlite3_step(pStmt) ){ 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return sqlite3_finalize(pStmt); 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Implementation of the scalar function fts3_tokenizer_internal_test(). 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function is used for testing only, it is not included in the 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** build unless SQLITE_TEST is defined. 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The purpose of this is to test that the fts3_tokenizer() function 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** can be used as designed by the C-code in the queryTokenizer and 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** registerTokenizer() functions above. These two functions are repeated 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** in the README.tokenizer file as an example, so it is important to 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** test them. 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** function with no arguments. An assert() will fail if a problem is 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** detected. i.e.: 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SELECT fts3_tokenizer_internal_test(); 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void intTestFunc( 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_context *context, 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int argc, 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **argv 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const sqlite3_tokenizer_module *p1; 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const sqlite3_tokenizer_module *p2; 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UNUSED_PARAMETER(argc); 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UNUSED_PARAMETER(argv); 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Test the query function */ 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3SimpleTokenizerModule(&p1); 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = queryTokenizer(db, "simple", &p2); 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc==SQLITE_OK ); 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( p1==p2 ); 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = queryTokenizer(db, "nosuchtokenizer", &p2); 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc==SQLITE_ERROR ); 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( p2==0 ); 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Test the storage function */ 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = registerTokenizer(db, "nosuchtokenizer", p1); 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc==SQLITE_OK ); 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = queryTokenizer(db, "nosuchtokenizer", &p2); 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc==SQLITE_OK ); 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( p2==p1 ); 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 4325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Set up SQL objects in database db used to access the contents of 4335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the hash table pointed to by argument pHash. The hash table must 4345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** been initialised to use string keys, and to take a private copy 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** of the key when a value is inserted. i.e. by a call similar to: 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function adds a scalar function (see header comment above 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** scalarFunc() in this file for details) and, if ENABLE_TABLE is 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** defined at compilation time, a temporary virtual table (see header 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** comment above struct HashTableVtab) to the database schema. Both 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** provide read/write access to the contents of *pHash. 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The third argument to this function, zName, is used as the name 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** of both the scalar and, if created, the virtual table. 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3InitHashTable( 4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3 *db, 4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Hash *pHash, 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zName 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *p = (void *)pHash; 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int any = SQLITE_ANY; 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef SQLITE_TEST 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zTest = 0; 4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zTest2 = 0; 4605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) void *pdb = (void *)db; 4615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zTest = sqlite3_mprintf("%s_test", zName); 4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zTest2 = sqlite3_mprintf("%s_internal_test", zName); 4635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !zTest || !zTest2 ){ 4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_NOMEM; 4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK==rc ){ 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK==rc ){ 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef SQLITE_TEST 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK==rc ){ 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0); 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK==rc ){ 4795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0); 4805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK==rc ){ 4825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); 4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef SQLITE_TEST 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(zTest); 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(zTest2); 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ 495