15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2006 Oct 10 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The author disclaims copyright to this source code. In place of 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a legal notice, here is a blessing: 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you do good and not evil. 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you find forgiveness for yourself and forgive others. 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you share freely, never taking more than you give. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)****************************************************************************** 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Implementation of the "simple" full-text-search tokenizer. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The code in this file is only compiled if: 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** * The FTS2 module is being built as an extension 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** (in which case SQLITE_CORE is not defined), or 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** * The FTS2 module is being built into the core of 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SQLite (in which case SQLITE_ENABLE_FTS2 is defined). 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <assert.h> 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdlib.h> 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdio.h> 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string.h> 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fts2_tokenizer.h" 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef struct simple_tokenizer { 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer base; 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char delim[128]; /* flag ASCII delimiters */ 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} simple_tokenizer; 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef struct simple_tokenizer_cursor { 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_cursor base; 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *pInput; /* input we are tokenizing */ 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nBytes; /* size of the input */ 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iOffset; /* current position in pInput */ 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iToken; /* index of next token to be returned */ 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pToken; /* storage for current token */ 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTokenAllocated; /* space allocated to zToken buffer */ 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} simple_tokenizer_cursor; 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* Forward declaration */ 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const sqlite3_tokenizer_module simpleTokenizerModule; 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int simpleDelim(simple_tokenizer *t, unsigned char c){ 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return c<0x80 && t->delim[c]; 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Create a new tokenizer instance. 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int simpleCreate( 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int argc, const char * const *argv, 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer **ppTokenizer 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simple_tokenizer *t; 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( t==NULL ) return SQLITE_NOMEM; 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(t, 0, sizeof(*t)); 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* TODO(shess) Delimiters need to remain the same from run to run, 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** else we need to reindex. One solution would be a meta-table to 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** track such information in the database, then we'd only want this 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** information on the initial create. 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( argc>1 ){ 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i, n = strlen(argv[1]); 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<n; i++){ 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) unsigned char ch = argv[1][i]; 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* We explicitly don't support UTF-8 delimiters for now. */ 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( ch>=0x80 ){ 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(t); 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_ERROR; 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t->delim[ch] = 1; 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } else { 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Mark non-alphanumeric ASCII characters as delimiters */ 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=1; i<0x80; i++){ 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') || 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (i>='a' && i<='z')); 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppTokenizer = &t->base; 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Destroy a tokenizer 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pTokenizer); 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Prepare to begin tokenizing a particular string. The input 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** string to be tokenized is pInput[0..nBytes-1]. A cursor 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** used to incrementally tokenize this string is returned in 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** *ppCursor. 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int simpleOpen( 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer *pTokenizer, /* The tokenizer */ 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *pInput, int nBytes, /* String to be tokenized */ 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simple_tokenizer_cursor *c; 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( c==NULL ) return SQLITE_NOMEM; 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->pInput = pInput; 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pInput==0 ){ 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->nBytes = 0; 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else if( nBytes<0 ){ 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->nBytes = (int)strlen(pInput); 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->nBytes = nBytes; 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->iOffset = 0; /* start tokenizing at the beginning */ 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->iToken = 0; 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->pToken = NULL; /* no space allocated, yet. */ 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->nTokenAllocated = 0; 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppCursor = &c->base; 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Close a tokenization cursor previously opened by a call to 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** simpleOpen() above. 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(c->pToken); 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(c); 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Extract the next token from a tokenization cursor. The cursor must 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** have been opened by a prior call to simpleOpen(). 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int simpleNext( 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char **ppToken, /* OUT: *ppToken is the token text */ 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pnBytes, /* OUT: Number of bytes in token */ 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *piStartOffset, /* OUT: Starting offset of token */ 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *piEndOffset, /* OUT: Ending offset of token */ 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *piPosition /* OUT: Position integer of token */ 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) unsigned char *p = (unsigned char *)c->pInput; 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( c->iOffset<c->nBytes ){ 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iStartOffset; 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Scan past delimiter characters */ 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){ 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->iOffset++; 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Count non-delimiter characters. */ 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iStartOffset = c->iOffset; 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){ 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->iOffset++; 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( c->iOffset>iStartOffset ){ 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i, n = c->iOffset-iStartOffset; 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( n>c->nTokenAllocated ){ 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->nTokenAllocated = n+20; 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated); 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( c->pToken==NULL ) return SQLITE_NOMEM; 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<n; i++){ 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* TODO(shess) This needs expansion to handle UTF-8 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** case-insensitivity. 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) unsigned char ch = p[iStartOffset+i]; 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch; 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppToken = c->pToken; 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnBytes = n; 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *piStartOffset = iStartOffset; 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *piEndOffset = c->iOffset; 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *piPosition = c->iToken++; 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_DONE; 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The set of routines that implement the simple tokenizer 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static const sqlite3_tokenizer_module simpleTokenizerModule = { 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 0, 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simpleCreate, 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simpleDestroy, 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simpleOpen, 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simpleClose, 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simpleNext, 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Allocate a new simple tokenizer. Return a pointer to the new 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** tokenizer in *ppModule 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts2SimpleTokenizerModule( 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_module const**ppModule 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppModule = &simpleTokenizerModule; 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */ 231