1/* 2** 2006 Oct 10 3** 4** The author disclaims copyright to this source code. In place of 5** a legal notice, here is a blessing: 6** 7** May you do good and not evil. 8** May you find forgiveness for yourself and forgive others. 9** May you share freely, never taking more than you give. 10** 11****************************************************************************** 12** 13** Implementation of the "simple" full-text-search tokenizer. 14*/ 15 16/* 17** The code in this file is only compiled if: 18** 19** * The FTS3 module is being built as an extension 20** (in which case SQLITE_CORE is not defined), or 21** 22** * The FTS3 module is being built into the core of 23** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). 24*/ 25#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) 26 27#include "fts3Int.h" 28 29#include <assert.h> 30#include <stdlib.h> 31#include <stdio.h> 32#include <string.h> 33 34#include "fts3_tokenizer.h" 35 36typedef struct simple_tokenizer { 37 sqlite3_tokenizer base; 38 char delim[128]; /* flag ASCII delimiters */ 39} simple_tokenizer; 40 41typedef struct simple_tokenizer_cursor { 42 sqlite3_tokenizer_cursor base; 43 const char *pInput; /* input we are tokenizing */ 44 int nBytes; /* size of the input */ 45 int iOffset; /* current position in pInput */ 46 int iToken; /* index of next token to be returned */ 47 char *pToken; /* storage for current token */ 48 int nTokenAllocated; /* space allocated to zToken buffer */ 49} simple_tokenizer_cursor; 50 51 52static int simpleDelim(simple_tokenizer *t, unsigned char c){ 53 return c<0x80 && t->delim[c]; 54} 55static int fts3_isalnum(int x){ 56 return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); 57} 58 59/* 60** Create a new tokenizer instance. 61*/ 62static int simpleCreate( 63 int argc, const char * const *argv, 64 sqlite3_tokenizer **ppTokenizer 65){ 66 simple_tokenizer *t; 67 68 t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); 69 if( t==NULL ) return SQLITE_NOMEM; 70 memset(t, 0, sizeof(*t)); 71 72 /* TODO(shess) Delimiters need to remain the same from run to run, 73 ** else we need to reindex. One solution would be a meta-table to 74 ** track such information in the database, then we'd only want this 75 ** information on the initial create. 76 */ 77 if( argc>1 ){ 78 int i, n = (int)strlen(argv[1]); 79 for(i=0; i<n; i++){ 80 unsigned char ch = argv[1][i]; 81 /* We explicitly don't support UTF-8 delimiters for now. */ 82 if( ch>=0x80 ){ 83 sqlite3_free(t); 84 return SQLITE_ERROR; 85 } 86 t->delim[ch] = 1; 87 } 88 } else { 89 /* Mark non-alphanumeric ASCII characters as delimiters */ 90 int i; 91 for(i=1; i<0x80; i++){ 92 t->delim[i] = !fts3_isalnum(i) ? -1 : 0; 93 } 94 } 95 96 *ppTokenizer = &t->base; 97 return SQLITE_OK; 98} 99 100/* 101** Destroy a tokenizer 102*/ 103static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ 104 sqlite3_free(pTokenizer); 105 return SQLITE_OK; 106} 107 108/* 109** Prepare to begin tokenizing a particular string. The input 110** string to be tokenized is pInput[0..nBytes-1]. A cursor 111** used to incrementally tokenize this string is returned in 112** *ppCursor. 113*/ 114static int simpleOpen( 115 sqlite3_tokenizer *pTokenizer, /* The tokenizer */ 116 const char *pInput, int nBytes, /* String to be tokenized */ 117 sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ 118){ 119 simple_tokenizer_cursor *c; 120 121 UNUSED_PARAMETER(pTokenizer); 122 123 c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); 124 if( c==NULL ) return SQLITE_NOMEM; 125 126 c->pInput = pInput; 127 if( pInput==0 ){ 128 c->nBytes = 0; 129 }else if( nBytes<0 ){ 130 c->nBytes = (int)strlen(pInput); 131 }else{ 132 c->nBytes = nBytes; 133 } 134 c->iOffset = 0; /* start tokenizing at the beginning */ 135 c->iToken = 0; 136 c->pToken = NULL; /* no space allocated, yet. */ 137 c->nTokenAllocated = 0; 138 139 *ppCursor = &c->base; 140 return SQLITE_OK; 141} 142 143/* 144** Close a tokenization cursor previously opened by a call to 145** simpleOpen() above. 146*/ 147static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ 148 simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 149 sqlite3_free(c->pToken); 150 sqlite3_free(c); 151 return SQLITE_OK; 152} 153 154/* 155** Extract the next token from a tokenization cursor. The cursor must 156** have been opened by a prior call to simpleOpen(). 157*/ 158static int simpleNext( 159 sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ 160 const char **ppToken, /* OUT: *ppToken is the token text */ 161 int *pnBytes, /* OUT: Number of bytes in token */ 162 int *piStartOffset, /* OUT: Starting offset of token */ 163 int *piEndOffset, /* OUT: Ending offset of token */ 164 int *piPosition /* OUT: Position integer of token */ 165){ 166 simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 167 simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; 168 unsigned char *p = (unsigned char *)c->pInput; 169 170 while( c->iOffset<c->nBytes ){ 171 int iStartOffset; 172 173 /* Scan past delimiter characters */ 174 while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){ 175 c->iOffset++; 176 } 177 178 /* Count non-delimiter characters. */ 179 iStartOffset = c->iOffset; 180 while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){ 181 c->iOffset++; 182 } 183 184 if( c->iOffset>iStartOffset ){ 185 int i, n = c->iOffset-iStartOffset; 186 if( n>c->nTokenAllocated ){ 187 char *pNew; 188 c->nTokenAllocated = n+20; 189 pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); 190 if( !pNew ) return SQLITE_NOMEM; 191 c->pToken = pNew; 192 } 193 for(i=0; i<n; i++){ 194 /* TODO(shess) This needs expansion to handle UTF-8 195 ** case-insensitivity. 196 */ 197 unsigned char ch = p[iStartOffset+i]; 198 c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); 199 } 200 *ppToken = c->pToken; 201 *pnBytes = n; 202 *piStartOffset = iStartOffset; 203 *piEndOffset = c->iOffset; 204 *piPosition = c->iToken++; 205 206 return SQLITE_OK; 207 } 208 } 209 return SQLITE_DONE; 210} 211 212/* 213** The set of routines that implement the simple tokenizer 214*/ 215static const sqlite3_tokenizer_module simpleTokenizerModule = { 216 0, 217 simpleCreate, 218 simpleDestroy, 219 simpleOpen, 220 simpleClose, 221 simpleNext, 222}; 223 224/* 225** Allocate a new simple tokenizer. Return a pointer to the new 226** tokenizer in *ppModule 227*/ 228void sqlite3Fts3SimpleTokenizerModule( 229 sqlite3_tokenizer_module const**ppModule 230){ 231 *ppModule = &simpleTokenizerModule; 232} 233 234#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ 235