1/* 2** The author disclaims copyright to this source code. 3** 4************************************************************************* 5** Implementation of the "simple" full-text-search tokenizer. 6*/ 7 8/* 9** The code in this file is only compiled if: 10** 11** * The FTS1 module is being built as an extension 12** (in which case SQLITE_CORE is not defined), or 13** 14** * The FTS1 module is being built into the core of 15** SQLite (in which case SQLITE_ENABLE_FTS1 is defined). 16*/ 17#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) 18 19 20#include <assert.h> 21#include <stdlib.h> 22#include <stdio.h> 23#include <string.h> 24#include <ctype.h> 25 26#include "fts1_tokenizer.h" 27 28typedef struct simple_tokenizer { 29 sqlite3_tokenizer base; 30 char delim[128]; /* flag ASCII delimiters */ 31} simple_tokenizer; 32 33typedef struct simple_tokenizer_cursor { 34 sqlite3_tokenizer_cursor base; 35 const char *pInput; /* input we are tokenizing */ 36 int nBytes; /* size of the input */ 37 int iOffset; /* current position in pInput */ 38 int iToken; /* index of next token to be returned */ 39 char *pToken; /* storage for current token */ 40 int nTokenAllocated; /* space allocated to zToken buffer */ 41} simple_tokenizer_cursor; 42 43 44/* Forward declaration */ 45static const sqlite3_tokenizer_module simpleTokenizerModule; 46 47static int isDelim(simple_tokenizer *t, unsigned char c){ 48 return c<0x80 && t->delim[c]; 49} 50 51/* 52** Create a new tokenizer instance. 53*/ 54static int simpleCreate( 55 int argc, const char * const *argv, 56 sqlite3_tokenizer **ppTokenizer 57){ 58 simple_tokenizer *t; 59 60 t = (simple_tokenizer *) calloc(sizeof(*t), 1); 61 if( t==NULL ) return SQLITE_NOMEM; 62 63 /* TODO(shess) Delimiters need to remain the same from run to run, 64 ** else we need to reindex. One solution would be a meta-table to 65 ** track such information in the database, then we'd only want this 66 ** information on the initial create. 67 */ 68 if( argc>1 ){ 69 int i, n = strlen(argv[1]); 70 for(i=0; i<n; i++){ 71 unsigned char ch = argv[1][i]; 72 /* We explicitly don't support UTF-8 delimiters for now. */ 73 if( ch>=0x80 ){ 74 free(t); 75 return SQLITE_ERROR; 76 } 77 t->delim[ch] = 1; 78 } 79 } else { 80 /* Mark non-alphanumeric ASCII characters as delimiters */ 81 int i; 82 for(i=1; i<0x80; i++){ 83 t->delim[i] = !isalnum(i); 84 } 85 } 86 87 *ppTokenizer = &t->base; 88 return SQLITE_OK; 89} 90 91/* 92** Destroy a tokenizer 93*/ 94static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ 95 free(pTokenizer); 96 return SQLITE_OK; 97} 98 99/* 100** Prepare to begin tokenizing a particular string. The input 101** string to be tokenized is pInput[0..nBytes-1]. A cursor 102** used to incrementally tokenize this string is returned in 103** *ppCursor. 104*/ 105static int simpleOpen( 106 sqlite3_tokenizer *pTokenizer, /* The tokenizer */ 107 const char *pInput, int nBytes, /* String to be tokenized */ 108 sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ 109){ 110 simple_tokenizer_cursor *c; 111 112 c = (simple_tokenizer_cursor *) malloc(sizeof(*c)); 113 if( c==NULL ) return SQLITE_NOMEM; 114 115 c->pInput = pInput; 116 if( pInput==0 ){ 117 c->nBytes = 0; 118 }else if( nBytes<0 ){ 119 c->nBytes = (int)strlen(pInput); 120 }else{ 121 c->nBytes = nBytes; 122 } 123 c->iOffset = 0; /* start tokenizing at the beginning */ 124 c->iToken = 0; 125 c->pToken = NULL; /* no space allocated, yet. */ 126 c->nTokenAllocated = 0; 127 128 *ppCursor = &c->base; 129 return SQLITE_OK; 130} 131 132/* 133** Close a tokenization cursor previously opened by a call to 134** simpleOpen() above. 135*/ 136static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ 137 simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 138 free(c->pToken); 139 free(c); 140 return SQLITE_OK; 141} 142 143/* 144** Extract the next token from a tokenization cursor. The cursor must 145** have been opened by a prior call to simpleOpen(). 146*/ 147static int simpleNext( 148 sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ 149 const char **ppToken, /* OUT: *ppToken is the token text */ 150 int *pnBytes, /* OUT: Number of bytes in token */ 151 int *piStartOffset, /* OUT: Starting offset of token */ 152 int *piEndOffset, /* OUT: Ending offset of token */ 153 int *piPosition /* OUT: Position integer of token */ 154){ 155 simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; 156 simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; 157 unsigned char *p = (unsigned char *)c->pInput; 158 159 while( c->iOffset<c->nBytes ){ 160 int iStartOffset; 161 162 /* Scan past delimiter characters */ 163 while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){ 164 c->iOffset++; 165 } 166 167 /* Count non-delimiter characters. */ 168 iStartOffset = c->iOffset; 169 while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){ 170 c->iOffset++; 171 } 172 173 if( c->iOffset>iStartOffset ){ 174 int i, n = c->iOffset-iStartOffset; 175 if( n>c->nTokenAllocated ){ 176 c->nTokenAllocated = n+20; 177 c->pToken = realloc(c->pToken, c->nTokenAllocated); 178 if( c->pToken==NULL ) return SQLITE_NOMEM; 179 } 180 for(i=0; i<n; i++){ 181 /* TODO(shess) This needs expansion to handle UTF-8 182 ** case-insensitivity. 183 */ 184 unsigned char ch = p[iStartOffset+i]; 185 c->pToken[i] = ch<0x80 ? tolower(ch) : ch; 186 } 187 *ppToken = c->pToken; 188 *pnBytes = n; 189 *piStartOffset = iStartOffset; 190 *piEndOffset = c->iOffset; 191 *piPosition = c->iToken++; 192 193 return SQLITE_OK; 194 } 195 } 196 return SQLITE_DONE; 197} 198 199/* 200** The set of routines that implement the simple tokenizer 201*/ 202static const sqlite3_tokenizer_module simpleTokenizerModule = { 203 0, 204 simpleCreate, 205 simpleDestroy, 206 simpleOpen, 207 simpleClose, 208 simpleNext, 209}; 210 211/* 212** Allocate a new simple tokenizer. Return a pointer to the new 213** tokenizer in *ppModule 214*/ 215void sqlite3Fts1SimpleTokenizerModule( 216 sqlite3_tokenizer_module const**ppModule 217){ 218 *ppModule = &simpleTokenizerModule; 219} 220 221#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */ 222