15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2009 Oct 23 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The author disclaims copyright to this source code. In place of 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a legal notice, here is a blessing: 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you do good and not evil. 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you find forgiveness for yourself and forgive others. 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** May you share freely, never taking more than you give. 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)****************************************************************************** 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This file is part of the SQLite FTS3 extension module. Specifically, 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** this file contains code to insert, update and delete rows from FTS3 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** tables. It also contains code to merge FTS3 b-tree segments. Some 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** of the sub-routines used to merge segments are also used by the query 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** code in fts3.c. 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include "fts3Int.h" 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <string.h> 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <assert.h> 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#include <stdlib.h> 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** When full-text index nodes are loaded from disk, the buffer that they 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** are loaded into has the following number of bytes of padding at the end 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** of 920 bytes is allocated for it. 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This means that if we have a pointer into a buffer containing node data, 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** it is always safe to read up to two varints from it without risking an 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** overread, even if the node data is corrupted. 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef struct PendingList PendingList; 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef struct SegmentNode SegmentNode; 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)typedef struct SegmentWriter SegmentWriter; 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Data structure used while accumulating terms in the pending-terms hash 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** table. The hash table entry maps from term (a string) to a malloc'd 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** instance of this structure. 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct PendingList { 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nData; 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aData; 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSpace; 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLastDocid; 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLastCol; 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLastPos; 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Each cursor has a (possibly empty) linked list of the following objects. 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct Fts3DeferredToken { 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iCol; /* Column token must occur in */ 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PendingList *pList; /* Doclist is assembled here */ 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** An instance of this structure is used to iterate through the terms on 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a contiguous set of segment b-tree leaf nodes. Although the details of 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** this structure are only manipulated by code in this file, opaque handles 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** of type Fts3SegReader* are also used by code in fts3.c to iterate through 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** terms when querying the full-text index. See functions: 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** sqlite3Fts3SegReaderNew() 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** sqlite3Fts3SegReaderFree() 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** sqlite3Fts3SegReaderCost() 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** sqlite3Fts3SegReaderIterate() 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Methods used to manipulate Fts3SegReader structures: 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3SegReaderNext() 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3SegReaderFirstDocid() 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3SegReaderNextDocid() 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct Fts3SegReader { 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aNode; /* Pointer to node data (or NULL) */ 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nNode; /* Size of buffer at aNode (or 0) */ 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3HashElem **ppNextElem; 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Variables set by fts3SegReaderNext(). These may be read directly 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** by the caller. They are valid from the time SegmentReaderNew() returns 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** until SegmentReaderNext() returns something other than SQLITE_OK 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** (i.e. SQLITE_DONE). 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm; /* Number of bytes in current term */ 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zTerm; /* Pointer to current term */ 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTermAlloc; /* Allocated size of zTerm buffer */ 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aDoclist; /* Pointer to doclist of current entry */ 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nDoclist; /* Size of doclist in current entry */ 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* The following variables are used to iterate through the current doclist */ 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pOffsetList; 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDocid; 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** An instance of this structure is used to create a segment b-tree in the 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** database. The internal details of this type are only accessed by the 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** following functions: 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3SegWriterAdd() 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3SegWriterFlush() 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3SegWriterFree() 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct SegmentWriter { 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pTree; /* Pointer to interior tree structure */ 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iFirst; /* First slot in %_segments written */ 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iFree; /* Next free slot in %_segments */ 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zTerm; /* Pointer to previous term buffer */ 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm; /* Number of bytes in zTerm */ 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nMalloc; /* Size of malloc'd buffer at zMalloc */ 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSize; /* Size of allocation at aData */ 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nData; /* Bytes of data in aData */ 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aData; /* Pointer to block from malloc() */ 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Type SegmentNode is used by the following three functions to create 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the interior part of the segment b+-tree structures (everything except 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the leaf nodes). These functions and type are only ever used by code 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** within the fts3SegWriterXXX() family of functions described above. 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3NodeAddTerm() 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3NodeWrite() 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3NodeFree() 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)struct SegmentNode { 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pParent; /* Parent node (or NULL for root node) */ 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pRight; /* Pointer to right-sibling */ 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nEntry; /* Number of terms written to node so far */ 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zTerm; /* Pointer to previous term buffer */ 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm; /* Number of bytes in zTerm */ 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nMalloc; /* Size of malloc'd buffer at zMalloc */ 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nData; /* Bytes of valid data so far */ 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aData; /* Node data */ 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)}; 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Valid values for the second argument to fts3SqlStmt(). 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_CONTENT 0 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_IS_EMPTY 1 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_ALL_CONTENT 2 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_ALL_SEGMENTS 3 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_ALL_SEGDIR 4 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_ALL_DOCSIZE 5 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_ALL_STAT 6 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_SELECT_CONTENT_BY_ROWID 7 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_NEXT_SEGMENT_INDEX 8 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_INSERT_SEGMENTS 9 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_NEXT_SEGMENTS_ID 10 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_INSERT_SEGDIR 11 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_SELECT_LEVEL 12 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_SELECT_ALL_LEVEL 13 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_SELECT_LEVEL_COUNT 14 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_SELECT_SEGDIR_COUNT_MAX 15 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_SEGDIR_BY_LEVEL 16 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_SEGMENTS_RANGE 17 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_CONTENT_INSERT 18 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_DELETE_DOCSIZE 19 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_REPLACE_DOCSIZE 20 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_SELECT_DOCSIZE 21 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_SELECT_DOCTOTAL 22 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#define SQL_REPLACE_DOCTOTAL 23 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function is used to obtain an SQLite prepared statement handle 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** for the statement identified by the second argument. If successful, 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** *pp is set to the requested statement handle and SQLITE_OK returned. 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Otherwise, an SQLite error code is returned and *pp is set to 0. 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If argument apVal is not NULL, then it must point to an array with 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** at least as many entries as the requested statement has bound 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** parameters. The values are bound to the statements parameters before 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** returning. 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SqlStmt( 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int eStmt, /* One of the SQL_XXX constants above */ 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt **pp, /* OUT: Statement handle */ 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **apVal /* Values to bind to statement */ 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *azSql[] = { 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 2 */ "DELETE FROM %Q.'%q_content'", 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 3 */ "DELETE FROM %Q.'%q_segments'", 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 4 */ "DELETE FROM %Q.'%q_segdir'", 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 5 */ "DELETE FROM %Q.'%q_docsize'", 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 6 */ "DELETE FROM %Q.'%q_stat'", 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Return segments in order from oldest to newest.*/ 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }; 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pStmt = p->aStmt[eStmt]; 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pStmt ){ 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zSql; 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( eStmt==SQL_CONTENT_INSERT ){ 2495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); 2505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ 2515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); 2525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 2535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !zSql ){ 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_NOMEM; 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(zSql); 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc==SQLITE_OK || pStmt==0 ); 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->aStmt[eStmt] = pStmt; 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( apVal ){ 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nParam = sqlite3_bind_parameter_count(pStmt); 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; rc==SQLITE_OK && i<nParam; i++){ 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_bind_value(pStmt, i+1, apVal[i]); 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pp = pStmt; 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SelectDocsize( 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *pTab, /* FTS3 table handle */ 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt **ppStmt /* OUT: Statement handle */ 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return code */ 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL ); 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( eStmt==SQL_SELECT_DOCSIZE ){ 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pStmt, 1, iDocid); 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_step(pStmt); 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pStmt = 0; 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_OK; 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppStmt = pStmt; 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3SelectDoctotal( 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *pTab, /* Fts3 table handle */ 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt **ppStmt /* OUT: Statement handle */ 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt); 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3SelectDocsize( 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *pTab, /* Fts3 table handle */ 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDocid, /* Docid to read size data for */ 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt **ppStmt /* OUT: Statement handle */ 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt); 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Similar to fts3SqlStmt(). Except, after binding the parameters in 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** array apVal[] to the SQL statement identified by eStmt, the statement 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** is executed. 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Returns SQLITE_OK if the statement is successfully executed, or an 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SQLite error code otherwise. 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3SqlExec( 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pRC, /* Result code */ 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* The FTS3 table */ 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int eStmt, /* Index of statement to evaluate */ 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **apVal /* Parameters to bind */ 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( *pRC ) return; 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pStmt); 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = rc; 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function ensures that the caller has obtained a shared-cache 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** table-lock on the %_content table. This is required before reading 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** data from the fts3 table. If this lock is not acquired first, then 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the caller may end up holding read-locks on the %_segments and %_segdir 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** tables, but no read-lock on the %_content table. If this happens 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a second connection will be able to write to the fts3 table, but 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** attempting to commit those writes might return SQLITE_LOCKED or 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** write-locks on the %_segments and %_segdir ** tables). 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** We try to avoid this because if FTS3 returns any error when committing 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a transaction, the whole transaction will be rolled back. And this is 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** still happen if the user reads data directly from the %_segments or 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** %_segdir tables instead of going through FTS3 though. 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3ReadLock(Fts3Table *p){ 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return code */ 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; /* Statement used to obtain lock */ 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_null(pStmt, 1); 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pStmt); 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Set *ppStmt to a statement handle that may be used to iterate through 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** all rows in the %_segdir table, from oldest to newest. If successful, 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** return SQLITE_OK. If an error occurs while preparing the statement, 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** return an SQLite error code. 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** There is only ever one instance of this SQL statement compiled for 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** each FTS3 table. 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The statement returns the following columns from the %_segdir table: 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 0: idx 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 1: start_block 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2: leaves_end_block 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3: end_block 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 4: root 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt = 0; 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iLevel<0 ){ 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppStmt = pStmt; 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Append a single varint to a PendingList buffer. SQLITE_OK is returned 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** if successful, or an SQLite error code otherwise. 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function also serves to allocate the PendingList structure itself. 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** For example, to create a new PendingList structure containing two 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** varints: 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** PendingList *p = 0; 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3PendingListAppendVarint(&p, 1); 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3PendingListAppendVarint(&p, 2); 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3PendingListAppendVarint( 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 i /* Value to append to data */ 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PendingList *p = *pp; 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Allocate or grow the PendingList as required. */ 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !p ){ 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p = sqlite3_malloc(sizeof(*p) + 100); 4285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !p ){ 4295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 4305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nSpace = 100; 4325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->aData = (char *)&p[1]; 4335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nData = 0; 4345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nNew = p->nSpace * 2; 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p = sqlite3_realloc(p, sizeof(*p) + nNew); 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !p ){ 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(*pp); 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pp = 0; 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nSpace = nNew; 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->aData = (char *)&p[1]; 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Append the new serialized varint to the end of the list. */ 4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); 4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->aData[p->nData] = '\0'; 4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pp = p; 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Add a docid/column/position entry to a PendingList structure. Non-zero 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** is returned if the structure is sqlite3_realloced as part of adding 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the entry. Otherwise, zero. 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. 4605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Zero is always returned in this case. Otherwise, if no OOM error occurs, 4615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** it is set to SQLITE_OK. 4625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 4635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3PendingListAppend( 4645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PendingList **pp, /* IN/OUT: PendingList structure */ 4655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDocid, /* Docid for entry to add */ 4665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iCol, /* Column for entry to add */ 4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iPos, /* Position of term for entry to add */ 4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pRc /* OUT: Return code */ 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PendingList *p = *pp; 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( !p || p->iLastDocid<=iDocid ); 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !p || p->iLastDocid!=iDocid ){ 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p ){ 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( p->nData<p->nSpace ); 4795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( p->aData[p->nData]==0 ); 4805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nData++; 4815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ 4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) goto pendinglistappend_out; 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->iLastCol = -1; 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->iLastPos = 0; 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->iLastDocid = iDocid; 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iCol>0 && p->iLastCol!=iCol ){ 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 4935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) goto pendinglistappend_out; 4945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->iLastCol = iCol; 4965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->iLastPos = 0; 4975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 4985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iCol>=0 ){ 4995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); 5005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); 5015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 5025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->iLastPos = iPos; 5035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pendinglistappend_out: 5075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRc = rc; 5085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p!=*pp ){ 5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pp = p; 5105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 1; 5115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 5135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 5165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Tokenize the nul-terminated string zText and add all tokens to the 5175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** pending-terms hash-table. The docid used is that currently stored in 5185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** p->iPrevDocid, and the column is specified by argument iCol. 5195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 5205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. 5215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 5225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3PendingTermsAdd( 5235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Table into which text will be inserted */ 5245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zText, /* Text of document to be inserted */ 5255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iCol, /* Column into which text is being inserted */ 5265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *pnWord /* OUT: Number of tokens inserted */ 5275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 5285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iStart; 5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iEnd; 5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iPos; 5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nWord = 0; 5335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char const *zToken; 5355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nToken; 5365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer *pTokenizer = p->pTokenizer; 5385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; 5395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_cursor *pCsr; 5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int (*xNext)(sqlite3_tokenizer_cursor *pCursor, 5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char**,int*,int*,int*,int*); 5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pTokenizer && pModule ); 5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); 5465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 5475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 5485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->pTokenizer = pTokenizer; 5505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) xNext = pModule->xNext; 5525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( SQLITE_OK==rc 5535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) 5545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PendingList *pList; 5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iPos>=nWord ) nWord = iPos+1; 5585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Positions cannot be negative; we use -1 as a terminator internally. 5605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** Tokens must have a non-zero length. 5615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 5625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iPos<0 || !zToken || nToken<=0 ){ 5635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_ERROR; 5645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 5655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); 5685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pList ){ 5695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); 5705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ 5725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ 5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Malloc failed while inserting the new entry. This can only 5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** happen if there was no previous entry for this token. 5755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 5765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); 5775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pList); 5785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_NOMEM; 5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 5825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); 5835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 5855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pModule->xClose(pCsr); 5875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnWord = nWord; 5885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (rc==SQLITE_DONE ? SQLITE_OK : rc); 5895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 5905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 5925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Calling this function indicates that subsequent calls to 5935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3PendingTermsAdd() are to add term/position-list pairs for the 5945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** contents of the document with docid iDocid. 5955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 5965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ 5975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* TODO(shess) Explore whether partially flushing the buffer on 5985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** forced-flush would provide better performance. I suspect that if 5995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** we ordered the doclists by size and flushed the largest until the 6005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** buffer was half empty, that would let the less frequent terms 6015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** generate longer doclists. 6025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 6035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ 6045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = sqlite3Fts3PendingTermsFlush(p); 6055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 6065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->iPrevDocid = iDocid; 6085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 6095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 6125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Discard the contents of the pending-terms hash table. 6135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 6145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts3PendingTermsClear(Fts3Table *p){ 6155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3HashElem *pElem; 6165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ 6175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(fts3HashData(pElem)); 6185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3HashClear(&p->pendingTerms); 6205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nPendingData = 0; 6215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 6245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function is called by the xUpdate() method as part of an INSERT 6255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** operation. It adds entries for each term in the new record to the 6265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** pendingTerms hash table. 6275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 6285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Argument apVal is the same as the similarly named argument passed to 6295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3InsertData(). Parameter iDocid is the docid of the new row. 6305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 6315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ 6325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; /* Iterator variable */ 6335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=2; i<p->nColumn+2; i++){ 6345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zText = (const char *)sqlite3_value_text(apVal[i]); 6355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( zText ){ 6365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); 6375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 6385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 6395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); 6425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 6445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 6455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 6475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function is called by the xUpdate() method for an INSERT operation. 6485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The apVal parameter is passed a copy of the apVal argument passed by 6495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SQLite to the xUpdate() method. i.e: 6505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 6515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** apVal[0] Not used for INSERT. 6525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** apVal[1] rowid 6535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** apVal[2] Left-most user-defined column 6545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** ... 6555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** apVal[p->nColumn+1] Right-most user-defined column 6565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** apVal[p->nColumn+2] Hidden column with same name as table 6575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) 6585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 6595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3InsertData( 6605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Full-text table */ 6615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **apVal, /* Array of values to insert */ 6625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ 6635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 6645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return code */ 6655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ 6665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Locate the statement handle used to insert data into the %_content 6685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** table. The SQL for this statement is: 6695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 6705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** INSERT INTO %_content VALUES(?, ?, ?, ...) 6715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 6725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** The statement features N '?' variables, where N is the number of user 6735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** defined columns in the FTS3 table, plus one for the docid field. 6745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 6755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); 6765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 6775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 6785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* There is a quirk here. The users INSERT statement may have specified 6815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** a value for the "rowid" field, for the "docid" field, or for both. 6825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** Which is a problem, since "rowid" and "docid" are aliases for the 6835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** same value. For example: 6845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 6855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); 6865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 6875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** In FTS3, this is an error. It is an error to specify non-NULL values 6885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** for both docid and some other rowid alias. 6895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 6905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ 6915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_NULL==sqlite3_value_type(apVal[0]) 6925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && SQLITE_NULL!=sqlite3_value_type(apVal[1]) 6935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 6945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* A rowid/docid conflict. */ 6955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_ERROR; 6965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 6975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); 6985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 6995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Execute the statement to insert the record. Set *piDocid to the 7025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** new docid value. 7035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 7045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pContentInsert); 7055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pContentInsert); 7065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *piDocid = sqlite3_last_insert_rowid(p->db); 7085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 7095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 7105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 7145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Remove all data from the FTS3 table. Clear the hash table containing 7155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** pending terms. 7165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 7175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3DeleteAll(Fts3Table *p){ 7185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; /* Return code */ 7195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Discard the contents of the pending-terms hash table. */ 7215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3PendingTermsClear(p); 7225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Delete everything from the %_content, %_segments and %_segdir tables. */ 7245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); 7255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); 7265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); 7275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->bHasDocsize ){ 7285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); 7295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->bHasStat ){ 7315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); 7325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 7345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 7355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 7375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The first element in the apVal[] array is assumed to contain the docid 7385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** (an integer) of a row about to be deleted. Remove all terms from the 7395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** full-text index. 7405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 7415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3DeleteTerms( 7425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pRC, /* Result code */ 7435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* The FTS table to delete from */ 7445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ 7455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *aSz /* Sizes of deleted document written here */ 7465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 7475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 7485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pSelect; 7495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( *pRC ) return; 7515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); 7525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 7535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_ROW==sqlite3_step(pSelect) ){ 7545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; 7555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=1; i<=p->nColumn; i++){ 7565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zText = (const char *)sqlite3_column_text(pSelect, i); 7575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); 7585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 7595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_reset(pSelect); 7605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = rc; 7615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 7625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); 7645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pSelect); 7675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 7685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_reset(pSelect); 7695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 7705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = rc; 7715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 7725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 7745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Forward declaration to account for the circular dependency between 7755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). 7765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 7775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegmentMerge(Fts3Table *, int); 7785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 7805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function allocates a new level iLevel index in the segdir table. 7815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Usually, indexes are allocated within a level sequentially starting 7825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** with 0, so the allocated index is one greater than the value returned 7835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** by: 7845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 7855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SELECT max(idx) FROM %_segdir WHERE level = :iLevel 7865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 7875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** However, if there are already FTS3_MERGE_COUNT indexes at the requested 7885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** level, they are merged into a single level (iLevel+1) segment and the 7895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** allocated index is 0. 7905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 7915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If successful, *piIdx is set to the allocated index slot and SQLITE_OK 7925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** returned. Otherwise, an SQLite error code is returned. 7935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 7945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ 7955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return Code */ 7965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ 7975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iNext = 0; /* Result of query pNextIdx */ 7985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Set variable iNext to the next available segdir index at level iLevel. */ 8005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); 8015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 8025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int(pNextIdx, 1, iLevel); 8035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ 8045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iNext = sqlite3_column_int(pNextIdx, 0); 8055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pNextIdx); 8075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 8105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already 8115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** full, merge all segments in level iLevel into a single iLevel+1 8125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, 8135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. 8145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 8155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iNext>=FTS3_MERGE_COUNT ){ 8165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SegmentMerge(p, iLevel); 8175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *piIdx = 0; 8185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 8195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *piIdx = iNext; 8205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 8245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 8255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 8275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The %_segments table is declared as follows: 8285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 8295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) 8305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 8315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function reads data from a single row of the %_segments table. The 8325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** specific row is identified by the iBlockid parameter. If paBlob is not 8335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** NULL, then a buffer is allocated using sqlite3_malloc() and populated 8345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** with the contents of the blob stored in the "block" column of the 8355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** identified table row is. Whether or not paBlob is NULL, *pnBlob is set 8365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** to the size of the blob in bytes before returning. 8375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 8385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If an error occurs, or the table does not contain the specified row, 8395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If 8405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** paBlob is non-NULL, then it is the responsibility of the caller to 8415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** eventually free the returned buffer. 8425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 8435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function may leave an open sqlite3_blob* handle in the 8445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Fts3Table.pSegments variable. This handle is reused by subsequent calls 8455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** to this function. The handle may be closed by calling the 8465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy 8475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** performance improvement, but the blob handle should always be closed 8485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** before control is returned to the user (to prevent a lock being held 8495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** on the database file for longer than necessary). Thus, any virtual table 8505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** method (xFilter etc.) that may directly or indirectly call this function 8515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** must call sqlite3Fts3SegmentsClose() before returning. 8525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 8535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3ReadBlock( 8545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* FTS3 table handle */ 8555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ 8565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char **paBlob, /* OUT: Blob data in malloc'd buffer */ 8575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pnBlob /* OUT: Size of blob data */ 8585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 8595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return code */ 8605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ 8625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pnBlob); 8635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->pSegments ){ 8655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_blob_reopen(p->pSegments, iBlockid); 8665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 8675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( 0==p->zSegmentsTbl ){ 8685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); 8695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; 8705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_blob_open( 8725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments 8735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ); 8745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 8775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nByte = sqlite3_blob_bytes(p->pSegments); 8785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( paBlob ){ 8795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); 8805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !aByte ){ 8815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_NOMEM; 8825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 8835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); 8845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(&aByte[nByte], 0, FTS3_NODE_PADDING); 8855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 8865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(aByte); 8875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aByte = 0; 8885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *paBlob = aByte; 8915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnBlob = nByte; 8935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 8945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 8965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 8975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 8995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Close the blob handle at p->pSegments, if it is open. See comments above 9005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the sqlite3Fts3ReadBlock() function for details. 9015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 9025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts3SegmentsClose(Fts3Table *p){ 9035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_blob_close(p->pSegments); 9045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->pSegments = 0; 9055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 9065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 9085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Move the iterator passed as the first argument to the next term in the 9095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** segment. If successful, SQLITE_OK is returned. If there is no next term, 9105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SQLITE_DONE. Otherwise, an SQLite error code. 9115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 9125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ 9135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pNext; /* Cursor variable */ 9145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nPrefix; /* Number of bytes in term prefix */ 9155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSuffix; /* Number of bytes in term suffix */ 9165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pReader->aDoclist ){ 9185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext = pReader->aNode; 9195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 9205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext = &pReader->aDoclist[pReader->nDoclist]; 9215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ 9245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return code from Fts3ReadBlock() */ 9255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( fts3SegReaderIsPending(pReader) ){ 9275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3HashElem *pElem = *(pReader->ppNextElem); 9285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pElem==0 ){ 9295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->aNode = 0; 9305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 9315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PendingList *pList = (PendingList *)fts3HashData(pElem); 9325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->zTerm = (char *)fts3HashKey(pElem); 9335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->nTerm = fts3HashKeysize(pElem); 9345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->nNode = pReader->nDoclist = pList->nData + 1; 9355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->aNode = pReader->aDoclist = pList->aData; 9365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->ppNextElem++; 9375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pReader->aNode ); 9385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 9405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !fts3SegReaderIsRootOnly(pReader) ){ 9435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pReader->aNode); 9445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->aNode = 0; 9465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 9485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** blocks have already been traversed. */ 9495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); 9505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ 9515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 9525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3Fts3ReadBlock( 9555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode 9565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ); 9575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 9585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext = pReader->aNode; 9595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Because of the FTS3_NODE_PADDING bytes of padding, the following is 9625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** safe (no risk of overread) even if the node data is corrupted. 9635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 9645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); 9655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); 9665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nPrefix<0 || nSuffix<=0 9675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] 9685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 9695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_CORRUPT; 9705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nPrefix+nSuffix>pReader->nTermAlloc ){ 9735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nNew = (nPrefix+nSuffix)*2; 9745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zNew = sqlite3_realloc(pReader->zTerm, nNew); 9755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !zNew ){ 9765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 9775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->zTerm = zNew; 9795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->nTermAlloc = nNew; 9805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); 9825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->nTerm = nPrefix+nSuffix; 9835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext += nSuffix; 9845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); 9855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->aDoclist = pNext; 9865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->pOffsetList = 0; 9875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Check that the doclist does not appear to extend past the end of the 9895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** b-tree node. And that the final byte of the doclist is 0x00. If either 9905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** of these statements is untrue, then the data structure is corrupt. 9915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 9925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 9935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) || pReader->aDoclist[pReader->nDoclist-1] 9945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 9955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_CORRUPT; 9965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 9975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 9985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 9995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 10015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Set the SegReader to point to the first docid in the doclist associated 10025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** with the current term. 10035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 10045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ 10055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n; 10065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pReader->aDoclist ); 10075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( !pReader->pOffsetList ); 10085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); 10095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->pOffsetList = &pReader->aDoclist[n]; 10105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 10135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Advance the SegReader to point to the next docid in the doclist 10145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** associated with the current term. 10155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 10165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If arguments ppOffsetList and pnOffsetList are not NULL, then 10175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** *ppOffsetList is set to point to the first column-offset list 10185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** in the doclist entry (i.e. immediately past the docid varint). 10195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** *pnOffsetList is set to the length of the set of column-offset 10205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** lists, not including the nul-terminator byte. For example: 10215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 10225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3SegReaderNextDocid( 10235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pReader, 10245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char **ppOffsetList, 10255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pnOffsetList 10265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 10275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *p = pReader->pOffsetList; 10285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char c = 0; 10295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Pointer p currently points at the first byte of an offset list. The 10315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** following two lines advance it to point one byte past the end of 10325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** the same offset list. 10335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 10345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( *p | c ) c = *p++ & 0x80; 10355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p++; 10365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If required, populate the output variables with a pointer to and the 10385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** size of the previous offset-list. 10395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 10405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( ppOffsetList ){ 10415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppOffsetList = pReader->pOffsetList; 10425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnOffsetList = (int)(p - pReader->pOffsetList - 1); 10435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 10445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If there are no more entries in the doclist, set pOffsetList to 10465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and 10475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** Fts3SegReader.pOffsetList to point to the next offset list before 10485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** returning. 10495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 10505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p>=&pReader->aDoclist[pReader->nDoclist] ){ 10515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->pOffsetList = 0; 10525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 10535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDelta; 10545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); 10555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->iDocid += iDelta; 10565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 10575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 10585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 10605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function is called to estimate the amount of data that will be 10615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** loaded from the disk If SegReaderIterate() is called on this seg-reader, 10625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** in units of average document size. 10635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 10645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This can be used as follows: If the caller has a small doclist that 10655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** contains references to N documents, and is considering merging it with 10665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a large doclist (size X "average documents"), it may opt not to load 10675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the large doclist if X>N. 10685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 10695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3SegReaderCost( 10705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Cursor *pCsr, /* FTS3 cursor handle */ 10715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pReader, /* Segment-reader handle */ 10725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pnCost /* IN/OUT: Number of bytes read */ 10735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 10745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; 10755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; /* Return code */ 10765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nCost = 0; /* Cost in bytes to return */ 10775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int pgsz = p->nPgsz; /* Database page size */ 10785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If this seg-reader is reading the pending-terms table, or if all data 10805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** for the segment is stored on the root page of the b-tree, then the cost 10815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** is zero. In this case all required data is already in main memory. 10825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 10835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->bHasStat 10845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && !fts3SegReaderIsPending(pReader) 10855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && !fts3SegReaderIsRootOnly(pReader) 10865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 10875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nBlob = 0; 10885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iBlock; 10895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pCsr->nRowAvg==0 ){ 10915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* The average document size, which is required to calculate the cost 10925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** of each doclist, has not yet been determined. Read the required 10935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** data from the %_stat table to calculate it. 10945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 10955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 10965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** varints, where nCol is the number of columns in the FTS3 table. 10975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** The first varint is the number of documents currently stored in 10985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** the table. The following nCol varints contain the total amount of 10995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** data stored in all rows of each column of the table, from left 11005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** to right. 11015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 11025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 11035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 nDoc = 0; 11045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 nByte = 0; 11055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *pEnd; 11065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *a; 11075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3Fts3SelectDoctotal(p, &pStmt); 11095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 11105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a = sqlite3_column_blob(pStmt, 0); 11115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( a ); 11125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; 11145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a += sqlite3Fts3GetVarint(a, &nDoc); 11155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( a<pEnd ){ 11165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a += sqlite3Fts3GetVarint(a, &nByte); 11175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nDoc==0 || nByte==0 ){ 11195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_reset(pStmt); 11205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_CORRUPT; 11215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); 11245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pCsr->nRowAvg>0 ); 11255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 11265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 11275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Assume that a blob flows over onto overflow pages if it is larger 11305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** than (pgsz-35) bytes in size (the file-format documentation 11315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** confirms this). 11325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 11335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ 11345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); 11355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) break; 11365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( (nBlob+35)>pgsz ){ 11375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nOvfl = (nBlob + 34)/pgsz; 11385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); 11395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnCost += nCost; 11445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 11455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 11465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 11485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Free all allocations associated with the iterator passed as the 11495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** second argument. 11505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 11515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ 11525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pReader && !fts3SegReaderIsPending(pReader) ){ 11535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pReader->zTerm); 11545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !fts3SegReaderIsRootOnly(pReader) ){ 11555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pReader->aNode); 11565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pReader); 11595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 11605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 11625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Allocate a new SegReader object. 11635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 11645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3SegReaderNew( 11655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iAge, /* Segment "age". */ 11665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iStartLeaf, /* First leaf to traverse */ 11675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ 11685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iEndBlock, /* Final block of segment */ 11695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zRoot, /* Buffer containing root node */ 11705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nRoot, /* Size of buffer containing root node */ 11715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ 11725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 11735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; /* Return code */ 11745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pReader; /* Newly allocated SegReader object */ 11755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nExtra = 0; /* Bytes to allocate segment root node */ 11765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( iStartLeaf<=iEndLeaf ); 11785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iStartLeaf==0 ){ 11795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nExtra = nRoot + FTS3_NODE_PADDING; 11805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); 11835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pReader ){ 11845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 11855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 11865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(pReader, 0, sizeof(Fts3SegReader)); 11875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->iIdx = iAge; 11885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->iStartBlock = iStartLeaf; 11895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->iLeafEndBlock = iEndLeaf; 11905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->iEndBlock = iEndBlock; 11915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nExtra ){ 11935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* The entire segment is stored in the root node. */ 11945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->aNode = (char *)&pReader[1]; 11955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->nNode = nRoot; 11965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(pReader->aNode, zRoot, nRoot); 11975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); 11985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 11995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->iCurrentBlock = iStartLeaf-1; 12005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 12035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppReader = pReader; 12045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 12055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3SegReaderFree(pReader); 12065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 12085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 12095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 12115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This is a comparison function used as a qsort() callback when sorting 12125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** an array of pending terms by term. This occurs as part of flushing 12135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the contents of the pending-terms hash table to the database. 12145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 12155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ 12165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); 12175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); 12185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); 12195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); 12205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n = (n1<n2 ? n1 : n2); 12225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int c = memcmp(z1, z2, n); 12235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( c==0 ){ 12245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) c = n1 - n2; 12255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return c; 12275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 12285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 12305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function is used to allocate an Fts3SegReader that iterates through 12315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a subset of the terms stored in the Fts3Table.pendingTerms array. 12325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 12335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3SegReaderPending( 12345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 12355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zTerm, /* Term to search for */ 12365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm, /* Size of buffer zTerm */ 12375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isPrefix, /* True for a term-prefix query */ 12385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ 12395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 12405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ 12415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3HashElem *pE; /* Iterator variable */ 12425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ 12435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nElem = 0; /* Size of array at aElem */ 12445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; /* Return Code */ 12455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( isPrefix ){ 12475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nAlloc = 0; /* Size of allocated array at aElem */ 12485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ 12505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zKey = (char *)fts3HashKey(pE); 12515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nKey = fts3HashKeysize(pE); 12525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ 12535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nElem==nAlloc ){ 12545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3HashElem **aElem2; 12555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nAlloc += 16; 12565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aElem2 = (Fts3HashElem **)sqlite3_realloc( 12575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aElem, nAlloc*sizeof(Fts3HashElem *) 12585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ); 12595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !aElem2 ){ 12605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_NOMEM; 12615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nElem = 0; 12625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 12635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aElem = aElem2; 12655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aElem[nElem++] = pE; 12675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If more than one term matches the prefix, sort the Fts3HashElem 12715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** objects in term order using qsort(). This uses the same comparison 12725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** callback as is used when flushing terms to disk. 12735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 12745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nElem>1 ){ 12755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); 12765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 12795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); 12805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pE ){ 12815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aElem = &pE; 12825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nElem = 1; 12835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nElem>0 ){ 12875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); 12885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader = (Fts3SegReader *)sqlite3_malloc(nByte); 12895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pReader ){ 12905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_NOMEM; 12915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 12925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(pReader, 0, nByte); 12935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->iIdx = 0x7FFFFFFF; 12945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; 12955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); 12965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 12985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( isPrefix ){ 13005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(aElem); 13015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppReader = pReader; 13035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 13045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 13055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 13075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Compare the entries pointed to by two Fts3SegReader structures. 13085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Comparison is as follows: 13095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 13105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 1) EOF is greater than not EOF. 13115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 13125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2) The current terms (if any) are compared using memcmp(). If one 13135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** term is a prefix of another, the longer term is considered the 13145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** larger. 13155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 13165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3) By segment age. An older segment is considered larger. 13175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 13185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ 13195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 13205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pLhs->aNode && pRhs->aNode ){ 13215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc2 = pLhs->nTerm - pRhs->nTerm; 13225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc2<0 ){ 13235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); 13245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 13255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); 13265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==0 ){ 13285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = rc2; 13295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 13315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = (pLhs->aNode==0) - (pRhs->aNode==0); 13325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==0 ){ 13345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = pRhs->iIdx - pLhs->iIdx; 13355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc!=0 ); 13375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 13385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 13395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 13415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** A different comparison function for SegReader structures. In this 13425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** version, it is assumed that each SegReader points to an entry in 13435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a doclist for identical terms. Comparison is made as follows: 13445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 13455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 1) EOF (end of doclist in this case) is greater than not EOF. 13465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 13475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2) By current docid. 13485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 13495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 3) By segment age. An older segment is considered larger. 13505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 13515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ 13525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); 13535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==0 ){ 13545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pLhs->iDocid==pRhs->iDocid ){ 13555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = pRhs->iIdx - pLhs->iIdx; 13565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 13575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; 13585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pLhs->aNode && pRhs->aNode ); 13615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 13625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 13635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 13655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Compare the term that the Fts3SegReader object passed as the first argument 13665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** points to with the term specified by arguments zTerm and nTerm. 13675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 13685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If the pSeg iterator is already at EOF, return 0. Otherwise, return 13695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are 13705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** equal, or +ve if the pSeg term is greater than zTerm/nTerm. 13715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 13725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegReaderTermCmp( 13735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pSeg, /* Segment reader object */ 13745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zTerm, /* Term to compare to */ 13755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm /* Size of term zTerm in bytes */ 13765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 13775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int res = 0; 13785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pSeg->aNode ){ 13795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pSeg->nTerm>nTerm ){ 13805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) res = memcmp(pSeg->zTerm, zTerm, nTerm); 13815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 13825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); 13835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( res==0 ){ 13855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) res = pSeg->nTerm-nTerm; 13865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 13885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return res; 13895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 13905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 13925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Argument apSegment is an array of nSegment elements. It is known that 13935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the final (nSegment-nSuspect) members are already in sorted order 13945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** (according to the comparison function provided). This function shuffles 13955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the array around until all entries are in sorted order. 13965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 13975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3SegReaderSort( 13985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader **apSegment, /* Array to sort entries of */ 13995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSegment, /* Size of apSegment array */ 14005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSuspect, /* Unsorted entry count */ 14015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ 14025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 14035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; /* Iterator variable */ 14045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( nSuspect<=nSegment ); 14065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nSuspect==nSegment ) nSuspect--; 14085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=nSuspect-1; i>=0; i--){ 14095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int j; 14105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(j=i; j<(nSegment-1); j++){ 14115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pTmp; 14125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; 14135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTmp = apSegment[j+1]; 14145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) apSegment[j+1] = apSegment[j]; 14155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) apSegment[j] = pTmp; 14165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifndef NDEBUG 14205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Check that the list really is sorted now. */ 14215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<(nSuspect-1); i++){ 14225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( xCmp(apSegment[i], apSegment[i+1])<0 ); 14235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 14255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 14265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 14285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Insert a record into the %_segments table. 14295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 14305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3WriteSegment( 14315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 14325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iBlock, /* Block id for new block */ 14335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *z, /* Pointer to buffer containing block data */ 14345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n /* Size of buffer z in bytes */ 14355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 14365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 14375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); 14385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 14395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pStmt, 1, iBlock); 14405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); 14415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pStmt); 14425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 14435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 14455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 14465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 14485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Insert a record into the %_segdir table. 14495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 14505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3WriteSegdir( 14515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 14525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iLevel, /* Value for "level" field */ 14535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iIdx, /* Value for "idx" field */ 14545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iStartBlock, /* Value for "start_block" field */ 14555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ 14565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iEndBlock, /* Value for "end_block" field */ 14575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zRoot, /* Blob value for "root" field */ 14585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nRoot /* Number of bytes in buffer zRoot */ 14595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 14605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 14615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); 14625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 14635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int(pStmt, 1, iLevel); 14645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int(pStmt, 2, iIdx); 14655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pStmt, 3, iStartBlock); 14665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); 14675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pStmt, 5, iEndBlock); 14685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); 14695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pStmt); 14705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 14715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 14725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 14735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 14745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 14765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Return the size of the common prefix (if any) shared by zPrev and 14775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** zNext, in bytes. For example, 14785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 14795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 14805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 14815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 14825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 14835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3PrefixCompress( 14845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zPrev, /* Buffer containing previous term */ 14855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nPrev, /* Size of buffer zPrev in bytes */ 14865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zNext, /* Buffer containing next term */ 14875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nNext /* Size of buffer zNext in bytes */ 14885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 14895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int n; 14905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UNUSED_PARAMETER(nNext); 14915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++); 14925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return n; 14935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 14945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 14965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger 14975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** (according to memcmp) than the previous term. 14985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 14995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3NodeAddTerm( 15005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 15015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ 15025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isCopyTerm, /* True if zTerm/nTerm is transient */ 15035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zTerm, /* Pointer to buffer containing term */ 15045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm /* Size of term in bytes */ 15055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 15065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pTree = *ppTree; 15075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 15085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pNew; 15095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* First try to append the term to the current node. Return early if 15115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** this is possible. 15125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 15135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pTree ){ 15145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nData = pTree->nData; /* Current size of node in bytes */ 15155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nReq = nData; /* Required space after adding zTerm */ 15165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nPrefix; /* Number of bytes of prefix compression */ 15175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSuffix; /* Suffix length */ 15185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); 15205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nSuffix = nTerm-nPrefix; 15215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; 15235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nReq<=p->nNodeSize || !pTree->zTerm ){ 15245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nReq>p->nNodeSize ){ 15265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* An unusual case: this is the first term to be added to the node 15275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** and the static node buffer (p->nNodeSize bytes) is not large 15285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** enough. Use a separately malloced buffer instead This wastes 15295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** p->nNodeSize bytes, but since this scenario only comes about when 15305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** the database contain two terms that share a prefix of almost 2KB, 15315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** this is not expected to be a serious problem. 15325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 15335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pTree->aData==(char *)&pTree[1] ); 15345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->aData = (char *)sqlite3_malloc(nReq); 15355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pTree->aData ){ 15365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 15375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pTree->zTerm ){ 15415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* There is no prefix-length field for first term in a node */ 15425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); 15435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); 15465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); 15475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->nData = nData + nSuffix; 15485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->nEntry++; 15495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( isCopyTerm ){ 15515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pTree->nMalloc<nTerm ){ 15525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2); 15535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !zNew ){ 15545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 15555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->nMalloc = nTerm*2; 15575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->zMalloc = zNew; 15585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->zTerm = pTree->zMalloc; 15605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(pTree->zTerm, zTerm, nTerm); 15615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->nTerm = nTerm; 15625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 15635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->zTerm = (char *)zTerm; 15645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->nTerm = nTerm; 15655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 15675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If control flows to here, it was not possible to append zTerm to the 15715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** current node. Create a new node (a right-sibling of the current node). 15725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** If this is the first node in the tree, the term is added to it. 15735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 15745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** Otherwise, the term is not added to the new node, it is left empty for 15755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** now. Instead, the term is inserted into the parent of pTree. If pTree 15765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** has no parent, one is created here. 15775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 15785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); 15795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pNew ){ 15805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 15815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(pNew, 0, sizeof(SegmentNode)); 15835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew->nData = 1 + FTS3_VARINT_MAX; 15845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew->aData = (char *)&pNew[1]; 15855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 15865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pTree ){ 15875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pParent = pTree->pParent; 15885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); 15895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pTree->pParent==0 ){ 15905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->pParent = pParent; 15915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 15925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->pRight = pNew; 15935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew->pLeftmost = pTree->pLeftmost; 15945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew->pParent = pParent; 15955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew->zMalloc = pTree->zMalloc; 15965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew->nMalloc = pTree->nMalloc; 15975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->zMalloc = 0; 15985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 15995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNew->pLeftmost = pNew; 16005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); 16015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppTree = pNew; 16045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 16055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 16085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Helper function for fts3NodeWrite(). 16095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 16105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3TreeFinishNode( 16115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pTree, 16125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iHeight, 16135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLeftChild 16145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 16155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nStart; 16165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( iHeight>=1 && iHeight<128 ); 16175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); 16185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTree->aData[nStart] = (char)iHeight; 16195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); 16205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return nStart; 16215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 16245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Write the buffer for the segment node pTree and all of its peers to the 16255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** database. Then call this function recursively to write the parent of 16265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** pTree and its peers to the database. 16275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 16285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Except, if pTree is a root node, do not write it to the database. Instead, 16295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** set output variables *paRoot and *pnRoot to contain the root node. 16305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 16315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If successful, SQLITE_OK is returned and output variable *piLast is 16325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** set to the largest blockid written to the database (or zero if no 16335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** blocks were written to the db). Otherwise, an SQLite error code is 16345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** returned. 16355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 16365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3NodeWrite( 16375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 16385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pTree, /* SegmentNode handle */ 16395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iHeight, /* Height of this node in tree */ 16405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLeaf, /* Block id of first leaf node */ 16415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ 16425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ 16435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char **paRoot, /* OUT: Data for root node */ 16445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pnRoot /* OUT: Size of root node in bytes */ 16455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 16465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; 16475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pTree->pParent ){ 16495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Root node of the tree. */ 16505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); 16515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *piLast = iFree-1; 16525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnRoot = pTree->nData - nStart; 16535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *paRoot = &pTree->aData[nStart]; 16545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 16555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pIter; 16565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iNextFree = iFree; 16575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iNextLeaf = iLeaf; 16585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ 16595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); 16605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nWrite = pIter->nData - nStart; 16615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); 16635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iNextFree++; 16645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iNextLeaf += (pIter->nEntry+1); 16655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 16675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( iNextLeaf==iFree ); 16685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3NodeWrite( 16695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot 16705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ); 16715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 16755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 16785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Free all memory allocations associated with the tree pTree. 16795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 16805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3NodeFree(SegmentNode *pTree){ 16815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pTree ){ 16825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *p = pTree->pLeftmost; 16835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3NodeFree(p->pParent); 16845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( p ){ 16855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentNode *pRight = p->pRight; 16865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->aData!=(char *)&p[1] ){ 16875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(p->aData); 16885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pRight==0 || p->zMalloc==0 ); 16905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(p->zMalloc); 16915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(p); 16925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p = pRight; 16935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 16955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 16965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 16975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 16985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Add a term to the segment being constructed by the SegmentWriter object 16995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** *ppWriter. When adding the first term to a segment, *ppWriter should 17005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** be passed NULL. This function will allocate a new SegmentWriter object 17015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** and return it via the input/output variable *ppWriter in this case. 17025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 17035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. 17045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 17055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegWriterAdd( 17065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 17075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ 17085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isCopyTerm, /* True if buffer zTerm must be copied */ 17095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zTerm, /* Pointer to buffer containing term */ 17105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm, /* Size of term in bytes */ 17115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *aDoclist, /* Pointer to buffer containing doclist */ 17125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nDoclist /* Size of doclist in bytes */ 17135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 17145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nPrefix; /* Size of term prefix in bytes */ 17155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSuffix; /* Size of term suffix in bytes */ 17165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nReq; /* Number of bytes required on leaf page */ 17175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nData; 17185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentWriter *pWriter = *ppWriter; 17195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pWriter ){ 17215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 17225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 17235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Allocate the SegmentWriter structure */ 17255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); 17265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pWriter ) return SQLITE_NOMEM; 17275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(pWriter, 0, sizeof(SegmentWriter)); 17285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppWriter = pWriter; 17295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Allocate a buffer in which to accumulate data */ 17315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); 17325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pWriter->aData ) return SQLITE_NOMEM; 17335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->nSize = p->nNodeSize; 17345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Find the next free blockid in the %_segments table */ 17365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); 17375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 17385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_ROW==sqlite3_step(pStmt) ){ 17395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->iFree = sqlite3_column_int64(pStmt, 0); 17405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->iFirst = pWriter->iFree; 17415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 17425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 17435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 17445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 17455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData = pWriter->nData; 17465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); 17485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nSuffix = nTerm-nPrefix; 17495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Figure out how many bytes are required by this new entry */ 17515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ 17525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ 17535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nSuffix + /* Term suffix */ 17545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ 17555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nDoclist; /* Doclist data */ 17565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nData>0 && nData+nReq>p->nNodeSize ){ 17585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 17595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* The current leaf node is full. Write it out to the database. */ 17615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); 17625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 17635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Add the current term to the interior node tree. The term added to 17655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** the interior tree must: 17665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 17675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** a) be greater than the largest term on the leaf node just written 17685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** to the database (still available in pWriter->zTerm), and 17695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 17705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** b) be less than or equal to the term about to be added to the new 17715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** leaf node (zTerm/nTerm). 17725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 17735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** In other words, it must be the prefix of zTerm 1 byte longer than 17745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** the common prefix (if any) of zTerm and pWriter->zTerm. 17755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 17765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( nPrefix<nTerm ); 17775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1); 17785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 17795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData = 0; 17815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->nTerm = 0; 17825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nPrefix = 0; 17845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nSuffix = nTerm; 17855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nReq = 1 + /* varint containing prefix size */ 17865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ 17875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nTerm + /* Term suffix */ 17885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ 17895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nDoclist; /* Doclist data */ 17905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 17915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 17925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If the buffer currently allocated is too small for this entry, realloc 17935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** the buffer to make it large enough. 17945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 17955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nReq>pWriter->nSize ){ 17965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aNew = sqlite3_realloc(pWriter->aData, nReq); 17975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !aNew ) return SQLITE_NOMEM; 17985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->aData = aNew; 17995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->nSize = nReq; 18005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( nData+nReq<=pWriter->nSize ); 18025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Append the prefix-compressed term and doclist to the buffer. */ 18045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); 18055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); 18065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); 18075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData += nSuffix; 18085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); 18095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(&pWriter->aData[nData], aDoclist, nDoclist); 18105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->nData = nData + nDoclist; 18115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Save the current term so that it can be used to prefix-compress the next. 18135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** If the isCopyTerm parameter is true, then the buffer pointed to by 18145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** zTerm is transient, so take a copy of the term data. Otherwise, just 18155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** store a copy of the pointer. 18165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 18175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( isCopyTerm ){ 18185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nTerm>pWriter->nMalloc ){ 18195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); 18205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !zNew ){ 18215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 18225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->nMalloc = nTerm*2; 18245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->zMalloc = zNew; 18255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->zTerm = zNew; 18265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pWriter->zTerm==pWriter->zMalloc ); 18285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(pWriter->zTerm, zTerm, nTerm); 18295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 18305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->zTerm = (char *)zTerm; 18315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->nTerm = nTerm; 18335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 18355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 18385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Flush all data associated with the SegmentWriter object pWriter to the 18395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** database. This function must be called after all terms have been added 18405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is 18415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** returned. Otherwise, an SQLite error code. 18425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 18435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegWriterFlush( 18445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 18455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ 18465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iLevel, /* Value for 'level' column of %_segdir */ 18475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iIdx /* Value for 'idx' column of %_segdir */ 18485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 18495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return code */ 18505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pWriter->pTree ){ 18515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLast = 0; /* Largest block id written to database */ 18525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ 18535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zRoot = NULL; /* Pointer to buffer containing root node */ 18545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nRoot = 0; /* Size of buffer zRoot */ 18555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iLastLeaf = pWriter->iFree; 18575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); 18585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 18595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3NodeWrite(p, pWriter->pTree, 1, 18605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); 18615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 18635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3WriteSegdir( 18645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); 18655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 18675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* The entire tree fits on the root node. Write it to the segdir table. */ 18685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3WriteSegdir( 18695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); 18705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 18725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 18755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Release all memory held by the SegmentWriter object passed as the 18765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** first argument. 18775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 18785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3SegWriterFree(SegmentWriter *pWriter){ 18795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pWriter ){ 18805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pWriter->aData); 18815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pWriter->zMalloc); 18825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3NodeFree(pWriter->pTree); 18835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pWriter); 18845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 18855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 18865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 18875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 18885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** The first value in the apVal[] array is assumed to contain an integer. 18895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function tests if there exist any documents with docid values that 18905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** are different from that integer. i.e. if deleting the document with docid 18915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** apVal[0] would mean the FTS3 table were empty. 18925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 18935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If successful, *pisEmpty is set to true if the table is empty except for 18945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an 18955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** error occurs, an SQLite error code is returned. 18965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 18975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ 18985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 18995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 19005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); 19015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 19025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_ROW==sqlite3_step(pStmt) ){ 19035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pisEmpty = sqlite3_column_int(pStmt, 0); 19045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pStmt); 19065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 19085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 19095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 19105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 19115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Set *pnSegment to the total number of segments in the database. Set 19125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** *pnMax to the largest segment level in the database (segment levels 19135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** are stored in the 'level' column of the %_segdir table). 19145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 19155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Return SQLITE_OK if successful, or an SQLite error code if not. 19165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 19175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ 19185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; 19195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 19205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 19215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); 19225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 19235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( SQLITE_ROW==sqlite3_step(pStmt) ){ 19245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnSegment = sqlite3_column_int(pStmt, 0); 19255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnMax = sqlite3_column_int(pStmt, 1); 19265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return sqlite3_reset(pStmt); 19285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 19295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 19305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 19315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function is used after merging multiple segments into a single large 19325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** segment to delete the old, now redundant, segment b-trees. Specifically, 19335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** it: 19345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 19355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 1) Deletes all %_segments entries for the segments associated with 19365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** each of the SegReader objects in the array passed as the third 19375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** argument, and 19385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 19395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 2) deletes all %_segdir entries with level iLevel, or all %_segdir 19405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** entries regardless of level if (iLevel<0). 19415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 19425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** SQLITE_OK is returned if successful, otherwise an SQLite error code. 19435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 19445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3DeleteSegdir( 19455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 19465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iLevel, /* Level of %_segdir entries to delete */ 19475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader **apSegment, /* Array of SegReader objects */ 19485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nReader /* Size of array apSegment */ 19495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 19505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return Code */ 19515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; /* Iterator variable */ 19525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pDelete; /* SQL statement to delete rows */ 19535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 19545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); 19555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; rc==SQLITE_OK && i<nReader; i++){ 19565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pSegment = apSegment[i]; 19575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pSegment->iStartBlock ){ 19585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); 19595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); 19605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pDelete); 19615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pDelete); 19625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ){ 19655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 19665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 19685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iLevel==FTS3_SEGCURSOR_ALL ){ 19695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); 19705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ 19715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3PendingTermsClear(p); 19725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 19735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( iLevel>=0 ); 19745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); 19755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 19765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int(pDelete, 1, iLevel); 19775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pDelete); 19785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_reset(pDelete); 19795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 19815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 19825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 19835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 19845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 19855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 19865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** When this function is called, buffer *ppList (size *pnList bytes) contains 19875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a position list that may (or may not) feature multiple columns. This 19885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** function adjusts the pointer *ppList and the length *pnList so that they 19895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** identify the subset of the position list that corresponds to column iCol. 19905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 19915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If there are no entries in the input position list for column iCol, then 19925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** *pnList is set to zero before returning. 19935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 19945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3ColumnFilter( 19955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iCol, /* Column to filter on */ 19965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char **ppList, /* IN/OUT: Pointer to position list */ 19975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ 19985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 19995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pList = *ppList; 20005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nList = *pnList; 20015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pEnd = &pList[nList]; 20025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iCurrent = 0; 20035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *p = pList; 20045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( iCol>=0 ); 20065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( 1 ){ 20075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char c = 0; 20085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80; 20095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iCol==iCurrent ){ 20115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nList = (int)(p - pList); 20125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 20135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 20145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nList -= (int)(p - pList); 20165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pList = p; 20175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nList==0 ){ 20185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 20195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 20205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p = &pList[1]; 20215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p += sqlite3Fts3GetVarint32(p, &iCurrent); 20225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 20235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *ppList = pList; 20255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnList = nList; 20265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 20275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3SegReaderStart( 20295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 20305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReaderCursor *pCsr, /* Cursor object */ 20315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegFilter *pFilter /* Restrictions on range of iteration */ 20325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 20335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; 20345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Initialize the cursor object */ 20365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->pFilter = pFilter; 20375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If the Fts3SegFilter defines a specific term (or term prefix) to search 20395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** for, then advance each segment iterator until it points to a term of 20405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** equal or greater value than the specified term. This prevents many 20415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** unnecessary merge/sort operations for the case where single segment 20425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** b-tree leaf nodes contain more than one term. 20435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 20445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<pCsr->nSegment; i++){ 20455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nTerm = pFilter->nTerm; 20465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zTerm = pFilter->zTerm; 20475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader *pSeg = pCsr->apSegment[i]; 20485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do { 20495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = fts3SegReaderNext(p, pSeg); 20505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 20515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); 20525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 20535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegReaderSort( 20545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); 20555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 20575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 20585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3SegReaderStep( 20605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Virtual table handle */ 20615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReaderCursor *pCsr /* Cursor object */ 20625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 20635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; 20645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); 20665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); 20675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); 20685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); 20695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); 20705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReader **apSegment = pCsr->apSegment; 20725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nSegment = pCsr->nSegment; 20735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegFilter *pFilter = pCsr->pFilter; 20745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pCsr->nSegment==0 ) return SQLITE_OK; 20765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) do { 20785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nMerge; 20795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; 20805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Advance the first pCsr->nAdvance entries in the apSegment[] array 20825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** forward. Then sort the list in order of current term again. 20835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 20845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<pCsr->nAdvance; i++){ 20855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SegReaderNext(p, apSegment[i]); 20865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) return rc; 20875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 20885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); 20895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nAdvance = 0; 20905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ 20925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( rc==SQLITE_OK ); 20935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( apSegment[0]->aNode==0 ) break; 20945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nTerm = apSegment[0]->nTerm; 20965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->zTerm = apSegment[0]->zTerm; 20975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 20985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If this is a prefix-search, and if the term that apSegment[0] points 20995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** to does not share a suffix with pFilter->zTerm/nTerm, then all 21005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** required callbacks have been made. In this case exit early. 21015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** 21025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** Similarly, if this is a search for an exact match, and the first term 21035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** of segment apSegment[0] is not a match, exit early. 21045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 21055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pFilter->zTerm && !isScan ){ 21065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pCsr->nTerm<pFilter->nTerm 21075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) || (!isPrefix && pCsr->nTerm>pFilter->nTerm) 21085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) 21095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 21105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) break; 21115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nMerge = 1; 21155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( nMerge<nSegment 21165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && apSegment[nMerge]->aNode 21175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && apSegment[nMerge]->nTerm==pCsr->nTerm 21185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) 21195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 21205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nMerge++; 21215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); 21245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nMerge==1 && !isIgnoreEmpty ){ 21255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->aDoclist = apSegment[0]->aDoclist; 21265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nDoclist = apSegment[0]->nDoclist; 21275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_ROW; 21285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 21295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nDoclist = 0; /* Size of doclist */ 21305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ 21315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* The current term of the first nMerge entries in the array 21335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** of Fts3SegReader objects is the same. The doclists must be merged 21345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** and a single term returned with the merged doclist. 21355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 21365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<nMerge; i++){ 21375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegReaderFirstDocid(apSegment[i]); 21385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); 21405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( apSegment[0]->pOffsetList ){ 21415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int j; /* Number of segments that share a docid */ 21425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pList; 21435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nList; 21445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nByte; 21455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDocid = apSegment[0]->iDocid; 21465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegReaderNextDocid(apSegment[0], &pList, &nList); 21475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) j = 1; 21485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( j<nMerge 21495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && apSegment[j]->pOffsetList 21505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && apSegment[j]->iDocid==iDocid 21515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 21525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegReaderNextDocid(apSegment[j], 0, 0); 21535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) j++; 21545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( isColFilter ){ 21575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3ColumnFilter(pFilter->iCol, &pList, &nList); 21585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !isIgnoreEmpty || nList>0 ){ 21615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); 21625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nDoclist+nByte>pCsr->nBuffer ){ 21635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *aNew; 21645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nBuffer = (nDoclist+nByte)*2; 21655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); 21665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !aNew ){ 21675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 21685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->aBuffer = aNew; 21705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nDoclist += sqlite3Fts3PutVarint( 21725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) &pCsr->aBuffer[nDoclist], iDocid-iPrev 21735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ); 21745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iPrev = iDocid; 21755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( isRequirePos ){ 21765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memcpy(&pCsr->aBuffer[nDoclist], pList, nList); 21775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nDoclist += nList; 21785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->aBuffer[nDoclist++] = '\0'; 21795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); 21835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nDoclist>0 ){ 21855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->aDoclist = pCsr->aBuffer; 21865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nDoclist = nDoclist; 21875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_ROW; 21885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 21905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nAdvance = nMerge; 21915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }while( rc==SQLITE_OK ); 21925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 21945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 21955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 21965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts3SegReaderFinish( 21975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReaderCursor *pCsr /* Cursor object */ 21985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 21995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pCsr ){ 22005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; 22015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<pCsr->nSegment; i++){ 22025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); 22035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 22045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pCsr->apSegment); 22055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pCsr->aBuffer); 22065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->nSegment = 0; 22085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->apSegment = 0; 22095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->aBuffer = 0; 22105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 22115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 22125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 22145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Merge all level iLevel segments in the database into a single 22155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** iLevel+1 segment. Or, if iLevel<0, merge all segments into a 22165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** single segment with a level equal to the numerically largest level 22175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** currently present in the database. 22185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 22195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** If this function is called with iLevel<0, but there is only one 22205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** segment in the database, SQLITE_DONE is returned immediately. 22215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 22225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** an SQLite error code is returned. 22235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 22245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SegmentMerge(Fts3Table *p, int iLevel){ 22255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return code */ 22265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iIdx = 0; /* Index of new segment */ 22275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iNewLevel = 0; /* Level to create new segment at */ 22285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ 22295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegFilter filter; /* Segment term filter condition */ 22305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ 22315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); 22335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; 22345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( iLevel==FTS3_SEGCURSOR_ALL ){ 22365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* This call is to merge all segments in the database to a single 22375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** segment. The level of the new segment is equal to the the numerically 22385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** greatest segment level currently present in the database. The index 22395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** of the new segment is always 0. */ 22405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nDummy; /* TODO: Remove this */ 22415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( csr.nSegment==1 ){ 22425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_DONE; 22435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) goto finished; 22445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 22455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); 22465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 22475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* This call is to merge all segments at level iLevel. Find the next 22485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** available segment index at level iLevel+1. The call to 22495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to 22505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** a single iLevel+2 segment if necessary. */ 22515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iNewLevel = iLevel+1; 22525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); 22535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 22545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) goto finished; 22555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( csr.nSegment>0 ); 22565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( iNewLevel>=0 ); 22575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(&filter, 0, sizeof(Fts3SegFilter)); 22595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filter.flags = FTS3_SEGMENT_REQUIRE_POS; 22605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); 22615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); 22635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( SQLITE_OK==rc ){ 22645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3Fts3SegReaderStep(p, &csr); 22655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_ROW ) break; 22665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SegWriterAdd(p, &pWriter, 1, 22675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); 22685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 22695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) goto finished; 22705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pWriter ); 22715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); 22735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc!=SQLITE_OK ) goto finished; 22745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); 22755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finished: 22775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SegWriterFree(pWriter); 22785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3SegReaderFinish(&csr); 22795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 22805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 22815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 22845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Flush the contents of pendingTerms to a level 0 segment. 22855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 22865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ 22875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); 22885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 22895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 22905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 22915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Encode N integers as varints into a blob. 22925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 22935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3EncodeIntArray( 22945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int N, /* The number of integers to encode */ 22955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *a, /* The integer values */ 22965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *zBuf, /* Write the BLOB here */ 22975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pNBuf /* Write number of bytes if zBuf[] used here */ 22985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 22995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i, j; 23005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=j=0; i<N; i++){ 23015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) j += sqlite3Fts3PutVarint(&zBuf[j], (sqlite3_int64)a[i]); 23025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 23035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pNBuf = j; 23045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 23055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 23065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 23075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Decode a blob of varints into N integers 23085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 23095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3DecodeIntArray( 23105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int N, /* The number of integers to decode */ 23115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *a, /* Write the integer values */ 23125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zBuf, /* The BLOB containing the varints */ 23135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nBuf /* size of the BLOB */ 23145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 23155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i, j; 23165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) UNUSED_PARAMETER(nBuf); 23175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=j=0; i<N; i++){ 23185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 x; 23195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) j += sqlite3Fts3GetVarint(&zBuf[j], &x); 23205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert(j<=nBuf); 23215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a[i] = (u32)(x & 0xffffffff); 23225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 23235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 23245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 23255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 23265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Insert the sizes (in tokens) for each column of the document 23275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** with docid equal to p->iPrevDocid. The sizes are encoded as 23285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** a blob of varints. 23295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 23305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3InsertDocsize( 23315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pRC, /* Result code */ 23325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Table into which to insert */ 23335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *aSz /* Sizes of each column */ 23345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 23355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pBlob; /* The BLOB encoding of the document size */ 23365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nBlob; /* Number of bytes in the BLOB */ 23375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ 23385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Result code from subfunctions */ 23395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 23405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( *pRC ) return; 23415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pBlob = sqlite3_malloc( 10*p->nColumn ); 23425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pBlob==0 ){ 23435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = SQLITE_NOMEM; 23445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 23455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 23465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); 23475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); 23485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc ){ 23495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pBlob); 23505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = rc; 23515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 23525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 23535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); 23545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); 23555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pStmt); 23565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = sqlite3_reset(pStmt); 23575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 23585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 23595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 23605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Record 0 of the %_stat table contains a blob consisting of N varints, 23615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** where N is the number of user defined columns in the fts3 table plus 23625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** two. If nCol is the number of user defined columns, then values of the 23635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** varints are set as follows: 23645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 23655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Varint 0: Total number of rows in the table. 23665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 23675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Varint 1..nCol: For each column, the total number of tokens stored in 23685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** the column for all rows of the table. 23695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 23705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Varint 1+nCol: The total size, in bytes, of all text values in all 23715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** columns of all rows of the table. 23725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 23735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 23745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3UpdateDocTotals( 23755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int *pRC, /* The result code */ 23765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p, /* Table being updated */ 23775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *aSzIns, /* Size increases */ 23785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *aSzDel, /* Size decreases */ 23795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nChng /* Change in the number of documents */ 23805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 23815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char *pBlob; /* Storage for BLOB written into %_stat */ 23825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nBlob; /* Size of BLOB written into %_stat */ 23835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *a; /* Array of integers that becomes the BLOB */ 23845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_stmt *pStmt; /* Statement for reading and writing */ 23855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; /* Loop counter */ 23865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Result code from subfunctions */ 23875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 23885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const int nStat = p->nColumn+2; 23895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 23905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( *pRC ) return; 23915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); 23925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( a==0 ){ 23935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = SQLITE_NOMEM; 23945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 23955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 23965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pBlob = (char*)&a[nStat]; 23975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); 23985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc ){ 23995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(a); 24005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = rc; 24015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 24025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( sqlite3_step(pStmt)==SQLITE_ROW ){ 24045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3DecodeIntArray(nStat, a, 24055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_column_blob(pStmt, 0), 24065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_column_bytes(pStmt, 0)); 24075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 24085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(a, 0, sizeof(u32)*(nStat) ); 24095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_reset(pStmt); 24115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nChng<0 && a[0]<(u32)(-nChng) ){ 24125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a[0] = 0; 24135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 24145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a[0] += nChng; 24155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<p->nColumn+1; i++){ 24175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 x = a[i+1]; 24185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( x+aSzIns[i] < aSzDel[i] ){ 24195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) x = 0; 24205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 24215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) x = x + aSzIns[i] - aSzDel[i]; 24225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a[i+1] = x; 24245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3EncodeIntArray(nStat, a, pBlob, &nBlob); 24265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); 24275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc ){ 24285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(a); 24295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = rc; 24305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return; 24315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); 24335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_step(pStmt); 24345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pRC = sqlite3_reset(pStmt); 24355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(a); 24365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 24375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 24385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 24395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Handle a 'special' INSERT of the form: 24405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 24415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** "INSERT INTO tbl(tbl) VALUES(<expr>)" 24425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 24435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Argument pVal contains the result of <expr>. Currently the only 24445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** meaningful value to insert is the text 'optimize'. 24455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 24465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ 24475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; /* Return Code */ 24485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zVal = (const char *)sqlite3_value_text(pVal); 24495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nVal = sqlite3_value_bytes(pVal); 24505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 24515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !zVal ){ 24525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 24535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ 24545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); 24555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_DONE ){ 24565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_OK; 24575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 24585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3PendingTermsClear(p); 24595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#ifdef SQLITE_TEST 24615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ 24625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nNodeSize = atoi(&zVal[9]); 24635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_OK; 24645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ 24655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) p->nMaxPendingData = atoi(&zVal[11]); 24665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_OK; 24675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 24685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 24695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = SQLITE_ERROR; 24705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 24725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3SegmentsClose(p); 24735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 24745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 24755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 24765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 24775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Return the deferred doclist associated with deferred token pDeferred. 24785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already 24795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** been called to allocate and populate the doclist. 24805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 24815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ 24825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pDeferred->pList ){ 24835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnByte = pDeferred->pList->nData; 24845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return pDeferred->pList->aData; 24855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 24865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) *pnByte = 0; 24875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 0; 24885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 24895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 24905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 24915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Helper fucntion for FreeDeferredDoclists(). This function removes all 24925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** references to deferred doclists from within the tree of Fts3Expr 24935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** structures headed by 24945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 24955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ 24965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pExpr ){ 24975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3DeferredDoclistClear(pExpr->pLeft); 24985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3DeferredDoclistClear(pExpr->pRight); 24995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pExpr->isLoaded ){ 25005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pExpr->aDoclist); 25015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pExpr->isLoaded = 0; 25025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pExpr->aDoclist = 0; 25035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pExpr->nDoclist = 0; 25045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pExpr->pCurrent = 0; 25055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pExpr->iCurrent = 0; 25065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 25095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 25115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Delete all cached deferred doclists. Deferred doclists are cached 25125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. 25135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 25145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ 25155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3DeferredToken *pDef; 25165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ 25175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pDef->pList); 25185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pDef->pList = 0; 25195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pCsr->pDeferred ){ 25215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3DeferredDoclistClear(pCsr->pExpr); 25225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 25245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 25265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Free all entries in the pCsr->pDeffered list. Entries are added to 25275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** this list using sqlite3Fts3DeferToken(). 25285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 25295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ 25305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3DeferredToken *pDef; 25315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3DeferredToken *pNext; 25325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ 25335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pNext = pDef->pNext; 25345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pDef->pList); 25355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(pDef); 25365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->pDeferred = 0; 25385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 25395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 25415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Generate deferred-doclists for all tokens in the pCsr->pDeferred list 25425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** based on the row that pCsr currently points to. 25435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** 25445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** A deferred-doclist is like any other doclist with position information 25455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** included, except that it only contains entries for a single row of the 25465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** table, not for all rows. 25475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 25485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ 25495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; /* Return code */ 25505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pCsr->pDeferred ){ 25515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int i; /* Used to iterate through table columns */ 25525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ 25535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ 25545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; 25565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer *pT = p->pTokenizer; 25575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_module const *pModule = pT->pModule; 25585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pCsr->isRequireSeek==0 ); 25605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iDocid = sqlite3_column_int64(pCsr->pStmt, 0); 25615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){ 25635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); 25645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_tokenizer_cursor *pTC = 0; 25655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = pModule->xOpen(pT, zText, -1, &pTC); 25675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while( rc==SQLITE_OK ){ 25685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) char const *zToken; /* Buffer containing token */ 25695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nToken; /* Number of bytes in token */ 25705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iDum1, iDum2; /* Dummy variables */ 25715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iPos; /* Position of token in zText */ 25725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pTC->pTokenizer = pT; 25745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); 25755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ 25765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3PhraseToken *pPT = pDef->pToken; 25775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( (pDef->iCol>=p->nColumn || pDef->iCol==i) 25785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken)) 25795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) && (0==memcmp(zToken, pPT->z, pPT->n)) 25805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ){ 25815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); 25825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pTC ) pModule->xClose(pTC); 25865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_DONE ) rc = SQLITE_OK; 25875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ 25905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( pDef->pList ){ 25915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3PendingListAppendVarint(&pDef->pList, 0); 25925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 25955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 25975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 25985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 25995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 26005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Add an entry for token pToken to the pCsr->pDeferred list. 26015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 26025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3DeferToken( 26035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Cursor *pCsr, /* Fts3 table cursor */ 26045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3PhraseToken *pToken, /* Token to defer */ 26055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int iCol /* Column that token must appear in (or -1) */ 26065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 26075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3DeferredToken *pDeferred; 26085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pDeferred = sqlite3_malloc(sizeof(*pDeferred)); 26095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( !pDeferred ){ 26105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_NOMEM; 26115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(pDeferred, 0, sizeof(*pDeferred)); 26135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pDeferred->pToken = pToken; 26145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pDeferred->pNext = pCsr->pDeferred; 26155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pDeferred->iCol = iCol; 26165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pCsr->pDeferred = pDeferred; 26175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( pToken->pDeferred==0 ); 26195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) pToken->pDeferred = pDeferred; 26205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SQLITE_OK; 26225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 26235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 26265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** This function does the work for the xUpdate method of FTS3 virtual 26275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** tables. 26285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 26295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3UpdateMethod( 26305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_vtab *pVtab, /* FTS3 vtab object */ 26315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nArg, /* Size of argument array */ 26325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_value **apVal, /* Array of arguments */ 26335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ 26345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)){ 26355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Fts3Table *p = (Fts3Table *)pVtab; 26365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc = SQLITE_OK; /* Return Code */ 26375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isRemove = 0; /* True for an UPDATE or DELETE */ 26385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ 26395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *aSzIns; /* Sizes of inserted documents */ 26405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) u32 *aSzDel; /* Sizes of deleted documents */ 26415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int nChng = 0; /* Net change in number of documents */ 26425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) assert( p->pSegments==0 ); 26445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Allocate space to hold the change in document sizes */ 26465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); 26475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( aSzIns==0 ) return SQLITE_NOMEM; 26485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) aSzDel = &aSzIns[p->nColumn+1]; 26495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); 26505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If this is a DELETE or UPDATE operation, remove the old record. */ 26525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ 26535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int isEmpty = 0; 26545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3IsEmpty(p, apVal, &isEmpty); 26555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 26565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( isEmpty ){ 26575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* Deleting this row means the whole table is empty. In this case 26585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** delete the contents of all three tables and throw away any 26595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ** data in the pendingTerms hash table. 26605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) */ 26615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3DeleteAll(p); 26625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 26635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) isRemove = 1; 26645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iRemove = sqlite3_value_int64(apVal[0]); 26655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3PendingTermsDocid(p, iRemove); 26665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3DeleteTerms(&rc, p, apVal, aSzDel); 26675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); 26685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->bHasDocsize ){ 26695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); 26705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nChng--; 26725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ 26755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(aSzIns); 26765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return fts3SpecialInsert(p, apVal[p->nColumn+2]); 26775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) /* If this is an INSERT or UPDATE operation, insert the new record. */ 26805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( nArg>1 && rc==SQLITE_OK ){ 26815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3InsertData(p, apVal, pRowid); 26825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ 26835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3PendingTermsDocid(p, *pRowid); 26845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 26865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3InsertTerms(p, apVal, aSzIns); 26875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->bHasDocsize ){ 26895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3InsertDocsize(&rc, p, aSzIns); 26905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nChng++; 26925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( p->bHasStat ){ 26955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); 26965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 26975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 26985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_free(aSzIns); 26995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3SegmentsClose(p); 27005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 27015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 27025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 27035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)/* 27045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** Flush any data in the pending-terms hash table to disk. If successful, 27055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** merge all segments in the database (including the new segment, if 27065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)** there was any data to flush) into a single segment. 27075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)*/ 27085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)int sqlite3Fts3Optimize(Fts3Table *p){ 27095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) int rc; 27105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); 27115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 27125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); 27135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 27145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); 27155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if( rc==SQLITE_OK ){ 27165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3PendingTermsClear(p); 27175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 27185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) }else{ 27195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); 27205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); 27215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 27225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 27235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sqlite3Fts3SegmentsClose(p); 27245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return rc; 27255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)} 27265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 27275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#endif 2728