Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -276,24 +276,24 @@ #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) # define SQLITE_CORE 1 #endif +#include "fts3Int.h" + #include #include +#include #include #include -#include #include "fts3.h" -#include "fts3_expr.h" -#include "fts3_hash.h" -#include "fts3_tokenizer.h" #ifndef SQLITE_CORE # include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #endif + /* TODO(shess) MAN, this thing needs some refactoring. At minimum, it ** would be nice to order the file better, perhaps something along the ** lines of: @@ -311,29 +311,10 @@ # define FTSTRACE(A) printf A; fflush(stdout) #else # define FTSTRACE(A) #endif -/* It is not safe to call isspace(), tolower(), or isalnum() on -** hi-bit-set characters. This is the same solution used in the -** tokenizer. -*/ -/* TODO(shess) The snippet-generation code should be using the -** tokenizer-generated tokens rather than doing its own local -** tokenization. -*/ -/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */ -static int safe_isspace(char c){ - return (c&0x80)==0 ? isspace(c) : 0; -} -static int safe_tolower(char c){ - return (c&0x80)==0 ? tolower(c) : c; -} -static int safe_isalnum(char c){ - return (c&0x80)==0 ? isalnum(c) : 0; -} - typedef enum DocListType { DL_DOCIDS, /* docids only */ DL_POSITIONS, /* docids + positions */ DL_POSITIONS_OFFSETS /* docids + positions + offsets */ } DocListType; @@ -355,15 +336,10 @@ POS_END = 0, /* end of this position list */ POS_COLUMN, /* followed by new column number */ POS_BASE }; -/* MERGE_COUNT controls how often we merge segments (see comment at -** top of file). -*/ -#define MERGE_COUNT 16 - /* utility functions */ /* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single ** record to prevent errors of the form: ** @@ -378,2225 +354,71 @@ # define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b))) #else # define SCRAMBLE(b) #endif -/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */ -#define VARINT_MAX 10 - -/* Write a 64-bit variable-length integer to memory starting at p[0]. - * The length of data written will be between 1 and VARINT_MAX bytes. - * The number of bytes written is returned. */ -static int fts3PutVarint(char *p, sqlite_int64 v){ +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ unsigned char *q = (unsigned char *) p; sqlite_uint64 vu = v; do{ *q++ = (unsigned char) ((vu & 0x7f) | 0x80); vu >>= 7; }while( vu!=0 ); q[-1] &= 0x7f; /* turn off high bit in final byte */ - assert( q - (unsigned char *)p <= VARINT_MAX ); + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); return (int) (q - (unsigned char *)p); } -/* Read a 64-bit variable-length integer from memory starting at p[0]. - * Return the number of bytes read, or 0 on error. - * The value is stored in *v. */ -static int fts3GetVarint(const char *p, sqlite_int64 *v){ +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ const unsigned char *q = (const unsigned char *) p; sqlite_uint64 x = 0, y = 1; while( (*q & 0x80) == 0x80 ){ x += y * (*q++ & 0x7f); y <<= 7; - if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */ + if( q - (unsigned char *)p >= FTS3_VARINT_MAX ){ /* bad data */ assert( 0 ); return 0; } } x += y * (*q++); *v = (sqlite_int64) x; return (int) (q - (unsigned char *)p); } -static int fts3GetVarint32(const char *p, int *pi){ +/* +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a +** 32-bit integer before it is returned. +*/ +int sqlite3Fts3GetVarint32(const char *p, int *pi){ sqlite_int64 i; - int ret = fts3GetVarint(p, &i); + int ret = sqlite3Fts3GetVarint(p, &i); *pi = (int) i; assert( *pi==i ); return ret; } -/*******************************************************************/ -/* DataBuffer is used to collect data into a buffer in piecemeal -** fashion. It implements the usual distinction between amount of -** data currently stored (nData) and buffer capacity (nCapacity). -** -** dataBufferInit - create a buffer with given initial capacity. -** dataBufferReset - forget buffer's data, retaining capacity. -** dataBufferDestroy - free buffer's data. -** dataBufferSwap - swap contents of two buffers. -** dataBufferExpand - expand capacity without adding data. -** dataBufferAppend - append data. -** dataBufferAppend2 - append two pieces of data at once. -** dataBufferReplace - replace buffer's data. -*/ -typedef struct DataBuffer { - char *pData; /* Pointer to malloc'ed buffer. */ - int nCapacity; /* Size of pData buffer. */ - int nData; /* End of data loaded into pData. */ -} DataBuffer; - -static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){ - assert( nCapacity>=0 ); - pBuffer->nData = 0; - pBuffer->nCapacity = nCapacity; - pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity); -} -static void dataBufferReset(DataBuffer *pBuffer){ - pBuffer->nData = 0; -} -static void dataBufferDestroy(DataBuffer *pBuffer){ - if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData); - SCRAMBLE(pBuffer); -} -static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){ - DataBuffer tmp = *pBuffer1; - *pBuffer1 = *pBuffer2; - *pBuffer2 = tmp; -} -static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ - assert( nAddCapacity>0 ); - /* TODO(shess) Consider expanding more aggressively. Note that the - ** underlying malloc implementation may take care of such things for - ** us already. - */ - if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){ - pBuffer->nCapacity = pBuffer->nData+nAddCapacity; - pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity); - } -} -static void dataBufferAppend(DataBuffer *pBuffer, - const char *pSource, int nSource){ - assert( nSource>0 && pSource!=NULL ); - dataBufferExpand(pBuffer, nSource); - memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource); - pBuffer->nData += nSource; -} -static void dataBufferAppend2(DataBuffer *pBuffer, - const char *pSource1, int nSource1, - const char *pSource2, int nSource2){ - assert( nSource1>0 && pSource1!=NULL ); - assert( nSource2>0 && pSource2!=NULL ); - dataBufferExpand(pBuffer, nSource1+nSource2); - memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1); - memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2); - pBuffer->nData += nSource1+nSource2; -} -static void dataBufferReplace(DataBuffer *pBuffer, - const char *pSource, int nSource){ - dataBufferReset(pBuffer); - dataBufferAppend(pBuffer, pSource, nSource); -} - -/* StringBuffer is a null-terminated version of DataBuffer. */ -typedef struct StringBuffer { - DataBuffer b; /* Includes null terminator. */ -} StringBuffer; - -static void initStringBuffer(StringBuffer *sb){ - dataBufferInit(&sb->b, 100); - dataBufferReplace(&sb->b, "", 1); -} -static int stringBufferLength(StringBuffer *sb){ - return sb->b.nData-1; -} -static char *stringBufferData(StringBuffer *sb){ - return sb->b.pData; -} -static void stringBufferDestroy(StringBuffer *sb){ - dataBufferDestroy(&sb->b); -} - -static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){ - assert( sb->b.nData>0 ); - if( nFrom>0 ){ - sb->b.nData--; - dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1); - } -} -static void append(StringBuffer *sb, const char *zFrom){ - nappend(sb, zFrom, strlen(zFrom)); -} - -/* Append a list of strings separated by commas. */ -static void appendList(StringBuffer *sb, int nString, char **azString){ - int i; - for(i=0; i0 ) append(sb, ", "); - append(sb, azString[i]); - } -} - -static int endsInWhiteSpace(StringBuffer *p){ - return stringBufferLength(p)>0 && - safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]); -} - -/* If the StringBuffer ends in something other than white space, add a -** single space character to the end. -*/ -static void appendWhiteSpace(StringBuffer *p){ - if( stringBufferLength(p)==0 ) return; - if( !endsInWhiteSpace(p) ) append(p, " "); -} - -/* Remove white space from the end of the StringBuffer */ -static void trimWhiteSpace(StringBuffer *p){ - while( endsInWhiteSpace(p) ){ - p->b.pData[--p->b.nData-1] = '\0'; - } -} - -/*******************************************************************/ -/* DLReader is used to read document elements from a doclist. The -** current docid is cached, so dlrDocid() is fast. DLReader does not -** own the doclist buffer. -** -** dlrAtEnd - true if there's no more data to read. -** dlrDocid - docid of current document. -** dlrDocData - doclist data for current document (including docid). -** dlrDocDataBytes - length of same. -** dlrAllDataBytes - length of all remaining data. -** dlrPosData - position data for current document. -** dlrPosDataLen - length of pos data for current document (incl POS_END). -** dlrStep - step to current document. -** dlrInit - initial for doclist of given type against given data. -** dlrDestroy - clean up. -** -** Expected usage is something like: -** -** DLReader reader; -** dlrInit(&reader, pData, nData); -** while( !dlrAtEnd(&reader) ){ -** // calls to dlrDocid() and kin. -** dlrStep(&reader); -** } -** dlrDestroy(&reader); -*/ -typedef struct DLReader { - DocListType iType; - const char *pData; - int nData; - - sqlite_int64 iDocid; - int nElement; -} DLReader; - -static int dlrAtEnd(DLReader *pReader){ - assert( pReader->nData>=0 ); - return pReader->nData==0; -} -static sqlite_int64 dlrDocid(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->iDocid; -} -static const char *dlrDocData(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->pData; -} -static int dlrDocDataBytes(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->nElement; -} -static int dlrAllDataBytes(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - return pReader->nData; -} -/* TODO(shess) Consider adding a field to track iDocid varint length -** to make these two functions faster. This might matter (a tiny bit) -** for queries. -*/ -static const char *dlrPosData(DLReader *pReader){ - sqlite_int64 iDummy; - int n = fts3GetVarint(pReader->pData, &iDummy); - assert( !dlrAtEnd(pReader) ); - return pReader->pData+n; -} -static int dlrPosDataLen(DLReader *pReader){ - sqlite_int64 iDummy; - int n = fts3GetVarint(pReader->pData, &iDummy); - assert( !dlrAtEnd(pReader) ); - return pReader->nElement-n; -} -static void dlrStep(DLReader *pReader){ - assert( !dlrAtEnd(pReader) ); - - /* Skip past current doclist element. */ - assert( pReader->nElement<=pReader->nData ); - pReader->pData += pReader->nElement; - pReader->nData -= pReader->nElement; - - /* If there is more data, read the next doclist element. */ - if( pReader->nData!=0 ){ - sqlite_int64 iDocidDelta; - int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta); - pReader->iDocid += iDocidDelta; - if( pReader->iType>=DL_POSITIONS ){ - assert( nnData ); - while( 1 ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( n<=pReader->nData ); - if( iDummy==POS_END ) break; - if( iDummy==POS_COLUMN ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( nnData ); - }else if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += fts3GetVarint32(pReader->pData+n, &iDummy); - n += fts3GetVarint32(pReader->pData+n, &iDummy); - assert( nnData ); - } - } - } - pReader->nElement = n; - assert( pReader->nElement<=pReader->nData ); - } -} -static void dlrInit(DLReader *pReader, DocListType iType, - const char *pData, int nData){ - assert( pData!=NULL && nData!=0 ); - pReader->iType = iType; - pReader->pData = pData; - pReader->nData = nData; - pReader->nElement = 0; - pReader->iDocid = 0; - - /* Load the first element's data. There must be a first element. */ - dlrStep(pReader); -} -static void dlrDestroy(DLReader *pReader){ - SCRAMBLE(pReader); -} - -#ifndef NDEBUG -/* Verify that the doclist can be validly decoded. Also returns the -** last docid found because it is convenient in other assertions for -** DLWriter. -*/ -static void docListValidate(DocListType iType, const char *pData, int nData, - sqlite_int64 *pLastDocid){ - sqlite_int64 iPrevDocid = 0; - assert( nData>0 ); - assert( pData!=0 ); - assert( pData+nData>pData ); - while( nData!=0 ){ - sqlite_int64 iDocidDelta; - int n = fts3GetVarint(pData, &iDocidDelta); - iPrevDocid += iDocidDelta; - if( iType>DL_DOCIDS ){ - int iDummy; - while( 1 ){ - n += fts3GetVarint32(pData+n, &iDummy); - if( iDummy==POS_END ) break; - if( iDummy==POS_COLUMN ){ - n += fts3GetVarint32(pData+n, &iDummy); - }else if( iType>DL_POSITIONS ){ - n += fts3GetVarint32(pData+n, &iDummy); - n += fts3GetVarint32(pData+n, &iDummy); - } - assert( n<=nData ); - } - } - assert( n<=nData ); - pData += n; - nData -= n; - } - if( pLastDocid ) *pLastDocid = iPrevDocid; -} -#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o) -#else -#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 ) -#endif - -/*******************************************************************/ -/* DLWriter is used to write doclist data to a DataBuffer. DLWriter -** always appends to the buffer and does not own it. -** -** dlwInit - initialize to write a given type doclistto a buffer. -** dlwDestroy - clear the writer's memory. Does not free buffer. -** dlwAppend - append raw doclist data to buffer. -** dlwCopy - copy next doclist from reader to writer. -** dlwAdd - construct doclist element and append to buffer. -** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter). -*/ -typedef struct DLWriter { - DocListType iType; - DataBuffer *b; - sqlite_int64 iPrevDocid; -#ifndef NDEBUG - int has_iPrevDocid; -#endif -} DLWriter; - -static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){ - pWriter->b = b; - pWriter->iType = iType; - pWriter->iPrevDocid = 0; -#ifndef NDEBUG - pWriter->has_iPrevDocid = 0; -#endif -} -static void dlwDestroy(DLWriter *pWriter){ - SCRAMBLE(pWriter); -} -/* iFirstDocid is the first docid in the doclist in pData. It is -** needed because pData may point within a larger doclist, in which -** case the first item would be delta-encoded. -** -** iLastDocid is the final docid in the doclist in pData. It is -** needed to create the new iPrevDocid for future delta-encoding. The -** code could decode the passed doclist to recreate iLastDocid, but -** the only current user (docListMerge) already has decoded this -** information. -*/ -/* TODO(shess) This has become just a helper for docListMerge. -** Consider a refactor to make this cleaner. -*/ -static void dlwAppend(DLWriter *pWriter, - const char *pData, int nData, - sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){ - sqlite_int64 iDocid = 0; - char c[VARINT_MAX]; - int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */ -#ifndef NDEBUG - sqlite_int64 iLastDocidDelta; -#endif - - /* Recode the initial docid as delta from iPrevDocid. */ - nFirstOld = fts3GetVarint(pData, &iDocid); - assert( nFirstOldiType==DL_DOCIDS) ); - nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid); - - /* Verify that the incoming doclist is valid AND that it ends with - ** the expected docid. This is essential because we'll trust this - ** docid in future delta-encoding. - */ - ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta); - assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta ); - - /* Append recoded initial docid and everything else. Rest of docids - ** should have been delta-encoded from previous initial docid. - */ - if( nFirstOldb, c, nFirstNew, - pData+nFirstOld, nData-nFirstOld); - }else{ - dataBufferAppend(pWriter->b, c, nFirstNew); - } - pWriter->iPrevDocid = iLastDocid; -} -static void dlwCopy(DLWriter *pWriter, DLReader *pReader){ - dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader), - dlrDocid(pReader), dlrDocid(pReader)); -} -static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){ - char c[VARINT_MAX]; - int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid); - - /* Docids must ascend. */ - assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid ); - assert( pWriter->iType==DL_DOCIDS ); - - dataBufferAppend(pWriter->b, c, n); - pWriter->iPrevDocid = iDocid; -#ifndef NDEBUG - pWriter->has_iPrevDocid = 1; -#endif -} - -/*******************************************************************/ -/* PLReader is used to read data from a document's position list. As -** the caller steps through the list, data is cached so that varints -** only need to be decoded once. -** -** plrInit, plrDestroy - create/destroy a reader. -** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors -** plrAtEnd - at end of stream, only call plrDestroy once true. -** plrStep - step to the next element. -*/ -typedef struct PLReader { - /* These refer to the next position's data. nData will reach 0 when - ** reading the last position, so plrStep() signals EOF by setting - ** pData to NULL. - */ - const char *pData; - int nData; - - DocListType iType; - int iColumn; /* the last column read */ - int iPosition; /* the last position read */ - int iStartOffset; /* the last start offset read */ - int iEndOffset; /* the last end offset read */ -} PLReader; - -static int plrAtEnd(PLReader *pReader){ - return pReader->pData==NULL; -} -static int plrColumn(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iColumn; -} -static int plrPosition(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iPosition; -} -static int plrStartOffset(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iStartOffset; -} -static int plrEndOffset(PLReader *pReader){ - assert( !plrAtEnd(pReader) ); - return pReader->iEndOffset; -} -static void plrStep(PLReader *pReader){ - int i, n; - - assert( !plrAtEnd(pReader) ); - - if( pReader->nData==0 ){ - pReader->pData = NULL; - return; - } - - n = fts3GetVarint32(pReader->pData, &i); - if( i==POS_COLUMN ){ - n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn); - pReader->iPosition = 0; - pReader->iStartOffset = 0; - n += fts3GetVarint32(pReader->pData+n, &i); - } - /* Should never see adjacent column changes. */ - assert( i!=POS_COLUMN ); - - if( i==POS_END ){ - pReader->nData = 0; - pReader->pData = NULL; - return; - } - - pReader->iPosition += i-POS_BASE; - if( pReader->iType==DL_POSITIONS_OFFSETS ){ - n += fts3GetVarint32(pReader->pData+n, &i); - pReader->iStartOffset += i; - n += fts3GetVarint32(pReader->pData+n, &i); - pReader->iEndOffset = pReader->iStartOffset+i; - } - assert( n<=pReader->nData ); - pReader->pData += n; - pReader->nData -= n; -} - -static void plrInit(PLReader *pReader, DLReader *pDLReader){ - pReader->pData = dlrPosData(pDLReader); - pReader->nData = dlrPosDataLen(pDLReader); - pReader->iType = pDLReader->iType; - pReader->iColumn = 0; - pReader->iPosition = 0; - pReader->iStartOffset = 0; - pReader->iEndOffset = 0; - plrStep(pReader); -} -static void plrDestroy(PLReader *pReader){ - SCRAMBLE(pReader); -} - -/*******************************************************************/ -/* PLWriter is used in constructing a document's position list. As a -** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op. -** PLWriter writes to the associated DLWriter's buffer. -** -** plwInit - init for writing a document's poslist. -** plwDestroy - clear a writer. -** plwAdd - append position and offset information. -** plwCopy - copy next position's data from reader to writer. -** plwTerminate - add any necessary doclist terminator. -** -** Calling plwAdd() after plwTerminate() may result in a corrupt -** doclist. -*/ -/* TODO(shess) Until we've written the second item, we can cache the -** first item's information. Then we'd have three states: -** -** - initialized with docid, no positions. -** - docid and one position. -** - docid and multiple positions. -** -** Only the last state needs to actually write to dlw->b, which would -** be an improvement in the DLCollector case. -*/ -typedef struct PLWriter { - DLWriter *dlw; - - int iColumn; /* the last column written */ - int iPos; /* the last position written */ - int iOffset; /* the last start offset written */ -} PLWriter; - -/* TODO(shess) In the case where the parent is reading these values -** from a PLReader, we could optimize to a copy if that PLReader has -** the same type as pWriter. -*/ -static void plwAdd(PLWriter *pWriter, int iColumn, int iPos, - int iStartOffset, int iEndOffset){ - /* Worst-case space for POS_COLUMN, iColumn, iPosDelta, - ** iStartOffsetDelta, and iEndOffsetDelta. - */ - char c[5*VARINT_MAX]; - int n = 0; - - /* Ban plwAdd() after plwTerminate(). */ - assert( pWriter->iPos!=-1 ); - - if( pWriter->dlw->iType==DL_DOCIDS ) return; - - if( iColumn!=pWriter->iColumn ){ - n += fts3PutVarint(c+n, POS_COLUMN); - n += fts3PutVarint(c+n, iColumn); - pWriter->iColumn = iColumn; - pWriter->iPos = 0; - pWriter->iOffset = 0; - } - assert( iPos>=pWriter->iPos ); - n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos)); - pWriter->iPos = iPos; - if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){ - assert( iStartOffset>=pWriter->iOffset ); - n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset); - pWriter->iOffset = iStartOffset; - assert( iEndOffset>=iStartOffset ); - n += fts3PutVarint(c+n, iEndOffset-iStartOffset); - } - dataBufferAppend(pWriter->dlw->b, c, n); -} -static void plwCopy(PLWriter *pWriter, PLReader *pReader){ - plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader), - plrStartOffset(pReader), plrEndOffset(pReader)); -} -static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){ - char c[VARINT_MAX]; - int n; - - pWriter->dlw = dlw; - - /* Docids must ascend. */ - assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid ); - n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid); - dataBufferAppend(pWriter->dlw->b, c, n); - pWriter->dlw->iPrevDocid = iDocid; -#ifndef NDEBUG - pWriter->dlw->has_iPrevDocid = 1; -#endif - - pWriter->iColumn = 0; - pWriter->iPos = 0; - pWriter->iOffset = 0; -} -/* TODO(shess) Should plwDestroy() also terminate the doclist? But -** then plwDestroy() would no longer be just a destructor, it would -** also be doing work, which isn't consistent with the overall idiom. -** Another option would be for plwAdd() to always append any necessary -** terminator, so that the output is always correct. But that would -** add incremental work to the common case with the only benefit being -** API elegance. Punt for now. -*/ -static void plwTerminate(PLWriter *pWriter){ - if( pWriter->dlw->iType>DL_DOCIDS ){ - char c[VARINT_MAX]; - int n = fts3PutVarint(c, POS_END); - dataBufferAppend(pWriter->dlw->b, c, n); - } -#ifndef NDEBUG - /* Mark as terminated for assert in plwAdd(). */ - pWriter->iPos = -1; -#endif -} -static void plwDestroy(PLWriter *pWriter){ - SCRAMBLE(pWriter); -} - -/*******************************************************************/ -/* DLCollector wraps PLWriter and DLWriter to provide a -** dynamically-allocated doclist area to use during tokenization. -** -** dlcNew - malloc up and initialize a collector. -** dlcDelete - destroy a collector and all contained items. -** dlcAddPos - append position and offset information. -** dlcAddDoclist - add the collected doclist to the given buffer. -** dlcNext - terminate the current document and open another. -*/ -typedef struct DLCollector { - DataBuffer b; - DLWriter dlw; - PLWriter plw; -} DLCollector; - -/* TODO(shess) This could also be done by calling plwTerminate() and -** dataBufferAppend(). I tried that, expecting nominal performance -** differences, but it seemed to pretty reliably be worth 1% to code -** it this way. I suspect it is the incremental malloc overhead (some -** percentage of the plwTerminate() calls will cause a realloc), so -** this might be worth revisiting if the DataBuffer implementation -** changes. -*/ -static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){ - if( pCollector->dlw.iType>DL_DOCIDS ){ - char c[VARINT_MAX]; - int n = fts3PutVarint(c, POS_END); - dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n); - }else{ - dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData); - } -} -static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){ - plwTerminate(&pCollector->plw); - plwDestroy(&pCollector->plw); - plwInit(&pCollector->plw, &pCollector->dlw, iDocid); -} -static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos, - int iStartOffset, int iEndOffset){ - plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset); -} - -static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){ - DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector)); - dataBufferInit(&pCollector->b, 0); - dlwInit(&pCollector->dlw, iType, &pCollector->b); - plwInit(&pCollector->plw, &pCollector->dlw, iDocid); - return pCollector; -} -static void dlcDelete(DLCollector *pCollector){ - plwDestroy(&pCollector->plw); - dlwDestroy(&pCollector->dlw); - dataBufferDestroy(&pCollector->b); - SCRAMBLE(pCollector); - sqlite3_free(pCollector); -} - - -/* Copy the doclist data of iType in pData/nData into *out, trimming -** unnecessary data as we go. Only columns matching iColumn are -** copied, all columns copied if iColumn is -1. Elements with no -** matching columns are dropped. The output is an iOutType doclist. -*/ -/* NOTE(shess) This code is only valid after all doclists are merged. -** If this is run before merges, then doclist items which represent -** deletion will be trimmed, and will thus not effect a deletion -** during the merge. -*/ -static void docListTrim(DocListType iType, const char *pData, int nData, - int iColumn, DocListType iOutType, DataBuffer *out){ - DLReader dlReader; - DLWriter dlWriter; - - assert( iOutType<=iType ); - - dlrInit(&dlReader, iType, pData, nData); - dlwInit(&dlWriter, iOutType, out); - - while( !dlrAtEnd(&dlReader) ){ - PLReader plReader; - PLWriter plWriter; - int match = 0; - - plrInit(&plReader, &dlReader); - - while( !plrAtEnd(&plReader) ){ - if( iColumn==-1 || plrColumn(&plReader)==iColumn ){ - if( !match ){ - plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader)); - match = 1; - } - plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader), - plrStartOffset(&plReader), plrEndOffset(&plReader)); - } - plrStep(&plReader); - } - if( match ){ - plwTerminate(&plWriter); - plwDestroy(&plWriter); - } - - plrDestroy(&plReader); - dlrStep(&dlReader); - } - dlwDestroy(&dlWriter); - dlrDestroy(&dlReader); -} - -/* Used by docListMerge() to keep doclists in the ascending order by -** docid, then ascending order by age (so the newest comes first). -*/ -typedef struct OrderedDLReader { - DLReader *pReader; - - /* TODO(shess) If we assume that docListMerge pReaders is ordered by - ** age (which we do), then we could use pReader comparisons to break - ** ties. - */ - int idx; -} OrderedDLReader; - -/* Order eof to end, then by docid asc, idx desc. */ -static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){ - if( dlrAtEnd(r1->pReader) ){ - if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */ - return 1; /* Only r1 atEnd(). */ - } - if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */ - - if( dlrDocid(r1->pReader)pReader) ) return -1; - if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1; - - /* Descending on idx. */ - return r2->idx-r1->idx; -} - -/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that -** p[1..n-1] is already sorted. -*/ -/* TODO(shess) Is this frequent enough to warrant a binary search? -** Before implementing that, instrument the code to check. In most -** current usage, I expect that p[0] will be less than p[1] a very -** high proportion of the time. -*/ -static void orderedDLReaderReorder(OrderedDLReader *p, int n){ - while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){ - OrderedDLReader tmp = p[0]; - p[0] = p[1]; - p[1] = tmp; - n--; - p++; - } -} - -/* Given an array of doclist readers, merge their doclist elements -** into out in sorted order (by docid), dropping elements from older -** readers when there is a duplicate docid. pReaders is assumed to be -** ordered by age, oldest first. -*/ -/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably -** be fixed. -*/ -static void docListMerge(DataBuffer *out, - DLReader *pReaders, int nReaders){ - OrderedDLReader readers[MERGE_COUNT]; - DLWriter writer; - int i, n; - const char *pStart = 0; - int nStart = 0; - sqlite_int64 iFirstDocid = 0, iLastDocid = 0; - - assert( nReaders>0 ); - if( nReaders==1 ){ - dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders)); - return; - } - - assert( nReaders<=MERGE_COUNT ); - n = 0; - for(i=0; i0 ){ - orderedDLReaderReorder(readers+i, nReaders-i); - } - - dlwInit(&writer, pReaders[0].iType, out); - while( !dlrAtEnd(readers[0].pReader) ){ - sqlite_int64 iDocid = dlrDocid(readers[0].pReader); - - /* If this is a continuation of the current buffer to copy, extend - ** that buffer. memcpy() seems to be more efficient if it has a - ** lots of data to copy. - */ - if( dlrDocData(readers[0].pReader)==pStart+nStart ){ - nStart += dlrDocDataBytes(readers[0].pReader); - }else{ - if( pStart!=0 ){ - dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); - } - pStart = dlrDocData(readers[0].pReader); - nStart = dlrDocDataBytes(readers[0].pReader); - iFirstDocid = iDocid; - } - iLastDocid = iDocid; - dlrStep(readers[0].pReader); - - /* Drop all of the older elements with the same docid. */ - for(i=1; i0 ){ - orderedDLReaderReorder(readers+i, nReaders-i); - } - } - - /* Copy over any remaining elements. */ - if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid); - dlwDestroy(&writer); -} - -/* Helper function for posListUnion(). Compares the current position -** between left and right, returning as standard C idiom of <0 if -** left0 if left>right, and 0 if left==right. "End" always -** compares greater. -*/ -static int posListCmp(PLReader *pLeft, PLReader *pRight){ - assert( pLeft->iType==pRight->iType ); - if( pLeft->iType==DL_DOCIDS ) return 0; - - if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1; - if( plrAtEnd(pRight) ) return -1; - - if( plrColumn(pLeft)plrColumn(pRight) ) return 1; - - if( plrPosition(pLeft)plrPosition(pRight) ) return 1; - if( pLeft->iType==DL_POSITIONS ) return 0; - - if( plrStartOffset(pLeft)plrStartOffset(pRight) ) return 1; - - if( plrEndOffset(pLeft)plrEndOffset(pRight) ) return 1; - - return 0; -} - -/* Write the union of position lists in pLeft and pRight to pOut. -** "Union" in this case meaning "All unique position tuples". Should -** work with any doclist type, though both inputs and the output -** should be the same type. -*/ -static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){ - PLReader left, right; - PLWriter writer; - - assert( dlrDocid(pLeft)==dlrDocid(pRight) ); - assert( pLeft->iType==pRight->iType ); - assert( pLeft->iType==pOut->iType ); - - plrInit(&left, pLeft); - plrInit(&right, pRight); - plwInit(&writer, pOut, dlrDocid(pLeft)); - - while( !plrAtEnd(&left) || !plrAtEnd(&right) ){ - int c = posListCmp(&left, &right); - if( c<0 ){ - plwCopy(&writer, &left); - plrStep(&left); - }else if( c>0 ){ - plwCopy(&writer, &right); - plrStep(&right); - }else{ - plwCopy(&writer, &left); - plrStep(&left); - plrStep(&right); - } - } - - plwTerminate(&writer); - plwDestroy(&writer); - plrDestroy(&left); - plrDestroy(&right); -} - -/* Write the union of doclists in pLeft and pRight to pOut. For -** docids in common between the inputs, the union of the position -** lists is written. Inputs and outputs are always type DL_DEFAULT. -*/ -static void docListUnion( - const char *pLeft, int nLeft, - const char *pRight, int nRight, - DataBuffer *pOut /* Write the combined doclist here */ -){ - DLReader left, right; - DLWriter writer; - - if( nLeft==0 ){ - if( nRight!=0) dataBufferAppend(pOut, pRight, nRight); - return; - } - if( nRight==0 ){ - dataBufferAppend(pOut, pLeft, nLeft); - return; - } - - dlrInit(&left, DL_DEFAULT, pLeft, nLeft); - dlrInit(&right, DL_DEFAULT, pRight, nRight); - dlwInit(&writer, DL_DEFAULT, pOut); - - while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){ - if( dlrAtEnd(&right) ){ - dlwCopy(&writer, &left); - dlrStep(&left); - }else if( dlrAtEnd(&left) ){ - dlwCopy(&writer, &right); - dlrStep(&right); - }else if( dlrDocid(&left)dlrDocid(&right) ){ - dlwCopy(&writer, &right); - dlrStep(&right); - }else{ - posListUnion(&left, &right, &writer); - dlrStep(&left); - dlrStep(&right); - } - } - - dlrDestroy(&left); - dlrDestroy(&right); - dlwDestroy(&writer); -} - -/* -** This function is used as part of the implementation of phrase and -** NEAR matching. -** -** pLeft and pRight are DLReaders positioned to the same docid in -** lists of type DL_POSITION. This function writes an entry to the -** DLWriter pOut for each position in pRight that is less than -** (nNear+1) greater (but not equal to or smaller) than a position -** in pLeft. For example, if nNear is 0, and the positions contained -** by pLeft and pRight are: -** -** pLeft: 5 10 15 20 -** pRight: 6 9 17 21 -** -** then the docid is added to pOut. If pOut is of type DL_POSITIONS, -** then a positionids "6" and "21" are also added to pOut. -** -** If boolean argument isSaveLeft is true, then positionids are copied -** from pLeft instead of pRight. In the example above, the positions "5" -** and "20" would be added instead of "6" and "21". -*/ -static void posListPhraseMerge( - DLReader *pLeft, - DLReader *pRight, - int nNear, - int isSaveLeft, - DLWriter *pOut -){ - PLReader left, right; - PLWriter writer; - int match = 0; - - assert( dlrDocid(pLeft)==dlrDocid(pRight) ); - assert( pOut->iType!=DL_POSITIONS_OFFSETS ); - - plrInit(&left, pLeft); - plrInit(&right, pRight); - - while( !plrAtEnd(&left) && !plrAtEnd(&right) ){ - if( plrColumn(&left)plrColumn(&right) ){ - plrStep(&right); - }else if( plrPosition(&left)>=plrPosition(&right) ){ - plrStep(&right); - }else{ - if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){ - if( !match ){ - plwInit(&writer, pOut, dlrDocid(pLeft)); - match = 1; - } - if( !isSaveLeft ){ - plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0); - }else{ - plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0); - } - plrStep(&right); - }else{ - plrStep(&left); - } - } - } - - if( match ){ - plwTerminate(&writer); - plwDestroy(&writer); - } - - plrDestroy(&left); - plrDestroy(&right); -} - -/* -** Compare the values pointed to by the PLReaders passed as arguments. -** Return -1 if the value pointed to by pLeft is considered less than -** the value pointed to by pRight, +1 if it is considered greater -** than it, or 0 if it is equal. i.e. -** -** (*pLeft - *pRight) -** -** A PLReader that is in the EOF condition is considered greater than -** any other. If neither argument is in EOF state, the return value of -** plrColumn() is used. If the plrColumn() values are equal, the -** comparison is on the basis of plrPosition(). -*/ -static int plrCompare(PLReader *pLeft, PLReader *pRight){ - assert(!plrAtEnd(pLeft) || !plrAtEnd(pRight)); - - if( plrAtEnd(pRight) || plrAtEnd(pLeft) ){ - return (plrAtEnd(pRight) ? -1 : 1); - } - if( plrColumn(pLeft)!=plrColumn(pRight) ){ - return ((plrColumn(pLeft)0) -** and write the results into pOut. -** -** A phrase intersection means that two documents only match -** if pLeft.iPos+1==pRight.iPos. -** -** A NEAR intersection means that two documents only match if -** (abs(pLeft.iPos-pRight.iPos) 0", - /* SEGDIR_DELETE */ "delete from %_segdir where level = ?", - - /* NOTE(shess): The first three results of the following two - ** statements must match. - */ - /* SEGDIR_SELECT_SEGMENT */ - "select start_block, leaves_end_block, root from %_segdir " - " where level = ? and idx = ?", - /* SEGDIR_SELECT_ALL */ - "select start_block, leaves_end_block, root from %_segdir " - " order by level desc, idx asc", - /* SEGDIR_DELETE_ALL */ "delete from %_segdir", - /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir", -}; - -/* -** A connection to a fulltext index is an instance of the following -** structure. The xCreate and xConnect methods create an instance -** of this structure and xDestroy and xDisconnect free that instance. -** All other methods receive a pointer to the structure as one of their -** arguments. -*/ -struct fulltext_vtab { - sqlite3_vtab base; /* Base class used by SQLite core */ - sqlite3 *db; /* The database connection */ - const char *zDb; /* logical database name */ - const char *zName; /* virtual table name */ - int nColumn; /* number of columns in virtual table */ - char **azColumn; /* column names. malloced */ - char **azContentColumn; /* column names in content table; malloced */ - sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ - - /* Precompiled statements which we keep as long as the table is - ** open. - */ - sqlite3_stmt *pFulltextStatements[MAX_STMT]; - - /* Precompiled statements used for segment merges. We run a - ** separate select across the leaf level of each tree being merged. - */ - sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT]; - /* The statement used to prepare pLeafSelectStmts. */ -#define LEAF_SELECT \ - "select block from %_segments where blockid between ? and ? order by blockid" - - /* These buffer pending index updates during transactions. - ** nPendingData estimates the memory size of the pending data. It - ** doesn't include the hash-bucket overhead, nor any malloc - ** overhead. When nPendingData exceeds kPendingThreshold, the - ** buffer is flushed even before the transaction closes. - ** pendingTerms stores the data, and is only valid when nPendingData - ** is >=0 (nPendingData<0 means pendingTerms has not been - ** initialized). iPrevDocid is the last docid written, used to make - ** certain we're inserting in sorted order. - */ - int nPendingData; -#define kPendingThreshold (1*1024*1024) - sqlite_int64 iPrevDocid; - fts3Hash pendingTerms; -}; - -/* -** When the core wants to do a query, it create a cursor using a -** call to xOpen. This structure is an instance of a cursor. It -** is destroyed by xClose. -*/ -typedef struct fulltext_cursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */ - sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ - int eof; /* True if at End Of Results */ - Fts3Expr *pExpr; /* Parsed MATCH query string */ - Snippet snippet; /* Cached snippet for the current row */ - int iColumn; /* Column being searched */ - DataBuffer result; /* Doclist results from fulltextQuery */ - DLReader reader; /* Result reader if result not empty */ -} fulltext_cursor; - -static fulltext_vtab *cursor_vtab(fulltext_cursor *c){ - return (fulltext_vtab *) c->base.pVtab; -} - -static const sqlite3_module fts3Module; /* forward declaration */ - -/* Return a dynamically generated statement of the form - * insert into %_content (docid, ...) values (?, ...) - */ -static const char *contentInsertStatement(fulltext_vtab *v){ - StringBuffer sb; - int i; - - initStringBuffer(&sb); - append(&sb, "insert into %_content (docid, "); - appendList(&sb, v->nColumn, v->azContentColumn); - append(&sb, ") values (?"); - for(i=0; inColumn; ++i) - append(&sb, ", ?"); - append(&sb, ")"); - return stringBufferData(&sb); -} - -/* Return a dynamically generated statement of the form - * select from %_content where docid = ? - */ -static const char *contentSelectStatement(fulltext_vtab *v){ - StringBuffer sb; - initStringBuffer(&sb); - append(&sb, "SELECT "); - appendList(&sb, v->nColumn, v->azContentColumn); - append(&sb, " FROM %_content WHERE docid = ?"); - return stringBufferData(&sb); -} - -/* Return a dynamically generated statement of the form - * update %_content set [col_0] = ?, [col_1] = ?, ... - * where docid = ? - */ -static const char *contentUpdateStatement(fulltext_vtab *v){ - StringBuffer sb; - int i; - - initStringBuffer(&sb); - append(&sb, "update %_content set "); - for(i=0; inColumn; ++i) { - if( i>0 ){ - append(&sb, ", "); - } - append(&sb, v->azContentColumn[i]); - append(&sb, " = ?"); - } - append(&sb, " where docid = ?"); - return stringBufferData(&sb); -} - -/* Puts a freshly-prepared statement determined by iStmt in *ppStmt. -** If the indicated statement has never been prepared, it is prepared -** and cached, otherwise the cached version is reset. -*/ -static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt, - sqlite3_stmt **ppStmt){ - assert( iStmtpFulltextStatements[iStmt]==NULL ){ - const char *zStmt; - int rc; - switch( iStmt ){ - case CONTENT_INSERT_STMT: - zStmt = contentInsertStatement(v); break; - case CONTENT_SELECT_STMT: - zStmt = contentSelectStatement(v); break; - case CONTENT_UPDATE_STMT: - zStmt = contentUpdateStatement(v); break; - default: - zStmt = fulltext_zStatement[iStmt]; - } - rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt], - zStmt); - if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt); - if( rc!=SQLITE_OK ) return rc; - } else { - int rc = sqlite3_reset(v->pFulltextStatements[iStmt]); - if( rc!=SQLITE_OK ) return rc; - } - - *ppStmt = v->pFulltextStatements[iStmt]; - return SQLITE_OK; -} - -/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and -** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE, -** where we expect no results. -*/ -static int sql_single_step(sqlite3_stmt *s){ - int rc = sqlite3_step(s); - return (rc==SQLITE_DONE) ? SQLITE_OK : rc; -} - -/* Like sql_get_statement(), but for special replicated LEAF_SELECT -** statements. idx -1 is a special case for an uncached version of -** the statement (used in the optimize implementation). -*/ -/* TODO(shess) Write version for generic statements and then share -** that between the cached-statement functions. -*/ -static int sql_get_leaf_statement(fulltext_vtab *v, int idx, - sqlite3_stmt **ppStmt){ - assert( idx>=-1 && idxdb, v->zDb, v->zName, ppStmt, LEAF_SELECT); - }else if( v->pLeafSelectStmts[idx]==NULL ){ - int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx], - LEAF_SELECT); - if( rc!=SQLITE_OK ) return rc; - }else{ - int rc = sqlite3_reset(v->pLeafSelectStmts[idx]); - if( rc!=SQLITE_OK ) return rc; - } - - *ppStmt = v->pLeafSelectStmts[idx]; - return SQLITE_OK; -} - -/* insert into %_content (docid, ...) values ([docid], [pValues]) -** If the docid contains SQL NULL, then a unique docid will be -** generated. -*/ -static int content_insert(fulltext_vtab *v, sqlite3_value *docid, - sqlite3_value **pValues){ - sqlite3_stmt *s; - int i; - int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_value(s, 1, docid); - if( rc!=SQLITE_OK ) return rc; - - for(i=0; inColumn; ++i){ - rc = sqlite3_bind_value(s, 2+i, pValues[i]); - if( rc!=SQLITE_OK ) return rc; - } - - return sql_single_step(s); -} - -/* update %_content set col0 = pValues[0], col1 = pValues[1], ... - * where docid = [iDocid] */ -static int content_update(fulltext_vtab *v, sqlite3_value **pValues, - sqlite_int64 iDocid){ - sqlite3_stmt *s; - int i; - int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - for(i=0; inColumn; ++i){ - rc = sqlite3_bind_value(s, 1+i, pValues[i]); - if( rc!=SQLITE_OK ) return rc; - } - - rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -static void freeStringArray(int nString, const char **pString){ - int i; - - for (i=0 ; i < nString ; ++i) { - if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]); - } - sqlite3_free((void *) pString); -} - -/* select * from %_content where docid = [iDocid] - * The caller must delete the returned array and all strings in it. - * null fields will be NULL in the returned array. - * - * TODO: Perhaps we should return pointer/length strings here for consistency - * with other code which uses pointer/length. */ -static int content_select(fulltext_vtab *v, sqlite_int64 iDocid, - const char ***pValues){ - sqlite3_stmt *s; - const char **values; - int i; - int rc; - - *pValues = NULL; - - rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iDocid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc!=SQLITE_ROW ) return rc; - - values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *)); - for(i=0; inColumn; ++i){ - if( sqlite3_column_type(s, i)==SQLITE_NULL ){ - values[i] = NULL; - }else{ - values[i] = string_dup((char*)sqlite3_column_text(s, i)); - } - } - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ){ - *pValues = values; - return SQLITE_OK; - } - - freeStringArray(v->nColumn, values); - return rc; -} - -/* delete from %_content where docid = [iDocid ] */ -static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iDocid); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if -** no rows exist, and any error in case of failure. -*/ -static int content_exists(fulltext_vtab *v){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc!=SQLITE_ROW ) return rc; - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_ROW; - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - return rc; -} - -/* insert into %_segments values ([pData]) -** returns assigned blockid in *piBlockid -*/ -static int block_insert(fulltext_vtab *v, const char *pData, int nData, - sqlite_int64 *piBlockid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - - /* blockid column is an alias for rowid. */ - *piBlockid = sqlite3_last_insert_rowid(v->db); - return SQLITE_OK; -} - -/* delete from %_segments -** where blockid between [iStartBlockid] and [iEndBlockid] -** -** Deletes the range of blocks, inclusive, used to delete the blocks -** which form a segment. -*/ -static int block_delete(fulltext_vtab *v, - sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iStartBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 2, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found -** at iLevel. Returns SQLITE_DONE if there are no segments at -** iLevel. Otherwise returns an error. -*/ -static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - /* Should always get at least one row due to how max() works. */ - if( rc==SQLITE_DONE ) return SQLITE_DONE; - if( rc!=SQLITE_ROW ) return rc; - - /* NULL means that there were no inputs to max(). */ - if( SQLITE_NULL==sqlite3_column_type(s, 0) ){ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - return rc; - } - - *pidx = sqlite3_column_int(s, 0); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - return SQLITE_ROW; -} - -/* insert into %_segdir values ( -** [iLevel], [idx], -** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid], -** [pRootData] -** ) -*/ -static int segdir_set(fulltext_vtab *v, int iLevel, int idx, - sqlite_int64 iStartBlockid, - sqlite_int64 iLeavesEndBlockid, - sqlite_int64 iEndBlockid, - const char *pRootData, int nRootData){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 2, idx); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 3, iStartBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 5, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Queries %_segdir for the block span of the segments in level -** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel, -** SQLITE_ROW if there are blocks, else an error. -*/ -static int segdir_span(fulltext_vtab *v, int iLevel, - sqlite_int64 *piStartBlockid, - sqlite_int64 *piEndBlockid){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */ - if( rc!=SQLITE_ROW ) return rc; - - /* This happens if all segments at this level are entirely inline. */ - if( SQLITE_NULL==sqlite3_column_type(s, 0) ){ - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - int rc2 = sqlite3_step(s); - if( rc2==SQLITE_ROW ) return SQLITE_ERROR; - return rc2; - } - - *piStartBlockid = sqlite3_column_int64(s, 0); - *piEndBlockid = sqlite3_column_int64(s, 1); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - return SQLITE_ROW; -} - -/* Delete the segment blocks and segment directory records for all -** segments at iLevel. -*/ -static int segdir_delete(fulltext_vtab *v, int iLevel){ - sqlite3_stmt *s; - sqlite_int64 iStartBlockid, iEndBlockid; - int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid); - if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc; - - if( rc==SQLITE_ROW ){ - rc = block_delete(v, iStartBlockid, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - } - - /* Delete the segment directory itself. */ - rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Delete entire fts index, SQLITE_OK on success, relevant error on -** failure. -*/ -static int segdir_delete_all(fulltext_vtab *v){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sql_single_step(s); - if( rc!=SQLITE_OK ) return rc; - - rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - return sql_single_step(s); -} - -/* Returns SQLITE_OK with *pnSegments set to the number of entries in -** %_segdir and *piMaxLevel set to the highest level which has a -** segment. Otherwise returns the SQLite error which caused failure. -*/ -static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){ - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - /* TODO(shess): This case should not be possible? Should stronger - ** measures be taken if it happens? - */ - if( rc==SQLITE_DONE ){ - *pnSegments = 0; - *piMaxLevel = 0; - return SQLITE_OK; - } - if( rc!=SQLITE_ROW ) return rc; - - *pnSegments = sqlite3_column_int(s, 0); - *piMaxLevel = sqlite3_column_int(s, 1); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_OK; - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - return rc; -} - -/* TODO(shess) clearPendingTerms() is far down the file because -** writeZeroSegment() is far down the file because LeafWriter is far -** down the file. Consider refactoring the code to move the non-vtab -** code above the vtab code so that we don't need this forward -** reference. -*/ -static int clearPendingTerms(fulltext_vtab *v); - -/* -** Free the memory used to contain a fulltext_vtab structure. -*/ -static void fulltext_vtab_destroy(fulltext_vtab *v){ - int iStmt, i; - - FTSTRACE(("FTS3 Destroy %p\n", v)); - for( iStmt=0; iStmtpFulltextStatements[iStmt]!=NULL ){ - sqlite3_finalize(v->pFulltextStatements[iStmt]); - v->pFulltextStatements[iStmt] = NULL; - } - } - - for( i=0; ipLeafSelectStmts[i]!=NULL ){ - sqlite3_finalize(v->pLeafSelectStmts[i]); - v->pLeafSelectStmts[i] = NULL; - } - } - - if( v->pTokenizer!=NULL ){ - v->pTokenizer->pModule->xDestroy(v->pTokenizer); - v->pTokenizer = NULL; - } - - clearPendingTerms(v); - - sqlite3_free(v->azColumn); - for(i = 0; i < v->nColumn; ++i) { - sqlite3_free(v->azContentColumn[i]); - } - sqlite3_free(v->azContentColumn); - sqlite3_free(v); -} - -/* -** Token types for parsing the arguments to xConnect or xCreate. -*/ -#define TOKEN_EOF 0 /* End of file */ -#define TOKEN_SPACE 1 /* Any kind of whitespace */ -#define TOKEN_ID 2 /* An identifier */ -#define TOKEN_STRING 3 /* A string literal */ -#define TOKEN_PUNCT 4 /* A single punctuation character */ - -/* -** If X is a character that can be used in an identifier then -** ftsIdChar(X) will be true. Otherwise it is false. -** -** For ASCII, any character with the high-order bit set is -** allowed in an identifier. For 7-bit characters, -** isFtsIdChar[X] must be 1. -** -** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identfiers. But many SQL implementations do. -** SQLite will allow '$' in identifiers for compatibility. -** But the feature is undocumented. -*/ -static const char isFtsIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ -}; -#define ftsIdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isFtsIdChar[c-0x20])) - - -/* -** Return the length of the token that begins at z[0]. -** Store the token type in *tokenType before returning. -*/ -static int ftsGetToken(const char *z, int *tokenType){ - int i, c; - switch( *z ){ - case 0: { - *tokenType = TOKEN_EOF; - return 0; - } - case ' ': case '\t': case '\n': case '\f': case '\r': { - for(i=1; safe_isspace(z[i]); i++){} - *tokenType = TOKEN_SPACE; - return i; - } - case '`': - case '\'': - case '"': { - int delim = z[0]; - for(i=1; (c=z[i])!=0; i++){ - if( c==delim ){ - if( z[i+1]==delim ){ - i++; - }else{ - break; - } - } - } - *tokenType = TOKEN_STRING; - return i + (c!=0); - } - case '[': { - for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} - *tokenType = TOKEN_ID; - return i; - } - default: { - if( !ftsIdChar(*z) ){ - break; - } - for(i=1; ftsIdChar(z[i]); i++){} - *tokenType = TOKEN_ID; - return i; - } - } - *tokenType = TOKEN_PUNCT; - return 1; -} - -/* -** A token extracted from a string is an instance of the following -** structure. -*/ -typedef struct FtsToken { - const char *z; /* Pointer to token text. Not '\000' terminated */ - short int n; /* Length of the token text in bytes. */ -} FtsToken; - -/* -** Given a input string (which is really one of the argv[] parameters -** passed into xConnect or xCreate) split the string up into tokens. -** Return an array of pointers to '\000' terminated strings, one string -** for each non-whitespace token. -** -** The returned array is terminated by a single NULL pointer. -** -** Space to hold the returned array is obtained from a single -** malloc and should be freed by passing the return value to free(). -** The individual strings within the token list are all a part of -** the single memory allocation and will all be freed at once. -*/ -static char **tokenizeString(const char *z, int *pnToken){ - int nToken = 0; - FtsToken *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) ); - int n = 1; - int e, i; - int totalSize = 0; - char **azToken; - char *zCopy; - while( n>0 ){ - n = ftsGetToken(z, &e); - if( e!=TOKEN_SPACE ){ - aToken[nToken].z = z; - aToken[nToken].n = n; - nToken++; - totalSize += n+1; - } - z += n; - } - azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize ); - zCopy = (char*)&azToken[nToken]; - nToken--; - for(i=0; i>= 7; + }while( v!=0 ); + return i; } /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the @@ -2608,14 +430,14 @@ ** "abc" becomes abc ** 'xyz' becomes xyz ** [pqr] becomes pqr ** `mno` becomes mno */ -static void dequoteString(char *z){ +void sqlite3Fts3Dequote(char *z){ int quote; int i, j; - if( z==0 ) return; + quote = z[0]; switch( quote ){ case '\'': break; case '"': break; case '`': break; /* For MySQL compatibility */ @@ -2635,1351 +457,1370 @@ z[j++] = z[i]; } } } -/* -** The input azIn is a NULL-terminated list of tokens. Remove the first -** token and all punctuation tokens. Remove the quotes from -** around string literal tokens. -** -** Example: -** -** input: tokenize chinese ( 'simplifed' , 'mixed' ) -** output: chinese simplifed mixed -** -** Another example: -** -** input: delimiters ( '[' , ']' , '...' ) -** output: [ ] ... -*/ -static void tokenListToIdList(char **azIn){ - int i, j; - if( azIn ){ - for(i=0, j=-1; azIn[i]; i++){ - if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){ - dequoteString(azIn[i]); - if( j>=0 ){ - azIn[j] = azIn[i]; - } - j++; - } - } - azIn[j] = 0; - } -} - - -/* -** Find the first alphanumeric token in the string zIn. Null-terminate -** this token. Remove any quotation marks. And return a pointer to -** the result. -*/ -static char *firstToken(char *zIn, char **pzTail){ - int n, ttype; - while(1){ - n = ftsGetToken(zIn, &ttype); - if( ttype==TOKEN_SPACE ){ - zIn += n; - }else if( ttype==TOKEN_EOF ){ - *pzTail = zIn; - return 0; - }else{ - zIn[n] = 0; - *pzTail = &zIn[1]; - dequoteString(zIn); - return zIn; - } - } - /*NOTREACHED*/ -} - -/* Return true if... -** -** * s begins with the string t, ignoring case -** * s is longer than t -** * The first character of s beyond t is not a alphanumeric -** -** Ignore leading space in *s. -** -** To put it another way, return true if the first token of -** s[] is t[]. -*/ -static int startsWith(const char *s, const char *t){ - while( safe_isspace(*s) ){ s++; } - while( *t ){ - if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0; - } - return *s!='_' && !safe_isalnum(*s); -} - -/* -** An instance of this structure defines the "spec" of a -** full text index. This structure is populated by parseSpec -** and use by fulltextConnect and fulltextCreate. -*/ -typedef struct TableSpec { - const char *zDb; /* Logical database name */ - const char *zName; /* Name of the full-text index */ - int nColumn; /* Number of columns to be indexed */ - char **azColumn; /* Original names of columns to be indexed */ - char **azContentColumn; /* Column names for %_content */ - char **azTokenizer; /* Name of tokenizer and its arguments */ -} TableSpec; - -/* -** Reclaim all of the memory used by a TableSpec -*/ -static void clearTableSpec(TableSpec *p) { - sqlite3_free(p->azColumn); - sqlite3_free(p->azContentColumn); - sqlite3_free(p->azTokenizer); -} - -/* Parse a CREATE VIRTUAL TABLE statement, which looks like this: - * - * CREATE VIRTUAL TABLE email - * USING fts3(subject, body, tokenize mytokenizer(myarg)) - * - * We return parsed information in a TableSpec structure. - * - */ -static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv, - char**pzErr){ - int i, n; - char *z, *zDummy; - char **azArg; - const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */ - - assert( argc>=3 ); - /* Current interface: - ** argv[0] - module name - ** argv[1] - database name - ** argv[2] - table name - ** argv[3..] - columns, optionally followed by tokenizer specification - ** and snippet delimiters specification. - */ - - /* Make a copy of the complete argv[][] array in a single allocation. - ** The argv[][] array is read-only and transient. We can write to the - ** copy in order to modify things and the copy is persistent. - */ - CLEAR(pSpec); - for(i=n=0; izDb = azArg[1]; - pSpec->zName = azArg[2]; - pSpec->nColumn = 0; - pSpec->azColumn = azArg; - zTokenizer = "tokenize simple"; - for(i=3; inColumn] = firstToken(azArg[i], &zDummy); - pSpec->nColumn++; - } - } - if( pSpec->nColumn==0 ){ - azArg[0] = "content"; - pSpec->nColumn = 1; - } - - /* - ** Construct the list of content column names. - ** - ** Each content column name will be of the form cNNAAAA - ** where NN is the column number and AAAA is the sanitized - ** column name. "sanitized" means that special characters are - ** converted to "_". The cNN prefix guarantees that all column - ** names are unique. - ** - ** The AAAA suffix is not strictly necessary. It is included - ** for the convenience of people who might examine the generated - ** %_content table and wonder what the columns are used for. - */ - pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) ); - if( pSpec->azContentColumn==0 ){ - clearTableSpec(pSpec); - return SQLITE_NOMEM; - } - for(i=0; inColumn; i++){ - char *p; - pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]); - for (p = pSpec->azContentColumn[i]; *p ; ++p) { - if( !safe_isalnum(*p) ) *p = '_'; - } - } - - /* - ** Parse the tokenizer specification string. - */ - pSpec->azTokenizer = tokenizeString(zTokenizer, &n); - tokenListToIdList(pSpec->azTokenizer); - - return SQLITE_OK; -} - -/* -** Generate a CREATE TABLE statement that describes the schema of -** the virtual table. Return a pointer to this schema string. -** -** Space is obtained from sqlite3_mprintf() and should be freed -** using sqlite3_free(). -*/ -static char *fulltextSchema( - int nColumn, /* Number of columns */ - const char *const* azColumn, /* List of columns */ - const char *zTableName /* Name of the table */ -){ - int i; - char *zSchema, *zNext; - const char *zSep = "("; - zSchema = sqlite3_mprintf("CREATE TABLE x"); - for(i=0; ibase */ - v->db = db; - v->zDb = spec->zDb; /* Freed when azColumn is freed */ - v->zName = spec->zName; /* Freed when azColumn is freed */ - v->nColumn = spec->nColumn; - v->azContentColumn = spec->azContentColumn; - spec->azContentColumn = 0; - v->azColumn = spec->azColumn; - spec->azColumn = 0; - - if( spec->azTokenizer==0 ){ - return SQLITE_NOMEM; - } - - zTok = spec->azTokenizer[0]; - if( !zTok ){ - zTok = "simple"; - } - nTok = strlen(zTok)+1; - - m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok); - if( !m ){ - *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]); - rc = SQLITE_ERROR; - goto err; - } - - for(n=0; spec->azTokenizer[n]; n++){} - if( n ){ - rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1], - &v->pTokenizer); - }else{ - rc = m->xCreate(0, 0, &v->pTokenizer); - } - if( rc!=SQLITE_OK ) goto err; - v->pTokenizer->pModule = m; - - /* TODO: verify the existence of backing tables foo_content, foo_term */ - - schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn, - spec->zName); - rc = sqlite3_declare_vtab(db, schema); - sqlite3_free(schema); - if( rc!=SQLITE_OK ) goto err; - - memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements)); - - /* Indicate that the buffer is not live. */ - v->nPendingData = -1; - - *ppVTab = &v->base; - FTSTRACE(("FTS3 Connect %p\n", v)); - - return rc; - -err: - fulltext_vtab_destroy(v); - return rc; -} - -static int fulltextConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVTab, - char **pzErr -){ - TableSpec spec; - int rc = parseSpec(&spec, argc, argv, pzErr); - if( rc!=SQLITE_OK ) return rc; - - rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr); - clearTableSpec(&spec); - return rc; -} - -/* The %_content table holds the text of each document, with -** the docid column exposed as the SQLite rowid for the table. -*/ -/* TODO(shess) This comment needs elaboration to match the updated -** code. Work it into the top-of-file comment at that time. -*/ -static int fulltextCreate(sqlite3 *db, void *pAux, - int argc, const char * const *argv, - sqlite3_vtab **ppVTab, char **pzErr){ - int rc; - TableSpec spec; - StringBuffer schema; - FTSTRACE(("FTS3 Create\n")); - - rc = parseSpec(&spec, argc, argv, pzErr); - if( rc!=SQLITE_OK ) return rc; - - initStringBuffer(&schema); - append(&schema, "CREATE TABLE %_content("); - append(&schema, " docid INTEGER PRIMARY KEY,"); - appendList(&schema, spec.nColumn, spec.azContentColumn); - append(&schema, ")"); - rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema)); - stringBufferDestroy(&schema); - if( rc!=SQLITE_OK ) goto out; - - rc = sql_exec(db, spec.zDb, spec.zName, - "create table %_segments(" - " blockid INTEGER PRIMARY KEY," - " block blob" - ");" - ); - if( rc!=SQLITE_OK ) goto out; - - rc = sql_exec(db, spec.zDb, spec.zName, - "create table %_segdir(" - " level integer," - " idx integer," - " start_block integer," - " leaves_end_block integer," - " end_block integer," - " root blob," - " primary key(level, idx)" - ");"); - if( rc!=SQLITE_OK ) goto out; - - rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr); - -out: - clearTableSpec(&spec); - return rc; -} - -/* Decide how to handle an SQL query. */ -static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ - fulltext_vtab *v = (fulltext_vtab *)pVTab; - int i; - FTSTRACE(("FTS3 BestIndex\n")); - - for(i=0; inConstraint; ++i){ - const struct sqlite3_index_constraint *pConstraint; - pConstraint = &pInfo->aConstraint[i]; - if( pConstraint->usable ) { - if( (pConstraint->iColumn==-1 || pConstraint->iColumn==v->nColumn+1) && - pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - pInfo->idxNum = QUERY_DOCID; /* lookup by docid */ - FTSTRACE(("FTS3 QUERY_DOCID\n")); - } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn && - pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - /* full-text search */ - pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn; - FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn)); - } else continue; - - pInfo->aConstraintUsage[i].argvIndex = 1; - pInfo->aConstraintUsage[i].omit = 1; - - /* An arbitrary value for now. - * TODO: Perhaps docid matches should be considered cheaper than - * full-text searches. */ - pInfo->estimatedCost = 1.0; - - return SQLITE_OK; - } - } - pInfo->idxNum = QUERY_GENERIC; - return SQLITE_OK; -} - -static int fulltextDisconnect(sqlite3_vtab *pVTab){ - FTSTRACE(("FTS3 Disconnect %p\n", pVTab)); - fulltext_vtab_destroy((fulltext_vtab *)pVTab); - return SQLITE_OK; -} - -static int fulltextDestroy(sqlite3_vtab *pVTab){ - fulltext_vtab *v = (fulltext_vtab *)pVTab; - int rc; - - FTSTRACE(("FTS3 Destroy %p\n", pVTab)); - rc = sql_exec(v->db, v->zDb, v->zName, - "drop table if exists %_content;" - "drop table if exists %_segments;" - "drop table if exists %_segdir;" - ); - if( rc!=SQLITE_OK ) return rc; - - fulltext_vtab_destroy((fulltext_vtab *)pVTab); - return SQLITE_OK; -} - -static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - fulltext_cursor *c; - - c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor)); - if( c ){ - memset(c, 0, sizeof(fulltext_cursor)); - /* sqlite will initialize c->base */ - *ppCursor = &c->base; - FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c)); - return SQLITE_OK; - }else{ - return SQLITE_NOMEM; - } -} - -/* Free all of the dynamically allocated memory held by the -** Snippet -*/ -static void snippetClear(Snippet *p){ - sqlite3_free(p->aMatch); - sqlite3_free(p->zOffset); - sqlite3_free(p->zSnippet); - CLEAR(p); -} - -/* -** Append a single entry to the p->aMatch[] log. -*/ -static void snippetAppendMatch( - Snippet *p, /* Append the entry to this snippet */ - int iCol, int iTerm, /* The column and query term */ - int iToken, /* Matching token in document */ - int iStart, int nByte /* Offset and size of the match */ -){ - int i; - struct snippetMatch *pMatch; - if( p->nMatch+1>=p->nAlloc ){ - p->nAlloc = p->nAlloc*2 + 10; - p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); - if( p->aMatch==0 ){ - p->nMatch = 0; - p->nAlloc = 0; - return; - } - } - i = p->nMatch++; - pMatch = &p->aMatch[i]; - pMatch->iCol = iCol; - pMatch->iTerm = iTerm; - pMatch->iToken = iToken; - pMatch->iStart = iStart; - pMatch->nByte = nByte; -} - -/* -** Sizing information for the circular buffer used in snippetOffsetsOfColumn() -*/ -#define FTS3_ROTOR_SZ (32) -#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1) - -/* -** Function to iterate through the tokens of a compiled expression. -** -** Except, skip all tokens on the right-hand side of a NOT operator. -** This function is used to find tokens as part of snippet and offset -** generation and we do nt want snippets and offsets to report matches -** for tokens on the RHS of a NOT. -*/ -static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){ - Fts3Expr *p = *ppExpr; - int iToken = *piToken; - if( iToken<0 ){ - /* In this case the expression p is the root of an expression tree. - ** Move to the first token in the expression tree. - */ - while( p->pLeft ){ - p = p->pLeft; - } - iToken = 0; - }else{ - assert(p && p->eType==FTSQUERY_PHRASE ); - if( iToken<(p->pPhrase->nToken-1) ){ - iToken++; - }else{ - iToken = 0; - while( p->pParent && p->pParent->pLeft!=p ){ - assert( p->pParent->pRight==p ); - p = p->pParent; - } - p = p->pParent; - if( p ){ - assert( p->pRight!=0 ); - p = p->pRight; - while( p->pLeft ){ - p = p->pLeft; - } - } - } - } - - *ppExpr = p; - *piToken = iToken; - return p?1:0; -} - -/* -** Return TRUE if the expression node pExpr is located beneath the -** RHS of a NOT operator. -*/ -static int fts3ExprBeneathNot(Fts3Expr *p){ - Fts3Expr *pParent; - while( p ){ - pParent = p->pParent; - if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){ - return 1; - } - p = pParent; - } - return 0; -} - -/* -** Add entries to pSnippet->aMatch[] for every match that occurs against -** document zDoc[0..nDoc-1] which is stored in column iColumn. -*/ -static void snippetOffsetsOfColumn( - fulltext_cursor *pCur, /* The fulltest search cursor */ - Snippet *pSnippet, /* The Snippet object to be filled in */ - int iColumn, /* Index of fulltext table column */ - const char *zDoc, /* Text of the fulltext table column */ - int nDoc /* Length of zDoc in bytes */ -){ - const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ - sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ - sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ - fulltext_vtab *pVtab; /* The full text index */ - int nColumn; /* Number of columns in the index */ - int i, j; /* Loop counters */ - int rc; /* Return code */ - unsigned int match, prevMatch; /* Phrase search bitmasks */ - const char *zToken; /* Next token from the tokenizer */ - int nToken; /* Size of zToken */ - int iBegin, iEnd, iPos; /* Offsets of beginning and end */ - - /* The following variables keep a circular buffer of the last - ** few tokens */ - unsigned int iRotor = 0; /* Index of current token */ - int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */ - int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */ - - pVtab = cursor_vtab(pCur); - nColumn = pVtab->nColumn; - pTokenizer = pVtab->pTokenizer; - pTModule = pTokenizer->pModule; - rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); - if( rc ) return; - pTCursor->pTokenizer = pTokenizer; - - prevMatch = 0; - while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){ - Fts3Expr *pIter = pCur->pExpr; - int iIter = -1; - iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin; - iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin; - match = 0; - for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){ - int nPhrase; /* Number of tokens in current phrase */ - struct PhraseToken *pToken; /* Current token */ - int iCol; /* Column index */ - - if( fts3ExprBeneathNot(pIter) ) continue; - nPhrase = pIter->pPhrase->nToken; - pToken = &pIter->pPhrase->aToken[iIter]; - iCol = pIter->pPhrase->iColumn; - if( iCol>=0 && iColn>nToken ) continue; - if( !pToken->isPrefix && pToken->nn<=nToken ); - if( memcmp(pToken->z, zToken, pToken->n) ) continue; - if( iIter>0 && (prevMatch & (1<=0; j--){ - int k = (iRotor-j) & FTS3_ROTOR_MASK; - snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j, - iRotorBegin[k], iRotorLen[k]); - } - } - } - prevMatch = match<<1; - iRotor++; - } - pTModule->xClose(pTCursor); -} - -/* -** Remove entries from the pSnippet structure to account for the NEAR -** operator. When this is called, pSnippet contains the list of token -** offsets produced by treating all NEAR operators as AND operators. -** This function removes any entries that should not be present after -** accounting for the NEAR restriction. For example, if the queried -** document is: -** -** "A B C D E A" -** -** and the query is: -** -** A NEAR/0 E -** -** then when this function is called the Snippet contains token offsets -** 0, 4 and 5. This function removes the "0" entry (because the first A -** is not near enough to an E). -** -** When this function is called, the value pointed to by parameter piLeft is -** the integer id of the left-most token in the expression tree headed by -** pExpr. This function increments *piLeft by the total number of tokens -** in the expression tree headed by pExpr. -** -** Return 1 if any trimming occurs. Return 0 if no trimming is required. -*/ -static int trimSnippetOffsets( - Fts3Expr *pExpr, /* The search expression */ - Snippet *pSnippet, /* The set of snippet offsets to be trimmed */ - int *piLeft /* Index of left-most token in pExpr */ -){ - if( pExpr ){ - if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){ - return 1; - } - - switch( pExpr->eType ){ - case FTSQUERY_PHRASE: - *piLeft += pExpr->pPhrase->nToken; - break; - case FTSQUERY_NEAR: { - /* The right-hand-side of a NEAR operator is always a phrase. The - ** left-hand-side is either a phrase or an expression tree that is - ** itself headed by a NEAR operator. The following initializations - ** set local variable iLeft to the token number of the left-most - ** token in the right-hand phrase, and iRight to the right most - ** token in the same phrase. For example, if we had: - ** - ** MATCH '"abc def" NEAR/2 "ghi jkl"' - ** - ** then iLeft will be set to 2 (token number of ghi) and nToken will - ** be set to 4. - */ - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - int iLeft = *piLeft; - int nNear = pExpr->nNear; - int nToken = pRight->pPhrase->nToken; - int jj, ii; - if( pLeft->eType==FTSQUERY_NEAR ){ - pLeft = pLeft->pRight; - } - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - nToken += pLeft->pPhrase->nToken; - - for(ii=0; iinMatch; ii++){ - struct snippetMatch *p = &pSnippet->aMatch[ii]; - if( p->iTerm==iLeft ){ - int isOk = 0; - /* Snippet ii is an occurence of query term iLeft in the document. - ** It occurs at position (p->iToken) of the document. We now - ** search for an instance of token (iLeft-1) somewhere in the - ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within - ** the set of snippetMatch structures. If one is found, proceed. - ** If one cannot be found, then remove snippets ii..(ii+N-1) - ** from the matching snippets, where N is the number of tokens - ** in phrase pRight->pPhrase. - */ - for(jj=0; isOk==0 && jjnMatch; jj++){ - struct snippetMatch *p2 = &pSnippet->aMatch[jj]; - if( p2->iTerm==(iLeft-1) ){ - if( p2->iToken>=(p->iToken-nNear-1) - && p2->iToken<(p->iToken+nNear+nToken) - ){ - isOk = 1; - } - } - } - if( !isOk ){ - int kk; - for(kk=0; kkpPhrase->nToken; kk++){ - pSnippet->aMatch[kk+ii].iTerm = -2; - } - return 1; - } - } - if( p->iTerm==(iLeft-1) ){ - int isOk = 0; - for(jj=0; isOk==0 && jjnMatch; jj++){ - struct snippetMatch *p2 = &pSnippet->aMatch[jj]; - if( p2->iTerm==iLeft ){ - if( p2->iToken<=(p->iToken+nNear+1) - && p2->iToken>(p->iToken-nNear-nToken) - ){ - isOk = 1; - } - } - } - if( !isOk ){ - int kk; - for(kk=0; kkpPhrase->nToken; kk++){ - pSnippet->aMatch[ii-kk].iTerm = -2; - } - return 1; - } - } - } - break; - } - } - - if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){ - return 1; - } - } - return 0; -} - -/* -** Compute all offsets for the current row of the query. -** If the offsets have already been computed, this routine is a no-op. -*/ -static void snippetAllOffsets(fulltext_cursor *p){ - int nColumn; - int iColumn, i; - int iFirst, iLast; - int iTerm = 0; - fulltext_vtab *pFts = cursor_vtab(p); - - if( p->snippet.nMatch || p->pExpr==0 ){ - return; - } - nColumn = pFts->nColumn; - iColumn = (p->iCursorType - QUERY_FULLTEXT); - if( iColumn<0 || iColumn>=nColumn ){ - /* Look for matches over all columns of the full-text index */ - iFirst = 0; - iLast = nColumn-1; - }else{ - /* Look for matches in the iColumn-th column of the index only */ - iFirst = iColumn; - iLast = iColumn; - } - for(i=iFirst; i<=iLast; i++){ - const char *zDoc; - int nDoc; - zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1); - nDoc = sqlite3_column_bytes(p->pStmt, i+1); - snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc); - } - - while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){ - iTerm = 0; - } -} - -/* -** Convert the information in the aMatch[] array of the snippet -** into the string zOffset[0..nOffset-1]. This string is used as -** the return of the SQL offsets() function. -*/ -static void snippetOffsetText(Snippet *p){ - int i; - int cnt = 0; - StringBuffer sb; - char zBuf[200]; - if( p->zOffset ) return; - initStringBuffer(&sb); - for(i=0; inMatch; i++){ - struct snippetMatch *pMatch = &p->aMatch[i]; - if( pMatch->iTerm>=0 ){ - /* If snippetMatch.iTerm is less than 0, then the match was - ** discarded as part of processing the NEAR operator (see the - ** trimSnippetOffsetsForNear() function for details). Ignore - ** it in this case - */ - zBuf[0] = ' '; - sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", - pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); - append(&sb, zBuf); - cnt++; - } - } - p->zOffset = stringBufferData(&sb); - p->nOffset = stringBufferLength(&sb); -} - -/* -** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set -** of matching words some of which might be in zDoc. zDoc is column -** number iCol. -** -** iBreak is suggested spot in zDoc where we could begin or end an -** excerpt. Return a value similar to iBreak but possibly adjusted -** to be a little left or right so that the break point is better. -*/ -static int wordBoundary( - int iBreak, /* The suggested break point */ - const char *zDoc, /* Document text */ - int nDoc, /* Number of bytes in zDoc[] */ - struct snippetMatch *aMatch, /* Matching words */ - int nMatch, /* Number of entries in aMatch[] */ - int iCol /* The column number for zDoc[] */ -){ - int i; - if( iBreak<=10 ){ - return 0; - } - if( iBreak>=nDoc-10 ){ - return nDoc; - } - for(i=0; i0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){ - return aMatch[i-1].iStart; - } - } - for(i=1; i<=10; i++){ - if( safe_isspace(zDoc[iBreak-i]) ){ - return iBreak - i + 1; - } - if( safe_isspace(zDoc[iBreak+i]) ){ - return iBreak + i + 1; - } - } - return iBreak; -} - - - -/* -** Allowed values for Snippet.aMatch[].snStatus -*/ -#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */ -#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */ - -/* -** Generate the text of a snippet. -*/ -static void snippetText( - fulltext_cursor *pCursor, /* The cursor we need the snippet for */ - const char *zStartMark, /* Markup to appear before each match */ - const char *zEndMark, /* Markup to appear after each match */ - const char *zEllipsis /* Ellipsis mark */ -){ - int i, j; - struct snippetMatch *aMatch; - int nMatch; - int nDesired; - StringBuffer sb; - int tailCol; - int tailOffset; - int iCol; - int nDoc; - const char *zDoc; - int iStart, iEnd; - int tailEllipsis = 0; - int iMatch; - - - sqlite3_free(pCursor->snippet.zSnippet); - pCursor->snippet.zSnippet = 0; - aMatch = pCursor->snippet.aMatch; - nMatch = pCursor->snippet.nMatch; - initStringBuffer(&sb); - - for(i=0; i0; i++){ - if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue; - nDesired--; - iCol = aMatch[i].iCol; - zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1); - nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1); - iStart = aMatch[i].iStart - 40; - iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol); - if( iStart<=10 ){ - iStart = 0; - } - if( iCol==tailCol && iStart<=tailOffset+20 ){ - iStart = tailOffset; - } - if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){ - trimWhiteSpace(&sb); - appendWhiteSpace(&sb); - append(&sb, zEllipsis); - appendWhiteSpace(&sb); - } - iEnd = aMatch[i].iStart + aMatch[i].nByte + 40; - iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol); - if( iEnd>=nDoc-10 ){ - iEnd = nDoc; - tailEllipsis = 0; - }else{ - tailEllipsis = 1; - } - while( iMatchsnippet.zSnippet = stringBufferData(&sb); - pCursor->snippet.nSnippet = stringBufferLength(&sb); -} +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ + if( *pp>=pEnd ){ + *pp = 0; + }else{ + fts3GetDeltaVarint(pp, pVal); + } +} + + +/* +** The Fts3Cursor.eType member is always set to one of the following. +*/ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +static Fts3Table *cursor_vtab(Fts3Cursor *c){ + return (Fts3Table *) c->base.pVtab; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + sqlite3_free(p->zSelectLeaves); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + int rc; /* Return code */ + Fts3Table *p = (Fts3Table *)pVtab; + + /* Create a script to drop the underlying three storage tables. */ + char *zSql = sqlite3_mprintf( + "DROP TABLE IF EXISTS %Q.'%q_content';" + "DROP TABLE IF EXISTS %Q.'%q_segments';" + "DROP TABLE IF EXISTS %Q.'%q_segdir';", + p->zDb, p->zName, p->zDb, p->zName, p->zDb, p->zName + ); + + /* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to + ** execute the SQL script created above. + */ + if( zSql ){ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} + + +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +*/ +static int fts3DeclareVtab(Fts3Table *p){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + ); + + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + return rc; +} + +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc; /* Return code */ + int i; /* Iterator variable */ + char *zContentCols; /* Columns of %_content table */ + char *zSql; /* SQL script to create required tables */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + + /* Create the whole SQL script */ + zSql = sqlite3_mprintf( + "CREATE TABLE %Q.'%q_content'(%s);" + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);" + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName + ); + + /* Unless a malloc() failure has occurred, execute the SQL script to + ** create the tables used to store data for this FTS3 virtual table. + */ + if( zContentCols==0 || zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + } + + sqlite3_free(zSql); + sqlite3_free(zContentCols); + return rc; +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the FTS3 virtual table. +** +** The argv[] array contains the following: +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" fields... +*/ +int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p; /* Pointer to allocated vtab */ + int rc; /* Return code */ + int i; + int nByte; + int iCol; + int nString = 0; + int nCol = 0; + char *zCsr; + int nDb; + int nName; + + const char *zTokenizer = 0; + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this table */ + + nDb = strlen(argv[1]) + 1; + nName = strlen(argv[2]) + 1; + for(i=3; idb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + zCsr = (char *)&p->azColumn[nCol]; + + fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + + /* Fill in the zName and zDb fields of the vtab structure. */ + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + iCol = 0; + for(i=3; iazColumn[iCol++] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + } + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + if( rc!=SQLITE_OK ) goto fts3_init_out; + } + + rc = fts3DeclareVtab(p); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + *ppVTab = &p->base; + +fts3_init_out: + if( rc!=SQLITE_OK ){ + if( p ) fts3DisconnectMethod((sqlite3_vtab *)p); + else if( pTokenizer ) pTokenizer->pModule->xDestroy(pTokenizer); + } + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 500000; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ) continue; + + /* A direct lookup on the rowid or docid column. This is the best + ** strategy in all cases. Assign a cost of 1.0 and return early. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) + ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + break; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the rowid/docid strategy. + */ + if( iCons<0 + && pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + } + + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = 1; + pInfo->aConstraintUsage[iCons].omit = 1; + } + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/****************************************************************/ +/****************************************************************/ +/****************************************************************/ +/****************************************************************/ /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fulltextClose(sqlite3_vtab_cursor *pCursor){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - FTSTRACE(("FTS3 Close %p\n", c)); - sqlite3_finalize(c->pStmt); - sqlite3Fts3ExprFree(c->pExpr); - snippetClear(&c->snippet); - if( c->result.nData!=0 ){ - dlrDestroy(&c->reader); - } - dataBufferDestroy(&c->result); - sqlite3_free(c); - return SQLITE_OK; -} - -static int fulltextNext(sqlite3_vtab_cursor *pCursor){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - int rc; - - FTSTRACE(("FTS3 Next %p\n", pCursor)); - snippetClear(&c->snippet); - if( c->iCursorType < QUERY_FULLTEXT ){ - /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ - rc = sqlite3_step(c->pStmt); - switch( rc ){ - case SQLITE_ROW: - c->eof = 0; - return SQLITE_OK; - case SQLITE_DONE: - c->eof = 1; - return SQLITE_OK; - default: - c->eof = 1; - return rc; - } - } else { /* full-text query */ - rc = sqlite3_reset(c->pStmt); - if( rc!=SQLITE_OK ) return rc; - - if( c->result.nData==0 || dlrAtEnd(&c->reader) ){ - c->eof = 1; - return SQLITE_OK; - } - rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader)); - dlrStep(&c->reader); - if( rc!=SQLITE_OK ) return rc; - /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ - rc = sqlite3_step(c->pStmt); - if( rc==SQLITE_ROW ){ /* the case we expect */ - c->eof = 0; - return SQLITE_OK; - } - /* an error occurred; abort */ - return rc==SQLITE_DONE ? SQLITE_ERROR : rc; - } -} - - -/* TODO(shess) If we pushed LeafReader to the top of the file, or to -** another file, term_select() could be pushed above -** docListOfTerm(). -*/ -static int termSelect(fulltext_vtab *v, int iColumn, - const char *pTerm, int nTerm, int isPrefix, - DocListType iType, DataBuffer *out); + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + sqlite3_finalize(pCsr->pStmt); + sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ + int rc; /* Return code */ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + if( pCsr->aDoclist==0 ){ + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + } + }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + pCsr->isEof = 1; + rc = SQLITE_OK; + }else{ + sqlite3_reset(pCsr->pStmt); + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){ + rc = SQLITE_ERROR; + } + } + } + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains the +** root node of a b-tree segment. The segment is guaranteed to be at least +** one level high (i.e. the root node is not also a leaf). If successful, +** this function locates the leaf node of the segment that may contain the +** term specified by arguments zTerm and nTerm and writes its block number +** to *piLeaf. +** +** It is possible that the returned leaf node does not contain the specified +** term. However, if the segment does contain said term, it is stored on +** the identified leaf node. Because this function only inspects interior +** segment nodes (and never loads leaf nodes into memory), it is not possible +** to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + + while( 1 ){ + int iHeight; /* Height of this node in tree */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + int nBlock; /* Size of child node in bytes */ + + zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + + while( zCsrnAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + sqlite3_free(zBuffer); + return SQLITE_NOMEM; + } + zBuffer = zNew; + } + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break; + iChild++; + }; + + /* If (iHeight==1), the children of this interior node are leaves. The + ** specified term may be present on leaf node iChild. + */ + if( iHeight==1 ){ + *piLeaf = iChild; + break; + } + + /* Descend to interior node iChild. */ + rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock); + if( rc!=SQLITE_OK ) break; + zEnd = &zCsr[nBlock]; + } + sqlite3_free(zBuffer); + return rc; +} + +static void fts3PutDeltaVarint( + char **pp, + sqlite3_int64 *piPrev, + sqlite3_int64 iVal +){ + assert( iVal-*piPrev > 0 ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + while( *pEnd | c ) c = *pEnd++ & 0x80; + pEnd++; + if( pp ){ + int n = pEnd - *ppPoslist; + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + while( 0xFE & (*pEnd | c) ) c = *pEnd++ & 0x80; + if( pp ){ + int n = pEnd - *ppPoslist; + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** +*/ +static void fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 && *p2 ){ + int iCol1 = 0; + int iCol2 = 0; + if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); + if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + if( iCol1!=0 ){ + int n; + *p++ = 0x01; + n = sqlite3Fts3PutVarint(p, iCol1); + p += n; + p1 += 1 + n; + p2 += 1 + n; + } + while( (*p1&0xFE) || (*p2&0xFE) ){ + if( i1==i2 ){ + fts3GetDeltaVarint(&p1, &i1); i1 -= 2; + fts3GetDeltaVarint(&p2, &i2); i2 -= 2; + }else if( i1iPos1 && iPos2<=iPos1+nToken ){ + sqlite3_int64 iSave; + if( !pp ){ + fts3PoslistCopy(0, &p2); + fts3PoslistCopy(0, &p1); + *pp1 = p1; + *pp2 = p2; + return 1; + } + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + } + if( iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + if( pSave && pp ){ + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p2++; + p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1 D */ +#define MERGE_AND 3 /* D + D -> D */ +#define MERGE_OR 4 /* D + D -> D */ +#define MERGE_POS_OR 5 /* P + P -> P */ +#define MERGE_PHRASE 6 /* P + P -> D */ +#define MERGE_POS_PHRASE 7 /* P + P -> P */ +#define MERGE_NEAR 8 /* P + P -> D */ +#define MERGE_POS_NEAR 9 /* P + P -> P */ + +static int fts3DoclistMerge( + int mergetype, /* One of the MERGE_XXX constants */ + int nParam1, + int nParam2, + char *aBuffer, /* Pre-allocated output buffer */ + int *pnBuffer, /* OUT: Bytes written to aBuffer */ + char *a1, /* Buffer containing first doclist */ + int n1, /* Size of buffer a1 */ + char *a2, /* Buffer containing second doclist */ + int n2 /* Size of buffer a2 */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + + char *p = aBuffer; + char *p1 = a1; + char *p2 = a2; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + + assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR + || mergetype==MERGE_AND || mergetype==MERGE_NOT + || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE + || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR + ); + + if( !aBuffer ){ + return SQLITE_NOMEM; + } + + /* Read the first docid from each doclist */ + fts3GetDeltaVarint2(&p1, pEnd1, &i1); + fts3GetDeltaVarint2(&p2, pEnd2, &i2); + + switch( mergetype ){ + case MERGE_OR: + case MERGE_POS_OR: + while( p1 || p2 ){ + if( p2 && p1 && i1==i2 ){ + fts3PutDeltaVarint(&p, &iPrev, i1); + if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint2(&p1, pEnd1, &i1); + fts3GetDeltaVarint2(&p2, pEnd2, &i2); + }else if( !p2 || (p1 && i1nOutput + nDoclist; + char *aNew = sqlite3_malloc(nNew); + + if( !aNew ){ + return SQLITE_NOMEM; + } + + if( pTS->nOutput==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). TODO: Add a way to transfer control of the + ** aDoclist buffer from the caller so as to avoid the memcpy(). + */ + memcpy(aNew, aDoclist, nDoclist); + }else{ + /* The output buffer is not empty. Merge doclist aDoclist with the + ** existing output. This can only happen with prefix-searches (as + ** searches for exact terms return exactly one doclist). + */ + int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); + fts3DoclistMerge(mergetype, 0, 0, + aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist + ); + } + + sqlite3_free(pTS->aOutput); + pTS->aOutput = aNew; + pTS->nOutput = nNew; + + return SQLITE_OK; +} + +/* +** This function retreives the doclist for the specified term (or term +** prefix) from the database. +** +** The returned doclist may be in one of two formats, depending on the +** value of parameter isReqPos. If isReqPos is zero, then the doclist is +** a sorted list of delta-compressed docids. If isReqPos is non-zero, +** then the returned list is in the same format as is stored in the +** database without the found length specifier at the start of on-disk +** doclists. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + int iColumn, /* Column to query (or -ve for all columns) */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isReqPos, /* True to include position lists in output */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int i; + TermSelect tsc; + Fts3SegFilter filter; /* Segment term filter configuration */ + Fts3SegReader **apSegment = 0; /* Array of segments to read data from */ + int nSegment = 0; /* Size of apSegment array */ + int nAlloc = 0; /* Allocated size of segment array */ + int rc; /* Return code */ + sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */ + int iAge = 0; /* Used to assign ages to segments */ + + /* Loop through the entire %_segdir table. For each segment, create a + ** Fts3SegReader to iterate through the subset of the segment leaves + ** that may contain a term that matches zTerm/nTerm. For non-prefix + ** searches, this is always a single leaf. For prefix searches, this + ** may be a contiguous block of leaves. + ** + ** The code in this loop does not actually load any leaves into memory + ** (unless the root node happens to be a leaf). It simply examines the + ** b-tree structure to determine which leaves need to be inspected. + */ + rc = sqlite3Fts3AllSegdirs(p, &pStmt); + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pNew = 0; + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + if( sqlite3_column_int64(pStmt, 1)==0 ){ + /* The entire segment is stored on the root node (which must be a + ** leaf). Do not bother inspecting any data in this case, just + ** create a Fts3SegReader to scan the single leaf. + */ + rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); + }else{ + sqlite3_int64 i1; + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1); + if( rc==SQLITE_OK ){ + sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2); + rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); + } + } + iAge++; + + /* If a new Fts3SegReader was allocated, add it to the apSegment array. */ + assert( (rc==SQLITE_OK)==(pNew!=0) ); + if( pNew ){ + if( nSegment==nAlloc ){ + nAlloc += 16; + Fts3SegReader **pArray = (Fts3SegReader **)sqlite3_realloc( + apSegment, nAlloc*sizeof(Fts3SegReader *) + ); + if( !pArray ){ + sqlite3Fts3SegReaderFree(pNew); + rc = SQLITE_NOMEM; + goto finished; + } + apSegment = pArray; + } + apSegment[nSegment++] = pNew; + } + } + if( rc!=SQLITE_DONE ){ + assert( rc!=SQLITE_OK ); + goto finished; + } + + memset(&tsc, 0, sizeof(TermSelect)); + tsc.isReqPos = isReqPos; + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY + | (isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = zTerm; + filter.nTerm = nTerm; + + rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter, + fts3TermSelectCb, (void *)&tsc + ); + + if( rc==SQLITE_OK ){ + *ppOut = tsc.aOutput; + *pnOut = tsc.nOutput; + }else{ + sqlite3_free(tsc.aOutput); + } + +finished: + sqlite3_reset(pStmt); + for(i=0; iiColumn; - DocListType eType = eListType; - assert( eType==DL_POSITIONS || eType==DL_DOCIDS ); - if( pPhrase->nToken>1 ){ - eType = DL_POSITIONS; - } - - /* This code should never be called with buffered updates. */ - assert( pTab->nPendingData<0 ); - - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - DataBuffer tmp; - struct PhraseToken *p = &pPhrase->aToken[ii]; - rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp); - if( rc==SQLITE_OK ){ - if( ii==0 ){ - *pResult = tmp; - }else{ - DataBuffer res = *pResult; - dataBufferInit(pResult, 0); - if( ii==(pPhrase->nToken-1) ){ - eType = eListType; - } - docListPhraseMerge( - res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult - ); - dataBufferDestroy(&res); - dataBufferDestroy(&tmp); - } - } - } - +static int fts3PhraseSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to return a doclist for */ + int isReqPos, /* True if output should contain positions */ + char **paOut, /* OUT: Pointer to malloc'd result buffer */ + int *pnOut /* OUT: Size of buffer at *paOut */ +){ + char *pOut = 0; + int nOut = 0; + int rc = SQLITE_OK; + int ii; + int iCol = pPhrase->iColumn; + int isTermPos = (pPhrase->nToken>1 || isReqPos); + + assert( p->nPendingData==0 ); + + for(ii=0; iinToken; ii++){ + struct PhraseToken *pTok = &pPhrase->aToken[ii]; + char *z = pTok->z; /* Next token of the phrase */ + int n = pTok->n; /* Size of z in bytes */ + int isPrefix = pTok->isPrefix;/* True if token is a prefix */ + char *pList; /* Pointer to token doclist */ + int nList; /* Size of buffer at pList */ + + rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList); + if( rc!=SQLITE_OK ) break; + + if( ii==0 ){ + pOut = pList; + nOut = nList; + }else{ + /* Merge the new term list and the current output. If this is the + ** last term in the phrase, and positions are not required in the + ** output of this function, the positions can be dropped as part + ** of this merge. Either way, the result of this merge will be + ** smaller than nList bytes. The code in fts3DoclistMerge() is written + ** so that it is safe to use pList as the output as well as an input + ** in this case. + */ + int mergetype = MERGE_POS_PHRASE; + if( ii==pPhrase->nToken-1 && !isReqPos ){ + mergetype = MERGE_PHRASE; + } + fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList); + sqlite3_free(pOut); + pOut = pList; + } + } + + if( rc==SQLITE_OK ){ + *paOut = pOut; + *pnOut = nOut; + }else{ + sqlite3_free(pOut); + } return rc; } /* ** Evaluate the full-text expression pExpr against fts3 table pTab. Write ** the results into pRes. */ static int evalFts3Expr( - fulltext_vtab *pTab, /* Fts3 Virtual table object */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - DataBuffer *pRes /* OUT: Write results of the expression here */ + Fts3Table *p, /* Virtual table handle */ + Fts3Expr *pExpr, /* Parsed fts3 expression */ + char **paOut, /* OUT: Pointer to malloc'd result buffer */ + int *pnOut /* OUT: Size of buffer at *paOut */ ){ - int rc = SQLITE_OK; + int rc = SQLITE_OK; /* Return code */ - /* Initialize the output buffer. If this is an empty query (pExpr==0), - ** this is all that needs to be done. Empty queries produce empty - ** result sets. - */ - dataBufferInit(pRes, 0); + /* Zero the output parameters. */ + *paOut = 0; + *pnOut = 0; if( pExpr ){ if( pExpr->eType==FTSQUERY_PHRASE ){ - DocListType eType = DL_DOCIDS; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - eType = DL_POSITIONS; - } - rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes); + int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR); + rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut); }else{ - DataBuffer lhs; - DataBuffer rhs; + char *aLeft; + char *aRight; + int nLeft; + int nRight; - dataBufferInit(&rhs, 0); - if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs)) - && SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs)) + if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight)) + && SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft)) ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: { - int nToken; Fts3Expr *pLeft; - DocListType eType = DL_DOCIDS; + Fts3Expr *pRight; + int mergetype = MERGE_NEAR; + int nParam1; + int nParam2; + char *aBuffer; + if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - eType = DL_POSITIONS; + mergetype = MERGE_POS_NEAR; } pLeft = pExpr->pLeft; while( pLeft->eType==FTSQUERY_NEAR ){ pLeft=pLeft->pRight; } - assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken; - docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, - pExpr->nNear+1, nToken, eType, pRes - ); - break; - } - case FTSQUERY_NOT: { - docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes); - break; - } - case FTSQUERY_AND: { - docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); - break; - } - case FTSQUERY_OR: { - docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes); - break; - } - } - } - dataBufferDestroy(&lhs); - dataBufferDestroy(&rhs); - } - } - - return rc; -} - -/* TODO(shess) Refactor the code to remove this forward decl. */ -static int flushPendingTerms(fulltext_vtab *v); - -/* Perform a full-text query using the search expression in -** zInput[0..nInput-1]. Return a list of matching documents -** in pResult. -** -** Queries must match column iColumn. Or if iColumn>=nColumn -** they are allowed to match against any column. -*/ -static int fulltextQuery( - fulltext_vtab *v, /* The full text index */ - int iColumn, /* Match against this column by default */ - const char *zInput, /* The query string */ - int nInput, /* Number of bytes in zInput[] */ - DataBuffer *pResult, /* Write the result doclist here */ - Fts3Expr **ppExpr /* Put parsed query string here */ -){ - int rc; - - /* TODO(shess) Instead of flushing pendingTerms, we could query for - ** the relevant term and merge the doclist into what we receive from - ** the database. Wait and see if this is a common issue, first. - ** - ** A good reason not to flush is to not generate update-related - ** error codes from here. - */ - - /* Flush any buffered updates before executing the query. */ - rc = flushPendingTerms(v); - if( rc!=SQLITE_OK ){ - return rc; - } - - /* Parse the query passed to the MATCH operator. */ - rc = sqlite3Fts3ExprParse(v->pTokenizer, - v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr - ); - if( rc!=SQLITE_OK ){ - assert( 0==(*ppExpr) ); - return rc; - } - - return evalFts3Expr(v, *ppExpr, pResult); + pRight = pExpr->pRight; + assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->eType==FTSQUERY_PHRASE ); + + nParam1 = pExpr->nNear+1; + nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2; + aBuffer = sqlite3_malloc(nLeft+nRight+1); + rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer, + pnOut, aLeft, nLeft, aRight, nRight + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aBuffer); + }else{ + *paOut = aBuffer; + } + sqlite3_free(aLeft); + break; + } + + case FTSQUERY_OR: { + /* Allocate a buffer for the output. The maximum size is the + ** sum of the sizes of the two input buffers. The +1 term is + ** so that a buffer of zero bytes is never allocated - this can + ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. + */ + char *aBuffer = sqlite3_malloc(nRight+nLeft+1); + rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, + aLeft, nLeft, aRight, nRight + ); + *paOut = aBuffer; + sqlite3_free(aLeft); + break; + } + + case FTSQUERY_AND: + case FTSQUERY_NOT: { + assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); + fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, + aLeft, nLeft, aRight, nRight + ); + *paOut = aLeft; + break; + } + } + } + sqlite3_free(aRight); + } + } + + return rc; } /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. ** -** If idxNum==QUERY_GENERIC then do a full table scan against +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against ** the %_content table. ** -** If idxNum==QUERY_DOCID then do a docid lookup for a single entry +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry ** in the %_content table. ** -** If idxNum>=QUERY_FULLTEXT then use the full text index. The +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The ** column on the left-hand side of the MATCH operator is column -** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand ** side of the MATCH operator. */ /* TODO(shess) Upgrade the cursor initialization and destruction to -** account for fulltextFilter() being called multiple times on the -** same cursor. The current solution is very fragile. Apply fix to +** account for fts3FilterMethod() being called multiple times on the +** same cursor. The current solution is very fragile. Apply fix to ** fts3 as appropriate. */ -static int fulltextFilter( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, const char *idxStr, /* Which indexing scheme to use */ - int argc, sqlite3_value **argv /* Arguments for the indexing scheme */ -){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - fulltext_vtab *v = cursor_vtab(c); - int rc; - - FTSTRACE(("FTS3 Filter %p\n",pCursor)); - - /* If the cursor has a statement that was not prepared according to - ** idxNum, clear it. I believe all calls to fulltextFilter with a - ** given cursor will have the same idxNum , but in this case it's - ** easy to be safe. - */ - if( c->pStmt && c->iCursorType!=idxNum ){ - sqlite3_finalize(c->pStmt); - c->pStmt = NULL; - } - - /* Get a fresh statement appropriate to idxNum. */ - /* TODO(shess): Add a prepared-statement cache in the vt structure. - ** The cache must handle multiple open cursors. Easier to cache the - ** statement variants at the vt to reduce malloc/realloc/free here. - ** Or we could have a StringBuffer variant which allowed stack - ** construction for small values. - */ - if( !c->pStmt ){ - StringBuffer sb; - initStringBuffer(&sb); - append(&sb, "SELECT docid, "); - appendList(&sb, v->nColumn, v->azContentColumn); - append(&sb, " FROM %_content"); - if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?"); - rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, - stringBufferData(&sb)); - stringBufferDestroy(&sb); - if( rc!=SQLITE_OK ) return rc; - c->iCursorType = idxNum; - }else{ - sqlite3_reset(c->pStmt); - assert( c->iCursorType==idxNum ); - } - - switch( idxNum ){ - case QUERY_GENERIC: - break; - - case QUERY_DOCID: - rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0])); - if( rc!=SQLITE_OK ) return rc; - break; - - default: /* full-text search */ - { - int iCol = idxNum-QUERY_FULLTEXT; - const char *zQuery = (const char *)sqlite3_value_text(argv[0]); - assert( idxNum<=QUERY_FULLTEXT+v->nColumn); - assert( argc==1 ); - if( c->result.nData!=0 ){ - /* This case happens if the same cursor is used repeatedly. */ - dlrDestroy(&c->reader); - dataBufferReset(&c->result); - }else{ - dataBufferInit(&c->result, 0); - } - rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr); - if( rc!=SQLITE_OK ) return rc; - if( c->result.nData!=0 ){ - dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData); - } - break; - } - } - - return fulltextNext(pCursor); -} - -/* This is the xEof method of the virtual table. The SQLite core -** calls this routine to find out if it has reached the end of -** a query's results set. -*/ -static int fulltextEof(sqlite3_vtab_cursor *pCursor){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - return c->eof; +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + const char *azSql[] = { + "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */ + "SELECT * FROM %Q.'%q_content'", /* full-table-scan */ + }; + int rc; /* Return code */ + char *zSql; /* SQL statement used to access %_content */ + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( nVal==0 || nVal==1 ); + assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + + /* In case the cursor has been used before, clear it now. */ + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->aDoclist); + memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + if( rc!=SQLITE_OK ) return rc; + pCsr->eType = idxNum; + + if( idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){ + int iCol = idxNum-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, + iCol, zQuery, -1, &pCsr->pExpr + ); + if( rc!=SQLITE_OK ) return rc; + + rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist); + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + + if( rc!=SQLITE_OK ) return rc; + return fts3NextMethod(pCursor); +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + return ((Fts3Cursor *)pCursor)->isEof; } /* This is the xColumn method of the virtual table. The SQLite ** core calls this method during a query when it needs the value ** of a column from the virtual table. This method needs to use @@ -3986,12 +1827,12 @@ ** one of the sqlite3_result_*() routines to store the requested ** value back in the pContext. */ static int fulltextColumn(sqlite3_vtab_cursor *pCursor, sqlite3_context *pContext, int idxCol){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - fulltext_vtab *v = cursor_vtab(c); + Fts3Cursor *c = (Fts3Cursor *) pCursor; + Fts3Table *v = cursor_vtab(c); if( idxColnColumn ){ sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1); sqlite3_result_value(pContext, pVal); }else if( idxCol==v->nColumn ){ @@ -4005,2884 +1846,223 @@ sqlite3_result_value(pContext, pVal); } return SQLITE_OK; } -/* This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts3 -** exposes %_content.docid as the rowid for the virtual table. The -** rowid should be written to *pRowid. -*/ -static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - fulltext_cursor *c = (fulltext_cursor *) pCursor; - - *pRowid = sqlite3_column_int64(c->pStmt, 0); - return SQLITE_OK; -} - -/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0, -** we also store positions and offsets in the hash table using that -** column number. -*/ -static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid, - const char *zText, int iColumn){ - sqlite3_tokenizer *pTokenizer = v->pTokenizer; - sqlite3_tokenizer_cursor *pCursor; - const char *pToken; - int nTokenBytes; - int iStartOffset, iEndOffset, iPosition; - int rc; - - rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor); - if( rc!=SQLITE_OK ) return rc; - - pCursor->pTokenizer = pTokenizer; - while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor, - &pToken, &nTokenBytes, - &iStartOffset, &iEndOffset, - &iPosition)) ){ - DLCollector *p; - int nData; /* Size of doclist before our update. */ - - /* Positions can't be negative; we use -1 as a terminator - * internally. Token can't be NULL or empty. */ - if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){ - rc = SQLITE_ERROR; - break; - } - - p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes); - if( p==NULL ){ - nData = 0; - p = dlcNew(iDocid, DL_DEFAULT); - fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p); - - /* Overhead for our hash table entry, the key, and the value. */ - v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes; - }else{ - nData = p->b.nData; - if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid); - } - if( iColumn>=0 ){ - dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset); - } - - /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */ - v->nPendingData += p->b.nData-nData; - } - - /* TODO(shess) Check return? Should this be able to cause errors at - ** this point? Actually, same question about sqlite3_finalize(), - ** though one could argue that failure there means that the data is - ** not durable. *ponder* - */ - pTokenizer->pModule->xClose(pCursor); - if( SQLITE_DONE == rc ) return SQLITE_OK; - return rc; -} - -/* Add doclists for all terms in [pValues] to pendingTerms table. */ -static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid, - sqlite3_value **pValues){ - int i; - for(i = 0; i < v->nColumn ; ++i){ - char *zText = (char*)sqlite3_value_text(pValues[i]); - int rc = buildTerms(v, iDocid, zText, i); - if( rc!=SQLITE_OK ) return rc; - } - return SQLITE_OK; -} - -/* Add empty doclists for all terms in the given row's content to -** pendingTerms. -*/ -static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){ - const char **pValues; - int i, rc; - - /* TODO(shess) Should we allow such tables at all? */ - if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR; - - rc = content_select(v, iDocid, &pValues); - if( rc!=SQLITE_OK ) return rc; - - for(i = 0 ; i < v->nColumn; ++i) { - rc = buildTerms(v, iDocid, pValues[i], -1); - if( rc!=SQLITE_OK ) break; - } - - freeStringArray(v->nColumn, pValues); - return SQLITE_OK; -} - -/* TODO(shess) Refactor the code to remove this forward decl. */ -static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid); - -/* Insert a row into the %_content table; set *piDocid to be the ID of the -** new row. Add doclists for terms to pendingTerms. -*/ -static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid, - sqlite3_value **pValues, sqlite_int64 *piDocid){ - int rc; - - rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */ - if( rc!=SQLITE_OK ) return rc; - - /* docid column is an alias for rowid. */ - *piDocid = sqlite3_last_insert_rowid(v->db); - rc = initPendingTerms(v, *piDocid); - if( rc!=SQLITE_OK ) return rc; - - return insertTerms(v, *piDocid, pValues); -} - -/* Delete a row from the %_content table; add empty doclists for terms -** to pendingTerms. -*/ -static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){ - int rc = initPendingTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - rc = deleteTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - return content_delete(v, iRow); /* execute an SQL DELETE */ -} - -/* Update a row in the %_content table; add delete doclists to -** pendingTerms for old terms not in the new data, add insert doclists -** to pendingTerms for terms in the new data. -*/ -static int index_update(fulltext_vtab *v, sqlite_int64 iRow, - sqlite3_value **pValues){ - int rc = initPendingTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - /* Generate an empty doclist for each term that previously appeared in this - * row. */ - rc = deleteTerms(v, iRow); - if( rc!=SQLITE_OK ) return rc; - - rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */ - if( rc!=SQLITE_OK ) return rc; - - /* Now add positions for terms which appear in the updated row. */ - return insertTerms(v, iRow, pValues); -} - -/*******************************************************************/ -/* InteriorWriter is used to collect terms and block references into -** interior nodes in %_segments. See commentary at top of file for -** format. -*/ - -/* How large interior nodes can grow. */ -#define INTERIOR_MAX 2048 - -/* Minimum number of terms per interior node (except the root). This -** prevents large terms from making the tree too skinny - must be >0 -** so that the tree always makes progress. Note that the min tree -** fanout will be INTERIOR_MIN_TERMS+1. -*/ -#define INTERIOR_MIN_TERMS 7 -#if INTERIOR_MIN_TERMS<1 -# error INTERIOR_MIN_TERMS must be greater than 0. -#endif - -/* ROOT_MAX controls how much data is stored inline in the segment -** directory. -*/ -/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's -** only here so that interiorWriterRootInfo() and leafWriterRootInfo() -** can both see it, but if the caller passed it in, we wouldn't even -** need a define. -*/ -#define ROOT_MAX 1024 -#if ROOT_MAXterm, 0); - dataBufferReplace(&block->term, pTerm, nTerm); - - n = fts3PutVarint(c, iHeight); - n += fts3PutVarint(c+n, iChildBlock); - dataBufferInit(&block->data, INTERIOR_MAX); - dataBufferReplace(&block->data, c, n); - } - return block; -} - -#ifndef NDEBUG -/* Verify that the data is readable as an interior node. */ -static void interiorBlockValidate(InteriorBlock *pBlock){ - const char *pData = pBlock->data.pData; - int nData = pBlock->data.nData; - int n, iDummy; - sqlite_int64 iBlockid; - - assert( nData>0 ); - assert( pData!=0 ); - assert( pData+nData>pData ); - - /* Must lead with height of node as a varint(n), n>0 */ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>0 ); - assert( n0 ); - assert( n<=nData ); - pData += n; - nData -= n; - - /* Zero or more terms of positive length */ - if( nData!=0 ){ - /* First term is not delta-encoded. */ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>0 ); - assert( n+iDummy>0); - assert( n+iDummy<=nData ); - pData += n+iDummy; - nData -= n+iDummy; - - /* Following terms delta-encoded. */ - while( nData!=0 ){ - /* Length of shared prefix. */ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>=0 ); - assert( n0 ); - assert( iDummy>0 ); - assert( n+iDummy>0); - assert( n+iDummy<=nData ); - pData += n+iDummy; - nData -= n+iDummy; - } - } -} -#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x) -#else -#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 ) -#endif - -typedef struct InteriorWriter { - int iHeight; /* from 0 at leaves. */ - InteriorBlock *first, *last; - struct InteriorWriter *parentWriter; - - DataBuffer term; /* Last term written to block "last". */ - sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */ -#ifndef NDEBUG - sqlite_int64 iLastChildBlock; /* for consistency checks. */ -#endif -} InteriorWriter; - -/* Initialize an interior node where pTerm[nTerm] marks the leftmost -** term in the tree. iChildBlock is the leftmost child block at the -** next level down the tree. -*/ -static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm, - sqlite_int64 iChildBlock, - InteriorWriter *pWriter){ - InteriorBlock *block; - assert( iHeight>0 ); - CLEAR(pWriter); - - pWriter->iHeight = iHeight; - pWriter->iOpeningChildBlock = iChildBlock; -#ifndef NDEBUG - pWriter->iLastChildBlock = iChildBlock; -#endif - block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm); - pWriter->last = pWriter->first = block; - ASSERT_VALID_INTERIOR_BLOCK(pWriter->last); - dataBufferInit(&pWriter->term, 0); -} - -/* Append the child node rooted at iChildBlock to the interior node, -** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree. -*/ -static void interiorWriterAppend(InteriorWriter *pWriter, - const char *pTerm, int nTerm, - sqlite_int64 iChildBlock){ - char c[VARINT_MAX+VARINT_MAX]; - int n, nPrefix = 0; - - ASSERT_VALID_INTERIOR_BLOCK(pWriter->last); - - /* The first term written into an interior node is actually - ** associated with the second child added (the first child was added - ** in interiorWriterInit, or in the if clause at the bottom of this - ** function). That term gets encoded straight up, with nPrefix left - ** at 0. - */ - if( pWriter->term.nData==0 ){ - n = fts3PutVarint(c, nTerm); - }else{ - while( nPrefixterm.nData && - pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){ - nPrefix++; - } - - n = fts3PutVarint(c, nPrefix); - n += fts3PutVarint(c+n, nTerm-nPrefix); - } - -#ifndef NDEBUG - pWriter->iLastChildBlock++; -#endif - assert( pWriter->iLastChildBlock==iChildBlock ); - - /* Overflow to a new block if the new term makes the current block - ** too big, and the current block already has enough terms. - */ - if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX && - iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){ - pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock, - pTerm, nTerm); - pWriter->last = pWriter->last->next; - pWriter->iOpeningChildBlock = iChildBlock; - dataBufferReset(&pWriter->term); - }else{ - dataBufferAppend2(&pWriter->last->data, c, n, - pTerm+nPrefix, nTerm-nPrefix); - dataBufferReplace(&pWriter->term, pTerm, nTerm); - } - ASSERT_VALID_INTERIOR_BLOCK(pWriter->last); -} - -/* Free the space used by pWriter, including the linked-list of -** InteriorBlocks, and parentWriter, if present. -*/ -static int interiorWriterDestroy(InteriorWriter *pWriter){ - InteriorBlock *block = pWriter->first; - - while( block!=NULL ){ - InteriorBlock *b = block; - block = block->next; - dataBufferDestroy(&b->term); - dataBufferDestroy(&b->data); - sqlite3_free(b); - } - if( pWriter->parentWriter!=NULL ){ - interiorWriterDestroy(pWriter->parentWriter); - sqlite3_free(pWriter->parentWriter); - } - dataBufferDestroy(&pWriter->term); - SCRAMBLE(pWriter); - return SQLITE_OK; -} - -/* If pWriter can fit entirely in ROOT_MAX, return it as the root info -** directly, leaving *piEndBlockid unchanged. Otherwise, flush -** pWriter to %_segments, building a new layer of interior nodes, and -** recursively ask for their root into. -*/ -static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter, - char **ppRootInfo, int *pnRootInfo, - sqlite_int64 *piEndBlockid){ - InteriorBlock *block = pWriter->first; - sqlite_int64 iBlockid = 0; - int rc; - - /* If we can fit the segment inline */ - if( block==pWriter->last && block->data.nDatadata.pData; - *pnRootInfo = block->data.nData; - return SQLITE_OK; - } - - /* Flush the first block to %_segments, and create a new level of - ** interior node. - */ - ASSERT_VALID_INTERIOR_BLOCK(block); - rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid); - if( rc!=SQLITE_OK ) return rc; - *piEndBlockid = iBlockid; - - pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter)); - interiorWriterInit(pWriter->iHeight+1, - block->term.pData, block->term.nData, - iBlockid, pWriter->parentWriter); - - /* Flush additional blocks and append to the higher interior - ** node. - */ - for(block=block->next; block!=NULL; block=block->next){ - ASSERT_VALID_INTERIOR_BLOCK(block); - rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid); - if( rc!=SQLITE_OK ) return rc; - *piEndBlockid = iBlockid; - - interiorWriterAppend(pWriter->parentWriter, - block->term.pData, block->term.nData, iBlockid); - } - - /* Parent node gets the chance to be the root. */ - return interiorWriterRootInfo(v, pWriter->parentWriter, - ppRootInfo, pnRootInfo, piEndBlockid); -} - -/****************************************************************/ -/* InteriorReader is used to read off the data from an interior node -** (see comment at top of file for the format). -*/ -typedef struct InteriorReader { - const char *pData; - int nData; - - DataBuffer term; /* previous term, for decoding term delta. */ - - sqlite_int64 iBlockid; -} InteriorReader; - -static void interiorReaderDestroy(InteriorReader *pReader){ - dataBufferDestroy(&pReader->term); - SCRAMBLE(pReader); -} - -/* TODO(shess) The assertions are great, but what if we're in NDEBUG -** and the blob is empty or otherwise contains suspect data? -*/ -static void interiorReaderInit(const char *pData, int nData, - InteriorReader *pReader){ - int n, nTerm; - - /* Require at least the leading flag byte */ - assert( nData>0 ); - assert( pData[0]!='\0' ); - - CLEAR(pReader); - - /* Decode the base blockid, and set the cursor to the first term. */ - n = fts3GetVarint(pData+1, &pReader->iBlockid); - assert( 1+n<=nData ); - pReader->pData = pData+1+n; - pReader->nData = nData-(1+n); - - /* A single-child interior node (such as when a leaf node was too - ** large for the segment directory) won't have any terms. - ** Otherwise, decode the first term. - */ - if( pReader->nData==0 ){ - dataBufferInit(&pReader->term, 0); - }else{ - n = fts3GetVarint32(pReader->pData, &nTerm); - dataBufferInit(&pReader->term, nTerm); - dataBufferReplace(&pReader->term, pReader->pData+n, nTerm); - assert( n+nTerm<=pReader->nData ); - pReader->pData += n+nTerm; - pReader->nData -= n+nTerm; - } -} - -static int interiorReaderAtEnd(InteriorReader *pReader){ - return pReader->term.nData==0; -} - -static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){ - return pReader->iBlockid; -} - -static int interiorReaderTermBytes(InteriorReader *pReader){ - assert( !interiorReaderAtEnd(pReader) ); - return pReader->term.nData; -} -static const char *interiorReaderTerm(InteriorReader *pReader){ - assert( !interiorReaderAtEnd(pReader) ); - return pReader->term.pData; -} - -/* Step forward to the next term in the node. */ -static void interiorReaderStep(InteriorReader *pReader){ - assert( !interiorReaderAtEnd(pReader) ); - - /* If the last term has been read, signal eof, else construct the - ** next term. - */ - if( pReader->nData==0 ){ - dataBufferReset(&pReader->term); - }else{ - int n, nPrefix, nSuffix; - - n = fts3GetVarint32(pReader->pData, &nPrefix); - n += fts3GetVarint32(pReader->pData+n, &nSuffix); - - /* Truncate the current term and append suffix data. */ - pReader->term.nData = nPrefix; - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); - - assert( n+nSuffix<=pReader->nData ); - pReader->pData += n+nSuffix; - pReader->nData -= n+nSuffix; - } - pReader->iBlockid++; -} - -/* Compare the current term to pTerm[nTerm], returning strcmp-style -** results. If isPrefix, equality means equal through nTerm bytes. -*/ -static int interiorReaderTermCmp(InteriorReader *pReader, - const char *pTerm, int nTerm, int isPrefix){ - const char *pReaderTerm = interiorReaderTerm(pReader); - int nReaderTerm = interiorReaderTermBytes(pReader); - int c, n = nReaderTerm0 ) return -1; - if( nTerm>0 ) return 1; - return 0; - } - - c = memcmp(pReaderTerm, pTerm, n); - if( c!=0 ) return c; - if( isPrefix && n==nTerm ) return 0; - return nReaderTerm - nTerm; -} - -/****************************************************************/ -/* LeafWriter is used to collect terms and associated doclist data -** into leaf blocks in %_segments (see top of file for format info). -** Expected usage is: -** -** LeafWriter writer; -** leafWriterInit(0, 0, &writer); -** while( sorted_terms_left_to_process ){ -** // data is doclist data for that term. -** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData); -** if( rc!=SQLITE_OK ) goto err; -** } -** rc = leafWriterFinalize(v, &writer); -**err: -** leafWriterDestroy(&writer); -** return rc; -** -** leafWriterStep() may write a collected leaf out to %_segments. -** leafWriterFinalize() finishes writing any buffered data and stores -** a root node in %_segdir. leafWriterDestroy() frees all buffers and -** InteriorWriters allocated as part of writing this segment. -** -** TODO(shess) Document leafWriterStepMerge(). -*/ - -/* Put terms with data this big in their own block. */ -#define STANDALONE_MIN 1024 - -/* Keep leaf blocks below this size. */ -#define LEAF_MAX 2048 - -typedef struct LeafWriter { - int iLevel; - int idx; - sqlite_int64 iStartBlockid; /* needed to create the root info */ - sqlite_int64 iEndBlockid; /* when we're done writing. */ - - DataBuffer term; /* previous encoded term */ - DataBuffer data; /* encoding buffer */ - - /* bytes of first term in the current node which distinguishes that - ** term from the last term of the previous node. - */ - int nTermDistinct; - - InteriorWriter parentWriter; /* if we overflow */ - int has_parent; -} LeafWriter; - -static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){ - CLEAR(pWriter); - pWriter->iLevel = iLevel; - pWriter->idx = idx; - - dataBufferInit(&pWriter->term, 32); - - /* Start out with a reasonably sized block, though it can grow. */ - dataBufferInit(&pWriter->data, LEAF_MAX); -} - -#ifndef NDEBUG -/* Verify that the data is readable as a leaf node. */ -static void leafNodeValidate(const char *pData, int nData){ - int n, iDummy; - - if( nData==0 ) return; - assert( nData>0 ); - assert( pData!=0 ); - assert( pData+nData>pData ); - - /* Must lead with a varint(0) */ - n = fts3GetVarint32(pData, &iDummy); - assert( iDummy==0 ); - assert( n>0 ); - assert( n0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy<=nData ); - ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL); - pData += n+iDummy; - nData -= n+iDummy; - - /* Verify that trailing terms and doclists also are readable. */ - while( nData!=0 ){ - n = fts3GetVarint32(pData, &iDummy); - assert( n>0 ); - assert( iDummy>=0 ); - assert( n0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy0 ); - assert( iDummy>0 ); - assert( n+iDummy>0 ); - assert( n+iDummy<=nData ); - ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL); - pData += n+iDummy; - nData -= n+iDummy; - } -} -#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n) -#else -#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 ) -#endif - -/* Flush the current leaf node to %_segments, and adding the resulting -** blockid and the starting term to the interior node which will -** contain it. -*/ -static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter, - int iData, int nData){ - sqlite_int64 iBlockid = 0; - const char *pStartingTerm; - int nStartingTerm, rc, n; - - /* Must have the leading varint(0) flag, plus at least some - ** valid-looking data. - */ - assert( nData>2 ); - assert( iData>=0 ); - assert( iData+nData<=pWriter->data.nData ); - ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData); - - rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid); - if( rc!=SQLITE_OK ) return rc; - assert( iBlockid!=0 ); - - /* Reconstruct the first term in the leaf for purposes of building - ** the interior node. - */ - n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm); - pStartingTerm = pWriter->data.pData+iData+1+n; - assert( pWriter->data.nData>iData+1+n+nStartingTerm ); - assert( pWriter->nTermDistinct>0 ); - assert( pWriter->nTermDistinct<=nStartingTerm ); - nStartingTerm = pWriter->nTermDistinct; - - if( pWriter->has_parent ){ - interiorWriterAppend(&pWriter->parentWriter, - pStartingTerm, nStartingTerm, iBlockid); - }else{ - interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid, - &pWriter->parentWriter); - pWriter->has_parent = 1; - } - - /* Track the span of this segment's leaf nodes. */ - if( pWriter->iEndBlockid==0 ){ - pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid; - }else{ - pWriter->iEndBlockid++; - assert( iBlockid==pWriter->iEndBlockid ); - } - - return SQLITE_OK; -} -static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){ - int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData); - if( rc!=SQLITE_OK ) return rc; - - /* Re-initialize the output buffer. */ - dataBufferReset(&pWriter->data); - - return SQLITE_OK; -} - -/* Fetch the root info for the segment. If the entire leaf fits -** within ROOT_MAX, then it will be returned directly, otherwise it -** will be flushed and the root info will be returned from the -** interior node. *piEndBlockid is set to the blockid of the last -** interior or leaf node written to disk (0 if none are written at -** all). -*/ -static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter, - char **ppRootInfo, int *pnRootInfo, - sqlite_int64 *piEndBlockid){ - /* we can fit the segment entirely inline */ - if( !pWriter->has_parent && pWriter->data.nDatadata.pData; - *pnRootInfo = pWriter->data.nData; - *piEndBlockid = 0; - return SQLITE_OK; - } - - /* Flush remaining leaf data. */ - if( pWriter->data.nData>0 ){ - int rc = leafWriterFlush(v, pWriter); - if( rc!=SQLITE_OK ) return rc; - } - - /* We must have flushed a leaf at some point. */ - assert( pWriter->has_parent ); - - /* Tenatively set the end leaf blockid as the end blockid. If the - ** interior node can be returned inline, this will be the final - ** blockid, otherwise it will be overwritten by - ** interiorWriterRootInfo(). - */ - *piEndBlockid = pWriter->iEndBlockid; - - return interiorWriterRootInfo(v, &pWriter->parentWriter, - ppRootInfo, pnRootInfo, piEndBlockid); -} - -/* Collect the rootInfo data and store it into the segment directory. -** This has the effect of flushing the segment's leaf data to -** %_segments, and also flushing any interior nodes to %_segments. -*/ -static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){ - sqlite_int64 iEndBlockid; - char *pRootInfo; - int rc, nRootInfo; - - rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - /* Don't bother storing an entirely empty segment. */ - if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK; - - return segdir_set(v, pWriter->iLevel, pWriter->idx, - pWriter->iStartBlockid, pWriter->iEndBlockid, - iEndBlockid, pRootInfo, nRootInfo); -} - -static void leafWriterDestroy(LeafWriter *pWriter){ - if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter); - dataBufferDestroy(&pWriter->term); - dataBufferDestroy(&pWriter->data); -} - -/* Encode a term into the leafWriter, delta-encoding as appropriate. -** Returns the length of the new term which distinguishes it from the -** previous term, which can be used to set nTermDistinct when a node -** boundary is crossed. -*/ -static int leafWriterEncodeTerm(LeafWriter *pWriter, - const char *pTerm, int nTerm){ - char c[VARINT_MAX+VARINT_MAX]; - int n, nPrefix = 0; - - assert( nTerm>0 ); - while( nPrefixterm.nData && - pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){ - nPrefix++; - /* Failing this implies that the terms weren't in order. */ - assert( nPrefixdata.nData==0 ){ - /* Encode the node header and leading term as: - ** varint(0) - ** varint(nTerm) - ** char pTerm[nTerm] - */ - n = fts3PutVarint(c, '\0'); - n += fts3PutVarint(c+n, nTerm); - dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm); - }else{ - /* Delta-encode the term as: - ** varint(nPrefix) - ** varint(nSuffix) - ** char pTermSuffix[nSuffix] - */ - n = fts3PutVarint(c, nPrefix); - n += fts3PutVarint(c+n, nTerm-nPrefix); - dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix); - } - dataBufferReplace(&pWriter->term, pTerm, nTerm); - - return nPrefix+1; -} - -/* Used to avoid a memmove when a large amount of doclist data is in -** the buffer. This constructs a node and term header before -** iDoclistData and flushes the resulting complete node using -** leafWriterInternalFlush(). -*/ -static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter, - const char *pTerm, int nTerm, - int iDoclistData){ - char c[VARINT_MAX+VARINT_MAX]; - int iData, n = fts3PutVarint(c, 0); - n += fts3PutVarint(c+n, nTerm); - - /* There should always be room for the header. Even if pTerm shared - ** a substantial prefix with the previous term, the entire prefix - ** could be constructed from earlier data in the doclist, so there - ** should be room. - */ - assert( iDoclistData>=n+nTerm ); - - iData = iDoclistData-(n+nTerm); - memcpy(pWriter->data.pData+iData, c, n); - memcpy(pWriter->data.pData+iData+n, pTerm, nTerm); - - return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData); -} - -/* Push pTerm[nTerm] along with the doclist data to the leaf layer of -** %_segments. -*/ -static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter, - const char *pTerm, int nTerm, - DLReader *pReaders, int nReaders){ - char c[VARINT_MAX+VARINT_MAX]; - int iTermData = pWriter->data.nData, iDoclistData; - int i, nData, n, nActualData, nActual, rc, nTermDistinct; - - ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData); - nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm); - - /* Remember nTermDistinct if opening a new node. */ - if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct; - - iDoclistData = pWriter->data.nData; - - /* Estimate the length of the merged doclist so we can leave space - ** to encode it. - */ - for(i=0, nData=0; idata, c, n); - - docListMerge(&pWriter->data, pReaders, nReaders); - ASSERT_VALID_DOCLIST(DL_DEFAULT, - pWriter->data.pData+iDoclistData+n, - pWriter->data.nData-iDoclistData-n, NULL); - - /* The actual amount of doclist data at this point could be smaller - ** than the length we encoded. Additionally, the space required to - ** encode this length could be smaller. For small doclists, this is - ** not a big deal, we can just use memmove() to adjust things. - */ - nActualData = pWriter->data.nData-(iDoclistData+n); - nActual = fts3PutVarint(c, nActualData); - assert( nActualData<=nData ); - assert( nActual<=n ); - - /* If the new doclist is big enough for force a standalone leaf - ** node, we can immediately flush it inline without doing the - ** memmove(). - */ - /* TODO(shess) This test matches leafWriterStep(), which does this - ** test before it knows the cost to varint-encode the term and - ** doclist lengths. At some point, change to - ** pWriter->data.nData-iTermData>STANDALONE_MIN. - */ - if( nTerm+nActualData>STANDALONE_MIN ){ - /* Push leaf node from before this term. */ - if( iTermData>0 ){ - rc = leafWriterInternalFlush(v, pWriter, 0, iTermData); - if( rc!=SQLITE_OK ) return rc; - - pWriter->nTermDistinct = nTermDistinct; - } - - /* Fix the encoded doclist length. */ - iDoclistData += n - nActual; - memcpy(pWriter->data.pData+iDoclistData, c, nActual); - - /* Push the standalone leaf node. */ - rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData); - if( rc!=SQLITE_OK ) return rc; - - /* Leave the node empty. */ - dataBufferReset(&pWriter->data); - - return rc; - } - - /* At this point, we know that the doclist was small, so do the - ** memmove if indicated. - */ - if( nActualdata.pData+iDoclistData+nActual, - pWriter->data.pData+iDoclistData+n, - pWriter->data.nData-(iDoclistData+n)); - pWriter->data.nData -= n-nActual; - } - - /* Replace written length with actual length. */ - memcpy(pWriter->data.pData+iDoclistData, c, nActual); - - /* If the node is too large, break things up. */ - /* TODO(shess) This test matches leafWriterStep(), which does this - ** test before it knows the cost to varint-encode the term and - ** doclist lengths. At some point, change to - ** pWriter->data.nData>LEAF_MAX. - */ - if( iTermData+nTerm+nActualData>LEAF_MAX ){ - /* Flush out the leading data as a node */ - rc = leafWriterInternalFlush(v, pWriter, 0, iTermData); - if( rc!=SQLITE_OK ) return rc; - - pWriter->nTermDistinct = nTermDistinct; - - /* Rebuild header using the current term */ - n = fts3PutVarint(pWriter->data.pData, 0); - n += fts3PutVarint(pWriter->data.pData+n, nTerm); - memcpy(pWriter->data.pData+n, pTerm, nTerm); - n += nTerm; - - /* There should always be room, because the previous encoding - ** included all data necessary to construct the term. - */ - assert( ndata.nData-iDoclistDatadata.pData+n, - pWriter->data.pData+iDoclistData, - pWriter->data.nData-iDoclistData); - pWriter->data.nData -= iDoclistData-n; - } - ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData); - - return SQLITE_OK; -} - -/* Push pTerm[nTerm] along with the doclist data to the leaf layer of -** %_segments. -*/ -/* TODO(shess) Revise writeZeroSegment() so that doclists are -** constructed directly in pWriter->data. -*/ -static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter, - const char *pTerm, int nTerm, - const char *pData, int nData){ - int rc; - DLReader reader; - - dlrInit(&reader, DL_DEFAULT, pData, nData); - rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1); - dlrDestroy(&reader); - - return rc; -} - - -/****************************************************************/ -/* LeafReader is used to iterate over an individual leaf node. */ -typedef struct LeafReader { - DataBuffer term; /* copy of current term. */ - - const char *pData; /* data for current term. */ - int nData; -} LeafReader; - -static void leafReaderDestroy(LeafReader *pReader){ - dataBufferDestroy(&pReader->term); - SCRAMBLE(pReader); -} - -static int leafReaderAtEnd(LeafReader *pReader){ - return pReader->nData<=0; -} - -/* Access the current term. */ -static int leafReaderTermBytes(LeafReader *pReader){ - return pReader->term.nData; -} -static const char *leafReaderTerm(LeafReader *pReader){ - assert( pReader->term.nData>0 ); - return pReader->term.pData; -} - -/* Access the doclist data for the current term. */ -static int leafReaderDataBytes(LeafReader *pReader){ - int nData; - assert( pReader->term.nData>0 ); - fts3GetVarint32(pReader->pData, &nData); - return nData; -} -static const char *leafReaderData(LeafReader *pReader){ - int n, nData; - assert( pReader->term.nData>0 ); - n = fts3GetVarint32(pReader->pData, &nData); - return pReader->pData+n; -} - -static void leafReaderInit(const char *pData, int nData, - LeafReader *pReader){ - int nTerm, n; - - assert( nData>0 ); - assert( pData[0]=='\0' ); - - CLEAR(pReader); - - /* Read the first term, skipping the header byte. */ - n = fts3GetVarint32(pData+1, &nTerm); - dataBufferInit(&pReader->term, nTerm); - dataBufferReplace(&pReader->term, pData+1+n, nTerm); - - /* Position after the first term. */ - assert( 1+n+nTermpData = pData+1+n+nTerm; - pReader->nData = nData-1-n-nTerm; -} - -/* Step the reader forward to the next term. */ -static void leafReaderStep(LeafReader *pReader){ - int n, nData, nPrefix, nSuffix; - assert( !leafReaderAtEnd(pReader) ); - - /* Skip previous entry's data block. */ - n = fts3GetVarint32(pReader->pData, &nData); - assert( n+nData<=pReader->nData ); - pReader->pData += n+nData; - pReader->nData -= n+nData; - - if( !leafReaderAtEnd(pReader) ){ - /* Construct the new term using a prefix from the old term plus a - ** suffix from the leaf data. - */ - n = fts3GetVarint32(pReader->pData, &nPrefix); - n += fts3GetVarint32(pReader->pData+n, &nSuffix); - assert( n+nSuffixnData ); - pReader->term.nData = nPrefix; - dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix); - - pReader->pData += n+nSuffix; - pReader->nData -= n+nSuffix; - } -} - -/* strcmp-style comparison of pReader's current term against pTerm. -** If isPrefix, equality means equal through nTerm bytes. -*/ -static int leafReaderTermCmp(LeafReader *pReader, - const char *pTerm, int nTerm, int isPrefix){ - int c, n = pReader->term.nDataterm.nData : nTerm; - if( n==0 ){ - if( pReader->term.nData>0 ) return -1; - if(nTerm>0 ) return 1; - return 0; - } - - c = memcmp(pReader->term.pData, pTerm, n); - if( c!=0 ) return c; - if( isPrefix && n==nTerm ) return 0; - return pReader->term.nData - nTerm; -} - - -/****************************************************************/ -/* LeavesReader wraps LeafReader to allow iterating over the entire -** leaf layer of the tree. -*/ -typedef struct LeavesReader { - int idx; /* Index within the segment. */ - - sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */ - int eof; /* we've seen SQLITE_DONE from pStmt. */ - - LeafReader leafReader; /* reader for the current leaf. */ - DataBuffer rootData; /* root data for inline. */ -} LeavesReader; - -/* Access the current term. */ -static int leavesReaderTermBytes(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderTermBytes(&pReader->leafReader); -} -static const char *leavesReaderTerm(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderTerm(&pReader->leafReader); -} - -/* Access the doclist data for the current term. */ -static int leavesReaderDataBytes(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderDataBytes(&pReader->leafReader); -} -static const char *leavesReaderData(LeavesReader *pReader){ - assert( !pReader->eof ); - return leafReaderData(&pReader->leafReader); -} - -static int leavesReaderAtEnd(LeavesReader *pReader){ - return pReader->eof; -} - -/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus -** leaving the statement handle open, which locks the table. -*/ -/* TODO(shess) This "solution" is not satisfactory. Really, there -** should be check-in function for all statement handles which -** arranges to call sqlite3_reset(). This most likely will require -** modification to control flow all over the place, though, so for now -** just punt. -** -** Note the the current system assumes that segment merges will run to -** completion, which is why this particular probably hasn't arisen in -** this case. Probably a brittle assumption. -*/ -static int leavesReaderReset(LeavesReader *pReader){ - return sqlite3_reset(pReader->pStmt); -} - -static void leavesReaderDestroy(LeavesReader *pReader){ - /* If idx is -1, that means we're using a non-cached statement - ** handle in the optimize() case, so we need to release it. - */ - if( pReader->pStmt!=NULL && pReader->idx==-1 ){ - sqlite3_finalize(pReader->pStmt); - } - leafReaderDestroy(&pReader->leafReader); - dataBufferDestroy(&pReader->rootData); - SCRAMBLE(pReader); -} - -/* Initialize pReader with the given root data (if iStartBlockid==0 -** the leaf data was entirely contained in the root), or from the -** stream of blocks between iStartBlockid and iEndBlockid, inclusive. -*/ -static int leavesReaderInit(fulltext_vtab *v, - int idx, - sqlite_int64 iStartBlockid, - sqlite_int64 iEndBlockid, - const char *pRootData, int nRootData, - LeavesReader *pReader){ - CLEAR(pReader); - pReader->idx = idx; - - dataBufferInit(&pReader->rootData, 0); - if( iStartBlockid==0 ){ - /* Entire leaf level fit in root data. */ - dataBufferReplace(&pReader->rootData, pRootData, nRootData); - leafReaderInit(pReader->rootData.pData, pReader->rootData.nData, - &pReader->leafReader); - }else{ - sqlite3_stmt *s; - int rc = sql_get_leaf_statement(v, idx, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iStartBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 2, iEndBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ){ - pReader->eof = 1; - return SQLITE_OK; - } - if( rc!=SQLITE_ROW ) return rc; - - pReader->pStmt = s; - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), - sqlite3_column_bytes(pReader->pStmt, 0), - &pReader->leafReader); - } - return SQLITE_OK; -} - -/* Step the current leaf forward to the next term. If we reach the -** end of the current leaf, step forward to the next leaf block. -*/ -static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){ - assert( !leavesReaderAtEnd(pReader) ); - leafReaderStep(&pReader->leafReader); - - if( leafReaderAtEnd(&pReader->leafReader) ){ - int rc; - if( pReader->rootData.pData ){ - pReader->eof = 1; - return SQLITE_OK; - } - rc = sqlite3_step(pReader->pStmt); - if( rc!=SQLITE_ROW ){ - pReader->eof = 1; - return rc==SQLITE_DONE ? SQLITE_OK : rc; - } - leafReaderDestroy(&pReader->leafReader); - leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0), - sqlite3_column_bytes(pReader->pStmt, 0), - &pReader->leafReader); - } - return SQLITE_OK; -} - -/* Order LeavesReaders by their term, ignoring idx. Readers at eof -** always sort to the end. -*/ -static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){ - if( leavesReaderAtEnd(lr1) ){ - if( leavesReaderAtEnd(lr2) ) return 0; - return 1; - } - if( leavesReaderAtEnd(lr2) ) return -1; - - return leafReaderTermCmp(&lr1->leafReader, - leavesReaderTerm(lr2), leavesReaderTermBytes(lr2), - 0); -} - -/* Similar to leavesReaderTermCmp(), with additional ordering by idx -** so that older segments sort before newer segments. -*/ -static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){ - int c = leavesReaderTermCmp(lr1, lr2); - if( c!=0 ) return c; - return lr1->idx-lr2->idx; -} - -/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its -** sorted position. -*/ -static void leavesReaderReorder(LeavesReader *pLr, int nLr){ - while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){ - LeavesReader tmp = pLr[0]; - pLr[0] = pLr[1]; - pLr[1] = tmp; - nLr--; - pLr++; - } -} - -/* Initializes pReaders with the segments from level iLevel, returning -** the number of segments in *piReaders. Leaves pReaders in sorted -** order. -*/ -static int leavesReadersInit(fulltext_vtab *v, int iLevel, - LeavesReader *pReaders, int *piReaders){ - sqlite3_stmt *s; - int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int(s, 1, iLevel); - if( rc!=SQLITE_OK ) return rc; - - i = 0; - while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - sqlite_int64 iStart = sqlite3_column_int64(s, 0); - sqlite_int64 iEnd = sqlite3_column_int64(s, 1); - const char *pRootData = sqlite3_column_blob(s, 2); - int nRootData = sqlite3_column_bytes(s, 2); - - assert( i0 ){ - leavesReaderDestroy(&pReaders[i]); - } - return rc; - } - - *piReaders = i; - - /* Leave our results sorted by term, then age. */ - while( i-- ){ - leavesReaderReorder(pReaders+i, *piReaders-i); - } - return SQLITE_OK; -} - -/* Merge doclists from pReaders[nReaders] into a single doclist, which -** is written to pWriter. Assumes pReaders is ordered oldest to -** newest. -*/ -/* TODO(shess) Consider putting this inline in segmentMerge(). */ -static int leavesReadersMerge(fulltext_vtab *v, - LeavesReader *pReaders, int nReaders, - LeafWriter *pWriter){ - DLReader dlReaders[MERGE_COUNT]; - const char *pTerm = leavesReaderTerm(pReaders); - int i, nTerm = leavesReaderTermBytes(pReaders); - - assert( nReaders<=MERGE_COUNT ); - - for(i=0; i0 ){ - rc = leavesReaderStep(v, lrs+i); - if( rc!=SQLITE_OK ) goto err; - - /* Reorder by term, then by age. */ - leavesReaderReorder(lrs+i, MERGE_COUNT-i); - } - } - - for(i=0; i0 ); - - for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader); - rc=leavesReaderStep(v, pReader)){ - /* TODO(shess) Really want leavesReaderTermCmp(), but that name is - ** already taken to compare the terms of two LeavesReaders. Think - ** on a better name. [Meanwhile, break encapsulation rather than - ** use a confusing name.] - */ - int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix); - if( c>0 ) break; /* Past any possible matches. */ - if( c==0 ){ - const char *pData = leavesReaderData(pReader); - int iBuffer, nData = leavesReaderDataBytes(pReader); - - /* Find the first empty buffer. */ - for(iBuffer=0; iBuffer0 ){ - assert(pBuffers!=NULL); - memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers)); - sqlite3_free(pBuffers); - } - pBuffers = p; - } - dataBufferInit(&(pBuffers[nBuffers]), 0); - nBuffers++; - } - - /* At this point, must have an empty at iBuffer. */ - assert(iBufferpData, p->nData); - - /* dataBufferReset() could allow a large doclist to blow up - ** our memory requirements. - */ - if( p->nCapacity<1024 ){ - dataBufferReset(p); - }else{ - dataBufferDestroy(p); - dataBufferInit(p, 0); - } - } - } - } - } - - /* Union all the doclists together into *out. */ - /* TODO(shess) What if *out is big? Sigh. */ - if( rc==SQLITE_OK && nBuffers>0 ){ - int iBuffer; - for(iBuffer=0; iBuffer0 ){ - if( out->nData==0 ){ - dataBufferSwap(out, &(pBuffers[iBuffer])); - }else{ - docListAccumulateUnion(out, pBuffers[iBuffer].pData, - pBuffers[iBuffer].nData); - } - } - } - } - - while( nBuffers-- ){ - dataBufferDestroy(&(pBuffers[nBuffers])); - } - if( pBuffers!=NULL ) sqlite3_free(pBuffers); - - return rc; -} - -/* Call loadSegmentLeavesInt() with pData/nData as input. */ -static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - LeavesReader reader; - int rc; - - assert( nData>1 ); - assert( *pData=='\0' ); - rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader); - if( rc!=SQLITE_OK ) return rc; - - rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out); - leavesReaderReset(&reader); - leavesReaderDestroy(&reader); - return rc; -} - -/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to -** iEndLeaf (inclusive) as input, and merge the resulting doclist into -** out. -*/ -static int loadSegmentLeaves(fulltext_vtab *v, - sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - int rc; - LeavesReader reader; - - assert( iStartLeaf<=iEndLeaf ); - rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader); - if( rc!=SQLITE_OK ) return rc; - - rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out); - leavesReaderReset(&reader); - leavesReaderDestroy(&reader); - return rc; -} - -/* Taking pData/nData as an interior node, find the sequence of child -** nodes which could include pTerm/nTerm/isPrefix. Note that the -** interior node terms logically come between the blocks, so there is -** one more blockid than there are terms (that block contains terms >= -** the last interior-node term). -*/ -/* TODO(shess) The calling code may already know that the end child is -** not worth calculating, because the end may be in a later sibling -** node. Consider whether breaking symmetry is worthwhile. I suspect -** it is not worthwhile. -*/ -static void getChildrenContaining(const char *pData, int nData, - const char *pTerm, int nTerm, int isPrefix, - sqlite_int64 *piStartChild, - sqlite_int64 *piEndChild){ - InteriorReader reader; - - assert( nData>1 ); - assert( *pData!='\0' ); - interiorReaderInit(pData, nData, &reader); - - /* Scan for the first child which could contain pTerm/nTerm. */ - while( !interiorReaderAtEnd(&reader) ){ - if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break; - interiorReaderStep(&reader); - } - *piStartChild = interiorReaderCurrentBlockid(&reader); - - /* Keep scanning to find a term greater than our term, using prefix - ** comparison if indicated. If isPrefix is false, this will be the - ** same blockid as the starting block. - */ - while( !interiorReaderAtEnd(&reader) ){ - if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break; - interiorReaderStep(&reader); - } - *piEndChild = interiorReaderCurrentBlockid(&reader); - - interiorReaderDestroy(&reader); - - /* Children must ascend, and if !prefix, both must be the same. */ - assert( *piEndChild>=*piStartChild ); - assert( isPrefix || *piStartChild==*piEndChild ); -} - -/* Read block at iBlockid and pass it with other params to -** getChildrenContaining(). -*/ -static int loadAndGetChildrenContaining( - fulltext_vtab *v, - sqlite_int64 iBlockid, - const char *pTerm, int nTerm, int isPrefix, - sqlite_int64 *piStartChild, sqlite_int64 *piEndChild -){ - sqlite3_stmt *s = NULL; - int rc; - - assert( iBlockid!=0 ); - assert( pTerm!=NULL ); - assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */ - assert( piStartChild!=NULL ); - assert( piEndChild!=NULL ); - - rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_bind_int64(s, 1, iBlockid); - if( rc!=SQLITE_OK ) return rc; - - rc = sqlite3_step(s); - if( rc==SQLITE_DONE ) return SQLITE_ERROR; - if( rc!=SQLITE_ROW ) return rc; - - getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0), - pTerm, nTerm, isPrefix, piStartChild, piEndChild); - - /* We expect only one row. We must execute another sqlite3_step() - * to complete the iteration; otherwise the table will remain - * locked. */ - rc = sqlite3_step(s); - if( rc==SQLITE_ROW ) return SQLITE_ERROR; - if( rc!=SQLITE_DONE ) return rc; - - return SQLITE_OK; -} - -/* Traverse the tree represented by pData[nData] looking for -** pTerm[nTerm], placing its doclist into *out. This is internal to -** loadSegment() to make error-handling cleaner. -*/ -static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData, - sqlite_int64 iLeavesEnd, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - /* Special case where root is a leaf. */ - if( *pData=='\0' ){ - return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out); - }else{ - int rc; - sqlite_int64 iStartChild, iEndChild; - - /* Process pData as an interior node, then loop down the tree - ** until we find the set of leaf nodes to scan for the term. - */ - getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix, - &iStartChild, &iEndChild); - while( iStartChild>iLeavesEnd ){ - sqlite_int64 iNextStart, iNextEnd; - rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix, - &iNextStart, &iNextEnd); - if( rc!=SQLITE_OK ) return rc; - - /* If we've branched, follow the end branch, too. */ - if( iStartChild!=iEndChild ){ - sqlite_int64 iDummy; - rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix, - &iDummy, &iNextEnd); - if( rc!=SQLITE_OK ) return rc; - } - - assert( iNextStart<=iNextEnd ); - iStartChild = iNextStart; - iEndChild = iNextEnd; - } - assert( iStartChild<=iLeavesEnd ); - assert( iEndChild<=iLeavesEnd ); - - /* Scan through the leaf segments for doclists. */ - return loadSegmentLeaves(v, iStartChild, iEndChild, - pTerm, nTerm, isPrefix, out); - } -} - -/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then -** merge its doclist over *out (any duplicate doclists read from the -** segment rooted at pData will overwrite those in *out). -*/ -/* TODO(shess) Consider changing this to determine the depth of the -** leaves using either the first characters of interior nodes (when -** ==1, we're one level above the leaves), or the first character of -** the root (which will describe the height of the tree directly). -** Either feels somewhat tricky to me. -*/ -/* TODO(shess) The current merge is likely to be slow for large -** doclists (though it should process from newest/smallest to -** oldest/largest, so it may not be that bad). It might be useful to -** modify things to allow for N-way merging. This could either be -** within a segment, with pairwise merges across segments, or across -** all segments at once. -*/ -static int loadSegment(fulltext_vtab *v, const char *pData, int nData, - sqlite_int64 iLeavesEnd, - const char *pTerm, int nTerm, int isPrefix, - DataBuffer *out){ - DataBuffer result; - int rc; - - assert( nData>1 ); - - /* This code should never be called with buffered updates. */ - assert( v->nPendingData<0 ); - - dataBufferInit(&result, 0); - rc = loadSegmentInt(v, pData, nData, iLeavesEnd, - pTerm, nTerm, isPrefix, &result); - if( rc==SQLITE_OK && result.nData>0 ){ - if( out->nData==0 ){ - DataBuffer tmp = *out; - *out = result; - result = tmp; - }else{ - DataBuffer merged; - DLReader readers[2]; - - dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData); - dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData); - dataBufferInit(&merged, out->nData+result.nData); - docListMerge(&merged, readers, 2); - dataBufferDestroy(out); - *out = merged; - dlrDestroy(&readers[0]); - dlrDestroy(&readers[1]); - } - } - dataBufferDestroy(&result); - return rc; -} - -/* Scan the database and merge together the posting lists for the term -** into *out. -*/ -static int termSelect( - fulltext_vtab *v, - int iColumn, - const char *pTerm, int nTerm, /* Term to query for */ - int isPrefix, /* True for a prefix search */ - DocListType iType, - DataBuffer *out /* Write results here */ -){ - DataBuffer doclist; - sqlite3_stmt *s; - int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); - if( rc!=SQLITE_OK ) return rc; - - /* This code should never be called with buffered updates. */ - assert( v->nPendingData<0 ); - - dataBufferInit(&doclist, 0); - dataBufferInit(out, 0); - - /* Traverse the segments from oldest to newest so that newer doclist - ** elements for given docids overwrite older elements. - */ - while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - const char *pData = sqlite3_column_blob(s, 2); - const int nData = sqlite3_column_bytes(s, 2); - const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); - rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, - &doclist); - if( rc!=SQLITE_OK ) goto err; - } - if( rc==SQLITE_DONE ){ - if( doclist.nData!=0 ){ - /* TODO(shess) The old term_select_all() code applied the column - ** restrict as we merged segments, leading to smaller buffers. - ** This is probably worthwhile to bring back, once the new storage - ** system is checked in. - */ - if( iColumn==v->nColumn) iColumn = -1; - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, - iColumn, iType, out); - } - rc = SQLITE_OK; - } - - err: - dataBufferDestroy(&doclist); - return rc; -} - -/****************************************************************/ -/* Used to hold hashtable data for sorting. */ -typedef struct TermData { - const char *pTerm; - int nTerm; - DLCollector *pCollector; -} TermData; - -/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0 -** for equal, >0 for greater-than). -*/ -static int termDataCmp(const void *av, const void *bv){ - const TermData *a = (const TermData *)av; - const TermData *b = (const TermData *)bv; - int n = a->nTermnTerm ? a->nTerm : b->nTerm; - int c = memcmp(a->pTerm, b->pTerm, n); - if( c!=0 ) return c; - return a->nTerm-b->nTerm; -} - -/* Order pTerms data by term, then write a new level 0 segment using -** LeafWriter. -*/ -static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){ - fts3HashElem *e; - int idx, rc, i, n; - TermData *pData; - LeafWriter writer; - DataBuffer dl; - - /* Determine the next index at level 0, merging as necessary. */ - rc = segdirNextIndex(v, 0, &idx); - if( rc!=SQLITE_OK ) return rc; - - n = fts3HashCount(pTerms); - pData = sqlite3_malloc(n*sizeof(TermData)); - - for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){ - assert( i1 ) qsort(pData, n, sizeof(*pData), termDataCmp); - - /* TODO(shess) Refactor so that we can write directly to the segment - ** DataBuffer, as happens for segment merges. - */ - leafWriterInit(0, idx, &writer); - dataBufferInit(&dl, 0); - for(i=0; inPendingData>=0 ){ - fts3HashElem *e; - for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){ - dlcDelete(fts3HashData(e)); - } - fts3HashClear(&v->pendingTerms); - v->nPendingData = -1; - } - return SQLITE_OK; -} - -/* If pendingTerms has data, flush it to a level-zero segment, and -** free it. -*/ -static int flushPendingTerms(fulltext_vtab *v){ - if( v->nPendingData>=0 ){ - int rc = writeZeroSegment(v, &v->pendingTerms); - if( rc==SQLITE_OK ) clearPendingTerms(v); - return rc; - } - return SQLITE_OK; -} - -/* If pendingTerms is "too big", or docid is out of order, flush it. -** Regardless, be certain that pendingTerms is initialized for use. -*/ -static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){ - /* TODO(shess) Explore whether partially flushing the buffer on - ** forced-flush would provide better performance. I suspect that if - ** we ordered the doclists by size and flushed the largest until the - ** buffer was half empty, that would let the less frequent terms - ** generate longer doclists. - */ - if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){ - int rc = flushPendingTerms(v); - if( rc!=SQLITE_OK ) return rc; - } - if( v->nPendingData<0 ){ - fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1); - v->nPendingData = 0; - } - v->iPrevDocid = iDocid; - return SQLITE_OK; -} - -/* This function implements the xUpdate callback; it is the top-level entry - * point for inserting, deleting or updating a row in a full-text table. */ -static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg, - sqlite_int64 *pRowid){ - fulltext_vtab *v = (fulltext_vtab *) pVtab; - int rc; - - FTSTRACE(("FTS3 Update %p\n", pVtab)); - - if( nArg<2 ){ - rc = index_delete(v, sqlite3_value_int64(ppArg[0])); - if( rc==SQLITE_OK ){ - /* If we just deleted the last row in the table, clear out the - ** index data. - */ - rc = content_exists(v); - if( rc==SQLITE_ROW ){ - rc = SQLITE_OK; - }else if( rc==SQLITE_DONE ){ - /* Clear the pending terms so we don't flush a useless level-0 - ** segment when the transaction closes. - */ - rc = clearPendingTerms(v); - if( rc==SQLITE_OK ){ - rc = segdir_delete_all(v); - } - } - } - } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){ - /* An update: - * ppArg[0] = old rowid - * ppArg[1] = new rowid - * ppArg[2..2+v->nColumn-1] = values - * ppArg[2+v->nColumn] = value for magic column (we ignore this) - * ppArg[2+v->nColumn+1] = value for docid - */ - sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]); - if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER || - sqlite3_value_int64(ppArg[1]) != rowid ){ - rc = SQLITE_ERROR; /* we don't allow changing the rowid */ - }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER || - sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){ - rc = SQLITE_ERROR; /* we don't allow changing the docid */ - }else{ - assert( nArg==2+v->nColumn+2); - rc = index_update(v, rowid, &ppArg[2]); - } - } else { - /* An insert: - * ppArg[1] = requested rowid - * ppArg[2..2+v->nColumn-1] = values - * ppArg[2+v->nColumn] = value for magic column (we ignore this) - * ppArg[2+v->nColumn+1] = value for docid - */ - sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1]; - assert( nArg==2+v->nColumn+2); - if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) && - SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){ - /* TODO(shess) Consider allowing this to work if the values are - ** identical. I'm inclined to discourage that usage, though, - ** given that both rowid and docid are special columns. Better - ** would be to define one or the other as the default winner, - ** but should it be fts3-centric (docid) or SQLite-centric - ** (rowid)? - */ - rc = SQLITE_ERROR; - }else{ - if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){ - pRequestDocid = ppArg[1]; - } - rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid); - } - } - - return rc; -} - -static int fulltextSync(sqlite3_vtab *pVtab){ - FTSTRACE(("FTS3 xSync()\n")); - return flushPendingTerms((fulltext_vtab *)pVtab); -} - -static int fulltextBegin(sqlite3_vtab *pVtab){ - fulltext_vtab *v = (fulltext_vtab *) pVtab; - FTSTRACE(("FTS3 xBegin()\n")); - - /* Any buffered updates should have been cleared by the previous - ** transaction. - */ - assert( v->nPendingData<0 ); - return clearPendingTerms(v); -} - -static int fulltextCommit(sqlite3_vtab *pVtab){ - fulltext_vtab *v = (fulltext_vtab *) pVtab; - FTSTRACE(("FTS3 xCommit()\n")); - - /* Buffered updates should have been cleared by fulltextSync(). */ - assert( v->nPendingData<0 ); - return clearPendingTerms(v); -} - -static int fulltextRollback(sqlite3_vtab *pVtab){ - FTSTRACE(("FTS3 xRollback()\n")); - return clearPendingTerms((fulltext_vtab *)pVtab); +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + return SQLITE_OK; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); +} + +/* +** Implementation of xBegin() method. This is a no-op. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + assert( ((Fts3Table *)pVtab)->nPendingData==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + assert( ((Fts3Table *)pVtab)->nPendingData==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab); + return SQLITE_OK; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + Fts3Cursor *pRet; + if( sqlite3_value_type(pVal)!=SQLITE_BLOB + && sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) + ){ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return SQLITE_ERROR; + } + memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); + *ppCsr = pRet; + return SQLITE_OK; } /* ** Implementation of the snippet() function for FTS3 */ -static void snippetFunc( +static void fts3SnippetFunc( sqlite3_context *pContext, int argc, sqlite3_value **argv ){ - fulltext_cursor *pCursor; - if( argc<1 ) return; - if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1); - }else{ - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipsis = "..."; - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - if( argc>=2 ){ - zStart = (const char*)sqlite3_value_text(argv[1]); - if( argc>=3 ){ - zEnd = (const char*)sqlite3_value_text(argv[2]); - if( argc>=4 ){ - zEllipsis = (const char*)sqlite3_value_text(argv[3]); - } - } - } - snippetAllOffsets(pCursor); - snippetText(pCursor, zStart, zEnd, zEllipsis); - sqlite3_result_text(pContext, pCursor->snippet.zSnippet, - pCursor->snippet.nSnippet, SQLITE_STATIC); - } + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + + if( argc<1 || argc>4 ) return; + if( fts3FunctionArg(pContext, "snippet", argv[0], &pCsr) ) return; + + switch( argc ){ + case 4: zEllipsis = (const char*)sqlite3_value_text(argv[3]); + case 3: zEnd = (const char*)sqlite3_value_text(argv[2]); + case 2: zStart = (const char*)sqlite3_value_text(argv[1]); + } + + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis); } /* ** Implementation of the offsets() function for FTS3 */ -static void snippetOffsetsFunc( - sqlite3_context *pContext, - int argc, - sqlite3_value **argv -){ - fulltext_cursor *pCursor; - if( argc<1 ) return; - if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - sqlite3_result_error(pContext, "illegal first argument to offsets",-1); - }else{ - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - snippetAllOffsets(pCursor); - snippetOffsetText(&pCursor->snippet); - sqlite3_result_text(pContext, - pCursor->snippet.zOffset, pCursor->snippet.nOffset, - SQLITE_STATIC); - } -} - -/* OptLeavesReader is nearly identical to LeavesReader, except that -** where LeavesReader is geared towards the merging of complete -** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader -** is geared towards implementation of the optimize() function, and -** can merge all segments simultaneously. This version may be -** somewhat less efficient than LeavesReader because it merges into an -** accumulator rather than doing an N-way merge, but since segment -** size grows exponentially (so segment count logrithmically) this is -** probably not an immediate problem. -*/ -/* TODO(shess): Prove that assertion, or extend the merge code to -** merge tree fashion (like the prefix-searching code does). -*/ -/* TODO(shess): OptLeavesReader and LeavesReader could probably be -** merged with little or no loss of performance for LeavesReader. The -** merged code would need to handle >MERGE_COUNT segments, and would -** also need to be able to optionally optimize away deletes. -*/ -typedef struct OptLeavesReader { - /* Segment number, to order readers by age. */ - int segment; - LeavesReader reader; -} OptLeavesReader; - -static int optLeavesReaderAtEnd(OptLeavesReader *pReader){ - return leavesReaderAtEnd(&pReader->reader); -} -static int optLeavesReaderTermBytes(OptLeavesReader *pReader){ - return leavesReaderTermBytes(&pReader->reader); -} -static const char *optLeavesReaderData(OptLeavesReader *pReader){ - return leavesReaderData(&pReader->reader); -} -static int optLeavesReaderDataBytes(OptLeavesReader *pReader){ - return leavesReaderDataBytes(&pReader->reader); -} -static const char *optLeavesReaderTerm(OptLeavesReader *pReader){ - return leavesReaderTerm(&pReader->reader); -} -static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){ - return leavesReaderStep(v, &pReader->reader); -} -static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){ - return leavesReaderTermCmp(&lr1->reader, &lr2->reader); -} -/* Order by term ascending, segment ascending (oldest to newest), with -** exhausted readers to the end. -*/ -static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){ - int c = optLeavesReaderTermCmp(lr1, lr2); - if( c!=0 ) return c; - return lr1->segment-lr2->segment; -} -/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that -** pLr[1..nLr-1] is already sorted. -*/ -static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){ - while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){ - OptLeavesReader tmp = pLr[0]; - pLr[0] = pLr[1]; - pLr[1] = tmp; - nLr--; - pLr++; - } -} - -/* optimize() helper function. Put the readers in order and iterate -** through them, merging doclists for matching terms into pWriter. -** Returns SQLITE_OK on success, or the SQLite error code which -** prevented success. -*/ -static int optimizeInternal(fulltext_vtab *v, - OptLeavesReader *readers, int nReaders, - LeafWriter *pWriter){ - int i, rc = SQLITE_OK; - DataBuffer doclist, merged, tmp; - - /* Order the readers. */ - i = nReaders; - while( i-- > 0 ){ - optLeavesReaderReorder(&readers[i], nReaders-i); - } - - dataBufferInit(&doclist, LEAF_MAX); - dataBufferInit(&merged, LEAF_MAX); - - /* Exhausted readers bubble to the end, so when the first reader is - ** at eof, all are at eof. - */ - while( !optLeavesReaderAtEnd(&readers[0]) ){ - - /* Figure out how many readers share the next term. */ - for(i=1; i 0 ){ - dlrDestroy(&dlReaders[nReaders]); - } - - /* Accumulated doclist to reader 0 for next pass. */ - dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData); - } - - /* Destroy reader that was left in the pipeline. */ - dlrDestroy(&dlReaders[0]); - - /* Trim deletions from the doclist. */ - dataBufferReset(&merged); - docListTrim(DL_DEFAULT, doclist.pData, doclist.nData, - -1, DL_DEFAULT, &merged); - } - - /* Only pass doclists with hits (skip if all hits deleted). */ - if( merged.nData>0 ){ - rc = leafWriterStep(v, pWriter, - optLeavesReaderTerm(&readers[0]), - optLeavesReaderTermBytes(&readers[0]), - merged.pData, merged.nData); - if( rc!=SQLITE_OK ) goto err; - } - - /* Step merged readers to next term and reorder. */ - while( i-- > 0 ){ - rc = optLeavesReaderStep(v, &readers[i]); - if( rc!=SQLITE_OK ) goto err; - - optLeavesReaderReorder(&readers[i], nReaders-i); - } - } - - err: - dataBufferDestroy(&doclist); - dataBufferDestroy(&merged); - return rc; -} - -/* Implement optimize() function for FTS3. optimize(t) merges all -** segments in the fts index into a single segment. 't' is the magic -** table-named column. -*/ -static void optimizeFunc(sqlite3_context *pContext, - int argc, sqlite3_value **argv){ - fulltext_cursor *pCursor; - if( argc>1 ){ - sqlite3_result_error(pContext, "excess arguments to optimize()",-1); - }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - sqlite3_result_error(pContext, "illegal first argument to optimize",-1); - }else{ - fulltext_vtab *v; - int i, rc, iMaxLevel; - OptLeavesReader *readers; - int nReaders; - LeafWriter writer; - sqlite3_stmt *s; - - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - v = cursor_vtab(pCursor); - - /* Flush any buffered updates before optimizing. */ - rc = flushPendingTerms(v); - if( rc!=SQLITE_OK ) goto err; - - rc = segdir_count(v, &nReaders, &iMaxLevel); - if( rc!=SQLITE_OK ) goto err; - if( nReaders==0 || nReaders==1 ){ - sqlite3_result_text(pContext, "Index already optimal", -1, - SQLITE_STATIC); - return; - } - - rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); - if( rc!=SQLITE_OK ) goto err; - - readers = sqlite3_malloc(nReaders*sizeof(readers[0])); - if( readers==NULL ) goto err; - - /* Note that there will already be a segment at this position - ** until we call segdir_delete() on iMaxLevel. - */ - leafWriterInit(iMaxLevel, 0, &writer); - - i = 0; - while( (rc = sqlite3_step(s))==SQLITE_ROW ){ - sqlite_int64 iStart = sqlite3_column_int64(s, 0); - sqlite_int64 iEnd = sqlite3_column_int64(s, 1); - const char *pRootData = sqlite3_column_blob(s, 2); - int nRootData = sqlite3_column_bytes(s, 2); - - assert( i 0 ){ - leavesReaderDestroy(&readers[i].reader); - } - sqlite3_free(readers); - - /* If we've successfully gotten to here, delete the old segments - ** and flush the interior structure of the new segment. - */ - if( rc==SQLITE_OK ){ - for( i=0; i<=iMaxLevel; i++ ){ - rc = segdir_delete(v, i); - if( rc!=SQLITE_OK ) break; - } - - if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer); - } - - leafWriterDestroy(&writer); - - if( rc!=SQLITE_OK ) goto err; - - sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); - return; - - /* TODO(shess): Error-handling needs to be improved along the - ** lines of the dump_ functions. - */ - err: - { - char buf[512]; - sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s", - sqlite3_errmsg(sqlite3_context_db_handle(pContext))); - sqlite3_result_error(pContext, buf, -1); - } - } -} - -#ifdef SQLITE_TEST -/* Generate an error of the form ": ". If msg is NULL, -** pull the error from the context's db handle. -*/ -static void generateError(sqlite3_context *pContext, - const char *prefix, const char *msg){ - char buf[512]; - if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext)); - sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg); - sqlite3_result_error(pContext, buf, -1); -} - -/* Helper function to collect the set of terms in the segment into -** pTerms. The segment is defined by the leaf nodes between -** iStartBlockid and iEndBlockid, inclusive, or by the contents of -** pRootData if iStartBlockid is 0 (in which case the entire segment -** fit in a leaf). -*/ -static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s, - fts3Hash *pTerms){ - const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0); - const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); - const char *pRootData = sqlite3_column_blob(s, 2); - const int nRootData = sqlite3_column_bytes(s, 2); - LeavesReader reader; - int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, - pRootData, nRootData, &reader); - if( rc!=SQLITE_OK ) return rc; - - while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ - const char *pTerm = leavesReaderTerm(&reader); - const int nTerm = leavesReaderTermBytes(&reader); - void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm); - void *newValue = (void *)((char *)oldValue+1); - - /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c, - ** the data value passed is returned in case of malloc failure. - */ - if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){ - rc = SQLITE_NOMEM; - }else{ - rc = leavesReaderStep(v, &reader); - } - } - - leavesReaderDestroy(&reader); - return rc; -} - -/* Helper function to build the result string for dump_terms(). */ -static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){ - int iTerm, nTerms, nResultBytes, iByte; - char *result; - TermData *pData; - fts3HashElem *e; - - /* Iterate pTerms to generate an array of terms in pData for - ** sorting. - */ - nTerms = fts3HashCount(pTerms); - assert( nTerms>0 ); - pData = sqlite3_malloc(nTerms*sizeof(TermData)); - if( pData==NULL ) return SQLITE_NOMEM; - - nResultBytes = 0; - for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){ - nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */ - assert( iTerm0 ); /* nTerms>0, nResultsBytes must be, too. */ - result = sqlite3_malloc(nResultBytes); - if( result==NULL ){ - sqlite3_free(pData); - return SQLITE_NOMEM; - } - - if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp); - - /* Read the terms in order to build the result. */ - iByte = 0; - for(iTerm=0; iTerm0 ){ - rc = generateTermsResult(pContext, &terms); - if( rc==SQLITE_NOMEM ){ - generateError(pContext, "dump_terms", "out of memory"); - }else{ - assert( rc==SQLITE_OK ); - } - }else if( argc==3 ){ - /* The specific segment asked for could not be found. */ - generateError(pContext, "dump_terms", "segment not found"); - }else{ - /* No segments found. */ - /* TODO(shess): It should be impossible to reach this. This - ** case can only happen for an empty table, in which case - ** SQLite has no rows to call this function on. - */ - sqlite3_result_null(pContext); - } - } - sqlite3Fts3HashClear(&terms); - } -} - -/* Expand the DL_DEFAULT doclist in pData into a text result in -** pContext. -*/ -static void createDoclistResult(sqlite3_context *pContext, - const char *pData, int nData){ - DataBuffer dump; - DLReader dlReader; - - assert( pData!=NULL && nData>0 ); - - dataBufferInit(&dump, 0); - dlrInit(&dlReader, DL_DEFAULT, pData, nData); - for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ - char buf[256]; - PLReader plReader; - - plrInit(&plReader, &dlReader); - if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ - sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); - dataBufferAppend(&dump, buf, strlen(buf)); - }else{ - int iColumn = plrColumn(&plReader); - - sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[", - dlrDocid(&dlReader), iColumn); - dataBufferAppend(&dump, buf, strlen(buf)); - - for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ - if( plrColumn(&plReader)!=iColumn ){ - iColumn = plrColumn(&plReader); - sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); - assert( dump.nData>0 ); - dump.nData--; /* Overwrite trailing space. */ - assert( dump.pData[dump.nData]==' '); - dataBufferAppend(&dump, buf, strlen(buf)); - } - if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){ - sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ", - plrPosition(&plReader), - plrStartOffset(&plReader), plrEndOffset(&plReader)); - }else if( DL_DEFAULT==DL_POSITIONS ){ - sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader)); - }else{ - assert( NULL=="Unhandled DL_DEFAULT value"); - } - dataBufferAppend(&dump, buf, strlen(buf)); - } - plrDestroy(&plReader); - - assert( dump.nData>0 ); - dump.nData--; /* Overwrite trailing space. */ - assert( dump.pData[dump.nData]==' '); - dataBufferAppend(&dump, "]] ", 3); - } - } - dlrDestroy(&dlReader); - - assert( dump.nData>0 ); - dump.nData--; /* Overwrite trailing space. */ - assert( dump.pData[dump.nData]==' '); - dump.pData[dump.nData] = '\0'; - assert( dump.nData>0 ); - - /* Passes ownership of dump's buffer to pContext. */ - sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); - dump.pData = NULL; - dump.nData = dump.nCapacity = 0; -} - -/* Implements dump_doclist() for use in inspecting the fts3 index from -** tests. TEXT result containing a string representation of the -** doclist for the indicated term. dump_doclist(t, term, level, idx) -** dumps the doclist for term from the segment specified by level, idx -** (in %_segdir), while dump_doclist(t, term) dumps the logical -** doclist for the term across all segments. The per-segment doclist -** can contain deletions, while the full-index doclist will not -** (deletions are omitted). -** -** Result formats differ with the setting of DL_DEFAULTS. Examples: -** -** DL_DOCIDS: [1] [3] [7] -** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]] -** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]] -** -** In each case the number after the outer '[' is the docid. In the -** latter two cases, the number before the inner '[' is the column -** associated with the values within. For DL_POSITIONS the numbers -** within are the positions, for DL_POSITIONS_OFFSETS they are the -** position, the start offset, and the end offset. -*/ -static void dumpDoclistFunc( - sqlite3_context *pContext, - int argc, sqlite3_value **argv -){ - fulltext_cursor *pCursor; - if( argc!=2 && argc!=4 ){ - generateError(pContext, "dump_doclist", "incorrect arguments"); - }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || - sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ - generateError(pContext, "dump_doclist", "illegal first argument"); - }else if( sqlite3_value_text(argv[1])==NULL || - sqlite3_value_text(argv[1])[0]=='\0' ){ - generateError(pContext, "dump_doclist", "empty second argument"); - }else{ - const char *pTerm = (const char *)sqlite3_value_text(argv[1]); - const int nTerm = strlen(pTerm); - fulltext_vtab *v; - int rc; - DataBuffer doclist; - - memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); - v = cursor_vtab(pCursor); - - dataBufferInit(&doclist, 0); - - /* termSelect() yields the same logical doclist that queries are - ** run against. - */ - if( argc==2 ){ - rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist); - }else{ - sqlite3_stmt *s = NULL; - - /* Get our specific segment's information. */ - rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2])); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3])); - } - } - - if( rc==SQLITE_OK ){ - rc = sqlite3_step(s); - - if( rc==SQLITE_DONE ){ - dataBufferDestroy(&doclist); - generateError(pContext, "dump_doclist", "segment not found"); - return; - } - - /* Found a segment, load it into doclist. */ - if( rc==SQLITE_ROW ){ - const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); - const char *pData = sqlite3_column_blob(s, 2); - const int nData = sqlite3_column_bytes(s, 2); - - /* loadSegment() is used by termSelect() to load each - ** segment's data. - */ - rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0, - &doclist); - if( rc==SQLITE_OK ){ - rc = sqlite3_step(s); - - /* Should not have more than one matching segment. */ - if( rc!=SQLITE_DONE ){ - sqlite3_reset(s); - dataBufferDestroy(&doclist); - generateError(pContext, "dump_doclist", "invalid segdir"); - return; - } - rc = SQLITE_OK; - } - } - } - - sqlite3_reset(s); - } - - if( rc==SQLITE_OK ){ - if( doclist.nData>0 ){ - createDoclistResult(pContext, doclist.pData, doclist.nData); - }else{ - /* TODO(shess): This can happen if the term is not present, or - ** if all instances of the term have been deleted and this is - ** an all-index dump. It may be interesting to distinguish - ** these cases. - */ - sqlite3_result_text(pContext, "", 0, SQLITE_STATIC); - } - }else if( rc==SQLITE_NOMEM ){ - /* Handle out-of-memory cases specially because if they are - ** generated in fts3 code they may not be reflected in the db - ** handle. - */ - /* TODO(shess): Handle this more comprehensively. - ** sqlite3ErrStr() has what I need, but is internal. - */ - generateError(pContext, "dump_doclist", "out of memory"); - }else{ - generateError(pContext, "dump_doclist", NULL); - } - - dataBufferDestroy(&doclist); - } -} -#endif +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + sqlite3Fts3Offsets(pContext, pCsr); +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ -static int fulltextFindFunction( - sqlite3_vtab *pVtab, - int nArg, - const char *zName, - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), - void **ppArg +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ ){ - if( strcmp(zName,"snippet")==0 ){ - *pxFunc = snippetFunc; - return 1; - }else if( strcmp(zName,"offsets")==0 ){ - *pxFunc = snippetOffsetsFunc; - return 1; - }else if( strcmp(zName,"optimize")==0 ){ - *pxFunc = optimizeFunc; - return 1; -#ifdef SQLITE_TEST - /* NOTE(shess): These functions are present only for testing - ** purposes. No particular effort is made to optimize their - ** execution or how they build their results. - */ - }else if( strcmp(zName,"dump_terms")==0 ){ - /* fprintf(stderr, "Found dump_terms\n"); */ - *pxFunc = dumpTermsFunc; - return 1; - }else if( strcmp(zName,"dump_doclist")==0 ){ - /* fprintf(stderr, "Found dump_doclist\n"); */ - *pxFunc = dumpDoclistFunc; - return 1; -#endif - } + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + }; + int i; /* Iterator variable */ + for(i=0; izDb, p->zName, zName , p->zDb, p->zName, zName @@ -6895,33 +2075,38 @@ return rc; } static const sqlite3_module fts3Module = { /* iVersion */ 0, - /* xCreate */ fulltextCreate, - /* xConnect */ fulltextConnect, - /* xBestIndex */ fulltextBestIndex, - /* xDisconnect */ fulltextDisconnect, - /* xDestroy */ fulltextDestroy, - /* xOpen */ fulltextOpen, + /* xCreate */ fts3CreateMethod, + /* xConnect */ fts3ConnectMethod, + /* xBestIndex */ fts3BestIndexMethod, + /* xDisconnect */ fts3DisconnectMethod, + /* xDestroy */ fts3DestroyMethod, + /* xOpen */ fts3OpenMethod, /* xClose */ fulltextClose, - /* xFilter */ fulltextFilter, - /* xNext */ fulltextNext, - /* xEof */ fulltextEof, + /* xFilter */ fts3FilterMethod, + /* xNext */ fts3NextMethod, + /* xEof */ fts3EofMethod, /* xColumn */ fulltextColumn, - /* xRowid */ fulltextRowid, - /* xUpdate */ fulltextUpdate, - /* xBegin */ fulltextBegin, - /* xSync */ fulltextSync, - /* xCommit */ fulltextCommit, - /* xRollback */ fulltextRollback, - /* xFindFunction */ fulltextFindFunction, - /* xRename */ fulltextRename, + /* xRowid */ fts3RowidMethod, + /* xUpdate */ fts3UpdateMethod, + /* xBegin */ fts3BeginMethod, + /* xSync */ fts3SyncMethod, + /* xCommit */ fts3CommitMethod, + /* xRollback */ fts3RollbackMethod, + /* xFindFunction */ fts3FindFunctionMethod, + /* xRename */ fts3RenameMethod, }; +/* +** This function is registered as the module destructor (called when an +** FTS3 enabled database connection is closed). It frees the memory +** allocated for the tokenizer hash table. +*/ static void hashDestroy(void *p){ - fts3Hash *pHash = (fts3Hash *)p; + Fts3Hash *pHash = (Fts3Hash *)p; sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } /* @@ -6937,21 +2122,19 @@ */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); -int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *); - /* ** Initialise the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. */ int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; - fts3Hash *pHash = 0; + Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3SimpleTokenizerModule(&pSimple); @@ -6959,11 +2142,11 @@ #ifdef SQLITE_ENABLE_ICU sqlite3Fts3IcuTokenizerModule(&pIcu); #endif /* Allocate and initialise the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(fts3Hash)); + pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; }else{ sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); } @@ -6987,16 +2170,12 @@ ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1)) -#ifdef SQLITE_TEST - && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1)) -#endif + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ return sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); } @@ -7019,6 +2198,6 @@ SQLITE_EXTENSION_INIT2(pApi) return sqlite3Fts3Init(db); } #endif -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif ADDED ext/fts3/fts3Int.h Index: ext/fts3/fts3Int.h ================================================================== --- /dev/null +++ ext/fts3/fts3Int.h @@ -0,0 +1,230 @@ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + +#ifndef _FTSINT_H +#define _FTSINT_H + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +#include +#include "fts3_tokenizer.h" +#include "fts3_hash.h" + +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 + +/* +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. +*/ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) + +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 + +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3SegFilter Fts3SegFilter; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of columns in virtual table */ + char **azColumn; /* column names. malloced */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[18]; + + /* Pointer to string containing the SQL: + ** + ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ? + ** ORDER BY blockid" + */ + char *zSelectLeaves; + + /* The following hash table is used to buffer pending index updates during + ** transactions. Variable nPendingData estimates the memory size of the + ** pending data, including hash table overhead, but not malloc overhead. + ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed + ** automatically. Variable iPrevDocid is the docid of the most recently + ** inserted record. + */ + int nPendingData; + sqlite_int64 iPrevDocid; + Fts3Hash pendingTerms; +}; + +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + int eType; /* Search strategy (see below) */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + int isEof; /* True if at End Of Results */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ +}; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in "...", nToken will be the number +** of tokens in the string. +*/ +struct Fts3Phrase { + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + int isNot; /* Phrase prefixed by unary not (-) operator */ + struct PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer pointed to by z */ + int isPrefix; /* True if token ends in with a "*" character */ + } aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_init.c */ +int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *); +int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, + sqlite3_vtab **, char **); + +/* fts3_write.c */ +int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +int sqlite3Fts3PendingTermsFlush(Fts3Table *); +void sqlite3Fts3PendingTermsClear(Fts3Table *); +int sqlite3Fts3Optimize(Fts3Table *); +int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +void sqlite3Fts3SegReaderFree(Fts3SegReader *); +int sqlite3Fts3SegReaderIterate( + Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, + int (*)(Fts3Table *, void *, char *, int, char *, int), void * +); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); +int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +/* fts3.c */ +int sqlite3Fts3PutVarint(char *, sqlite3_int64); +int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +int sqlite3Fts3GetVarint32(const char *, int *); +int sqlite3Fts3VarintLen(sqlite3_uint64); +void sqlite3Fts3Dequote(char *); + +/* fts3_tokenizer.c */ +const char *sqlite3Fts3NextToken(const char *, int *); +int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, + const char *, sqlite3_tokenizer **, const char **, char ** +); + +/* fts3_snippet.c */ +void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +void sqlite3Fts3Snippet(sqlite3_context*, Fts3Cursor*, + const char *, const char *, const char * +); + +/* fts3_expr.c */ +int sqlite3Fts3ExprParse(sqlite3_tokenizer *, + char **, int, int, const char *, int, Fts3Expr ** +); +void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +void sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +#endif + +#endif /* _FTSINT_H */ Index: ext/fts3/fts3_expr.c ================================================================== --- ext/fts3/fts3_expr.c +++ ext/fts3/fts3_expr.c @@ -11,12 +11,11 @@ ****************************************************************************** ** ** This module contains code that implements a parser for fts3 query strings ** (the right-hand argument to the MATCH operator). Because the supported ** syntax is relatively simple, the whole tokenizer/parser system is -** hand-coded. The public interface to this module is declared in source -** code file "fts3_expr.h". +** hand-coded. */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* ** By default, this module parses the legacy syntax that has been @@ -38,11 +37,33 @@ ** If compiled with SQLITE_TEST defined, then this module exports the ** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable ** to zero causes the module to use the old syntax. If it is set to ** non-zero the new syntax is activated. This is so both syntaxes can ** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". */ + #ifdef SQLITE_TEST int sqlite3_fts3_enable_parentheses = 0; #else # ifdef SQLITE_ENABLE_FTS3_PARENTHESIS # define sqlite3_fts3_enable_parentheses 1 @@ -54,12 +75,11 @@ /* ** Default span for NEAR operators. */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 -#include "fts3_expr.h" -#include "sqlite3.h" +#include "fts3Int.h" #include #include #include typedef struct ParseContext ParseContext; @@ -352,10 +372,13 @@ cNext = zInput[nKey]; if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } memset(pRet, 0, sizeof(Fts3Expr)); pRet->eType = pKey->eType; pRet->nNear = nNear; *ppExpr = pRet; *pnConsumed = (zInput - z) + nKey; DELETED ext/fts3/fts3_expr.h Index: ext/fts3/fts3_expr.h ================================================================== --- ext/fts3/fts3_expr.h +++ /dev/null @@ -1,96 +0,0 @@ -/* -** 2008 Nov 28 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -*/ - -#include "fts3_tokenizer.h" -#include "sqlite3.h" - -/* -** The following describes the syntax supported by the fts3 MATCH -** operator in a similar format to that used by the lemon parser -** generator. This module does not use actually lemon, it uses a -** custom parser. -** -** query ::= andexpr (OR andexpr)*. -** -** andexpr ::= notexpr (AND? notexpr)*. -** -** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. -** notexpr ::= LP query RP. -** -** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. -** -** distance_opt ::= . -** distance_opt ::= / INTEGER. -** -** phrase ::= TOKEN. -** phrase ::= COLUMN:TOKEN. -** phrase ::= "TOKEN TOKEN TOKEN...". -*/ - -typedef struct Fts3Expr Fts3Expr; -typedef struct Fts3Phrase Fts3Phrase; - -/* -** A "phrase" is a sequence of one or more tokens that must match in -** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in "...", nToken will be the number -** of tokens in the string. -*/ -struct Fts3Phrase { - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ - struct PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer pointed to by z */ - int isPrefix; /* True if token ends in with a "*" character */ - } aToken[1]; /* One entry for each token in the phrase */ -}; - -/* -** A tree of these objects forms the RHS of a MATCH operator. -*/ -struct Fts3Expr { - int eType; /* One of the FTSQUERY_XXX values defined below */ - int nNear; /* Valid if eType==FTSQUERY_NEAR */ - Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ - Fts3Expr *pLeft; /* Left operand */ - Fts3Expr *pRight; /* Right operand */ - Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ -}; - -int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, - const char *, int, Fts3Expr **); -void sqlite3Fts3ExprFree(Fts3Expr *); - -/* -** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For -** example, the following: -** -** "a OR b AND c NOT d NEAR e" -** -** is equivalent to: -** -** "a OR (b AND (c NOT (d NEAR e)))" -*/ -#define FTSQUERY_NEAR 1 -#define FTSQUERY_NOT 2 -#define FTSQUERY_AND 3 -#define FTSQUERY_OR 4 -#define FTSQUERY_PHRASE 5 - -#ifdef SQLITE_TEST -void sqlite3Fts3ExprInitTestInterface(sqlite3 *db); -#endif Index: ext/fts3/fts3_hash.c ================================================================== --- ext/fts3/fts3_hash.c +++ ext/fts3/fts3_hash.c @@ -54,11 +54,11 @@ ** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass ** determines what kind of key the hash table will use. "copyKey" is ** true if the hash table should make its own private copy of keys and ** false if it should just use the supplied pointer. */ -void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){ +void sqlite3Fts3HashInit(Fts3Hash *pNew, int keyClass, int copyKey){ assert( pNew!=0 ); assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); pNew->keyClass = keyClass; pNew->copyKey = copyKey; pNew->first = 0; @@ -69,21 +69,21 @@ /* Remove all entries from a hash table. Reclaim all memory. ** Call this routine to delete a hash table or to reset a hash table ** to the empty state. */ -void sqlite3Fts3HashClear(fts3Hash *pH){ - fts3HashElem *elem; /* For looping over all elements of the table */ +void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ assert( pH!=0 ); elem = pH->first; pH->first = 0; fts3HashFree(pH->ht); pH->ht = 0; pH->htsize = 0; while( elem ){ - fts3HashElem *next_elem = elem->next; + Fts3HashElem *next_elem = elem->next; if( pH->copyKey && elem->pKey ){ fts3HashFree(elem->pKey); } fts3HashFree(elem); elem = next_elem; @@ -162,15 +162,15 @@ } /* Link an element into the hash table */ static void fts3HashInsertElement( - fts3Hash *pH, /* The complete hash table */ + Fts3Hash *pH, /* The complete hash table */ struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ - fts3HashElem *pNew /* The element to be inserted */ + Fts3HashElem *pNew /* The element to be inserted */ ){ - fts3HashElem *pHead; /* First element already in pEntry */ + Fts3HashElem *pHead; /* First element already in pEntry */ pHead = pEntry->chain; if( pHead ){ pNew->next = pHead; pNew->prev = pHead->prev; if( pHead->prev ){ pHead->prev->next = pNew; } @@ -189,13 +189,13 @@ /* Resize the hash table so that it cantains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. */ -static void fts3Rehash(fts3Hash *pH, int new_size){ +static void fts3Rehash(Fts3Hash *pH, int new_size){ struct _fts3ht *new_ht; /* The new hash table */ - fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ int (*xHash)(const void*,int); /* The hash function */ assert( (new_size & (new_size-1))==0 ); new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); if( new_ht==0 ) return; @@ -212,17 +212,17 @@ /* This function (for internal use only) locates an element in an ** hash table that matches the given key. The hash for this key has ** already been computed and is passed as the 4th parameter. */ -static fts3HashElem *fts3FindElementByHash( - const fts3Hash *pH, /* The pH to be searched */ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ const void *pKey, /* The key we are searching for */ int nKey, int h /* The hash for this key. */ ){ - fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ int (*xCompare)(const void*,int,const void*,int); /* comparison function */ if( pH->ht ){ struct _fts3ht *pEntry = &pH->ht[h]; @@ -241,12 +241,12 @@ /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ static void fts3RemoveElementByHash( - fts3Hash *pH, /* The pH containing "elem" */ - fts3HashElem* elem, /* The element to be removed from the pH */ + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ int h /* Hash value for the element */ ){ struct _fts3ht *pEntry; if( elem->prev ){ elem->prev->next = elem->next; @@ -278,13 +278,13 @@ /* Attempt to locate an element of the hash table pH with a key ** that matches pKey,nKey. Return the data for this element if it is ** found, or NULL if there is no match. */ -void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){ +void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ int h; /* A hash on key */ - fts3HashElem *elem; /* The element that matches key */ + Fts3HashElem *elem; /* The element that matches key */ int (*xHash)(const void*,int); /* The hash function */ if( pH==0 || pH->ht==0 ) return 0; xHash = ftsHashFunction(pH->keyClass); assert( xHash!=0 ); @@ -308,19 +308,19 @@ ** ** If the "data" parameter to this function is NULL, then the ** element corresponding to "key" is removed from the hash table. */ void *sqlite3Fts3HashInsert( - fts3Hash *pH, /* The hash table to insert into */ + Fts3Hash *pH, /* The hash table to insert into */ const void *pKey, /* The key */ int nKey, /* Number of bytes in the key */ void *data /* The data */ ){ int hraw; /* Raw hash value of the key */ int h; /* the hash of the key modulo hash table size */ - fts3HashElem *elem; /* Used to loop thru the element list */ - fts3HashElem *new_elem; /* New element added to the pH */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ int (*xHash)(const void*,int); /* The hash function */ assert( pH!=0 ); xHash = ftsHashFunction(pH->keyClass); assert( xHash!=0 ); @@ -343,11 +343,11 @@ if( pH->htsize==0 ){ pH->count = 0; return data; } } - new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); if( new_elem==0 ) return data; if( pH->copyKey && pKey!=0 ){ new_elem->pKey = fts3HashMalloc( nKey ); if( new_elem->pKey==0 ){ fts3HashFree(new_elem); Index: ext/fts3/fts3_hash.h ================================================================== --- ext/fts3/fts3_hash.h +++ ext/fts3/fts3_hash.h @@ -16,41 +16,41 @@ */ #ifndef _FTS3_HASH_H_ #define _FTS3_HASH_H_ /* Forward declarations of structures. */ -typedef struct fts3Hash fts3Hash; -typedef struct fts3HashElem fts3HashElem; +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; /* A complete hash table is an instance of the following structure. ** The internals of this structure are intended to be opaque -- client ** code should not attempt to access or modify the fields of this structure ** directly. Change this structure only by using the routines below. ** However, many of the "procedures" and "functions" for modifying and ** accessing this structure are really macros, so we can't really make ** this structure opaque. */ -struct fts3Hash { +struct Fts3Hash { char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ char copyKey; /* True if copy of key made on insert */ int count; /* Number of entries in this table */ - fts3HashElem *first; /* The first element of the array */ + Fts3HashElem *first; /* The first element of the array */ int htsize; /* Number of buckets in the hash table */ struct _fts3ht { /* the hash table */ int count; /* Number of entries with this hash */ - fts3HashElem *chain; /* Pointer to first entry with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; /* Each element in the hash table is an instance of the following ** structure. All elements are stored on a single doubly-linked list. ** ** Again, this structure is intended to be opaque, but it can't really ** be opaque because it is used by macros. */ -struct fts3HashElem { - fts3HashElem *next, *prev; /* Next and previous elements in the table */ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ void *pKey; int nKey; /* Key associated with this element */ }; /* @@ -69,14 +69,14 @@ #define FTS3_HASH_BINARY 2 /* ** Access routines. To delete, insert a NULL pointer. */ -void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey); -void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData); -void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey); -void sqlite3Fts3HashClear(fts3Hash*); +void sqlite3Fts3HashInit(Fts3Hash*, int keytype, int copyKey); +void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +void sqlite3Fts3HashClear(Fts3Hash*); /* ** Shorthand for the functions above */ #define fts3HashInit sqlite3Fts3HashInit @@ -86,12 +86,12 @@ /* ** Macros for looping over all elements of a hash table. The idiom is ** like this: ** -** fts3Hash h; -** fts3HashElem *p; +** Fts3Hash h; +** Fts3HashElem *p; ** ... ** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ ** SomeStructure *pData = fts3HashData(p); ** // do something with pData ** } ADDED ext/fts3/fts3_snippet.c Index: ext/fts3/fts3_snippet.c ================================================================== --- /dev/null +++ ext/fts3/fts3_snippet.c @@ -0,0 +1,750 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +#include "fts3Int.h" +#include +#include +#include + +typedef struct Snippet Snippet; + +/* +** An instance of the following structure keeps track of generated +** matching-word offset information and snippets. +*/ +struct Snippet { + int nMatch; /* Total number of matches */ + int nAlloc; /* Space allocated for aMatch[] */ + struct snippetMatch { /* One entry for each matching term */ + char snStatus; /* Status flag for use while constructing snippets */ + short int iCol; /* The column that contains the match */ + short int iTerm; /* The index in Query.pTerms[] of the matching term */ + int iToken; /* The index of the matching document token */ + short int nByte; /* Number of bytes in the term */ + int iStart; /* The offset to the first character of the term */ + } *aMatch; /* Points to space obtained from malloc */ + char *zOffset; /* Text rendering of aMatch[] */ + int nOffset; /* strlen(zOffset) */ + char *zSnippet; /* Snippet text */ + int nSnippet; /* strlen(zSnippet) */ +}; + + +/* It is not safe to call isspace(), tolower(), or isalnum() on +** hi-bit-set characters. This is the same solution used in the +** tokenizer. +*/ +/* TODO(shess) The snippet-generation code should be using the +** tokenizer-generated tokens rather than doing its own local +** tokenization. +*/ +/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */ +static int safe_isspace(char c){ + return (c&0x80)==0 ? isspace(c) : 0; +} +static int safe_isalnum(char c){ + return (c&0x80)==0 ? isalnum(c) : 0; +} + +/*******************************************************************/ +/* DataBuffer is used to collect data into a buffer in piecemeal +** fashion. It implements the usual distinction between amount of +** data currently stored (nData) and buffer capacity (nCapacity). +** +** dataBufferInit - create a buffer with given initial capacity. +** dataBufferReset - forget buffer's data, retaining capacity. +** dataBufferSwap - swap contents of two buffers. +** dataBufferExpand - expand capacity without adding data. +** dataBufferAppend - append data. +** dataBufferAppend2 - append two pieces of data at once. +** dataBufferReplace - replace buffer's data. +*/ +typedef struct DataBuffer { + char *pData; /* Pointer to malloc'ed buffer. */ + int nCapacity; /* Size of pData buffer. */ + int nData; /* End of data loaded into pData. */ +} DataBuffer; + +static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){ + assert( nCapacity>=0 ); + pBuffer->nData = 0; + pBuffer->nCapacity = nCapacity; + pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity); +} +static void dataBufferReset(DataBuffer *pBuffer){ + pBuffer->nData = 0; +} +static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ + assert( nAddCapacity>0 ); + /* TODO(shess) Consider expanding more aggressively. Note that the + ** underlying malloc implementation may take care of such things for + ** us already. + */ + if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){ + pBuffer->nCapacity = pBuffer->nData+nAddCapacity; + pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity); + } +} +static void dataBufferAppend(DataBuffer *pBuffer, + const char *pSource, int nSource){ + assert( nSource>0 && pSource!=NULL ); + dataBufferExpand(pBuffer, nSource); + memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource); + pBuffer->nData += nSource; +} +static void dataBufferAppend2(DataBuffer *pBuffer, + const char *pSource1, int nSource1, + const char *pSource2, int nSource2){ + assert( nSource1>0 && pSource1!=NULL ); + assert( nSource2>0 && pSource2!=NULL ); + dataBufferExpand(pBuffer, nSource1+nSource2); + memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1); + memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2); + pBuffer->nData += nSource1+nSource2; +} +static void dataBufferReplace(DataBuffer *pBuffer, + const char *pSource, int nSource){ + dataBufferReset(pBuffer); + dataBufferAppend(pBuffer, pSource, nSource); +} + + +/* StringBuffer is a null-terminated version of DataBuffer. */ +typedef struct StringBuffer { + DataBuffer b; /* Includes null terminator. */ +} StringBuffer; + +static void initStringBuffer(StringBuffer *sb){ + dataBufferInit(&sb->b, 100); + dataBufferReplace(&sb->b, "", 1); +} +static int stringBufferLength(StringBuffer *sb){ + return sb->b.nData-1; +} +static char *stringBufferData(StringBuffer *sb){ + return sb->b.pData; +} + +static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){ + assert( sb->b.nData>0 ); + if( nFrom>0 ){ + sb->b.nData--; + dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1); + } +} +static void append(StringBuffer *sb, const char *zFrom){ + nappend(sb, zFrom, strlen(zFrom)); +} + +static int endsInWhiteSpace(StringBuffer *p){ + return stringBufferLength(p)>0 && + safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]); +} + +/* If the StringBuffer ends in something other than white space, add a +** single space character to the end. +*/ +static void appendWhiteSpace(StringBuffer *p){ + if( stringBufferLength(p)==0 ) return; + if( !endsInWhiteSpace(p) ) append(p, " "); +} + +/* Remove white space from the end of the StringBuffer */ +static void trimWhiteSpace(StringBuffer *p){ + while( endsInWhiteSpace(p) ){ + p->b.pData[--p->b.nData-1] = '\0'; + } +} + + +/* +** Release all memory associated with the Snippet structure passed as +** an argument. +*/ +static void fts3SnippetFree(Snippet *p){ + sqlite3_free(p->aMatch); + sqlite3_free(p->zOffset); + sqlite3_free(p->zSnippet); + sqlite3_free(p); +} + +/* +** Append a single entry to the p->aMatch[] log. +*/ +static void snippetAppendMatch( + Snippet *p, /* Append the entry to this snippet */ + int iCol, int iTerm, /* The column and query term */ + int iToken, /* Matching token in document */ + int iStart, int nByte /* Offset and size of the match */ +){ + int i; + struct snippetMatch *pMatch; + if( p->nMatch+1>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + 10; + p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); + if( p->aMatch==0 ){ + p->nMatch = 0; + p->nAlloc = 0; + return; + } + } + i = p->nMatch++; + pMatch = &p->aMatch[i]; + pMatch->iCol = iCol; + pMatch->iTerm = iTerm; + pMatch->iToken = iToken; + pMatch->iStart = iStart; + pMatch->nByte = nByte; +} + +/* +** Sizing information for the circular buffer used in snippetOffsetsOfColumn() +*/ +#define FTS3_ROTOR_SZ (32) +#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1) + +/* +** Function to iterate through the tokens of a compiled expression. +** +** Except, skip all tokens on the right-hand side of a NOT operator. +** This function is used to find tokens as part of snippet and offset +** generation and we do nt want snippets and offsets to report matches +** for tokens on the RHS of a NOT. +*/ +static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){ + Fts3Expr *p = *ppExpr; + int iToken = *piToken; + if( iToken<0 ){ + /* In this case the expression p is the root of an expression tree. + ** Move to the first token in the expression tree. + */ + while( p->pLeft ){ + p = p->pLeft; + } + iToken = 0; + }else{ + assert(p && p->eType==FTSQUERY_PHRASE ); + if( iToken<(p->pPhrase->nToken-1) ){ + iToken++; + }else{ + iToken = 0; + while( p->pParent && p->pParent->pLeft!=p ){ + assert( p->pParent->pRight==p ); + p = p->pParent; + } + p = p->pParent; + if( p ){ + assert( p->pRight!=0 ); + p = p->pRight; + while( p->pLeft ){ + p = p->pLeft; + } + } + } + } + + *ppExpr = p; + *piToken = iToken; + return p?1:0; +} + +/* +** Return TRUE if the expression node pExpr is located beneath the +** RHS of a NOT operator. +*/ +static int fts3ExprBeneathNot(Fts3Expr *p){ + Fts3Expr *pParent; + while( p ){ + pParent = p->pParent; + if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){ + return 1; + } + p = pParent; + } + return 0; +} + +/* +** Add entries to pSnippet->aMatch[] for every match that occurs against +** document zDoc[0..nDoc-1] which is stored in column iColumn. +*/ +static void snippetOffsetsOfColumn( + Fts3Cursor *pCur, /* The fulltest search cursor */ + Snippet *pSnippet, /* The Snippet object to be filled in */ + int iColumn, /* Index of fulltext table column */ + const char *zDoc, /* Text of the fulltext table column */ + int nDoc /* Length of zDoc in bytes */ +){ + const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ + sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ + sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ + Fts3Table *pVtab; /* The full text index */ + int nColumn; /* Number of columns in the index */ + int i, j; /* Loop counters */ + int rc; /* Return code */ + unsigned int match, prevMatch; /* Phrase search bitmasks */ + const char *zToken; /* Next token from the tokenizer */ + int nToken; /* Size of zToken */ + int iBegin, iEnd, iPos; /* Offsets of beginning and end */ + + /* The following variables keep a circular buffer of the last + ** few tokens */ + unsigned int iRotor = 0; /* Index of current token */ + int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */ + int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */ + + pVtab = (Fts3Table *)pCur->base.pVtab; + nColumn = pVtab->nColumn; + pTokenizer = pVtab->pTokenizer; + pTModule = pTokenizer->pModule; + rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); + if( rc ) return; + pTCursor->pTokenizer = pTokenizer; + + prevMatch = 0; + while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){ + Fts3Expr *pIter = pCur->pExpr; + int iIter = -1; + iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin; + iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin; + match = 0; + for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){ + int nPhrase; /* Number of tokens in current phrase */ + struct PhraseToken *pToken; /* Current token */ + int iCol; /* Column index */ + + if( fts3ExprBeneathNot(pIter) ) continue; + nPhrase = pIter->pPhrase->nToken; + pToken = &pIter->pPhrase->aToken[iIter]; + iCol = pIter->pPhrase->iColumn; + if( iCol>=0 && iColn>nToken ) continue; + if( !pToken->isPrefix && pToken->nn<=nToken ); + if( memcmp(pToken->z, zToken, pToken->n) ) continue; + if( iIter>0 && (prevMatch & (1<=0; j--){ + int k = (iRotor-j) & FTS3_ROTOR_MASK; + snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j, + iRotorBegin[k], iRotorLen[k]); + } + } + } + prevMatch = match<<1; + iRotor++; + } + pTModule->xClose(pTCursor); +} + +/* +** Remove entries from the pSnippet structure to account for the NEAR +** operator. When this is called, pSnippet contains the list of token +** offsets produced by treating all NEAR operators as AND operators. +** This function removes any entries that should not be present after +** accounting for the NEAR restriction. For example, if the queried +** document is: +** +** "A B C D E A" +** +** and the query is: +** +** A NEAR/0 E +** +** then when this function is called the Snippet contains token offsets +** 0, 4 and 5. This function removes the "0" entry (because the first A +** is not near enough to an E). +** +** When this function is called, the value pointed to by parameter piLeft is +** the integer id of the left-most token in the expression tree headed by +** pExpr. This function increments *piLeft by the total number of tokens +** in the expression tree headed by pExpr. +** +** Return 1 if any trimming occurs. Return 0 if no trimming is required. +*/ +static int trimSnippetOffsets( + Fts3Expr *pExpr, /* The search expression */ + Snippet *pSnippet, /* The set of snippet offsets to be trimmed */ + int *piLeft /* Index of left-most token in pExpr */ +){ + if( pExpr ){ + if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){ + return 1; + } + + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: + *piLeft += pExpr->pPhrase->nToken; + break; + case FTSQUERY_NEAR: { + /* The right-hand-side of a NEAR operator is always a phrase. The + ** left-hand-side is either a phrase or an expression tree that is + ** itself headed by a NEAR operator. The following initializations + ** set local variable iLeft to the token number of the left-most + ** token in the right-hand phrase, and iRight to the right most + ** token in the same phrase. For example, if we had: + ** + ** MATCH '"abc def" NEAR/2 "ghi jkl"' + ** + ** then iLeft will be set to 2 (token number of ghi) and nToken will + ** be set to 4. + */ + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + int iLeft = *piLeft; + int nNear = pExpr->nNear; + int nToken = pRight->pPhrase->nToken; + int jj, ii; + if( pLeft->eType==FTSQUERY_NEAR ){ + pLeft = pLeft->pRight; + } + assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->eType==FTSQUERY_PHRASE ); + nToken += pLeft->pPhrase->nToken; + + for(ii=0; iinMatch; ii++){ + struct snippetMatch *p = &pSnippet->aMatch[ii]; + if( p->iTerm==iLeft ){ + int isOk = 0; + /* Snippet ii is an occurence of query term iLeft in the document. + ** It occurs at position (p->iToken) of the document. We now + ** search for an instance of token (iLeft-1) somewhere in the + ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within + ** the set of snippetMatch structures. If one is found, proceed. + ** If one cannot be found, then remove snippets ii..(ii+N-1) + ** from the matching snippets, where N is the number of tokens + ** in phrase pRight->pPhrase. + */ + for(jj=0; isOk==0 && jjnMatch; jj++){ + struct snippetMatch *p2 = &pSnippet->aMatch[jj]; + if( p2->iTerm==(iLeft-1) ){ + if( p2->iToken>=(p->iToken-nNear-1) + && p2->iToken<(p->iToken+nNear+nToken) + ){ + isOk = 1; + } + } + } + if( !isOk ){ + int kk; + for(kk=0; kkpPhrase->nToken; kk++){ + pSnippet->aMatch[kk+ii].iTerm = -2; + } + return 1; + } + } + if( p->iTerm==(iLeft-1) ){ + int isOk = 0; + for(jj=0; isOk==0 && jjnMatch; jj++){ + struct snippetMatch *p2 = &pSnippet->aMatch[jj]; + if( p2->iTerm==iLeft ){ + if( p2->iToken<=(p->iToken+nNear+1) + && p2->iToken>(p->iToken-nNear-nToken) + ){ + isOk = 1; + } + } + } + if( !isOk ){ + int kk; + for(kk=0; kkpPhrase->nToken; kk++){ + pSnippet->aMatch[ii-kk].iTerm = -2; + } + return 1; + } + } + } + break; + } + } + + if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){ + return 1; + } + } + return 0; +} + +/* +** Compute all offsets for the current row of the query. +** If the offsets have already been computed, this routine is a no-op. +*/ +static int snippetAllOffsets(Fts3Cursor *pCsr, Snippet **ppSnippet){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + int nColumn; + int iColumn, i; + int iFirst, iLast; + int iTerm = 0; + Snippet *pSnippet; + + if( pCsr->pExpr==0 ){ + return SQLITE_OK; + } + + pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet)); + *ppSnippet = pSnippet; + if( !pSnippet ){ + return SQLITE_NOMEM; + } + memset(pSnippet, 0, sizeof(Snippet)); + + nColumn = p->nColumn; + iColumn = (pCsr->eType - 2); + if( iColumn<0 || iColumn>=nColumn ){ + /* Look for matches over all columns of the full-text index */ + iFirst = 0; + iLast = nColumn-1; + }else{ + /* Look for matches in the iColumn-th column of the index only */ + iFirst = iColumn; + iLast = iColumn; + } + for(i=iFirst; i<=iLast; i++){ + const char *zDoc; + int nDoc; + zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1); + snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc); + } + + while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){ + iTerm = 0; + } + + return SQLITE_OK; +} + +/* +** Convert the information in the aMatch[] array of the snippet +** into the string zOffset[0..nOffset-1]. This string is used as +** the return of the SQL offsets() function. +*/ +static void snippetOffsetText(Snippet *p){ + int i; + int cnt = 0; + StringBuffer sb; + char zBuf[200]; + if( p->zOffset ) return; + initStringBuffer(&sb); + for(i=0; inMatch; i++){ + struct snippetMatch *pMatch = &p->aMatch[i]; + if( pMatch->iTerm>=0 ){ + /* If snippetMatch.iTerm is less than 0, then the match was + ** discarded as part of processing the NEAR operator (see the + ** trimSnippetOffsetsForNear() function for details). Ignore + ** it in this case + */ + zBuf[0] = ' '; + sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", + pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); + append(&sb, zBuf); + cnt++; + } + } + p->zOffset = stringBufferData(&sb); + p->nOffset = stringBufferLength(&sb); +} + +/* +** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set +** of matching words some of which might be in zDoc. zDoc is column +** number iCol. +** +** iBreak is suggested spot in zDoc where we could begin or end an +** excerpt. Return a value similar to iBreak but possibly adjusted +** to be a little left or right so that the break point is better. +*/ +static int wordBoundary( + int iBreak, /* The suggested break point */ + const char *zDoc, /* Document text */ + int nDoc, /* Number of bytes in zDoc[] */ + struct snippetMatch *aMatch, /* Matching words */ + int nMatch, /* Number of entries in aMatch[] */ + int iCol /* The column number for zDoc[] */ +){ + int i; + if( iBreak<=10 ){ + return 0; + } + if( iBreak>=nDoc-10 ){ + return nDoc; + } + for(i=0; i0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){ + return aMatch[i-1].iStart; + } + } + for(i=1; i<=10; i++){ + if( safe_isspace(zDoc[iBreak-i]) ){ + return iBreak - i + 1; + } + if( safe_isspace(zDoc[iBreak+i]) ){ + return iBreak + i + 1; + } + } + return iBreak; +} + + + +/* +** Allowed values for Snippet.aMatch[].snStatus +*/ +#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */ +#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */ + +/* +** Generate the text of a snippet. +*/ +static void snippetText( + Fts3Cursor *pCursor, /* The cursor we need the snippet for */ + Snippet *pSnippet, + const char *zStartMark, /* Markup to appear before each match */ + const char *zEndMark, /* Markup to appear after each match */ + const char *zEllipsis /* Ellipsis mark */ +){ + int i, j; + struct snippetMatch *aMatch; + int nMatch; + int nDesired; + StringBuffer sb; + int tailCol; + int tailOffset; + int iCol; + int nDoc; + const char *zDoc; + int iStart, iEnd; + int tailEllipsis = 0; + int iMatch; + + + sqlite3_free(pSnippet->zSnippet); + pSnippet->zSnippet = 0; + aMatch = pSnippet->aMatch; + nMatch = pSnippet->nMatch; + initStringBuffer(&sb); + + for(i=0; i0; i++){ + if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue; + nDesired--; + iCol = aMatch[i].iCol; + zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1); + iStart = aMatch[i].iStart - 40; + iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol); + if( iStart<=10 ){ + iStart = 0; + } + if( iCol==tailCol && iStart<=tailOffset+20 ){ + iStart = tailOffset; + } + if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){ + trimWhiteSpace(&sb); + appendWhiteSpace(&sb); + append(&sb, zEllipsis); + appendWhiteSpace(&sb); + } + iEnd = aMatch[i].iStart + aMatch[i].nByte + 40; + iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol); + if( iEnd>=nDoc-10 ){ + iEnd = nDoc; + tailEllipsis = 0; + }else{ + tailEllipsis = 1; + } + while( iMatchzSnippet = stringBufferData(&sb); + pSnippet->nSnippet = stringBufferLength(&sb); +} + +void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Snippet *p; /* Snippet structure */ + int rc = snippetAllOffsets(pCsr, &p); + snippetOffsetText(p); + sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT); + fts3SnippetFree(p); +} + +void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis /* Snippet ellipsis text - "..." */ +){ + Snippet *p; /* Snippet structure */ + int rc = snippetAllOffsets(pCsr, &p); + snippetText(pCsr, p, zStart, zEnd, zEllipsis); + sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT); + fts3SnippetFree(p); +} + Index: ext/fts3/fts3_tokenizer.c ================================================================== --- ext/fts3/fts3_tokenizer.c +++ ext/fts3/fts3_tokenizer.c @@ -28,13 +28,18 @@ #include "sqlite3ext.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif -#include "fts3_hash.h" -#include "fts3_tokenizer.h" +#include "fts3Int.h" #include +#include +#include + +static int safe_isspace(char c){ + return (c&0x80)==0 ? isspace(c) : 0; +} /* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: ** @@ -57,18 +62,18 @@ static void scalarFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ - fts3Hash *pHash; + Fts3Hash *pHash; void *pPtr = 0; const unsigned char *zName; int nName; assert( argc==1 || argc==2 ); - pHash = (fts3Hash *)sqlite3_user_data(context); + pHash = (Fts3Hash *)sqlite3_user_data(context); zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ @@ -94,10 +99,132 @@ } } sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } + +static int fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + switch( *z1 ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = &z1[1]; + while( *z2 && (z2[0]!=*z1 || z2[1]==*z1) ) z2++; + if( *z2 ) z2++; + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (z2-z1); + return z1; +} + +int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Possible tokenizer specification */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + if( !z ){ + zCopy = sqlite3_mprintf("simple"); + }else{ + while( safe_isspace(*z) ) z++; + if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){ + return SQLITE_OK; + } + zCopy = sqlite3_mprintf("%s", &z[8]); + *pzTokenizer = zArg; + } + if( !zCopy ){ + return SQLITE_NOMEM; + } + + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, strlen(z)+1); + if( !m ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free(aArg); + } + + sqlite3_free(zCopy); + return rc; +} + #ifdef SQLITE_TEST #include #include @@ -131,11 +258,11 @@ static void testFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ - fts3Hash *pHash; + Fts3Hash *pHash; sqlite3_tokenizer_module *p; sqlite3_tokenizer *pTokenizer = 0; sqlite3_tokenizer_cursor *pCsr = 0; const char *zErr = 0; @@ -164,11 +291,11 @@ if( argc==3 ){ zArg = (const char *)sqlite3_value_text(argv[1]); } - pHash = (fts3Hash *)sqlite3_user_data(context); + pHash = (Fts3Hash *)sqlite3_user_data(context); p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); @@ -333,11 +460,11 @@ ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ int sqlite3Fts3InitHashTable( sqlite3 *db, - fts3Hash *pHash, + Fts3Hash *pHash, const char *zName ){ int rc = SQLITE_OK; void *p = (void *)pHash; const int any = SQLITE_ANY; ADDED ext/fts3/fts3_write.c Index: ext/fts3/fts3_write.c ================================================================== --- /dev/null +++ ext/fts3/fts3_write.c @@ -0,0 +1,1949 @@ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ + +#include "fts3Int.h" +#include +#include +#include + +#define INTERIOR_MAX 2048 /* Soft limit for segment node size */ +#define LEAF_MAX 2048 /* Soft limit for segment leaf size */ + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** Data structure used while accumulating terms in the pending-terms hash +** table. The hash table entry maps from term (a string) to a malloced +** instance of this structure. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level */ + sqlite3_int64 iStartBlock; + sqlite3_int64 iEndBlock; + sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */ + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used to iterate through the current doclist */ + char *pOffsetList; + sqlite3_int64 iDocid; +}; + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_SELECT_CONTENT_BY_ROWID 5 +#define SQL_NEXT_SEGMENT_INDEX 6 +#define SQL_INSERT_SEGMENTS 7 +#define SQL_NEXT_SEGMENTS_ID 8 +#define SQL_INSERT_SEGDIR 9 +#define SQL_SELECT_LEVEL 10 +#define SQL_SELECT_ALL_LEVEL 11 +#define SQL_SELECT_LEVEL_COUNT 12 +#define SQL_SELECT_SEGDIR_COUNT_MAX 13 +#define SQL_DELETE_SEGDIR_BY_LEVEL 14 +#define SQL_DELETE_SEGMENTS_RANGE 15 +#define SQL_CONTENT_INSERT 16 +#define SQL_GET_BLOCK 17 + +static int fts3SqlStmt( + Fts3Table *p, + int eStmt, + sqlite3_stmt **pp, + sqlite3_value **apVal +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?", +/* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?", +/* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'", +/* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + +/* 12 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 13 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", + +/* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 16 */ 0, /* CONTENT_INSERT - generated elsewhere */ +/* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + char *zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +static int fts3PendingListAppend( + PendingList **pp, + sqlite3_int64 iDocid, + sqlite3_int64 iCol, + sqlite3_int64 iPos, + int *pRc +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + p->iLastPos = iPos; + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){ + int rc; + int iStart; + int iEnd; + int iPos; + + char const *zToken; + int nToken; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + pCsr->pTokenizer = pTokenizer; + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + PendingList *pList; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + } + + pModule->xClose(pCsr); + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + return SQLITE_OK; +} + +void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + Fts3HashElem *pElem; + for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ + sqlite3_free(fts3HashData(pElem)); + } + fts3HashClear(&p->pendingTerms); + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + if( zText ){ + int rc = fts3PendingTermsAdd(p, zText, i-2); + if( rc!=SQLITE_OK ){ + return rc; + } + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + int i; /* Iterator variable */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + /* Locate the statement handle used to insert data into the %_content + ** table. If no such statement has been prepared, prepare a new one. + ** The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + pContentInsert = p->aStmt[SQL_CONTENT_INSERT]; + if( !pContentInsert ){ + char *zVarlist; /* The "?, ?, ..." string */ + char *zSql; /* The text of the INSERT statement */ + + /* Construct the SQL statement text. */ + zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2); + if( !zVarlist ){ + return SQLITE_NOMEM; + } + zVarlist[0] = '?'; + for(i=1; i<=p->nColumn; i++){ + zVarlist[i*2-1] = ','; + zVarlist[i*2] = '?'; + } + zVarlist[p->nColumn*2+1] = '\0'; + zSql = sqlite3_mprintf("INSERT INTO %Q.'%q_content' VALUES(%z)", + p->zDb, p->zName, zVarlist + ); + if( !zSql ) return SQLITE_NOMEM; + + /* Prepare the SQL statement. */ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pContentInsert, NULL); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ){ + return rc; + } + p->aStmt[SQL_CONTENT_INSERT] = pContentInsert; + } + + /* Bind values to the prepared statement. + ** + ** There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, if a non-NULL docid value is specified, it is the value + ** inserted. Otherwise, the rowid value is used. + */ + for(i=0; i<=p->nColumn; i++){ + rc = sqlite3_bind_value(pContentInsert, i+1, apVal[i+1]); + if( rc!=SQLITE_OK ) return rc; + } + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p){ + int rc; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the %_content, %_segments and %_segdir tables. */ + rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0); + if( rc==SQLITE_OK ){ + rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); + } + return rc; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static int fts3DeleteTerms(Fts3Table *p, sqlite3_value **apVal){ + int rc; + sqlite3_stmt *pSelect; + + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + for(i=1; i<=p->nColumn; i++){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, zText, -1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + return rc; + } + } + } + } + + return sqlite3_reset(pSelect); +} + +static int fts3SegmentMerge(Fts3Table *, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext; /* Result of query pNextIdx */ + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pNextIdx, 1, iLevel); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int64(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=FTS3_MERGE_COUNT ){ + rc = fts3SegmentMerge(p, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext(Fts3SegReader *pReader){ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + int rc; + if( !pReader->pStmt ){ + pReader->aNode = 0; + return SQLITE_OK; + } + rc = sqlite3_step(pReader->pStmt); + if( rc!=SQLITE_ROW ){ + pReader->aNode = 0; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); + } + pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0); + pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0); + pNext = pReader->aNode; + } + + pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); + pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); + + if( nPrefix+nSuffix>pReader->nTermAlloc ){ + int nNew = (nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); + assert( pNext<&pReader->aNode[pReader->nNode] ); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + return SQLITE_OK; +} + +static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ + int n; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; +} + +/* +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +** +** TODO: example. +** +*/ +static void fts3SegReaderNextDocid( + Fts3SegReader *pReader, + char **ppOffsetList, + int *pnOffsetList +){ + char *p = pReader->pOffsetList; + char c = 0; + + /* Pointer p currently points at the first byte of an offset list. The + ** following two lines advance it to point one byte past the end of + ** the same offset list. + */ + while( *p | c ) c = *p++ & 0x80; + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = p - pReader->pOffsetList - 1; + } + + /* If there are no more entries in the doclist, set pOffsetList to + ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and + ** Fts3SegReader.pOffsetList to point to the next offset list before + ** returning. + */ + if( p>=&pReader->aDoclist[pReader->nDoclist] ){ + pReader->pOffsetList = 0; + }else{ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + pReader->iDocid += iDelta; + } +} + +/* +** Free all allocations associated with the iterator passed as the first +** argument. +*/ +void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader ){ + sqlite3_finalize(pReader->pStmt); + sqlite3_free(pReader->zTerm); + sqlite3_free(pReader); + } +} + +int sqlite3Fts3SegReaderNew( + Fts3Table *p, /* Virtual table handle */ + int iAge, /* Segment "age". */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + int rc = SQLITE_OK; /* Return code */ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + if( iStartLeaf==0 ){ + nExtra = nRoot; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iStartBlock = iStartLeaf; + pReader->iIdx = iAge; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->nNode = nRoot; + memcpy(pReader->aNode, zRoot, nRoot); + }else{ + if( !p->zSelectLeaves ){ + p->zSelectLeaves = sqlite3_mprintf( + "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? " + "ORDER BY blockid", p->zDb, p->zName + ); + if( !p->zSelectLeaves ){ + rc = SQLITE_NOMEM; + goto finished; + } + } + rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0); + if( rc!=SQLITE_OK ){ + goto finished; + } + sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf); + sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf); + } + rc = fts3SegReaderNext(pReader); + + finished: + if( rc==SQLITE_OK ){ + *ppReader = pReader; + }else{ + sqlite3Fts3SegReaderFree(pReader); + } + return rc; +} + + +/* +** The second argument to this function is expected to be a statement of +** the form: +** +** SELECT +** idx, -- col 0 +** start_block, -- col 1 +** leaves_end_block, -- col 2 +** end_block, -- col 3 +** root -- col 4 +** FROM %_segdir ... +** +** This function allocates and initializes a Fts3SegReader structure to +** iterate through the terms stored in the segment identified by the +** current row that pStmt is pointing to. +** +** If successful, the Fts3SegReader is left pointing to the first term +** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error +** code is returned. +*/ +static int fts3SegReaderNew( + Fts3Table *p, /* Virtual table handle */ + sqlite3_stmt *pStmt, /* See above */ + int iAge, /* Segment "age". */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + return sqlite3Fts3SegReaderNew(p, iAge, + sqlite3_column_int64(pStmt, 1), + sqlite3_column_int64(pStmt, 2), + sqlite3_column_int64(pStmt, 3), + sqlite3_column_blob(pStmt, 4), + sqlite3_column_bytes(pStmt, 4), + ppReader + ); +} + +/* +** Compare the two Fts3SegReader structures. Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared with memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert( rc!=0 ); + return rc; +} + +static int fts3SegReaderCmp2(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx-pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, + sqlite3_int64 iBlock, + char *z, int n +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLevel, /* Value for "level" field */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + sqlite3_bind_int64(pStmt, 5, iEndBlock); + rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +static void fts3PrefixCompress( + const char *zPrev, + int nPrev, + const char *zNext, + int nNext, + int *pnPrefix +){ + int n; + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm, &nPrefix); + nSuffix = nTerm-nPrefix; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=INTERIOR_MAX || !pTree->zTerm ){ + + if( nReq>INTERIOR_MAX ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (INTERIOR_MAX bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** INTERIOR_MAX bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + INTERIOR_MAX); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(LEAF_MAX); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = LEAF_MAX; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm, &nPrefix); + nSuffix = nTerm-nPrefix; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>LEAF_MAX ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +static int fts3SegWriterFlush( + Fts3Table *p, + SegmentWriter *pWriter, + int iLevel, + int iIdx +){ + int rc; + if( pWriter->pTree ){ + sqlite3_int64 iLast; + char *zRoot; + int nRoot; + sqlite3_int64 iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir( + p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir( + p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + } + return rc; +} + +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** apVal[0] would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + return rc; +} + +static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){ + sqlite3_stmt *pStmt; + int rc; + + assert( iLevel>=0 ); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, iLevel); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnSegment = sqlite3_column_int(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ + sqlite3_stmt *pStmt; + int rc; + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnSegment = sqlite3_column_int(pStmt, 0); + *pnMax = sqlite3_column_int(pStmt, 1); + } + return sqlite3_reset(pStmt); +} + +static int fts3DeleteSegdir( + Fts3Table *p, + int iLevel, + Fts3SegReader **apSegment, + int nReader +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + for(i=0; rc==SQLITE_OK && iiStartBlock ){ + sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + + if( iLevel>=0 ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iLevel); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + }else{ + rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); + } + + return rc; +} + +static void fts3ColumnFilter(int iCol, char **ppList, int *pnList){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; + + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( pflags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + if( pFilter->zTerm ){ + int nTerm = pFilter->nTerm; + const char *zTerm = pFilter->zTerm; + for(i=0; iaNode ){ + int nTerm = apSegment[0]->nTerm; + char *zTerm = apSegment[0]->zTerm; + int nMerge = 1; + + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm ){ + if( nTermnTerm + || (!isPrefix && nTerm>pFilter->nTerm) + || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + goto finished; + } + } + + while( nMergeaNode + && apSegment[nMerge]->nTerm==nTerm + && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm) + ){ + nMerge++; + } + + if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){ + Fts3SegReader *p0 = apSegment[0]; + rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist); + if( rc!=SQLITE_OK ) goto finished; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + + /* The current term of the first nMerge entries in the linked list + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term added to the new segment. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList; + int nList; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(apSegment[j], 0, 0); + j++; + } + + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, &pList, &nList); + } + + if( !isIgnoreEmpty || nList>0 ){ + nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + if( nDoclist+nByte>nAlloc ){ + char *aNew; + nAlloc = nDoclist+nByte*2; + aNew = sqlite3_realloc(aBuffer, nAlloc); + if( !aNew ){ + rc = SQLITE_NOMEM; + goto finished; + } + aBuffer = aNew; + } + nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&aBuffer[nDoclist], pList, nList); + nDoclist += nList; + aBuffer[nDoclist++] = '\0'; + } + } + + fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderCmp2); + } + + if( nDoclist>0 ){ + rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist); + if( rc!=SQLITE_OK ) goto finished; + } + } + + /* If there is a term specified to filter on, and this is not a prefix + ** search, return now. The callback that corresponds to the required + ** term (if such a term exists in the index) has already been made. + */ + if( pFilter->zTerm && !isPrefix ){ + goto finished; + } + + for(i=0; i0 ); + assert( iNewLevel>=0 ); + + /* Allocate space for an array of pointers to segment iterators. */ + apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment); + if( !apSegment ){ + return SQLITE_NOMEM; + } + memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment); + + /* Allocate a Fts3SegReader structure for each segment being merged. A + ** Fts3SegReader stores the state data required to iterate through all + ** entries on all leaves of a single segment. + */ + assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0); + if( rc!=SQLITE_OK ) goto finished; + sqlite3_bind_int(pStmt, 1, iLevel); + for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){ + rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]); + if( rc!=SQLITE_OK ){ + goto finished; + } + } + rc = sqlite3_reset(pStmt); + pStmt = 0; + if( rc!=SQLITE_OK ) goto finished; + + memset(&filter, 0, sizeof(Fts3SegFilter)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS; + filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, + &filter, fts3MergeCallback, (void *)&pWriter + ); + if( rc!=SQLITE_OK ) goto finished; + + rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment); + if( rc==SQLITE_OK ){ + rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); + } + + finished: + fts3SegWriterFree(pWriter); + if( apSegment ){ + for(i=0; ipendingTerms); + if( nElem==0 ){ + assert( p->nPendingData==0 ); + return SQLITE_OK; + } + + /* Determine the next index at level 0, merging as necessary. */ + rc = fts3AllocateSegdirIdx(p, 0, &idx); + if( rc!=SQLITE_OK ){ + return rc; + } + + apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *)); + if( !apElem ){ + return SQLITE_NOMEM; + } + + i = 0; + for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ + apElem[i++] = pElem; + } + assert( i==nElem ); + + /* TODO(shess) Should we allow user-defined collation sequences, + ** here? I think we only need that once we support prefix searches. + ** Also, should we be using qsort()? + */ + if( nElem>1 ){ + qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare); + } + + + /* Write the segment tree into the database. */ + for(i=0; rc==SQLITE_OK && iaData, pList->nData+1); + } + if( rc==SQLITE_OK ){ + rc = fts3SegWriterFlush(p, pWriter, 0, idx); + } + + /* Free all allocated resources before returning */ + fts3SegWriterFree(pWriter); + sqlite3_free(apElem); + sqlite3Fts3PendingTermsClear(p); + return rc; +} + +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. +*/ +int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + int isRemove = 0; /* True for an UPDATE or DELETE */ + sqlite3_int64 iRemove; /* Rowid removed by UPDATE or DELETE */ + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + int isEmpty; + rc = fts3IsEmpty(p, apVal, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. + */ + rc = fts3DeleteAll(p); + }else{ + isRemove = 1; + iRemove = sqlite3_value_int64(apVal[0]); + rc = fts3PendingTermsDocid(p, iRemove); + if( rc==SQLITE_OK ){ + rc = fts3DeleteTerms(p, apVal); + if( rc==SQLITE_OK ){ + rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal); + } + } + } + } + } + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ + rc = fts3PendingTermsDocid(p, *pRowid); + } + if( rc==SQLITE_OK ){ + rc = fts3InsertTerms(p, apVal); + } + } + + return rc; +} + +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +int sqlite3Fts3Optimize(Fts3Table *p){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = fts3SegmentMerge(p, -1); + } + return rc; +} + Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -52,11 +52,11 @@ # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ - fts3_tokenizer.o fts3_tokenizer1.o \ + fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \ @@ -181,18 +181,18 @@ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_expr.h \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/fts3/fts3_tokenizer1.c + $(TOP)/ext/fts3/fts3_tokenizer1.c \ + $(TOP)/ext/fts3/fts3_write.c SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c SRC += \ $(TOP)/ext/rtree/rtree.h \ @@ -294,11 +294,11 @@ $(TOP)/ext/fts2/fts2.h \ $(TOP)/ext/fts2/fts2_hash.h \ $(TOP)/ext/fts2/fts2_tokenizer.h EXTHDR += \ $(TOP)/ext/fts3/fts3.h \ - $(TOP)/ext/fts3/fts3_expr.h \ + $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_tokenizer.h EXTHDR += \ $(TOP)/ext/rtree/rtree.h EXTHDR += \ @@ -432,19 +432,25 @@ fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c + +fts3_snippet.o: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c + +fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c Index: src/test_hexio.c ================================================================== --- src/test_hexio.c +++ src/test_hexio.c @@ -314,10 +314,52 @@ sqlite3_free(z); #endif return TCL_OK; } +static int getFts3Varint(const char *p, sqlite_int64 *v){ + const unsigned char *q = (const unsigned char *) p; + sqlite_uint64 x = 0, y = 1; + while( (*q & 0x80) == 0x80 ){ + x += y * (*q++ & 0x7f); + y <<= 7; + } + x += y * (*q++); + *v = (sqlite_int64) x; + return (int) (q - (unsigned char *)p); +} + + +/* +** USAGE: read_varint BLOB VARNAME +** +** Read a varint from the start of BLOB. Set variable VARNAME to contain +** the interpreted value. Return the number of bytes of BLOB consumed. +*/ +static int read_varint( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int nBlob; + unsigned char *zBlob; + sqlite3_int64 iVal; + int nVal; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BLOB VARNAME"); + return TCL_ERROR; + } + zBlob = Tcl_GetByteArrayFromObj(objv[1], &nBlob); + + nVal = getFts3Varint(zBlob, (sqlite3_uint64 *)(&iVal)); + Tcl_ObjSetVar2(interp, objv[2], 0, Tcl_NewWideIntObj(iVal), 0); + Tcl_SetObjResult(interp, Tcl_NewIntObj(nVal)); + return TCL_OK; +} + /* ** Register commands with the TCL interpreter. */ int Sqlitetest_hexio_Init(Tcl_Interp *interp){ @@ -329,12 +371,13 @@ { "hexio_write", hexio_write }, { "hexio_get_int", hexio_get_int }, { "hexio_render_int16", hexio_render_int16 }, { "hexio_render_int32", hexio_render_int32 }, { "utf8_to_utf8", utf8_to_utf8 }, + { "read_varint", read_varint }, }; int i; for(i=0; i0} { + set iCol 0 + set iPos 0 + set lPos [list] + set lCol [list] + + # First varint of a doclist-entry is the docid. Delta-compressed + # with respect to the docid of the previous entry. + # + incr docid [gobble_varint doclist] + if {[info exists D($zTerm,$docid)]} { + while {[set iDelta [gobble_varint doclist]] != 0} {} + continue + } + set D($zTerm,$docid) 1 + + # Gobble varints until the 0x00 that terminates the doclist-entry + # is found. + while {[set iDelta [gobble_varint doclist]] > 0} { + if {$iDelta == 1} { + set iCol [gobble_varint doclist] + set iPos 0 + } else { + incr iPos $iDelta + incr iPos -2 + set C($docid,$iCol,$iPos) $zTerm + } + } + } + } + } + + foreach key [array names C] { + #puts "$key -> $C($key)" + } + + + db eval "SELECT * FROM ${tbl}_content" E { + set iCol 0 + set iDoc $E(docid) + foreach col [lrange $E(*) 1 end] { + set c $E($col) + set sql {SELECT fts3_tokenizer_test('simple', $c)} + + foreach {pos term dummy} [db one $sql] { + if {![info exists C($iDoc,$iCol,$pos)]} { + set es "Error at docid=$iDoc col=$iCol pos=$pos. Index is missing" + lappend errors $es + } else { + if {$C($iDoc,$iCol,$pos) != "$term"} { + set es "Error at docid=$iDoc col=$iCol pos=$pos. Index " + append es "has \"$C($iDoc,$iCol,$pos)\", document has \"$term\"" + lappend errors $es + } + unset C($iDoc,$iCol,$pos) + } + } + incr iCol + } + } + + foreach c [array names C] { + lappend errors "Bad index entry: $c -> $C($c)" + } + + if {[info exists errors]} { return [join $errors "\n"] } + return "ok" +} + +# USAGE: fts3_terms TBL WHERE +# +# Argument TBL must be the name of an FTS3 table. Argument WHERE is an +# SQL expression that will be used as the WHERE clause when scanning +# the %_segdir table. As in the following query: +# +# "SELECT * FROM ${TBL}_segdir WHERE ${WHERE}" +# +# This function returns a list of all terms present in the segments +# selected by the statement above. +# +proc fts3_terms {tbl where} { + fts3_read $tbl $where a + return [lsort [array names a]] +} + + +# USAGE: fts3_doclist TBL TERM WHERE +# +# Argument TBL must be the name of an FTS3 table. TERM is a term that may +# or may not be present in the table. Argument WHERE is used to select a +# subset of the b-tree segments in the associated full-text index as +# described above for [fts3_terms]. +# +# This function returns the results of merging the doclists associated +# with TERM in the selected segments. Each doclist is an element of the +# returned list. Each doclist is formatted as follows: +# +# [$docid ?$col[$off1 $off2...]?...] +# +# The formatting is odd for a Tcl command in order to be compatible with +# the original C-language implementation. If argument WHERE is "1", then +# any empty doclists are omitted from the returned list. +# +proc fts3_doclist {tbl term where} { + fts3_read $tbl $where a + + + foreach doclist $a($term) { + set docid 0 + + while {[string length $doclist]>0} { + set iCol 0 + set iPos 0 + set lPos [list] + set lCol [list] + incr docid [gobble_varint doclist] + + while {[set iDelta [gobble_varint doclist]] > 0} { + if {$iDelta == 1} { + lappend lCol [list $iCol $lPos] + set iPos 0 + set lPos [list] + set iCol [gobble_varint doclist] + } else { + incr iPos $iDelta + incr iPos -2 + lappend lPos $iPos + } + } + + if {[llength $lPos]>0} { + lappend lCol [list $iCol $lPos] + } + + if {$where != "1" || [llength $lCol]>0} { + set ret($docid) $lCol + } else { + unset -nocomplain ret($docid) + } + } + } + + set lDoc [list] + foreach docid [lsort -integer [array names ret]] { + set lCol [list] + set cols "" + foreach col $ret($docid) { + foreach {iCol lPos} $col {} + append cols " $iCol\[[join $lPos { }]\]" + } + lappend lDoc "\[${docid}${cols}\]" + } + + join $lDoc " " +} + +########################################################################### + +proc gobble_varint {varname} { + upvar $varname blob + set n [read_varint $blob ret] + set blob [string range $blob $n end] + return $ret +} +proc gobble_string {varname nLength} { + upvar $varname blob + set ret [string range $blob 0 [expr $nLength-1]] + set blob [string range $blob $nLength end] + return $ret +} + +# The argument is a blob of data representing an FTS3 segment leaf. +# Return a list consisting of alternating terms (strings) and doclists +# (blobs of data). +# +proc fts3_readleaf {blob} { + set zPrev "" + set terms [list] + + while {[string length $blob] > 0} { + set nPrefix [gobble_varint blob] + set nSuffix [gobble_varint blob] + + set zTerm [string range $zPrev 0 [expr $nPrefix-1]] + append zTerm [gobble_string blob $nSuffix] + set doclist [gobble_string blob [gobble_varint blob]] + + lappend terms $zTerm $doclist + set zPrev $zTerm + } + + return $terms +} + +proc fts3_read2 {tbl where varname} { + upvar $varname a + array unset a + db eval " SELECT start_block, leaves_end_block, root + FROM ${tbl}_segdir WHERE $where + ORDER BY level ASC, idx DESC + " { + if {$start_block == 0} { + foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } + } else { + db eval " SELECT block + FROM ${tbl}_segments + WHERE blockid>=$start_block AND blockid<=$leaves_end_block + ORDER BY blockid + " { + foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } + + } + } + } +} + +proc fts3_read {tbl where varname} { + upvar $varname a + array unset a + db eval " SELECT start_block, leaves_end_block, root + FROM ${tbl}_segdir WHERE $where + ORDER BY level DESC, idx ASC + " { + if {$start_block == 0} { + foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } + } else { + db eval " SELECT block + FROM ${tbl}_segments + WHERE blockid>=$start_block AND blockid<$leaves_end_block + ORDER BY blockid + " { + foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } + + } + } + } +} + Index: test/fts3aa.test ================================================================== --- test/fts3aa.test +++ test/fts3aa.test @@ -144,10 +144,11 @@ } {1 5 9 13 17 21 25 29} do_test fts3aa-3.3 { execsql {SELECT rowid FROM t1 WHERE content MATCH '-two one'} } {1 5 9 13 17 21 25 29} +breakpoint do_test fts3aa-4.1 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one OR two'} } {1 2 3 5 6 7 9 10 11 13 14 15 17 18 19 21 22 23 25 26 27 29 30 31} do_test fts3aa-4.2 { execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two" OR three'} @@ -193,10 +194,11 @@ execsql {INSERT INTO t1(rowid, content) VALUES(-1, 'three four')} } {} do_test fts3aa-6.3 { execsql {SELECT content FROM t1 WHERE rowid = -1} } {{three four}} +breakpoint do_test fts3aa-6.4 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'four'} } {-1 0 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31} finish_test Index: test/fts3ab.test ================================================================== --- test/fts3ab.test +++ test/fts3ab.test @@ -113,10 +113,11 @@ for {set i 1} {$i<=15} {incr i} { set vset [list [wordset $i] [wordset [expr {$i+1}]] [wordset [expr {~$i}]]] db eval "INSERT INTO t4(norm,plusone,invert) VALUES([join $vset ,]);" } +breakpoint do_test fts3ab-4.1 { execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one'} } {1 3 5 7 9 11 13 15} do_test fts3ab-4.2 { execsql {SELECT rowid FROM t4 WHERE norm MATCH 'one'} Index: test/fts3ae.test ================================================================== --- test/fts3ae.test +++ test/fts3ae.test @@ -56,11 +56,11 @@ INSERT INTO t1 (rowid, content) VALUES(22, 'two three five'); DELETE FROM t1 WHERE rowid = 19; DELETE FROM t1 WHERE rowid = 22; } -do_test fts3af-1.1 { +do_test fts3ae-1.1 { execsql {SELECT COUNT(*) FROM t1} } {14} do_test fts3ae-2.1 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'} Index: test/fts3ag.test ================================================================== --- test/fts3ag.test +++ test/fts3ag.test @@ -76,10 +76,11 @@ } {1 {SQL logic error or missing database}} # Test that docListOrMerge() correctly handles reaching the end of one # doclist before it reaches the end of the other. do_test fts3ag-1.11 { +breakpoint execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this OR also'} } {1 2} do_test fts3ag-1.12 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'also OR this'} } {1 2} Index: test/fts3an.test ================================================================== --- test/fts3an.test +++ test/fts3an.test @@ -167,11 +167,11 @@ CREATE VIRTUAL TABLE t3 USING fts3(c); INSERT INTO t3(rowid, c) VALUES(1, $text); INSERT INTO t3(rowid, c) VALUES(2, 'Another lovely row'); } -for {set i 0} {$i<100} {incr i} { +for {set i 0} {$i<68} {incr i} { db eval {INSERT INTO t3(rowid, c) VALUES(3+$i, $bigtext)} lappend ret 192 } db eval {COMMIT;} Index: test/fts3c.test ================================================================== --- test/fts3c.test +++ test/fts3c.test @@ -10,142 +10,40 @@ #************************************************************************* # This file exercises some new testing functions in the FTS3 module, # and then uses them to do some basic tests that FTS3 is internally # working as expected. # -# $Id: fts3c.test,v 1.1 2008/07/03 19:53:22 shess Exp $ -# set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/fts3_common.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } -#************************************************************************* -# Probe to see if support for these functions is compiled in. -# TODO(shess): Change main.mk to do the right thing and remove this test. -db eval { - DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts3(c); - INSERT INTO t1 (docid, c) VALUES (1, 'x'); -} - -set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1} -set r {1 {unable to use function dump_terms in the requested context}} -if {[catchsql $s]==$r} { - finish_test - return -} - -#************************************************************************* -# Test that the new functions give appropriate errors. -do_test fts3c-0.0 { - catchsql { - SELECT dump_terms(t1, 1) FROM t1 LIMIT 1; - } -} {1 {dump_terms: incorrect arguments}} - -do_test fts3c-0.1 { - catchsql { - SELECT dump_terms(t1, 0, 0, 0) FROM t1 LIMIT 1; - } -} {1 {dump_terms: incorrect arguments}} - -do_test fts3c-0.2 { - catchsql { - SELECT dump_terms(1, t1) FROM t1 LIMIT 1; - } -} {1 {unable to use function dump_terms in the requested context}} - -do_test fts3c-0.3 { - catchsql { - SELECT dump_terms(t1, 16, 16) FROM t1 LIMIT 1; - } -} {1 {dump_terms: segment not found}} - -do_test fts3c-0.4 { - catchsql { - SELECT dump_doclist(t1) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: incorrect arguments}} - -do_test fts3c-0.5 { - catchsql { - SELECT dump_doclist(t1, NULL) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: empty second argument}} - -do_test fts3c-0.6 { - catchsql { - SELECT dump_doclist(t1, '') FROM t1 LIMIT 1; - } -} {1 {dump_doclist: empty second argument}} - -do_test fts3c-0.7 { - catchsql { - SELECT dump_doclist(t1, 'a', 0) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: incorrect arguments}} - -do_test fts3c-0.8 { - catchsql { - SELECT dump_doclist(t1, 'a', 0, 0, 0) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: incorrect arguments}} - -do_test fts3c-0.9 { - catchsql { - SELECT dump_doclist(t1, 'a', 16, 16) FROM t1 LIMIT 1; - } -} {1 {dump_doclist: segment not found}} - #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { - # TODO(shess): Figure out why uplevel in do_test can't catch - # $level and $index directly. - set ::level $level - set ::index $index - do_test $test.terms { - execsql { - SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $terms] + set where "level = $level AND idx = $index" + do_test $test.terms [list fts3_terms t1 $where] $terms } proc check_terms_all {test terms} { - do_test $test.terms { - execsql { - SELECT dump_terms(t1) FROM t1 LIMIT 1; - } - } [list $terms] + do_test $test.terms [list fts3_terms t1 1] $terms } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { - # TODO(shess): Again, why can't the non-:: versions work? - set ::term $term - set ::level $level - set ::index $index - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $doclist] + set where "level = $level AND idx = $index" + do_test $test [list fts3_doclist t1 $term $where] $doclist } proc check_doclist_all {test term doclist} { - set ::term $term - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1; - } - } [list $doclist] + do_test $test [list fts3_doclist t1 $term 1] $doclist } #************************************************************************* # Test the segments resulting from straight-forward inserts. db eval { Index: test/fts3d.test ================================================================== --- test/fts3d.test +++ test/fts3d.test @@ -9,80 +9,40 @@ # #************************************************************************* # This file implements regression tests for SQLite library. The focus # of this script is testing the FTS3 module's optimize() function. # -# $Id: fts3d.test,v 1.2 2008/07/15 21:32:07 shess Exp $ -# set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/fts3_common.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } -#************************************************************************* -# Probe to see if support for the FTS3 dump_* functions is compiled in. -# TODO(shess): Change main.mk to do the right thing and remove this test. -db eval { - DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts3(c); - INSERT INTO t1 (docid, c) VALUES (1, 'x'); -} - -set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1} -set r {1 {unable to use function dump_terms in the requested context}} -if {[catchsql $s]==$r} { - finish_test - return -} - #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { - # TODO(shess): Figure out why uplevel in do_test can't catch - # $level and $index directly. - set ::level $level - set ::index $index - do_test $test.terms { - execsql { - SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $terms] + set where "level = $level AND idx = $index" + do_test $test.terms [list fts3_terms t1 $where] $terms } proc check_terms_all {test terms} { - do_test $test.terms { - execsql { - SELECT dump_terms(t1) FROM t1 LIMIT 1; - } - } [list $terms] + do_test $test.terms [list fts3_terms t1 1] $terms } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { - # TODO(shess): Again, why can't the non-:: versions work? - set ::term $term - set ::level $level - set ::index $index - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1; - } - } [list $doclist] + set where "level = $level AND idx = $index" + do_test $test.doclist [list fts3_doclist t1 $term $where] $doclist } proc check_doclist_all {test term doclist} { - set ::term $term - do_test $test { - execsql { - SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1; - } - } [list $doclist] + do_test $test.doclist [list fts3_doclist t1 $term 1] $doclist } #************************************************************************* # Test results when all rows are deleted and one is added back. # Previously older segments would continue to exist, but now the index @@ -291,10 +251,11 @@ check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level # prior segment. +breakpoint do_test fts3d-4.5 { execsql { SELECT OPTIMIZE(t1) FROM t1 LIMIT 1; SELECT level, idx FROM t1_segdir ORDER BY level, idx; } ADDED test/fts3malloc.test Index: test/fts3malloc.test ================================================================== --- /dev/null +++ test/fts3malloc.test @@ -0,0 +1,410 @@ +# 2009 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests to verify that malloc() errors that occur +# within the FTS3 module code are handled correctly. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !fts3 { finish_test ; return } +source $testdir/malloc_common.tcl +source $testdir/fts3_common.tcl + +# Ensure the lookaside buffer is disabled for these tests. +# +sqlite3 db test.db +sqlite3_db_config_lookaside db 0 0 0 + +set sqlite_fts3_enable_parentheses 1 +set DO_MALLOC_TEST 1 + +# Test organization: +# +# fts3_malloc-1.*: Test CREATE and DROP table statements. +# +# + +#------------------------------------------------------------------------- +# This proc is used to test a single SELECT statement. Parameter $name is +# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter +# $sql is passed the text of the SELECT statement. Parameter $result is +# set to the expected output if the SELECT statement is successfully +# executed using [db eval]. +# +# Example: +# +# do_select_test testcase-1.1 "SELECT 1+1, 1+2" {1 2} +# +# If global variable DO_MALLOC_TEST is set to a non-zero value, or if +# it is not defined at all, then OOM testing is performed on the SELECT +# statement. Each OOM test case is said to pass if either (a) executing +# the SELECT statement succeeds and the results match those specified +# by parameter $result, or (b) TCL throws an "out of memory" error. +# +# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement +# is executed just once. In this case the test case passes if the results +# match the expected results passed via parameter $result. +# + +proc do_passive_test {name sql catchres} { + if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } + + if {$::DO_MALLOC_TEST} { + set answers [list {1 {out of memory}} $catchres] + set modes [list 100000 transient 1 persistent] + } else { + set answers [list $catchres] + set modes [list 0 nofail] + } + set str [join $answers " OR "] + + foreach {nRepeat zName} $modes { + for {set iFail 1} 1 {incr iFail} { + if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} + + set res [catchsql $sql] + if {[lsearch $answers $res]>=0} { + set res $str + } + do_test $name.$zName.$iFail [list set {} $res] $str + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] + if {$nFail==0} break + } + } +} + +proc do_select_test {name sql result} { + do_passive_test $name $sql [list 0 $result] +} + +proc do_error_test {name sql error} { + do_passive_test $name $sql [list 1 $error] +} + +#------------------------------------------------------------------------- +# Test a single write to the database. In this case a "write" is a +# DELETE, UPDATE or INSERT statement. +# +# If OOM testing is performed, there are several acceptable outcomes: +# +# 1) The write succeeds. No error is returned. +# +# 2) An "out of memory" exception is thrown and: +# +# a) The statement has no effect, OR +# b) The current transaction is rolled back, OR +# c) The statement succeeds. This can only happen if the connection +# is in auto-commit mode (after the statement is executed, so this +# includes COMMIT statements). +# +# If the write operation eventually succeeds, zero is returned. If a +# transaction is rolled back, non-zero is returned. +# +# Parameter $name is the name to use for the test case (or test cases). +# The second parameter, $tbl, should be the name of the database table +# being modified. Parameter $sql contains the SQL statement to test. +# +proc do_write_test {name tbl sql} { + if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } + + # Figure out an statement to get a checksum for table $tbl. + db eval "SELECT * FROM $tbl" V break + set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl" + + # Calculate the initial table checksum. + set cksum1 [db one $cksumsql] + + + if {$::DO_MALLOC_TEST } { + set answers [list {1 {out of memory}} {0 {}}] + set modes [list 100000 transient 1 persistent] + } else { + set answers [list {0 {}}] + set modes [list 0 nofail] + } + set str [join $answers " OR "] + + foreach {nRepeat zName} $modes { + for {set iFail 1} 1 {incr iFail} { + if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} + + set res [catchsql $sql] + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] + if {$nFail==0} { + do_test $name.$zName.$iFail [list set {} $res] {0 {}} + return + } else { + if {[lsearch $answers $res]>=0} { + set res $str + } + do_test $name.$zName.$iFail [list set {} $res] $str + set cksum2 [db one $cksumsql] + if {$cksum1 != $cksum2} return + } + } + } +} + +proc normal_list {l} { + set ret [list] + foreach elem $l {lappend ret $elem} + set ret +} + + +do_write_test fts3_malloc-1.1 sqlite_master { + CREATE VIRTUAL TABLE ft1 USING fts3(a, b) +} +do_write_test fts3_malloc-1.2 sqlite_master { + CREATE VIRTUAL TABLE ft2 USING fts3([a], [b]); +} +do_write_test fts3_malloc-1.3 sqlite_master { + CREATE VIRTUAL TABLE ft3 USING fts3('a', "b"); +} +do_write_test fts3_malloc-1.4 sqlite_master { + CREATE VIRTUAL TABLE ft4 USING fts3(`a`, 'fred''s column'); +} +do_error_test fts3_malloc-1.5 { + CREATE VIRTUAL TABLE ft5 USING fts3(a, b, tokenize unknown) +} {unknown tokenizer: unknown} +do_write_test fts3_malloc-1.6 sqlite_master { + CREATE VIRTUAL TABLE ft6 USING fts3(a, b, tokenize porter) +} + +# Test the xConnect/xDisconnect methods: +#db eval { ATTACH 'test2.db' AS aux } +#do_write_test fts3_malloc-1.6 aux.sqlite_master { +# CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); +#} +#do_write_test fts3_malloc-1.6 aux.sqlite_master { +# CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); +#} + + + +do_test fts3_malloc-2.0 { + execsql { + DROP TABLE ft1; + DROP TABLE ft2; + DROP TABLE ft3; + DROP TABLE ft4; + DROP TABLE ft6; + } + execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) } + for {set ii 1} {$ii < 32} {incr ii} { + set a [list] + set b [list] + if {$ii & 0x01} {lappend a one ; lappend b neung} + if {$ii & 0x02} {lappend a two ; lappend b song } + if {$ii & 0x04} {lappend a three ; lappend b sahm } + if {$ii & 0x08} {lappend a four ; lappend b see } + if {$ii & 0x10} {lappend a five ; lappend b hah } + execsql { INSERT INTO ft VALUES($a, $b) } + } +} {} + +foreach {tn sql result} { + 1 "SELECT count(*) FROM sqlite_master" {5} + 2 "SELECT * FROM ft WHERE docid = 1" {one neung} + 3 "SELECT * FROM ft WHERE docid = 2" {two song} + 4 "SELECT * FROM ft WHERE docid = 3" {{one two} {neung song}} + + 5 "SELECT a FROM ft" { + {one} {two} {one two} + {three} {one three} {two three} + {one two three} {four} {one four} + {two four} {one two four} {three four} + {one three four} {two three four} {one two three four} + {five} {one five} {two five} + {one two five} {three five} {one three five} + {two three five} {one two three five} {four five} + {one four five} {two four five} {one two four five} + {three four five} {one three four five} {two three four five} + {one two three four five} + } + + 6 "SELECT a FROM ft WHERE a MATCH 'one'" { + {one} {one two} {one three} {one two three} + {one four} {one two four} {one three four} {one two three four} + {one five} {one two five} {one three five} {one two three five} + {one four five} {one two four five} + {one three four five} {one two three four five} + } + + 7 "SELECT a FROM ft WHERE a MATCH 'o*'" { + {one} {one two} {one three} {one two three} + {one four} {one two four} {one three four} {one two three four} + {one five} {one two five} {one three five} {one two three five} + {one four five} {one two four five} + {one three four five} {one two three four five} + } + + 8 "SELECT a FROM ft WHERE a MATCH 'o* t*'" { + {one two} {one three} {one two three} + {one two four} {one three four} {one two three four} + {one two five} {one three five} {one two three five} + {one two four five} {one three four five} {one two three four five} + } + + 9 "SELECT a FROM ft WHERE a MATCH '\"o* t*\"'" { + {one two} {one three} {one two three} + {one two four} {one three four} {one two three four} + {one two five} {one three five} {one two three five} + {one two four five} {one three four five} {one two three four five} + } + + 10 {SELECT a FROM ft WHERE a MATCH '"o* f*"'} { + {one four} {one five} {one four five} + } + + 11 {SELECT a FROM ft WHERE a MATCH '"one two three"'} { + {one two three} + {one two three four} + {one two three five} + {one two three four five} + } + + 12 {SELECT a FROM ft WHERE a MATCH '"two three four"'} { + {two three four} + {one two three four} + {two three four five} + {one two three four five} + } + + 12 {SELECT a FROM ft WHERE a MATCH '"two three" five'} { + {two three five} {one two three five} + {two three four five} {one two three four five} + } + + 13 {SELECT a FROM ft WHERE ft MATCH '"song sahm" hah'} { + {two three five} {one two three five} + {two three four five} {one two three four five} + } + + 14 {SELECT a FROM ft WHERE b MATCH 'neung'} { + {one} {one two} + {one three} {one two three} + {one four} {one two four} + {one three four} {one two three four} + {one five} {one two five} + {one three five} {one two three five} + {one four five} {one two four five} + {one three four five} {one two three four five} + } + + 15 {SELECT a FROM ft WHERE b MATCH '"neung song sahm"'} { + {one two three} {one two three four} + {one two three five} {one two three four five} + } + + 16 {SELECT a FROM ft WHERE b MATCH 'hah "song sahm"'} { + {two three five} {one two three five} + {two three four five} {one two three four five} + } + + 17 {SELECT a FROM ft WHERE b MATCH 'song OR sahm'} { + {two} {one two} {three} + {one three} {two three} {one two three} + {two four} {one two four} {three four} + {one three four} {two three four} {one two three four} + {two five} {one two five} {three five} + {one three five} {two three five} {one two three five} + {two four five} {one two four five} {three four five} + {one three four five} {two three four five} {one two three four five} + } + + 18 {SELECT a FROM ft WHERE a MATCH 'three NOT two'} { + {three} {one three} {three four} + {one three four} {three five} {one three five} + {three four five} {one three four five} + } + + 19 {SELECT a FROM ft WHERE b MATCH 'sahm NOT song'} { + {three} {one three} {three four} + {one three four} {three five} {one three five} + {three four five} {one three four five} + } + + 20 {SELECT a FROM ft WHERE ft MATCH 'sahm NOT song'} { + {three} {one three} {three four} + {one three four} {three five} {one three five} + {three four five} {one three four five} + } + + 21 {SELECT a FROM ft WHERE b MATCH 'neung NEAR song NEAR sahm'} { + {one two three} {one two three four} + {one two three five} {one two three four five} + } + +} { + set result [normal_list $result] + do_select_test fts3_malloc-2.$tn $sql $result +} + +do_test fts3_malloc-3.0 { + execsql BEGIN + for {set ii 32} {$ii < 1024} {incr ii} { + set a [list] + set b [list] + if {$ii & 0x0001} {lappend a one ; lappend b neung } + if {$ii & 0x0002} {lappend a two ; lappend b song } + if {$ii & 0x0004} {lappend a three ; lappend b sahm } + if {$ii & 0x0008} {lappend a four ; lappend b see } + if {$ii & 0x0010} {lappend a five ; lappend b hah } + if {$ii & 0x0020} {lappend a six ; lappend b hok } + if {$ii & 0x0040} {lappend a seven ; lappend b jet } + if {$ii & 0x0080} {lappend a eight ; lappend b bairt } + if {$ii & 0x0100} {lappend a nine ; lappend b gow } + if {$ii & 0x0200} {lappend a ten ; lappend b sip } + execsql { INSERT INTO ft VALUES($a, $b) } + } + execsql COMMIT +} {} +foreach {tn sql result} { + 1 "SELECT count(*) FROM ft" {1023} + + 2 "SELECT a FROM ft WHERE a MATCH 'one two three four five six seven eight'" { + {one two three four five six seven eight} + {one two three four five six seven eight nine} + {one two three four five six seven eight ten} + {one two three four five six seven eight nine ten} + } + + 3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} { + 512 262144 + } + + 4 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH '"two three four"'} { + 128 66368 + } +} { + set result [normal_list $result] + do_select_test fts3_malloc-3.$tn $sql $result +} + +do_test fts3_malloc-4.0 { + execsql { DELETE FROM ft WHERE docid>=32 } +} {} +foreach {tn sql} { + 1 "DELETE FROM ft WHERE ft MATCH 'one'" + 2 "DELETE FROM ft WHERE ft MATCH 'three'" + 3 "DELETE FROM ft WHERE ft MATCH 'five'" +} { + do_write_test fts3_malloc-4.1.$tn ft_content $sql +} +do_test fts3_malloc-4.2 { + execsql { SELECT a FROM ft } +} {two four {two four}} + + +finish_test +