SQLite

/* DLCollector wraps PLWriter and DLWriter to provide a
** dynamically-allocated doclist area to use during tokenization.
**
** dlcNew - malloc up and initialize a collector.
** dlcDelete - destroy a collector and all contained items.
** dlcAddPos - append position and offset information.
** dlcAddDoclist - add the collected doclist to the given buffer.
** dlcNext - terminate the current document and open another.
*/
typedef struct DLCollector {
  DataBuffer b;
  DLWriter dlw;
  PLWriter plw;
} DLCollector;

  if( pCollector->dlw.iType>DL_DOCIDS ){
    char c[VARINT_MAX];
    int n = putVarint(c, POS_END);
    dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
  }else{
    dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
  }
}
static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
  plwTerminate(&pCollector->plw);
  plwDestroy(&pCollector->plw);
  plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
}
static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
                      int iStartOffset, int iEndOffset){
  plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
}

static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){

  /* Precompiled statements used for segment merges.  We run a
  ** separate select across the leaf level of each tree being merged.
  */
  sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
  /* The statement used to prepare pLeafSelectStmts. */
#define LEAF_SELECT \
  "select block from %_segments where rowid between ? and ? order by rowid"

  /* These buffer pending index updates during transactions.
  ** nPendingData estimates the memory size of the pending data.  It
  ** doesn't include the hash-bucket overhead, nor any malloc
  ** overhead.  When nPendingData exceeds kPendingThreshold, the
  ** buffer is flushed even before the transaction closes.
  ** pendingTerms stores the data, and is only valid when nPendingData
  ** is >=0 (nPendingData<0 means pendingTerms has not been
  ** initialized).  iPrevDocid is the last docid written, used to make
  ** certain we're inserting in sorted order.
  */
  int nPendingData;
#define kPendingThreshold (1*1024*1024)
  sqlite_int64 iPrevDocid;
  fts2Hash pendingTerms;
};

/*
** When the core wants to do a query, it create a cursor using a
** call to xOpen.  This structure is an instance of a cursor.  It
** is destroyed by xClose.
*/

  rc = sqlite3_bind_int64(s, 1, iLevel);
  if( rc!=SQLITE_OK ) return rc;

  return sql_single_step_statement(v, SEGDIR_DELETE_STMT, &s);
}

/* TODO(shess) clearPendingTerms() is far down the file because
** writeZeroSegment() is far down the file because LeafWriter is far
** down the file.  Consider refactoring the code to move the non-vtab
** code above the vtab code so that we don't need this forward
** reference.
*/
static int clearPendingTerms(fulltext_vtab *v);

/*
** Free the memory used to contain a fulltext_vtab structure.
*/
static void fulltext_vtab_destroy(fulltext_vtab *v){
  int iStmt, i;

  TRACE(("FTS2 Destroy %p\n", v));

    }
  }

  if( v->pTokenizer!=NULL ){
    v->pTokenizer->pModule->xDestroy(v->pTokenizer);
    v->pTokenizer = NULL;
  }

  clearPendingTerms(v);

  free(v->azColumn);
  for(i = 0; i < v->nColumn; ++i) {
    sqlite3_free(v->azContentColumn[i]);
  }
  free(v->azContentColumn);
  free(v);
}

  schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn,
                          spec->zName);
  rc = sqlite3_declare_vtab(db, schema);
  sqlite3_free(schema);
  if( rc!=SQLITE_OK ) goto err;

  memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));

  /* Indicate that the buffer is not live. */
  v->nPendingData = -1;

  *ppVTab = &v->base;
  TRACE(("FTS2 Connect %p\n", v));

  return rc;

err:

){
  DataBuffer left, right, new;
  int i, rc;

  /* No phrase search if no position info. */
  assert( pQTerm->nPhrase==0 || DL_DEFAULT!=DL_DOCIDS );

  /* This code should never be called with buffered updates. */
  assert( v->nPendingData<0 );

  dataBufferInit(&left, 0);
  rc = termSelect(v, iColumn, pQTerm->pTerm, pQTerm->nTerm,
                  0<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS, &left);
  if( rc ) return rc;
  for(i=1; i<=pQTerm->nPhrase && left.nData>0; i++){
    dataBufferInit(&right, 0);
    rc = termSelect(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm,

  if( inPhrase ){
    /* unmatched quote */
    queryClear(pQuery);
    return SQLITE_ERROR;
  }
  return SQLITE_OK;
}

/* TODO(shess) Refactor the code to remove this forward decl. */
static int flushPendingTerms(fulltext_vtab *v);

/* Perform a full-text query using the search expression in
** zInput[0..nInput-1].  Return a list of matching documents
** in pResult.
**
** Queries must match column iColumn.  Or if iColumn>=nColumn
** they are allowed to match against any column.
*/
static int fulltextQuery(
  fulltext_vtab *v,      /* The full text index */
  int iColumn,           /* Match against this column by default */
  const char *zInput,    /* The query string */
  int nInput,            /* Number of bytes in zInput[] */
  DataBuffer *pResult,   /* Write the result doclist here */
  Query *pQuery          /* Put parsed query string here */
){
  int i, iNext, rc;
  DataBuffer left, right, or, new;
  int nNot = 0;
  QueryTerm *aTerm;

  /* TODO(shess) Instead of flushing pendingTerms, we could query for
  ** the relevant term and merge the doclist into what we receive from
  ** the database.  Wait and see if this is a common issue, first.
  **
  ** A good reason not to flush is to not generate update-related
  ** error codes from here.
  */

  /* Flush any buffered updates before executing the query. */
  rc = flushPendingTerms(v);
  if( rc!=SQLITE_OK ) return rc;

  /* TODO(shess) I think that the queryClear() calls below are not
  ** necessary, because fulltextClose() already clears the query.
  */
  rc = parseQuery(v, zInput, nInput, iColumn, pQuery);
  if( rc!=SQLITE_OK ) return rc;

static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  fulltext_cursor *c = (fulltext_cursor *) pCursor;

  *pRowid = sqlite3_column_int64(c->pStmt, 0);
  return SQLITE_OK;
}

/* Add all terms in [zText] to pendingTerms table.  If [iColumn] > 0,
** we also store positions and offsets in the hash table using that
** column number.
*/
static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
                      const char *zText, int iColumn){
  sqlite3_tokenizer *pTokenizer = v->pTokenizer;
  sqlite3_tokenizer_cursor *pCursor;
  const char *pToken;
  int nTokenBytes;
  int iStartOffset, iEndOffset, iPosition;
  int rc;

  rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
  if( rc!=SQLITE_OK ) return rc;

  pCursor->pTokenizer = pTokenizer;
  while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
                                               &pToken, &nTokenBytes,
                                               &iStartOffset, &iEndOffset,
                                               &iPosition) ){
    DLCollector *p;
    int nData;                   /* Size of doclist before our update. */

    /* Positions can't be negative; we use -1 as a terminator internally. */
    if( iPosition<0 ){
      pTokenizer->pModule->xClose(pCursor);
      return SQLITE_ERROR;
    }

    p = fts2HashFind(&v->pendingTerms, pToken, nTokenBytes);
    if( p==NULL ){
      nData = 0;
      p = dlcNew(iDocid, DL_DEFAULT);
      fts2HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);

      /* Overhead for our hash table entry, the key, and the value. */
      v->nPendingData += sizeof(struct fts2HashElem)+sizeof(*p)+nTokenBytes;
    }else{
      nData = p->b.nData;
      if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
    }
    if( iColumn>=0 ){
      dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
    }

    /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
    v->nPendingData += p->b.nData-nData;
  }

  /* TODO(shess) Check return?  Should this be able to cause errors at
  ** this point?  Actually, same question about sqlite3_finalize(),
  ** though one could argue that failure there means that the data is
  ** not durable.  *ponder*
  */
  pTokenizer->pModule->xClose(pCursor);
  return rc;
}

/* Add doclists for all terms in [pValues] to pendingTerms table. */
static int insertTerms(fulltext_vtab *v, sqlite_int64 iRowid,
                       sqlite3_value **pValues){
  int i;
  for(i = 0; i < v->nColumn ; ++i){
    char *zText = (char*)sqlite3_value_text(pValues[i]);
    int rc = buildTerms(v, iRowid, zText, i);
    if( rc!=SQLITE_OK ) return rc;
  }
  return SQLITE_OK;
}

/* Add empty doclists for all terms in the given row's content to
** pendingTerms.
*/
static int deleteTerms(fulltext_vtab *v, sqlite_int64 iRowid){
  const char **pValues;
  int i, rc;

  /* TODO(shess) Should we allow such tables at all? */
  if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;

  rc = content_select(v, iRowid, &pValues);
  if( rc!=SQLITE_OK ) return rc;

  for(i = 0 ; i < v->nColumn; ++i) {
    rc = buildTerms(v, iRowid, pValues[i], -1);
    if( rc!=SQLITE_OK ) break;
  }

  freeStringArray(v->nColumn, pValues);
  return SQLITE_OK;
}

/* TODO(shess) Refactor the code to remove this forward decl. */
static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);

/* Insert a row into the %_content table; set *piRowid to be the ID of the
** new row.  Add doclists for terms to pendingTerms.
*/
static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestRowid,
                        sqlite3_value **pValues, sqlite_int64 *piRowid){

  int rc;

  rc = content_insert(v, pRequestRowid, pValues);  /* execute an SQL INSERT */
  if( rc!=SQLITE_OK ) return rc;

  *piRowid = sqlite3_last_insert_rowid(v->db);
  rc = initPendingTerms(v, *piRowid);
  if( rc!=SQLITE_OK ) return rc;

  return insertTerms(v, *piRowid, pValues);
}

/* Delete a row from the %_content table; add empty doclists for terms
** to pendingTerms.
*/
static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
  int rc = initPendingTerms(v, iRow);
  if( rc!=SQLITE_OK ) return rc;

  rc = deleteTerms(v, iRow);
  if( rc!=SQLITE_OK ) return rc;

  return content_delete(v, iRow);  /* execute an SQL DELETE */
}

/* Update a row in the %_content table; add delete doclists to
** pendingTerms for old terms not in the new data, add insert doclists
** to pendingTerms for terms in the new data.
*/
static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
                        sqlite3_value **pValues){
  int rc = initPendingTerms(v, iRow);
  if( rc!=SQLITE_OK ) return rc;

  /* Generate an empty doclist for each term that previously appeared in this
   * row. */
  rc = deleteTerms(v, iRow);
  if( rc!=SQLITE_OK ) return rc;

  rc = content_update(v, pValues, iRow);  /* execute an SQL UPDATE */
  if( rc!=SQLITE_OK ) return rc;

  /* Now add positions for terms which appear in the updated row. */
  return insertTerms(v, iRow, pValues);
}

/*******************************************************************/
/* InteriorWriter is used to collect terms and block references into
** interior nodes in %_segments.  See commentary at top of file for
** format.
*/

** read from pData will overwrite those in *out).
*/
static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
                           const char *pTerm, int nTerm, DataBuffer *out){
  LeafReader reader;
  assert( nData>1 );
  assert( *pData=='\0' );

  /* This code should never be called with buffered updates. */
  assert( v->nPendingData<0 );

  leafReaderInit(pData, nData, &reader);
  while( !leafReaderAtEnd(&reader) ){
    int c = leafReaderTermCmp(&reader, pTerm, nTerm);
    if( c==0 ){
      if( out->nData==0 ){
        dataBufferReplace(out,

*/
static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
                       const char *pTerm, int nTerm, DataBuffer *out){
  int rc;
  sqlite3_stmt *s = NULL;

  assert( nData>1 );

  /* This code should never be called with buffered updates. */
  assert( v->nPendingData<0 );

  /* Process data as an interior node until we reach a leaf. */
  while( *pData!='\0' ){
    sqlite_int64 iBlockid;
    InteriorReader reader;

    /* Scan the node data until we find a term greater than our term.

static int termSelect(fulltext_vtab *v, int iColumn,
                      const char *pTerm, int nTerm,
                      DocListType iType, DataBuffer *out){
  DataBuffer doclist;
  sqlite3_stmt *s;
  int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
  if( rc!=SQLITE_OK ) return rc;

  /* This code should never be called with buffered updates. */
  assert( v->nPendingData<0 );

  dataBufferInit(&doclist, 0);

  /* Traverse the segments from oldest to newest so that newer doclist
  ** elements for given docids overwrite older elements.
  */
  while( (rc=sql_step_statement(v, SEGDIR_SELECT_ALL_STMT, &s))==SQLITE_ROW ){

 err:
  dataBufferDestroy(&dl);
  free(pData);
  leafWriterDestroy(&writer);
  return rc;
}

/* If pendingTerms has data, free it. */
static int clearPendingTerms(fulltext_vtab *v){
  if( v->nPendingData>=0 ){
    fts2HashElem *e;
    for(e=fts2HashFirst(&v->pendingTerms); e; e=fts2HashNext(e)){
      dlcDelete(fts2HashData(e));
    }
    fts2HashClear(&v->pendingTerms);
    v->nPendingData = -1;
  }
  return SQLITE_OK;
}

/* If pendingTerms has data, flush it to a level-zero segment, and
** free it.
*/
static int flushPendingTerms(fulltext_vtab *v){
  if( v->nPendingData>=0 ){
    int rc = writeZeroSegment(v, &v->pendingTerms);
    clearPendingTerms(v);
    return rc;
  }
  return SQLITE_OK;
}

/* If pendingTerms is "too big", or docid is out of order, flush it.
** Regardless, be certain that pendingTerms is initialized for use.
*/
static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
  /* TODO(shess) Explore whether partially flushing the buffer on
  ** forced-flush would provide better performance.  I suspect that if
  ** we ordered the doclists by size and flushed the largest until the
  ** buffer was half empty, that would let the less frequent terms
  ** generate longer doclists.
  */
  if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
    int rc = flushPendingTerms(v);
    if( rc!=SQLITE_OK ) return rc;
  }
  if( v->nPendingData<0 ){
    fts2HashInit(&v->pendingTerms, FTS2_HASH_STRING, 1);
    v->nPendingData = 0;
  }
  v->iPrevDocid = iDocid;
  return SQLITE_OK;
}

/* This function implements the xUpdate callback; it's the top-level entry
 * point for inserting, deleting or updating a row in a full-text table. */
static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
                   sqlite_int64 *pRowid){
  fulltext_vtab *v = (fulltext_vtab *) pVtab;

  int rc;


  TRACE(("FTS2 Update %p\n", pVtab));



  if( nArg<2 ){
    rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
  } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
    /* An update:
     * ppArg[0] = old rowid
     * ppArg[1] = new rowid
     * ppArg[2..2+v->nColumn-1] = values
     * ppArg[2+v->nColumn] = value for magic column (we ignore this)
     */
    sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
    if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
      sqlite3_value_int64(ppArg[1]) != rowid ){
      rc = SQLITE_ERROR;  /* we don't allow changing the rowid */
    } else {
      assert( nArg==2+v->nColumn+1);
      rc = index_update(v, rowid, &ppArg[2]);
    }
  } else {
    /* An insert:
     * ppArg[1] = requested rowid
     * ppArg[2..2+v->nColumn-1] = values
     * ppArg[2+v->nColumn] = value for magic column (we ignore this)
     */
    assert( nArg==2+v->nColumn+1);
    rc = index_insert(v, ppArg[1], &ppArg[2], pRowid);
  }

  return rc;
}

static int fulltextSync(sqlite3_vtab *pVtab){
  TRACE(("FTS2 xSync()\n"));
  return flushPendingTerms((fulltext_vtab *)pVtab);
}

static int fulltextBegin(sqlite3_vtab *pVtab){
  fulltext_vtab *v = (fulltext_vtab *) pVtab;
  TRACE(("FTS2 xBegin()\n"));

  /* Any buffered updates should have been cleared by the previous
  ** transaction.
  */
  assert( v->nPendingData<0 );
  return clearPendingTerms(v);
}

static int fulltextCommit(sqlite3_vtab *pVtab){
  fulltext_vtab *v = (fulltext_vtab *) pVtab;
  TRACE(("FTS2 xCommit()\n"));

  /* Buffered updates should have been cleared by fulltextSync(). */
  assert( v->nPendingData<0 );
  return clearPendingTerms(v);
}

static int fulltextRollback(sqlite3_vtab *pVtab){
  TRACE(("FTS2 xRollback()\n"));
  return clearPendingTerms((fulltext_vtab *)pVtab);
}

/*
** Implementation of the snippet() function for FTS2
*/
static void snippetFunc(
  sqlite3_context *pContext,
  int argc,

  /* xClose        */ fulltextClose,
  /* xFilter       */ fulltextFilter,
  /* xNext         */ fulltextNext,
  /* xEof          */ fulltextEof,
  /* xColumn       */ fulltextColumn,
  /* xRowid        */ fulltextRowid,
  /* xUpdate       */ fulltextUpdate,
  /* xBegin        */ fulltextBegin,
  /* xSync         */ fulltextSync,
  /* xCommit       */ fulltextCommit,
  /* xRollback     */ fulltextRollback,
  /* xFindFunction */ fulltextFindFunction,
};

int sqlite3Fts2Init(sqlite3 *db){
  sqlite3_overload_function(db, "snippet", -1);
  sqlite3_overload_function(db, "offsets", -1);
  return sqlite3_create_module(db, "fts2", &fulltextModule, 0);

# 2007 March 9
#
# The author disclaims copyright to this source code.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  These
# make sure that inserted documents are visible to selects within the
# transaction.
#
# $Id: fts2k.test,v 1.1 2007/03/29 18:41:05 shess Exp $
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If SQLITE_ENABLE_FTS2 is defined, omit this file.
ifcapable !fts2 {
  finish_test
  return
}

db eval {
  CREATE VIRTUAL TABLE t1 USING fts2(content);
  INSERT INTO t1 (rowid, content) VALUES(1, "hello world");
  INSERT INTO t1 (rowid, content) VALUES(2, "hello there");
  INSERT INTO t1 (rowid, content) VALUES(3, "cruel world");
}

# Test that possibly-buffered inserts went through after commit.
do_test fts2k-1.1 {
  execsql {
    BEGIN TRANSACTION;
    INSERT INTO t1 (rowid, content) VALUES(4, "false world");
    INSERT INTO t1 (rowid, content) VALUES(5, "false door");
    COMMIT TRANSACTION;
    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
  }
} {1 3 4}

# Test that buffered inserts are seen by selects in the same
# transaction.
do_test fts2k-1.2 {
  execsql {
    BEGIN TRANSACTION;
    INSERT INTO t1 (rowid, content) VALUES(6, "another world");
    INSERT INTO t1 (rowid, content) VALUES(7, "another test");
    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
    COMMIT TRANSACTION;
  }
} {1 3 4 6}

# Test that buffered inserts are seen within a transaction.  This is
# really the same test as 1.2.
do_test fts2k-1.3 {
  execsql {
    BEGIN TRANSACTION;
    INSERT INTO t1 (rowid, content) VALUES(8, "second world");
    INSERT INTO t1 (rowid, content) VALUES(9, "second sight");
    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
    ROLLBACK TRANSACTION;
  }
} {1 3 4 6 8}

# Double-check that the previous result doesn't persist past the
# rollback!
do_test fts2k-1.4 {
  execsql {
    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
  }
} {1 3 4 6}

# Test it all together.
do_test fts2k-1.5 {
  execsql {
    BEGIN TRANSACTION;
    INSERT INTO t1 (rowid, content) VALUES(10, "second world");
    INSERT INTO t1 (rowid, content) VALUES(11, "second sight");
    ROLLBACK TRANSACTION;
    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
  }
} {1 3 4 6}

finish_test