SQLite

/*
** Set up a cursor object for iterating through a full-text index or a 
** single level therein.
*/
int sqlite3Fts3SegReaderCursor(
  Fts3Table *p,                   /* FTS3 table handle */
  int iLangid,                    /* Language-id to search */
  int iIndex,                     /* Index to search (from 0 to p->nIndex-1) */
  int iLevel,                     /* Level of segments to scan */
  const char *zTerm,              /* Term to query for */
  int nTerm,                      /* Size of zTerm in bytes */
  int isPrefix,                   /* True for a prefix search */
  int isScan,                     /* True to scan from zTerm to EOF */
  Fts3MultiSegReader *pCsr       /* Cursor object to populate */
){
  assert( iIndex>=0 && iIndex<p->nIndex );
  assert( iLevel==FTS3_SEGCURSOR_ALL
      ||  iLevel==FTS3_SEGCURSOR_PENDING 
      ||  iLevel>=0
  );
  assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
  assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
  assert( isPrefix==0 || isScan==0 );





  memset(pCsr, 0, sizeof(Fts3MultiSegReader));

  return fts3SegReaderCursor(
      p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
  );
}

/*
** In addition to its current configuration, have the Fts3MultiSegReader

    memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
    for(i=0; i<pPhrase->nToken; i++){
      fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
      pPhrase->aToken[i].pSegcsr = 0;
    }
  }
}


/*
** Return SQLITE_CORRUPT_VTAB.
*/
#ifdef SQLITE_DEBUG
int sqlite3Fts3Corrupt(){
  return SQLITE_CORRUPT_VTAB;

/*
** Internal types used by SQLite.
*/
typedef unsigned char u8;         /* 1-byte (or larger) unsigned integer */
typedef short int i16;            /* 2-byte (or larger) signed integer */
typedef unsigned int u32;         /* 4-byte unsigned integer */
typedef sqlite3_uint64 u64;       /* 8-byte unsigned integer */
typedef sqlite3_int64 i64;        /* 8-byte signed integer */

/*
** Macro used to suppress compiler warnings for unused parameters.
*/
#define UNUSED_PARAMETER(x) (void)(x)

/*

  sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
  sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
  sqlite3_step(pStmt);
  rc = sqlite3_reset(pStmt);
  return rc;
}

/*
** Return a 64-bit checksum for the FTS index entry specified by the
** arguments to this function.
*/
static i64 fts3ChecksumEntry(
  const char *zTerm,              /* Pointer to buffer containing term */
  int nTerm,                      /* Size of zTerm in bytes */
  int iLangid,                    /* Language id for current row */
  int iIndex,                     /* Index (0..Fts3Table.nIndex-1) */
  i64 iDocid,                     /* Docid for current row. */
  int iCol,                       /* Column number */
  int iPos                        /* Position */
){
  int i;
  i64 ret = iDocid;

  ret += (ret<<3) + iLangid;
  ret += (ret<<3) + iIndex;
  ret += (ret<<3) + iCol;
  ret += (ret<<3) + iPos;
  for(i=0; i<nTerm; i++) ret += (ret<<3) + zTerm[i];

  return ret;
}

/*
** Return a checksum of all entries in the FTS index that correspond to
** language id iLangid. The checksum is calculated by XORing the checksums
** of each individual entry (see fts3ChecksumEntry()) together.
**
** If successful, the checksum value is returned and *pRc set to SQLITE_OK.
** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The
** return value is undefined in this case.
*/
static i64 fts3ChecksumIndex(
  Fts3Table *p,                   /* FTS3 table handle */
  int iLangid,                    /* Language id to return cksum for */
  int iIndex,                     /* Index to cksum (0..p->nIndex-1) */
  int *pRc                        /* OUT: Return code */
){
  Fts3SegFilter filter;
  Fts3MultiSegReader csr;
  int rc;
  i64 cksum = 0;

  assert( *pRc==SQLITE_OK );

  memset(&filter, 0, sizeof(filter));
  memset(&csr, 0, sizeof(csr));
  filter.flags =  FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
  filter.flags |= FTS3_SEGMENT_SCAN;

  rc = sqlite3Fts3SegReaderCursor(
      p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr
  );
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
  }

  if( rc==SQLITE_OK ){
    while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){
      char *pCsr = csr.aDoclist;
      char *pEnd = &pCsr[csr.nDoclist];

      i64 iDocid = 0;
      i64 iCol = 0;
      i64 iPos = 0;

      pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
      while( pCsr<pEnd ){
        i64 iVal = 0;
        pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
        if( pCsr<pEnd ){
          if( iVal==0 || iVal==1 ){
            iCol = 0;
            iPos = 0;
            if( iVal ){
              pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
            }else{
              pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
              iDocid += iVal;
            }
          }else{
            iPos += (iVal - 2);
            cksum = cksum ^ fts3ChecksumEntry(
                csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, iCol, iPos
            );
          }
        }
      }
    }
  }
  sqlite3Fts3SegReaderFinish(&csr);

  *pRc = rc;
  return cksum;
}

/*
** Check if the contents of the FTS index match the current contents of the
** content table. If no error occurs and the contents do match, set *pbOk
** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk
** to false before returning.
**
** If an error occurs (e.g. an OOM or IO error), return an SQLite error 
** code. The final value of *pbOk is undefined in this case.
*/
static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
  int rc = SQLITE_OK;             /* Return code */
  i64 cksum1 = 0;                 /* Checksum based on FTS index contents */
  i64 cksum2 = 0;                 /* Checksum based on %_content contents */
  sqlite3_stmt *pAllLangid = 0;   /* Statement to return all language-ids */

  /* This block calculates the checksum according to the FTS index. */
  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pAllLangid, 1, p->nIndex);
    while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
      int iLangid = sqlite3_column_int(pAllLangid, 0);
      int i;
      for(i=0; i<p->nIndex; i++){
        cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc);
      }
    }
    rc2 = sqlite3_reset(pAllLangid);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  /* This block calculates the checksum according to the %_content table */
  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
    sqlite3_stmt *pStmt = 0;
    char *zSql;
   
    zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
    if( !zSql ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
      sqlite3_free(zSql);
    }

    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
      i64 iDocid = sqlite3_column_int64(pStmt, 0);
      int iLang = langidFromSelect(p, pStmt);
      int iCol;

      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
        const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
        int nText = sqlite3_column_bytes(pStmt, iCol+1);
        sqlite3_tokenizer_cursor *pT = 0;

        rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT);
        while( rc==SQLITE_OK ){
          char const *zToken;       /* Buffer containing token */
          int nToken;               /* Number of bytes in token */
          int iDum1, iDum2;         /* Dummy variables */
          int iPos;                 /* Position of token in zText */

          rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos);
          if( rc==SQLITE_OK ){
            int i;
            cksum2 = cksum2 ^ fts3ChecksumEntry(
                zToken, nToken, iLang, 0, iDocid, iCol, iPos
            );
            for(i=1; i<p->nIndex; i++){
              if( p->aIndex[i].nPrefix<=nToken ){
                cksum2 = cksum2 ^ fts3ChecksumEntry(
                  zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos
                );
              }
            }
          }
        }
        if( pT ) pModule->xClose(pT);
        if( rc==SQLITE_DONE ) rc = SQLITE_OK;
      }
    }

    sqlite3_finalize(pStmt);
  }

  *pbOk = (cksum1==cksum2);
  return rc;
}

/*
** Run the integrity-check. If no error occurs and the current contents of
** the FTS index are correct, return SQLITE_OK. Or, if the contents of the
** FTS index are incorrect, return SQLITE_CORRUPT_VTAB.
**
** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite 
** error code.
*/
static int fts3DoIntegrityCheck(
  Fts3Table *p                    /* FTS3 table handle */
){
  int rc;
  int bOk = 0;
  rc = fts3IntegrityCheck(p, &bOk);
  if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
  return rc;
}

/*
** Handle a 'special' INSERT of the form:
**
**   "INSERT INTO tbl(tbl) VALUES(<expr>)"
**
** Argument pVal contains the result of <expr>. Currently the only 
** meaningful value to insert is the text 'optimize'.
*/
static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
  int rc;                         /* Return Code */
  const char *zVal = (const char *)sqlite3_value_text(pVal);
  int nVal = sqlite3_value_bytes(pVal);

  if( !zVal ){
    return SQLITE_NOMEM;
  }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
    rc = fts3DoOptimize(p, 0);
  }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){
    rc = fts3DoRebuild(p);
  }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){
    rc = fts3DoIntegrityCheck(p);
  }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){
    rc = fts3DoIncrmerge(p, &zVal[6]);
  }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
    rc = fts3DoAutoincrmerge(p, &zVal[10]);
#ifdef SQLITE_TEST
  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
    p->nNodeSize = atoi(&zVal[9]);

# 2012 March 26
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS 'integrity-check' function,
# used to check if the current FTS index accurately reflects the content
# of the table.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/fts3_common.tcl
set ::testprefix fts4check

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

# Run the integrity-check on FTS table $tbl using database handle $db. If
# the integrity-check passes, return "ok". Otherwise, throw an exception.
#
proc fts_integrity {db tbl} {
  $db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')"
  return "ok"
}

#-------------------------------------------------------------------------
# Test cases 1.*
#
#   1.0: Build a reasonably sized FTS table (5000 rows).
#
#   1.1: Run the integrity check code to check it passes.
#
#   1.2: Make a series of minor changes to the underlying FTS data structures
#        (e.g. delete or insert a row from the %_content table). Check that
#        this causes the integrity-check code to fail.
#

# Build an FTS table and check the integrity-check passes.
#
do_test 1.0 { fts3_build_db_1 5000 } {}
do_test 1.1 { fts_integrity db t1 } {ok}

# Mess around with the underlying tables. Check that this causes the
# integrity-check test to fail.
#
foreach {tn disruption} {
  1 {
    INSERT INTO t1_content(docid, c0x, c1y) VALUES(NULL, 'a', 'b');
  }
  2 {
    DELETE FROM t1_content WHERE docid = (SELECT max(docid) FROM t1_content);
  }
  3 {
    DELETE FROM t1_segdir WHERE level=0 AND idx=(
      SELECT max(idx) FROM t1_segdir WHERE level=0
    );
  }
} {
  do_execsql_test  1.2.1.$tn "BEGIN; $disruption"
  do_catchsql_test 1.2.2.$tn {
    INSERT INTO t1 (t1) VALUES('integrity-check')
  } {1 {database disk image is malformed}}
  do_execsql_test  1.2.3.$tn "ROLLBACK"
}

do_test 1.3 { fts_integrity db t1 } {ok}

#-------------------------------------------------------------------------
# Test cases 2.*
#
#   2.0: Build a reasonably sized FTS table (20000 rows) that includes
#        prefix indexes.
#
#   2.1: Run the integrity check code to check it passes.
#
#   2.2: Make a series of minor changes to the underlying FTS data structures
#        (e.g. delete or insert a row from the %_content table). Check that
#        this causes the integrity-check code to fail.
#

do_test 2.0 { fts3_build_db_2 20000 {prefix="3,1"} } {}
do_test 2.1 { fts_integrity db t2 } {ok}
foreach {tn disruption} {
  1 {
    INSERT INTO t2_content VALUES(NULL, 'xyz')
  }
  3 {
    DELETE FROM t2_segdir WHERE level=0 AND idx=(
      SELECT max(idx) FROM t2_segdir WHERE level=1024
    );
  }
} {
  do_execsql_test  2.2.1.$tn "BEGIN; $disruption"
  do_catchsql_test 2.2.2.$tn {
    INSERT INTO t2 (t2) VALUES('integrity-check')
  } {1 {database disk image is malformed}}
  do_execsql_test  2.2.3.$tn "ROLLBACK"
}


#-------------------------------------------------------------------------
# Test cases 3.*
#
#   3.0: Build a reasonably sized FTS table (5000 rows) that includes
#        prefix indexes and uses the languageid= feature.
#
#   3.1: Run the integrity check code to check it passes.
#
#   3.2: Make a series of minor changes to the underlying FTS data structures
#        (e.g. delete or insert a row from the %_content table). Check that
#        this causes the integrity-check code to fail.
#
do_test 3.0 {
  reset_db
  fts3_build_db_1 5000
  execsql {
    CREATE VIRTUAL TABLE t3 USING fts4(x, y, prefix="2,3", languageid=langid);
  }
  foreach docid [execsql {SELECT docid FROM t1 ORDER BY 1 ASC}] {
    execsql {
      INSERT INTO t3(x, y, langid) 
      SELECT x, y, (docid%9)*4 FROM t1 WHERE docid=$docid;
    }
  }
} {}
do_test 3.1 { fts_integrity db t3 } {ok}

foreach {tn disruption} {
  1 {
    INSERT INTO t3_content(c0x, c1y, langid) VALUES(NULL, 'a', 0);
  }
  2 {
    UPDATE t3_content SET langid=langid+1 WHERE rowid = (
      SELECT max(rowid) FROM t3_content
    )
  }
} {
  do_execsql_test  3.2.1.$tn "BEGIN; $disruption"
  do_catchsql_test 3.2.2.$tn {
    INSERT INTO t3 (t3) VALUES('integrity-check')
  } {1 {database disk image is malformed}}
  do_execsql_test  3.2.3.$tn "ROLLBACK"
}



finish_test

set ::testprefix fts4merge

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

proc fts3_integrity_check {tbl} {
  db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')"
  return "ok"
}

#-------------------------------------------------------------------------
# Test cases 1.*
#
do_test 1.0 { fts3_build_db_1 1004 } {}
do_test 1.1 { fts3_integrity_check t1 } {ok}
do_execsql_test 1.1 { 

  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
  fts4check.test
}


lappend ::testsuitelist xxx
#-------------------------------------------------------------------------
# Define the coverage related test suites:
#