SQLite

  Fts5SegIter *pIter              /* Iterator to advance to next page */
){
  Fts5Data *pLeaf;
  Fts5StructureSegment *pSeg = pIter->pSeg;
  fts5DataRelease(pIter->pLeaf);
  pIter->iLeafPgno++;
  if( pIter->pNextLeaf ){

    pIter->pLeaf = pIter->pNextLeaf;
    pIter->pNextLeaf = 0;
  }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
    pIter->pLeaf = fts5DataRead(p, 
        FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)
    );
  }else{

    Fts5Data *pNew;
    pIter->iLeafPgno--;
    pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
          pIter->pSeg->iSegid, pIter->iLeafPgno
    ));
    if( pNew ){
      if( pIter->iLeafPgno==pIter->iTermLeafPgno ){

        pIter->pLeaf = pNew;
        pIter->iLeafOffset = pIter->iTermLeafOffset;

      }else{
        int iRowidOff;
        iRowidOff = fts5LeafFirstRowidOff(pNew);
        if( iRowidOff ){
          pIter->pLeaf = pNew;
          pIter->iLeafOffset = iRowidOff;
        }

  Fts5SegWriter *pWriter,
  int nTerm, const u8 *pTerm 
){
  int nPrefix;                    /* Bytes of prefix compression for term */
  Fts5PageWriter *pPage = &pWriter->writer;
  Fts5Buffer *pPgidx = &pWriter->writer.pgidx;

  assert( p->rc==SQLITE_OK );
  assert( pPage->buf.n>=4 );
  assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );

  /* If the current leaf page is full, flush it to disk. */
  if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
    if( pPage->buf.n>4 ){
      fts5WriteFlushLeaf(p, pWriter);

  Fts5Index *p, 
  Fts5SegWriter *pWriter,         /* Writer object */
  int *pnLeaf                     /* OUT: Number of leaf pages in b-tree */
){
  int i;
  Fts5PageWriter *pLeaf = &pWriter->writer;
  if( p->rc==SQLITE_OK ){
    assert( pLeaf->pgno>=1 );


    if( pLeaf->buf.n>4 ){
      fts5WriteFlushLeaf(p, pWriter);
    }
    *pnLeaf = pLeaf->pgno-1;

    fts5WriteFlushBtree(p, pWriter);

  }
  fts5BufferFree(&pLeaf->term);
  fts5BufferFree(&pLeaf->buf);
  fts5BufferFree(&pLeaf->pgidx);
  fts5BufferFree(&pWriter->btterm);

  for(i=0; i<pWriter->nDlidx; i++){

      const u8 *pDoclist;         /* Pointer to doclist for this term */
      int nDoclist;               /* Size of doclist in bytes */

      /* Write the term for this entry to disk. */
      sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
      fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm);

      assert( writer.bFirstRowidInPage==0 );
      if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){

        /* The entire doclist will fit on the current leaf. */
        fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
      }else{
        i64 iRowid = 0;
        i64 iDelta = 0;
        int iOff = 0;



        /* The entire doclist will not fit on this leaf. The following 
        ** loop iterates through the poslists that make up the current 
        ** doclist.  */
        while( p->rc==SQLITE_OK && iOff<nDoclist ){
          int nPos;
          int nCopy;
          int bDummy;

    for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
        fts5MultiIterEof(p, p1)==0;
        fts5MultiIterNext(p, p1, 0, 0)
    ){
      i64 iRowid = fts5MultiIterRowid(p1);
      int nTerm;
      const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
      assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
      if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;

      if( doclist.n>0 && iRowid<=iLastRowid ){
        for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
          assert( i<nBuf );
          if( aBuf[i].n==0 ){
            fts5BufferSwap(&doclist, &aBuf[i]);

    int nPos;
    int bDummy;
    iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
    iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos));
    if( iOff<n ){
      i64 iDelta;
      iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);

      iDocid += iDelta;
      sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
    }
  }

  return iOff;
}

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}
sqlite3_fts5_may_be_corrupt 1

proc create_t1 {} {
  expr srand(0)
  db func rnddoc fts5_rnddoc
  db eval {
    CREATE VIRTUAL TABLE t1 USING fts5(x);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
    WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
      INSERT INTO t1 SELECT rnddoc(10) FROM ii;
  }
}

if 1 {

# Create a simple FTS5 table containing 100 documents. Each document 
# contains 10 terms, each of which start with the character "x".
#


do_test 1.0 { create_t1 } {}







do_test 1.1 {
  # Pick out the rowid of the right-most b-tree leaf in the new segment.
  set rowid [db one {
    SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
  }]
  set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]

  UPDATE t2_data SET block = block || 'abcd' WHERE id=1;
  SELECT length(block) FROM t2_data WHERE id=1;
} {6}
do_execsql_test 2.2 {
  INSERT INTO t2 VALUES(rnddoc(10));
  SELECT length(block) FROM t2_data WHERE id=1;
} {2}


#-------------------------------------------------------------------------
# Test that missing leaf pages are recognized as corruption.
#
reset_db
do_test 3.0 { create_t1 } {}

do_execsql_test 3.1 {
  SELECT count(*) FROM t1_data;
} {105}

proc do_3_test {tn} {
  set i 0
  foreach ::rowid [db eval "SELECT rowid FROM t1_data WHERE rowid>100"] {
    incr i
    do_test $tn.$i {
      db eval BEGIN
      db eval {DELETE FROM t1_data WHERE rowid = $::rowid}
      list [
        catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg
      ] $msg
    } {1 {database disk image is malformed}}
    catch { db eval ROLLBACK }
  }
}

do_3_test 3.2

do_execsql_test 3.3 {
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  INSERT INTO t1 SELECT x FROM t1;
  INSERT INTO t1(t1) VALUES('optimize');
} {}

do_3_test 3.4

do_test 3.5 {
  execsql { 
    DELETE FROM t1;
    INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
  }
  for {set i 0} {$i < 1000} {incr i} {
    set rnd [expr int(rand() * 1000)]
    set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]]
    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
  }
} {}

do_3_test 3.6

do_test 3.7 {
  execsql {
    INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
    INSERT INTO t1 SELECT x FROM t1;
    INSERT INTO t1(t1) VALUES('optimize');
  }
} {}

do_3_test 3.8

do_test 3.9 {
  execsql { 
    DELETE FROM t1;
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }
  for {set i 0} {$i < 100} {incr i} {
    set rnd [expr int(rand() * 100)]
    set doc "x[string repeat $rnd 20]"
    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
  }
} {}

do_3_test 3.10

}

#-------------------------------------------------------------------------
# Test that segments that end unexpectedly are identified as corruption.
#
reset_db
do_test 4.0 {
  execsql { 
    CREATE VIRTUAL TABLE t1 USING fts5(x);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }
  for {set i 0} {$i < 100} {incr i} {
    set rnd [expr int(rand() * 100)]
    set doc "x[string repeat $rnd 20]"
    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
  }
  execsql { INSERT INTO t1(t1) VALUES('optimize') }
} {}

set nErr 0
for {set i 1} {1} {incr i} {
  set struct [db one {SELECT block FROM t1_data WHERE id=10}]
  binary scan $struct c* var
  set end [lindex $var end]
  if {$end<=$i} break
  lset var end [expr $end - $i]
  set struct [binary format c* $var]
  db eval {
    BEGIN;
    UPDATE t1_data SET block = $struct WHERE id=10;
  }
  do_test 4.1.$i {
    incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
    set {} {}
  } {}
  catch { db eval ROLLBACK }
}
do_test 4.1.x { expr $nErr>45 } 1

#-------------------------------------------------------------------------
#

# The first argument passed to this command must be a binary blob 
# containing an FTS5 leaf page. This command returns a copy of this
# blob, with the pgidx of the leaf page replaced by a single varint
# containing value $iVal.
#
proc rewrite_pgidx {blob iVal} {
  binary scan $blob SS off1 szLeaf
  if {$iVal<0 || $iVal>=128} {
    error "$iVal out of range!"
  } else {
    set pgidx [binary format c $iVal]
  }

  binary format a${szLeaf}a* $blob $pgidx
}

reset_db
do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 40);
  BEGIN;
  INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj');
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1 SELECT x FROM x1;
  INSERT INTO x1(x1) VALUES('optimize');
  COMMIT;
}

#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b }
#
db func rewrite_pgidx rewrite_pgidx  
set i 0
foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
  foreach val {2 100} {
    do_test 5.2.$val.[incr i] {
      catchsql {
        BEGIN;
        UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid;
        SELECT rowid FROM x1 WHERE x1 MATCH 'xa*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xb*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xc*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xd*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xe*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xf*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xg*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xh*';
        SELECT rowid FROM x1 WHERE x1 MATCH 'xi*';
      }
      set {} {}
    } {}
    catch { db eval ROLLBACK }
  }
}


sqlite3_fts5_may_be_corrupt 0
finish_test

  return
}

if { $tcl_platform(wordSize)<8 } {
  finish_test
  return
}

if 1 {

proc do_fb_test {tn sql res} {
  set res2 [lsort -integer -decr $res]
  uplevel [list do_execsql_test $tn.1 $sql $res]
  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}

  breakpoint
  do_execsql_test $tn.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
  } {1}
}

do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]

}

#--------------------------------------------------------------------
#
reset_db

set ::vocab [list \
  IteratorpItercurrentlypointstothefirstrowidofadoclist \
  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
  pageIfthecurrenttermisthelasttermonthepageloadthe \
  doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
  IteratorpItercurrentlypointstothefirstrowidofadoclist \
  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
  pageIfthecurrenttermisthelasttermonthepageloadthe \
  doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
]
proc rnddoc {} {
  global vocab
  set nVocab [llength $vocab]
  set ret [list]
  for {set i 0} {$i < 64} {incr i} {
    lappend ret [lindex $vocab [expr $i % $nVocab]]
  }
  set ret
}
db func rnddoc rnddoc

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a);
  INSERT INTO abc(abc, rank) VALUES('pgsz', 32);

  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );
  INSERT INTO abc VALUES ( rnddoc() );

  INSERT INTO abc SELECT rnddoc() FROM abc;
  INSERT INTO abc SELECT rnddoc() FROM abc;
}

do_execsql_test 3.2 {
  INSERT INTO abc(abc) VALUES('integrity-check');
  INSERT INTO abc(abc) VALUES('optimize');
  INSERT INTO abc(abc) VALUES('integrity-check');
}

set v [lindex $vocab 0]
set i 0
foreach v $vocab {
  do_execsql_test 3.3.[incr i] {
    SELECT rowid FROM abc WHERE abc MATCH $v
  } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}


finish_test

#*************************************************************************
#
# This file is focused on OOM errors.
#

source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault7

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

if 1 {

#-------------------------------------------------------------------------
# Test fault-injection on a query that uses xColumnSize() on columnsize=0
# table.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0);
  INSERT INTO t1 VALUES('a b c d e f g');
  INSERT INTO t1 VALUES('a b c d');
  INSERT INTO t1 VALUES('a b c d e f g h i j');
}


fts5_aux_test_functions db
do_faultsim_test 1 -faults oom* -body {
  execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' }
} -test {
  faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM}
}

}

#-------------------------------------------------------------------------
# Test fault-injection when a segment is promoted.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a);
  INSERT INTO t2(t2, rank) VALUES('automerge', 0);
  INSERT INTO t2(t2, rank) VALUES('crisismerge', 4);
  INSERT INTO t2(t2, rank) VALUES('pgsz', 40);

  INSERT INTO t2 VALUES('a b c');
  INSERT INTO t2 VALUES('d e f');
  INSERT INTO t2 VALUES('f e d');
  INSERT INTO t2 VALUES('c b a');

  INSERT INTO t2 VALUES('a b c');
  INSERT INTO t2 VALUES('d e f');
  INSERT INTO t2 VALUES('f e d');
  INSERT INTO t2 VALUES('c b a');
} {}

faultsim_save_and_close
do_faultsim_test 1 -faults oom-t* -prep {
  faultsim_restore_and_reopen
  db eval {
    BEGIN;
    INSERT INTO t2 VALUES('c d c g g f');
    INSERT INTO t2 VALUES('c d g b f d');
    INSERT INTO t2 VALUES('c c f d e d');
    INSERT INTO t2 VALUES('e a f c e f');
    INSERT INTO t2 VALUES('c g f b b d');
    INSERT INTO t2 VALUES('d a g a b b');
    INSERT INTO t2 VALUES('e f a b c e');
    INSERT INTO t2 VALUES('e c a g c d');
    INSERT INTO t2 VALUES('g b d d e b');
    INSERT INTO t2 VALUES('e a d a e d');
  }
} -body {
  db eval COMMIT
} -test {
  faultsim_test_result {0 {}}
}

finish_test