SQLite

         CREATE TABLE rbu.data_t1(a, b, rbu_control);
         INSERT INTO rbu.data_t1 VALUES(1, 2, 4);
       } {SQLITE_ERROR - invalid rbu_control value}

       9 {
         CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID;
         CREATE TABLE rbu.data_t1(a, b, rbu_control);
         INSERT INTO rbu.data_t1 VALUES(1, 2, 2);
         INSERT INTO rbu.data_t1 VALUES(1, 2, 3);
       } {SQLITE_ERROR - invalid rbu_control value}

       10 {
         CREATE TABLE t2(a, b);
         CREATE TABLE rbu.data_t1(a, b, rbu_control);
         INSERT INTO rbu.data_t1 VALUES(1, 2, 2);
       } {SQLITE_ERROR - no such table: t1}

    rbu close
    if {$rc != "SQLITE_OK"} break
  }
  set rc
}

proc apply_rbudiff {sql target} {
  test_rbucount $sql
  forcedelete rbu.db
  sqlite3 rbudb rbu.db
  rbudb eval $sql
  rbudb close
  step_rbu $target rbu.db
}

# The only argument is the output of an [sqldiff -rbu] run. This command
# tests that the contents of the rbu_count table is correct. An exception
# is thrown if it is not.
#
proc test_rbucount {sql} {
  sqlite3 tmpdb ""
  tmpdb eval $sql
  tmpdb eval {
    SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table'
  } {
    set a [tmpdb eval "SELECT count(*) FROM $name"]
    set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}]
    if {$a != $b} { 
      tmpdb close
      error "rbu_count error - tbl = $name" 
    }
  }
  tmpdb close
  return ""
}

proc rbudiff_cksum {db1} {
  set txt ""

  sqlite3 dbtmp $db1
  foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] {
    set cols [list]

# 2016 March 18
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbuprogress


proc create_db_file {filename sql} {
  forcedelete $filename
  sqlite3 tmpdb $filename  
  tmpdb eval $sql
  tmpdb close
}

# Create a simple RBU database. That expects to write to a table:
#
#   CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
#
proc create_rbu1 {filename} {
  create_db_file $filename {
    CREATE TABLE data_t1(a, b, c, rbu_control);
    INSERT INTO data_t1 VALUES(1, 2, 3, 0);
    INSERT INTO data_t1 VALUES(2, 'two', 'three', 0);
    INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0);

    CREATE TABLE rbu_count(tbl, cnt);
    INSERT INTO rbu_count VALUES('data_t1', 3);
  }
  return $filename
}


do_execsql_test 1.0 {
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}

do_test 1.1 {
  create_rbu1 rbu.db
  sqlite3rbu rbu test.db rbu.db
  rbu bp_progress
} {0 0}
do_test 1.2 { rbu step ; rbu bp_progress } {3333 0}
do_test 1.3 { rbu step ; rbu bp_progress } {6666 0}
do_test 1.4 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.5 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.6 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000}
do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000}
do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000}

do_test 1.10 {
  rbu close
} {SQLITE_DONE}

#-------------------------------------------------------------------------
#
proc do_sp_test {tn bReopen target rbu reslist} {
  uplevel [list do_test $tn [subst -nocommands {
    if {$bReopen==0} { sqlite3rbu rbu $target $rbu }
    set res [list]
    while 1 {
      if {$bReopen} { sqlite3rbu rbu $target $rbu }
      set rc [rbu step]
      if {[set rc] != "SQLITE_OK"} { rbu close ; error "error 1" }
      lappend res [lindex [rbu bp_progress] 0]
      if {[lindex [set res] end]==10000} break
      if {$bReopen} { rbu close }
    }
    if {[set res] != [list $reslist]} {
      rbu close
      error "1. reslist incorrect (expect=$reslist got=[set res])"
    }

    # One step to clean up the temporary tables used to update the only
    # target table in the rbu database. And one more to move the *-oal 
    # file to *-wal. After each of these steps, the progress remains
    # at "10000 0".
    #
    if {[lindex [list $reslist] 0]!=-1} {
      rbu step
      set res [rbu bp_progress]
      if {[set res] != [list 10000 0]} {
        rbu close
        error "2. reslist incorrect (expect=10000 0 got=[set res])"
      }
    }

    rbu step
    set res [rbu bp_progress]
    if {[set res] != [list 10000 0]} {
      rbu close
      error "3. reslist incorrect (expect=10000 0 got=[set res])"
    }

    # Do the checkpoint.
    while {[rbu step]=="SQLITE_OK"} { 
      foreach {a b} [rbu bp_progress] {}
      if {[set a]!=10000 || [set b]<=0 || [set b]>10000} {
        rbu close
        error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])"
      }
    }

    set res [rbu bp_progress]
    if {[set res] != [list 10000 10000]} {
      rbu close
      error "5. reslist is incorrect (expect=10000 10000 got=[set res])"
    }

    rbu close
  }] {SQLITE_DONE}]
}

foreach {bReopen} { 0 1 } {
  reset_db
  do_test 2.$bReopen.1.0 {
    execsql {
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, c, rbu_control);
      INSERT INTO data_t1 VALUES(4, 4, 4, 0);
      INSERT INTO data_t1 VALUES(5, 5, 5, 0);
  
      CREATE TABLE rbu_count(tbl, cnt);
      INSERT INTO rbu_count VALUES('data_t1', 2);
    }
  } {}
  do_sp_test 2.$bReopen.1.1 $bReopen test.db rbu.db {5000 10000}
  
  reset_db
  do_test 2.$bReopen.2.0 {
    execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) }
    create_rbu1 rbu.db
  } {rbu.db}
  do_sp_test 2.$bReopen.2.1 $bReopen test.db rbu.db {3333 6666 10000}
  
  reset_db
  do_test 2.$bReopen.3.0 {
    execsql { 
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
      CREATE INDEX i1 ON t1(b);
      INSERT INTO t1 VALUES(1, 1, 1);
      INSERT INTO t1 VALUES(2, 2, 2);
      INSERT INTO t1 VALUES(3, 3, 3);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, c, rbu_control);
      INSERT INTO data_t1 VALUES(4, 4, 4, 0);
      INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
      INSERT INTO data_t1 VALUES(5, NULL, NULL, 1);
  
      CREATE TABLE rbu_count(tbl, cnt);
      INSERT INTO rbu_count VALUES('data_t1', 3);
    }
  } {}
  do_sp_test 2.$bReopen.3.1 $bReopen test.db rbu.db {1666 3333 6000 8000 10000}
  
  reset_db
  do_test 2.$bReopen.4.0 {
    execsql { 
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
      CREATE INDEX i1 ON t1(b);
      INSERT INTO t1 VALUES(1, 1, 1);
      INSERT INTO t1 VALUES(2, 2, 2);
      INSERT INTO t1 VALUES(3, 3, 3);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, c, rbu_control);
      INSERT INTO data_t1 VALUES(2, 4, 4, '.xx');
  
      CREATE TABLE rbu_count(tbl, cnt);
      INSERT INTO rbu_count VALUES('data_t1', 1);
    }
  } {}
  do_sp_test 2.$bReopen.4.1 $bReopen test.db rbu.db {3333 6666 10000}
  
  reset_db
  do_test 2.$bReopen.5.0 {
    execsql { 
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
      CREATE INDEX i1 ON t1(b);
      INSERT INTO t1 VALUES(1, 1, 1);
      INSERT INTO t1 VALUES(2, 2, 2);
      INSERT INTO t1 VALUES(3, 3, 3);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, c, rbu_control);
      INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx');
  
      CREATE TABLE rbu_count(tbl, cnt);
      INSERT INTO rbu_count VALUES('data_t1', 1);
    }
  } {}
  do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000}

  reset_db
  do_test 2.$bReopen.6.0 {
    execsql { 
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
      CREATE INDEX i1 ON t1(b);
      INSERT INTO t1 VALUES(1, 1, 1);
      INSERT INTO t1 VALUES(2, 2, 2);
      INSERT INTO t1 VALUES(3, 3, 3);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, c, rbu_control);
      INSERT INTO data_t1 VALUES(4, 4, 4, 0);
      INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
      INSERT INTO data_t1 VALUES(5, NULL, NULL, 1);
    }
  } {}
  do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000}
}

#-------------------------------------------------------------------------
# The following tests verify that the API works when resuming an update
# during the incremental checkpoint stage.
#
proc do_phase2_test {tn bReopen target rbu nStep} {
  uplevel [list do_test $tn [subst -nocommands {

    # Build the OAL/WAL file:
    sqlite3rbu rbu $target $rbu
    while {[lindex [rbu bp_progress] 0]<10000} { 
      set rc [rbu step]
      if {"SQLITE_OK" != [set rc]} { rbu close }
    }

    # Clean up the temp tables and move the *-oal file to *-wal.
    rbu step
    rbu step

    for {set i 0} {[set i] < $nStep} {incr i} {
      if {$bReopen} {
        rbu close
        sqlite3rbu rbu $target $rbu
      }
      rbu step
      set res [rbu bp_progress]
      set expect [expr (1 + [set i]) * 10000 / $nStep]
      if {[lindex [set res] 1] != [set expect]} {
        error "Have [set res], expected 10000 [set expect]"
      }
    }

    set rc [rbu step]
    if {[set rc] != "SQLITE_DONE"} {
      error "Have [set rc], expected SQLITE_DONE" 
    }

    rbu close
  }] {SQLITE_DONE}]
}

foreach bReopen {0 1} {
  do_test 3.$bReopen.1.0 {
    reset_db
    execsql {
      CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
      CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
    }
    create_db_file rbu.db {
      CREATE TABLE data_t1(a, b, rbu_control);
      CREATE TABLE data_t2(a, b, rbu_control);
      CREATE TABLE data_t3(a, b, rbu_control);
      CREATE TABLE data_t4(a, b, rbu_control);
      INSERT INTO data_t1 VALUES(1, 2, 0);
      INSERT INTO data_t2 VALUES(1, 2, 0);
      INSERT INTO data_t3 VALUES(1, 2, 0);
      INSERT INTO data_t4 VALUES(1, 2, 0);
  
      CREATE TABLE rbu_count(tbl, cnt);
      INSERT INTO rbu_count VALUES('data_t1', 1);
      INSERT INTO rbu_count VALUES('data_t2', 1);
      INSERT INTO rbu_count VALUES('data_t3', 1);
      INSERT INTO rbu_count VALUES('data_t4', 1);
    }
  } {}
  do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5
}


foreach {bReopen} { 0 1 } {
  foreach {tn tbl} {
    ipk { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) }
    wr  { CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID }
    pk  { CREATE TABLE t1(a INT PRIMARY KEY, b, c) }
  } {

    foreach {tn2 rbusql r1 r3} {
      1 {
        CREATE TABLE data0_t1(a, b, c, rbu_control);
        INSERT INTO data0_t1 VALUES(15, 15, 15, 0);
        INSERT INTO data0_t1 VALUES(20, 20, 20, 0);
        CREATE TABLE rbu_count(tbl, cnt);
        INSERT INTO rbu_count VALUES('data0_t1', 2); 
      } 
      {2500 5000 7500 10000}
      {1666 3333 5000 6666 8333 10000}

      2 {
        CREATE TABLE data0_t1(a, b, c, rbu_control);
        INSERT INTO data0_t1 VALUES(10, 10, 10, 2);
        CREATE TABLE rbu_count(tbl, cnt);
        INSERT INTO rbu_count VALUES('data0_t1', 1); 
      } 
      {3333 6666 10000}
      {2000 4000 6000 8000 10000}

      3 {
        CREATE TABLE data0_t1(a, b, c, rbu_control);
        INSERT INTO data0_t1 VALUES(7, 7, 7, 2);
        INSERT INTO data0_t1 VALUES(10, 10, 10, 2);
        CREATE TABLE rbu_count(tbl, cnt);
        INSERT INTO rbu_count VALUES('data0_t1', 2); 
      } 
      {2500 4000 6000 8000 10000}
      {1666 2500 3750 5000 6250 7500 8750 10000}

    } {

      reset_db ; execsql $tbl
      do_test 4.$tn.$bReopen.$tn2.0 {
        execsql {
          CREATE INDEX t1c ON t1(c);
          INSERT INTO t1 VALUES(1, 1, 1);
          INSERT INTO t1 VALUES(5, 5, 5);
          INSERT INTO t1 VALUES(10, 10, 10);
        }
        create_db_file rbu.db $rbusql
      } {}

      set R(ipk) $r1
      set R(wr) $r1
      set R(pk) $r3
      do_sp_test 4.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn)
    }
  }
}

foreach {bReopen} { 0 1 } {
  foreach {tn tbl} {
    nopk { 
      CREATE TABLE t1(a, b, c);
      CREATE INDEX t1c ON t1(c);
    }
    vtab { 
      CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
    }
  } {

    foreach {tn2 rbusql r1 r2} {
      1 {
        CREATE TABLE data0_t1(a, b, c, rbu_rowid, rbu_control);
        INSERT INTO data0_t1 VALUES(15, 15, 15, 4, 0);
        INSERT INTO data0_t1 VALUES(20, 20, 20, 5, 0);
        CREATE TABLE rbu_count(tbl, cnt);
        INSERT INTO rbu_count VALUES('data0_t1', 2); 
      } 
      {2500 5000 7500 10000}
      {5000 10000}

      2 {
        CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control);
        INSERT INTO data0_t1 VALUES(0, 7, 7, 7, 2);
        INSERT INTO data0_t1 VALUES(2, 10, 10, 10, 2);
        CREATE TABLE rbu_count(tbl, cnt);
        INSERT INTO rbu_count VALUES('data0_t1', 2); 
      } 
      {2500 4000 6000 8000 10000}
      {5000 10000}

      3 {
        CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control);
        INSERT INTO data0_t1 VALUES(1, NULL, NULL, NULL, 1);
        INSERT INTO data0_t1 VALUES(2, NULL, NULL, 7, '..x');
        CREATE TABLE rbu_count(tbl, cnt);
        INSERT INTO rbu_count VALUES('data0_t1', 2); 
      } 
      {2500 4000 6000 8000 10000}
      {5000 10000}
    } {

      reset_db ; execsql $tbl
      do_test 5.$tn.$bReopen.$tn2.0 {
        execsql {
          INSERT INTO t1 VALUES(1, 1, 1);
          INSERT INTO t1 VALUES(5, 5, 5);
          INSERT INTO t1 VALUES(10, 10, 10);
        }
        create_db_file rbu.db $rbusql
      } {}

      set R(nopk) $r1
      set R(vtab) $r2
      do_sp_test 5.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn)
    }
  }
}


finish_test

** RBU_STATE_COOKIE:
**   Valid if STAGE==1. The current change-counter cookie value in the 
**   target db file.
**
** RBU_STATE_OALSZ:
**   Valid if STAGE==1. The size in bytes of the *-oal file.
*/
#define RBU_STATE_STAGE       1
#define RBU_STATE_TBL         2
#define RBU_STATE_IDX         3
#define RBU_STATE_ROW         4
#define RBU_STATE_PROGRESS    5
#define RBU_STATE_CKPT        6
#define RBU_STATE_COOKIE      7
#define RBU_STATE_OALSZ       8
#define RBU_STATE_STAGE        1
#define RBU_STATE_TBL          2
#define RBU_STATE_IDX          3
#define RBU_STATE_ROW          4
#define RBU_STATE_PROGRESS     5
#define RBU_STATE_CKPT         6
#define RBU_STATE_COOKIE       7
#define RBU_STATE_OALSZ        8
#define RBU_STATE_PHASEONESTEP 9

#define RBU_STAGE_OAL         1
#define RBU_STAGE_MOVE        2
#define RBU_STAGE_CAPTURE     3
#define RBU_STAGE_CKPT        4
#define RBU_STAGE_DONE        5

  char *zTbl;
  char *zIdx;
  i64 iWalCksum;
  int nRow;
  i64 nProgress;
  u32 iCookie;
  i64 iOalSz;
  i64 nPhaseOneStep;
};

struct RbuUpdateStmt {
  char *zMask;                    /* Copy of update mask used with pUpdate */
  sqlite3_stmt *pUpdate;          /* Last update statement (or NULL) */
  RbuUpdateStmt *pNext;
};

  int bCleanup;                   /* True in "cleanup" state */
  const char *zTbl;               /* Name of target db table */
  const char *zDataTbl;           /* Name of rbu db table (or null) */
  const char *zIdx;               /* Name of target db index (or null) */
  int iTnum;                      /* Root page of current object */
  int iPkTnum;                    /* If eType==EXTERNAL, root of PK index */
  int bUnique;                    /* Current index is unique */
  int nIndex;                     /* Number of aux. indexes on table zTbl */

  /* Statements created by rbuObjIterPrepareAll() */
  int nCol;                       /* Number of columns in current object */
  sqlite3_stmt *pSelect;          /* Source data */
  sqlite3_stmt *pInsert;          /* Statement for INSERT operations */
  sqlite3_stmt *pDelete;          /* Statement for DELETE ops */
  sqlite3_stmt *pTmpInsert;       /* Insert into rbu_tmp_$zDataTbl */

struct RbuFrame {
  u32 iDbPage;
  u32 iWalFrame;
};

/*
** RBU handle.
**
** nPhaseOneStep:
**   If the RBU database contains an rbu_count table, this value is set to
**   a running estimate of the number of b-tree operations required to 
**   finish populating the *-oal file. This allows the sqlite3_bp_progress()
**   API to calculate the permyriadage progress of populating the *-oal file
**   using the formula:
**
**     permyriadage = (10000 * nProgress) / nPhaseOneStep
**
**   nPhaseOneStep is initialized to the sum of:
**
**     nRow * (nIndex + 1)
**
**   for all source tables in the RBU database, where nRow is the number
**   of rows in the source table and nIndex the number of indexes on the
**   corresponding target database table.
**
**   This estimate is accurate if the RBU update consists entirely of
**   INSERT operations. However, it is inaccurate if:
**
**     * the RBU update contains any UPDATE operations. If the PK specified
**       for an UPDATE operation does not exist in the target table, then
**       no b-tree operations are required on index b-trees. Or if the 
**       specified PK does exist, then (nIndex*2) such operations are
**       required (one delete and one insert on each index b-tree).
**
**     * the RBU update contains any DELETE operations for which the specified
**       PK does not exist. In this case no operations are required on index
**       b-trees.
**
**     * the RBU update contains REPLACE operations. These are similar to
**       UPDATE operations.
**
**   nPhaseOneStep is updated to account for the conditions above during the
**   first pass of each source table. The updated nPhaseOneStep value is
**   stored in the rbu_state table if the RBU update is suspended.
*/
struct sqlite3rbu {
  int eStage;                     /* Value of RBU_STATE_STAGE field */
  sqlite3 *dbMain;                /* target database handle */
  sqlite3 *dbRbu;                 /* rbu database handle */
  char *zTarget;                  /* Path to target db */
  char *zRbu;                     /* Path to rbu db */
  char *zState;                   /* Path to state db (or NULL if zRbu) */
  char zStateDb[5];               /* Db name for state ("stat" or "main") */
  int rc;                         /* Value returned by last rbu_step() call */
  char *zErrmsg;                  /* Error message if rc!=SQLITE_OK */
  int nStep;                      /* Rows processed for current object */
  int nProgress;                  /* Rows processed for all objects */
  RbuObjIter objiter;             /* Iterator for skipping through tbl/idx */
  const char *zVfsName;           /* Name of automatically created rbu vfs */
  rbu_file *pTargetFd;            /* File handle open on target db */
  i64 iOalSz;
  i64 nPhaseOneStep;

  /* The following state variables are used as part of the incremental
  ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
  ** function rbuSetupCheckpoint() for details.  */
  u32 iMaxFrame;                  /* Largest iWalFrame value in aFrame[] */
  u32 mLock;
  int nFrame;                     /* Entries in aFrame[] array */

  if( p->rc==SQLITE_OK ){
    memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol);
    p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg,
        sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl)
    );
  }

  pIter->nIndex = 0;
  while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){
    const char *zIdx = (const char*)sqlite3_column_text(pList, 1);
    sqlite3_stmt *pXInfo = 0;
    if( zIdx==0 ) break;
    p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
        sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx)
    );
    while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
      int iCid = sqlite3_column_int(pXInfo, 1);
      if( iCid>=0 ) pIter->abIndexed[iCid] = 1;
    }
    rbuFinalize(p, pXInfo);
    bIndex = 1;
    pIter->nIndex++;
  }

  if( pIter->eType==RBU_PK_WITHOUT_ROWID ){
    /* "PRAGMA index_list" includes the main PK b-tree */
    pIter->nIndex--;
  }

  rbuFinalize(p, pList);
  if( bIndex==0 ) pIter->abIndexed = 0;
}

        iOrder++;
      }
    }

    rbuFinalize(p, pStmt);
    rbuObjIterCacheIndexedCols(p, pIter);
    assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 );
    assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 );
  }

  return p->rc;
}

/*
** This function constructs and returns a pointer to a nul-terminated 

  sqlite3_context *pCtx, 
  int nVal,
  sqlite3_value **apVal
){
  sqlite3rbu *p = sqlite3_user_data(pCtx);
  int rc = SQLITE_OK;
  int i;

  assert( sqlite3_value_int(apVal[0])!=0
      || p->objiter.eType==RBU_PK_EXTERNAL 
      || p->objiter.eType==RBU_PK_NONE 
  );
  if( sqlite3_value_int(apVal[0])!=0 ){
    p->nPhaseOneStep += p->objiter.nIndex;
  }

  for(i=0; rc==SQLITE_OK && i<nVal; i++){
    rc = sqlite3_bind_value(p->objiter.pTmpInsert, i+1, apVal[i]);
  }
  if( rc==SQLITE_OK ){
    sqlite3_step(p->objiter.pTmpInsert);
    rc = sqlite3_reset(p->objiter.pTmpInsert);

  RbuObjIter *pIter = &p->objiter;
  sqlite3_value *pVal;
  sqlite3_stmt *pWriter;
  int i;

  assert( p->rc==SQLITE_OK );
  assert( eType!=RBU_DELETE || pIter->zIdx==0 );
  assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE
       || eType==RBU_INSERT || eType==RBU_IDX_INSERT
  );

  /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE
  ** statement below does actually delete a row, nPhaseOneStep will be
  ** incremented by the same amount when SQL function rbu_tmp_insert()
  ** is invoked by the trigger.  */
  if( eType==RBU_DELETE ){
    p->nPhaseOneStep -= p->objiter.nIndex;
  }

  if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){
    pWriter = pIter->pDelete;
  }else{
    pWriter = pIter->pInsert;
  }

  if( eType ){
    assert( eType==RBU_INSERT     || eType==RBU_DELETE
         || eType==RBU_REPLACE    || eType==RBU_IDX_DELETE
         || eType==RBU_IDX_INSERT || eType==RBU_UPDATE
    );
    assert( eType!=RBU_UPDATE || pIter->zIdx==0 );

    if( pIter->zIdx==0 && eType==RBU_IDX_DELETE ){
    if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){
      rbuBadControlError(p);
    }
    else if( eType==RBU_REPLACE ){
      if( pIter->zIdx==0 ) rbuStepOneOp(p, RBU_DELETE);
      if( pIter->zIdx==0 ){
        p->nPhaseOneStep += p->objiter.nIndex;
        rbuStepOneOp(p, RBU_DELETE);
      }
      if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT);
    }
    else if( eType!=RBU_UPDATE ){
      rbuStepOneOp(p, eType);
    }
    else{
      sqlite3_value *pVal;
      sqlite3_stmt *pUpdate = 0;
      assert( eType==RBU_UPDATE );
      p->nPhaseOneStep -= p->objiter.nIndex;
      rbuGetUpdateStmt(p, pIter, zMask, &pUpdate);
      if( pUpdate ){
        int i;
        for(i=0; p->rc==SQLITE_OK && i<pIter->nCol; i++){
          char c = zMask[pIter->aiSrcOrder[i]];
          pVal = sqlite3_column_value(pIter->pSelect, i);
          if( pIter->abTblPk[i] || c!='.' ){

          "(%d, %d), "
          "(%d, %Q), "
          "(%d, %Q), "
          "(%d, %d), "
          "(%d, %d), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %lld), "
          "(%d, %lld) ",
          p->zStateDb,
          RBU_STATE_STAGE, eStage,
          RBU_STATE_TBL, p->objiter.zTbl, 
          RBU_STATE_IDX, p->objiter.zIdx, 
          RBU_STATE_ROW, p->nStep, 
          RBU_STATE_PROGRESS, p->nProgress,
          RBU_STATE_CKPT, p->iWalCksum,
          RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie,
          RBU_STATE_OALSZ, p->iOalSz
          RBU_STATE_OALSZ, p->iOalSz,
          RBU_STATE_PHASEONESTEP, p->nPhaseOneStep
      )
    );
    assert( pInsert==0 || rc==SQLITE_OK );

    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
      rc = sqlite3_finalize(pInsert);

        pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1);
        break;

      case RBU_STATE_OALSZ:
        pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
        break;

      case RBU_STATE_PHASEONESTEP:
        pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
        break;

      default:
        rc = SQLITE_CORRUPT;
        break;
    }
  }
  rc2 = sqlite3_finalize(pStmt);
  if( rc==SQLITE_OK ) rc = rc2;

*/
static void rbuDeleteVfs(sqlite3rbu *p){
  if( p->zVfsName ){
    sqlite3rbu_destroy_vfs(p->zVfsName);
    p->zVfsName = 0;
  }
}

/*
** This user-defined SQL function is invoked with a single argument - the
** name of a table expected to appear in the target database. It returns
** the number of auxilliary indexes on the table.
*/
static void rbuIndexCntFunc(
  sqlite3_context *pCtx, 
  int nVal,
  sqlite3_value **apVal
){
  sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx);
  sqlite3_stmt *pStmt = 0;
  char *zErrmsg = 0;
  int rc;

  assert( nVal==1 );
  
  rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, 
      sqlite3_mprintf("SELECT count(*) FROM sqlite_master "
        "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0]))
  );
  if( rc!=SQLITE_OK ){
    sqlite3_result_error(pCtx, zErrmsg, -1);
  }else{
    int nIndex = 0;
    if( SQLITE_ROW==sqlite3_step(pStmt) ){
      nIndex = sqlite3_column_int(pStmt, 0);
    }
    rc = sqlite3_finalize(pStmt);
    if( rc==SQLITE_OK ){
      sqlite3_result_int(pCtx, nIndex);
    }else{
      sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1);
    }
  }

  sqlite3_free(zErrmsg);
}

/*
** If the RBU database contains the rbu_count table, use it to initialize
** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table
** is assumed to contain the same columns as:
**
**   CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
**
** There should be one row in the table for each data_xxx table in the
** database. The 'tbl' column should contain the name of a data_xxx table,
** and the cnt column the number of rows it contains.
**
** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt
** for all rows in the rbu_count table, where nIndex is the number of 
** indexes on the corresponding target database table.
*/
static void rbuInitPhaseOneSteps(sqlite3rbu *p){
  if( p->rc==SQLITE_OK ){
    sqlite3_stmt *pStmt = 0;
    int bExists = 0;                /* True if rbu_count exists */

    p->nPhaseOneStep = -1;

    p->rc = sqlite3_create_function(p->dbRbu, 
        "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0
    );
  
    /* Check for the rbu_count table. If it does not exist, or if an error
    ** occurs, nPhaseOneStep will be left set to -1. */
    if( p->rc==SQLITE_OK ){
      p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
          "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'"
      );
    }
    if( p->rc==SQLITE_OK ){
      if( SQLITE_ROW==sqlite3_step(pStmt) ){
        bExists = 1;
      }
      p->rc = sqlite3_finalize(pStmt);
    }
  
    if( p->rc==SQLITE_OK && bExists ){
      p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
          "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))"
          "FROM rbu_count"
      );
      if( p->rc==SQLITE_OK ){
        if( SQLITE_ROW==sqlite3_step(pStmt) ){
          p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0);
        }
        p->rc = sqlite3_finalize(pStmt);
      }
    }
  }
}

/*
** Open and return a new RBU handle. 
*/
sqlite3rbu *sqlite3rbu_open(
  const char *zTarget, 
  const char *zRbu,

    if( p->rc==SQLITE_OK ){
      pState = rbuLoadState(p);
      assert( pState || p->rc!=SQLITE_OK );
      if( p->rc==SQLITE_OK ){

        if( pState->eStage==0 ){ 
          rbuDeleteOalFile(p);
          rbuInitPhaseOneSteps(p);
          p->eStage = RBU_STAGE_OAL;
        }else{
          p->eStage = pState->eStage;
          p->nPhaseOneStep = pState->nPhaseOneStep;
        }
        p->nProgress = pState->nProgress;
        p->iOalSz = pState->iOalSz;
      }
    }
    assert( p->rc!=SQLITE_OK || p->eStage!=0 );

** Return the total number of key-value operations (inserts, deletes or 
** updates) that have been performed on the target database since the
** current RBU update was started.
*/
sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){
  return pRbu->nProgress;
}

/*
** Return permyriadage progress indications for the two main stages of
** an RBU update.
*/
void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
  const int MAX_PROGRESS = 10000;
  switch( p->eStage ){
    case RBU_STAGE_OAL:
      if( p->nPhaseOneStep>0 ){
        *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep);
      }else{
        *pnOne = -1;
      }
      *pnTwo = 0;
      break;

    case RBU_STAGE_MOVE:
      *pnOne = MAX_PROGRESS;
      *pnTwo = 0;
      break;

    case RBU_STAGE_CKPT:
      *pnOne = MAX_PROGRESS;
      *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame);
      break;

    case RBU_STAGE_DONE:
      *pnOne = MAX_PROGRESS;
      *pnTwo = MAX_PROGRESS;
      break;

    default:
      assert( 0 );
  }
}

int sqlite3rbu_savestate(sqlite3rbu *p){
  int rc = p->rc;
  
  if( rc==SQLITE_DONE ) return SQLITE_OK;

  assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE );

/*
** Return the total number of key-value operations (inserts, deletes or 
** updates) that have been performed on the target database since the
** current RBU update was started.
*/
sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu);

/*
** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) 
** progress indications for the two stages of an RBU update. This API may
** be useful for driving GUI progress indicators and similar.
**
** An RBU update is divided into two stages:
**
**   * Stage 1, in which changes are accumulated in an oal/wal file, and
**   * Stage 2, in which the contents of the wal file are copied into the
**     main database.
**
** The update is visible to non-RBU clients during stage 2. During stage 1
** non-RBU reader clients may see the original database.
**
** If this API is called during stage 2 of the update, output variable 
** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo)
** to a value between 0 and 10000 to indicate the permyriadage progress of
** stage 2. A value of 5000 indicates that stage 2 is half finished, 
** 9000 indicates that it is 90% finished, and so on.
**
** If this API is called during stage 1 of the update, output variable 
** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The
** value to which (*pnOne) is set depends on whether or not the RBU 
** database contains an "rbu_count" table. The rbu_count table, if it 
** exists, must contain the same columns as the following:
**
**   CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
**
** There must be one row in the table for each source (data_xxx) table within
** the RBU database. The 'tbl' column should contain the name of the source
** table. The 'cnt' column should contain the number of rows within the
** source table.
**
** If the rbu_count table is present and populated correctly and this
** API is called during stage 1, the *pnOne output variable is set to the
** permyriadage progress of the same stage. If the rbu_count table does
** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count
** table exists but is not correctly populated, the value of the *pnOne
** output variable during stage 1 is undefined.
*/
void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);

/*
** Create an RBU VFS named zName that accesses the underlying file-system
** via existing VFS zParent. Or, if the zParent parameter is passed NULL, 
** then the new RBU VFS uses the default system VFS to access the file-system.
** The new object is registered as a non-default VFS with SQLite before 
** returning.
**

    const char *zUsage;
  } aCmd[] = {
    {"step", 2, ""},              /* 0 */
    {"close", 2, ""},             /* 1 */
    {"create_rbu_delta", 2, ""},  /* 2 */
    {"savestate", 2, ""},         /* 3 */
    {"dbMain_eval", 3, "SQL"},    /* 4 */
    {"bp_progress", 2, ""},    /* 5 */
    {0,0,0}
  };
  int iCmd;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
    return TCL_ERROR;

      int rc = sqlite3_exec(db, Tcl_GetString(objv[2]), 0, 0, 0);
      if( rc!=SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(db), -1));
        ret = TCL_ERROR;
      }
      break;
    }

    case 5: /* bp_progress */ {
      int one, two;
      Tcl_Obj *pObj;
      sqlite3rbu_bp_progress(pRbu, &one, &two);

      pObj = Tcl_NewObj();
      Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one));
      Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two));
      Tcl_SetObjResult(interp, pObj);
      break;
    }

    default: /* seems unlikely */
      assert( !"cannot happen" );
      break;
  }

  return ret;

  char **azCol;                   /* NULL terminated array of col names */
  int i;
  int nCol;
  Str ct = {0, 0, 0};             /* The "CREATE TABLE data_xxx" statement */
  Str sql = {0, 0, 0};            /* Query to find differences */
  Str insert = {0, 0, 0};         /* First part of output INSERT statement */
  sqlite3_stmt *pStmt = 0;
  int nRow = 0;                   /* Total rows in data_xxx table */

  /* --rbu mode must use real primary keys. */
  g.bSchemaPK = 1;

  /* Check that the schemas of the two tables match. Exit early otherwise. */
  checkSchemasMatch(zTab);

    if( ct.z ){
      fprintf(out, "%s\n", ct.z);
      strFree(&ct);
    }

    /* Output the first part of the INSERT statement */
    fprintf(out, "%s", insert.z);
    nRow++;

    if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
      for(i=0; i<=nCol; i++){
        if( i>0 ) fprintf(out, ", ");
        printQuoted(out, sqlite3_column_value(pStmt, i));
      }
    }else{

    }

    /* And the closing bracket of the insert statement */
    fprintf(out, ");\n");
  }

  sqlite3_finalize(pStmt);
  if( nRow>0 ){
    Str cnt = {0, 0, 0};
    strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
    fprintf(out, "%s\n", cnt.z);
    strFree(&cnt);
  }

  strFree(&ct);
  strFree(&sql);
  strFree(&insert);
}

/*

  }
  rc = sqlite3_exec(g.db, "SELECT * FROM aux.sqlite_master", 0, 0, &zErrMsg);
  if( rc || zErrMsg ){
    cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2);
  }

  if( neverUseTransaction ) useTransaction = 0;
  if( useTransaction ) printf("BEGIN TRANSACTION;\n");
  if( useTransaction ) fprintf(out, "BEGIN TRANSACTION;\n");
  if( xDiff==rbudiff_one_table ){
    fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count"
           "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) "
           "WITHOUT ROWID;\n"
    );
  }
  if( zTab ){
    xDiff(zTab, out);
  }else{
    /* Handle tables one by one */
    pStmt = db_prepare(
      "SELECT name FROM main.sqlite_master\n"
      " WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"