SQLite

# 2017 January 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests for resumption of RBU operations in the
# case where the previous RBU process crashed.
#

source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rburesume

forcedelete test.db-shm test.db-oal
do_execsql_test 1.0 {
  CREATE TABLE t1(a PRIMARY KEY, b, c);
  CREATE INDEX t1a ON t1(a);
  CREATE INDEX t1b ON t1(b);
  CREATE INDEX t1c ON t1(c);
  WITH s(i) AS (
    VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50
  )
  INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s;
}
db_save_and_close

do_test 1.1 {
  list [file exists test.db] \
       [file exists test.db-wal] \
       [file exists test.db-shm] \
       [file exists test.db-oal]
} {1 0 0 0}

# Each iteration of the following loop:
#
#   1. Restores the db to the state it was in following test case 1.0
#   2. Opens an RBU vacuum and steps it $n times.
#   3. Closes the RBU vacuum handled opened in (2).
#   4. Opens a second RBU vacuum handle, resumes and completes the vacuum op. 
#
# The loop runs until $n is large enough that step (2) vacuums the entire
# database.
#
for {set n 1} {$n < 5000} {incr n} {
  db_restore
  forcedelete state.db
  sqlite3rbu_vacuum rbu test.db state.db
  for {set i 0} {$i<$n} {incr i} {
    set rc [rbu step]
    if {$rc == "SQLITE_DONE"} break
  }
  rbu close
  if {$rc == "SQLITE_DONE"} break

  do_test 1.2.$n.1 {
    sqlite3rbu_vacuum rbu test.db state.db
    while {[rbu step]=="SQLITE_OK"} {}
    rbu close
  } {SQLITE_DONE}

  do_test 1.2.$n.2 {
    sqlite3 db2 test.db
    db2 eval { 
      SELECT count(*) FROM t1;
      PRAGMA integrity_check;
    }
  } {50 ok}
  db2 close
}

# Each iteration of this loop:
#
#   1. Restores the db to the state it was in following test case 1.0
#   2. Opens an RBU vacuum and steps it $n times.
#   3. Takes a copy of all database files and the state db.
#   4. Opens a second RBU vacuum handle on the copy, resumes and completes the
#      vacuum op. 
#
# The loop runs until $n is large enough that step (2) vacuums the entire
# database.
#
for {set n 1} {$n < 5000} {incr n} {
  db_restore
  forcedelete state.db state.db-shm state.db-oal state.db-wal
  sqlite3rbu_vacuum rbu test.db state.db
  for {set i 0} {$i<$n} {incr i} {
    set rc [rbu step]
    if {$rc == "SQLITE_DONE"} break
  }
  if {$rc == "SQLITE_DONE"} {
    rbu close
    break
  }

  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
    set f2 [string map [list test.db test.db2] $f]
    if {[file exists $f]} {
      forcecopy $f $f2
    } else {
      forcedelete $f2
    }
  }
  forcecopy state.db state.db2
  rbu close

  do_test 1.3.$n.1 {
    sqlite3rbu_vacuum rbu test.db2 state.db2
    while {[rbu step]=="SQLITE_OK"} {}
    rbu close
  } {SQLITE_DONE}

  do_test 1.3.$n.2 {
    sqlite3 db2 test.db2
    db2 eval { 
      SELECT count(*) FROM t1;
      PRAGMA integrity_check;
    }
  } {50 ok}
  db2 close
}

# Each iteration of this loop:
#
#   1. Restores the db to the state it was in following test case 1.0
#   2. Opens an RBU vacuum and steps it 10 times. Then closes it.
#   2. Opens an RBU vacuum and steps it $n times.
#   3. Takes a copy of all database files and the state db.
#   4. Opens a second RBU vacuum handle on the copy, resumes and completes the
#      vacuum op. 
#
# The loop runs until $n is large enough that step (3) vacuums the entire
# database.
#
for {set n 1} {$n < 5000} {incr n} {
  db_restore
  forcedelete state.db state.db-shm state.db-oal state.db-wal

  sqlite3rbu_vacuum rbu test.db state.db
  for {set i 0} {$i<10} {incr i} {
    rbu step
  }
  rbu close

  sqlite3rbu_vacuum rbu test.db state.db
  for {set i 0} {$i<$n} {incr i} {
    set rc [rbu step]
    if {$rc == "SQLITE_DONE"} break
  }
  if {$rc == "SQLITE_DONE"} {
    rbu close
    break
  }

  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
    set f2 [string map [list test.db test.db2] $f]
    if {[file exists $f]} {
      forcecopy $f $f2
    } else {
      forcedelete $f2
    }
  }
  forcecopy state.db state.db2
  rbu close

  do_test 1.4.$n.1 {
    sqlite3rbu_vacuum rbu test.db2 state.db2
    while {[rbu step]=="SQLITE_OK"} {}
    rbu close
  } {SQLITE_DONE}

  do_test 1.4.$n.2 {
    sqlite3 db2 test.db2
    db2 eval { 
      SELECT count(*) FROM t1;
      PRAGMA integrity_check;
    }
  } {50 ok}
  db2 close
}

forcedelete rbu.db
do_test 2.0 {
  sqlite3 db2 rbu.db
  db2 eval {
    CREATE TABLE data_t1(a, b, c, rbu_control);
    WITH s(i) AS (
        VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10
    )
    INSERT INTO data_t1 
      SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s;
  }
  db2 close
} {}

# Each iteration of this loop:
#
#   1. Restores the db to the state it was in following test case 1.0
#   2. Opens an RBU handle to apply the RBU update created in test case 2.0.
#   3. Steps the RBU handle $n times.
#   4. Takes a copy of all database files and the state db.
#   5. Opens a second RBU handle on the copy, resumes and completes the
#      RBU op. Checks it worked as expected.
#
# The loop runs until $n is large enough that step (3) applies the entire
# update.
#
for {set n 1} {$n < 5000} {incr n} {
  db_restore
  forcedelete state.db state.db-shm state.db-oal state.db-wal
  sqlite3rbu rbu test.db rbu.db state.db

  for {set i 0} {$i<$n} {incr i} {
    set rc [rbu step]
    if {$rc == "SQLITE_DONE"} break
  }
  if {$rc == "SQLITE_DONE"} {
    rbu close
    break
  }

  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
    set f2 [string map [list test.db test.db2] $f]
    if {[file exists $f]} {
      forcecopy $f $f2
    } else {
      forcedelete $f2
    }
  }
  forcecopy state.db state.db2
  rbu close

  do_test 2.$n.1 {
    sqlite3rbu rbu test.db2 rbu.db state.db2
    while {[rbu step]=="SQLITE_OK"} {}
    rbu close
  } {SQLITE_DONE}

  do_test 2.$n.2 {
    sqlite3 db2 test.db2
    db2 eval { 
      SELECT count(*) FROM t1;
      PRAGMA integrity_check;
    }
  } {60 ok}
  db2 close
}

finish_test

    rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p);
    if( rc!=SQLITE_NOTFOUND ) p->rc = rc;
    if( p->eStage>=RBU_STAGE_MOVE ){
      bOpen = 1;
    }else{
      RbuState *pState = rbuLoadState(p);
      if( pState ){
        bOpen = (pState->eStage>=RBU_STAGE_MOVE);
        rbuFreeState(pState);
      }
    }
    if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1);
  }

  p->eStage = 0;

    nCell = pPage->nCell;

    for(i=0; i<nCell; i++){
      u8 *pCell = findCell(pPage, i);
      if( eType==PTRMAP_OVERFLOW1 ){
        CellInfo info;
        pPage->xParseCell(pPage, pCell, &info);
        if( info.nLocal<info.nPayload ){
          if( pCell+info.nSize > pPage->aData+pPage->pBt->usableSize ){
            return SQLITE_CORRUPT_BKPT;
          }
          if( iFrom==get4byte(pCell+info.nSize-4) ){

            put4byte(pCell+info.nSize-4, iTo);
            break;
          }
        }
      }else{
        if( get4byte(pCell)==iFrom ){
          put4byte(pCell, iTo);
          break;
        }
      }

  ** usableSpace: Number of bytes of space available on each sibling.
  ** 
  */
  usableSpace = pBt->usableSize - 12 + leafCorrection;
  for(i=0; i<nOld; i++){
    MemPage *p = apOld[i];
    szNew[i] = usableSpace - p->nFree;

    for(j=0; j<p->nOverflow; j++){
      szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]);
    }
    cntNew[i] = cntOld[i];
  }
  k = nOld;
  for(i=0; i<k; i++){

** if pX->nMem is non-zero, then pX->aMem contains pointers to the unpacked
** key values and pX->aMem can be used instead of pX->pKey to avoid having
** to decode the key.
*/
int sqlite3BtreeInsert(
  BtCursor *pCur,                /* Insert data into the table of this cursor */
  const BtreePayload *pX,        /* Content of the row to be inserted */
  int flags,                     /* True if this is likely an append */
  int seekResult                 /* Result of prior MovetoUnpacked() call */
){
  int rc;
  int loc = seekResult;          /* -1: before desired location  +1: after */
  int szNew = 0;
  int idx;
  MemPage *pPage;
  Btree *p = pCur->pBtree;
  BtShared *pBt = p->pBt;
  unsigned char *oldCell;
  unsigned char *newCell = 0;

  assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND))==flags );

  if( pCur->eState==CURSOR_FAULT ){
    assert( pCur->skipNext!=SQLITE_OK );
    return pCur->skipNext;
  }

  assert( cursorOwnsBtShared(pCur) );

  if( pCur->pKeyInfo==0 ){
    assert( pX->pKey==0 );
    /* If this is an insert into a table b-tree, invalidate any incrblob 
    ** cursors open on the row being replaced */
    invalidateIncrblobCursors(p, pX->nKey, 0);

    /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing 
    ** to a row with the same key as the new entry being inserted.  */
    assert( (flags & BTREE_SAVEPOSITION)==0 || 
            ((pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey) );

    /* If the cursor is currently on the last row and we are appending a
    ** new row onto the end, set the "loc" to avoid an unnecessary
    ** btreeMoveto() call */
    if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){
      loc = 0;
    }else if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0
               && pCur->info.nKey==pX->nKey-1 ){
      loc = -1;
    }else if( loc==0 ){
      rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc);
      if( rc ) return rc;
    }
  }else if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){
    if( pX->nMem ){
      UnpackedRecord r;
      r.pKeyInfo = pCur->pKeyInfo;
      r.aMem = pX->aMem;
      r.nField = pX->nMem;
      r.default_rc = 0;
      r.errCode = 0;
      r.r1 = 0;
      r.r2 = 0;
      r.eqSeen = 0;
      rc = sqlite3BtreeMovetoUnpacked(pCur, &r, 0, flags!=0, &loc);
    }else{
      rc = btreeMoveto(pCur, pX->pKey, pX->nKey, flags!=0, &loc);
    }
    if( rc ) return rc;
  }
  assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) );

  pPage = pCur->apPage[pCur->iPage];
  assert( pPage->intKey || pX->nKey>=0 );

    /* Must make sure nOverflow is reset to zero even if the balance()
    ** fails. Internal data structure corruption will result otherwise. 
    ** Also, set the cursor state to invalid. This stops saveCursorPosition()
    ** from trying to save the current position of the cursor.  */
    pCur->apPage[pCur->iPage]->nOverflow = 0;
    pCur->eState = CURSOR_INVALID;
    if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){
      rc = moveToRoot(pCur);
      if( pCur->pKeyInfo ){
        assert( pCur->pKey==0 );
        pCur->pKey = sqlite3Malloc( pX->nKey );
        if( pCur->pKey==0 ){
          rc = SQLITE_NOMEM;
        }else{
          memcpy(pCur->pKey, pX->pKey, pX->nKey);
        }
      }
      pCur->eState = CURSOR_REQUIRESEEK;
      pCur->nKey = pX->nKey;
    }
  }
  assert( pCur->apPage[pCur->iPage]->nOverflow==0 );

end_insert:
  return rc;
}

  int bias,
  int *pRes
);
int sqlite3BtreeCursorHasMoved(BtCursor*);
int sqlite3BtreeCursorRestore(BtCursor*, int*);
int sqlite3BtreeDelete(BtCursor*, u8 flags);

/* Allowed flags for sqlite3BtreeDelete() and sqlite3BtreeInsert() */
#define BTREE_SAVEPOSITION 0x02  /* Leave cursor pointing at NEXT or PREV */
#define BTREE_AUXDELETE    0x04  /* not the primary delete operation */
#define BTREE_APPEND       0x08  /* Insert is likely an append */

/* An instance of the BtreePayload object describes the content of a single
** entry in either an index or table btree.
**
** Index btrees (used for indexes and also WITHOUT ROWID tables) contain
** an arbitrary key and no data.  These btrees have pKey,nKey set to their
** key and pData,nData,nZero set to zero.

  struct Mem *aMem;       /* First of nMem value in the unpacked pKey */
  u16 nMem;               /* Number of aMem[] value.  Might be zero */
  int nData;              /* Size of pData.  0 if none. */
  int nZero;              /* Extra zero data appended after pData,nData */
};

int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload,
                       int flags, int seekResult);
int sqlite3BtreeFirst(BtCursor*, int *pRes);
int sqlite3BtreeLast(BtCursor*, int *pRes);
int sqlite3BtreeNext(BtCursor*, int *pRes);
int sqlite3BtreeEof(BtCursor*);
int sqlite3BtreePrevious(BtCursor*, int *pRes);
i64 sqlite3BtreeIntegerKey(BtCursor*);
int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*);

void sqlite3CompleteInsertion(
  Parse *pParse,      /* The parser context */
  Table *pTab,        /* the table into which we are inserting */
  int iDataCur,       /* Cursor of the canonical data source */
  int iIdxCur,        /* First index cursor */
  int regNewData,     /* Range of content */
  int *aRegIdx,       /* Register used by each index.  0 for unused indices */
  int update_flags,   /* True for UPDATE, False for INSERT */
  int appendBias,     /* True if this is likely to be an append */
  int useSeekResult   /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */
){
  Vdbe *v;            /* Prepared statements under construction */
  Index *pIdx;        /* An index being inserted or updated */
  u8 pik_flags;       /* flag values passed to the btree insert */
  int regData;        /* Content registers (after the rowid) */
  int regRec;         /* Register holding assembled record for the table */
  int i;              /* Loop counter */
  u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */

  assert( update_flags==0
       || update_flags==OPFLAG_ISUPDATE
       || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION)
  );

  v = sqlite3GetVdbe(pParse);
  assert( v!=0 );
  assert( pTab->pSelect==0 );  /* This table is not a VIEW */
  for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){
    if( aRegIdx[i]==0 ) continue;
    bAffinityDone = 1;
    if( pIdx->pPartIdxWhere ){
      sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2);
      VdbeCoverage(v);
    }
    sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i],
                         aRegIdx[i]+1,
                         pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn);
    pik_flags = 0;
    if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT;
    if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){
      assert( pParse->nested==0 );
      pik_flags |= OPFLAG_NCHANGE;
      pik_flags |= (update_flags & OPFLAG_SAVEPOSITION);
    }
    sqlite3VdbeChangeP5(v, pik_flags);
  }
  if( !HasRowid(pTab) ) return;
  regData = regNewData + 1;
  regRec = sqlite3GetTempReg(pParse);
  sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec);
  if( !bAffinityDone ){
    sqlite3TableAffinity(v, pTab, 0);
    sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol);
  }
  if( pParse->nested ){
    pik_flags = 0;
  }else{
    pik_flags = OPFLAG_NCHANGE;
    pik_flags |= (update_flags?update_flags:OPFLAG_LASTROWID);
  }
  if( appendBias ){
    pik_flags |= OPFLAG_APPEND;
  }
  if( useSeekResult ){
    pik_flags |= OPFLAG_USESEEKRESULT;
  }

/*
** Do a memory allocation with statistics and alarms.  Assume the
** lock is already held.
*/
static void mallocWithAlarm(int n, void **pp){
  void *p;
  int nFull = 0;
  assert( sqlite3_mutex_held(mem0.mutex) );
  sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n);
  if( mem0.alarmThreshold>0 ){
    sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED);
    nFull = sqlite3GlobalConfig.m.xRoundup(n);
    if( nUsed >= mem0.alarmThreshold - nFull ){
      mem0.nearlyFull = 1;
      sqlite3MallocAlarm(nFull);
    }else{
      mem0.nearlyFull = 0;
    }
  }
  p = sqlite3GlobalConfig.m.xMalloc(n);
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
  if( p==0 && mem0.alarmThreshold>0 ){
    sqlite3MallocAlarm(nFull);
    p = sqlite3GlobalConfig.m.xMalloc(n);
  }
#endif
  if( p ){
    nFull = sqlite3MallocSize(p);
    sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull);
    sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1);
  }
  *pp = p;
}

/*

        }
  
        /* This case runs if the aggregate has no GROUP BY clause.  The
        ** processing is much simpler since there is only a single row
        ** of output.
        */
        resetAccumulator(pParse, &sAggInfo);
        pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax, 0,flag,0);
        if( pWInfo==0 ){
          sqlite3ExprListDelete(db, pDel);
          goto select_end;
        }
        updateAccumulator(pParse, &sAggInfo);
        assert( pMinMax==0 || pMinMax->nExpr==1 );
        if( sqlite3WhereIsOrdered(pWInfo)>0 ){

**    OPFLAG_FORDELETE    == BTREE_FORDELETE
**    OPFLAG_SAVEPOSITION == BTREE_SAVEPOSITION
**    OPFLAG_AUXDELETE    == BTREE_AUXDELETE
*/
#define OPFLAG_NCHANGE       0x01    /* OP_Insert: Set to update db->nChange */
                                     /* Also used in P2 (not P5) of OP_Delete */
#define OPFLAG_EPHEM         0x01    /* OP_Column: Ephemeral output is ok */
#define OPFLAG_LASTROWID     0x20    /* Set to update db->lastRowid */
#define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
#define OPFLAG_APPEND        0x08    /* This is likely to be an append */
#define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
#define OPFLAG_ISNOOP        0x40    /* OP_Delete does pre-update-hook only */
#endif
#define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
#define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
#define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
#define OPFLAG_SEEKEQ        0x02    /* OP_Open** cursor uses EQ seek only */
#define OPFLAG_FORDELETE     0x08    /* OP_Open should use BTREE_FORDELETE */
#define OPFLAG_P2ISREG       0x10    /* P2 to OP_Open** is a register number */
#define OPFLAG_PERMUTE       0x01    /* OP_Compare: use the permutation */
#define OPFLAG_SAVEPOSITION  0x02    /* OP_Delete/Insert: save cursor pos */
#define OPFLAG_AUXDELETE     0x04    /* OP_Delete: index in a DELETE op */

/*
 * Each trigger present in the database schema is stored as an instance of
 * struct Trigger.
 *
 * Pointers to instances of struct Trigger are stored in two ways.

    if( rc ){
      Tcl_AppendResult(interp, "Error: ", sqlite3_errmsg(pDb->db), (char*)0);
      sqlite3_finalize(pStmt);
      return TCL_ERROR;
    }
    in = fopen(zFile, "rb");
    if( in==0 ){
      Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, (char*)0);
      sqlite3_finalize(pStmt);
      return TCL_ERROR;
    }
    azCol = malloc( sizeof(azCol[0])*(nCol+1) );
    if( azCol==0 ) {
      Tcl_AppendResult(interp, "Error: can't malloc()", (char*)0);
      fclose(in);

      return TCL_ERROR;
    }
    for(i=3; i<(objc-1); i++){
      const char *z = Tcl_GetString(objv[i]);
      int n = strlen30(z);
      if( n>2 && strncmp(z, "-argcount",n)==0 ){
        if( i==(objc-2) ){
          Tcl_AppendResult(interp, "option requires an argument: ", z,(char*)0);
          return TCL_ERROR;
        }
        if( Tcl_GetIntFromObj(interp, objv[i+1], &nArg) ) return TCL_ERROR;
        if( nArg<0 ){
          Tcl_AppendResult(interp, "number of arguments must be non-negative",
                           (char*)0);
          return TCL_ERROR;
        }
        i++;
      }else
      if( n>2 && strncmp(z, "-deterministic",n)==0 ){
        flags |= SQLITE_DETERMINISTIC;
      }else{
        Tcl_AppendResult(interp, "bad option \"", z,
            "\": must be -argcount or -deterministic", (char*)0
        );
        return TCL_ERROR;
      }
    }

    pScript = objv[objc-1];
    zName = Tcl_GetStringFromObj(objv[2], 0);

        }

        if( rc==SQLITE_OK ){
          Tcl_Obj *pObj;
          pObj = Tcl_NewStringObj((char*)sqlite3_value_text(pValue), -1);
          Tcl_SetObjResult(interp, pObj);
        }else{
          Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), (char*)0);
          return TCL_ERROR;
        }
      }
    }
#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
    break;
  }

  Index *pIdx;           /* For looping over indices */
  Index *pPk;            /* The PRIMARY KEY index for WITHOUT ROWID tables */
  int nIdx;              /* Number of indices that need updating */
  int iBaseCur;          /* Base cursor number */
  int iDataCur;          /* Cursor for the canonical data btree */
  int iIdxCur;           /* Cursor for the first index */
  sqlite3 *db;           /* The database structure */
  int *aRegIdx = 0;      /* First register in array assigned to each index */
  int *aXRef = 0;        /* aXRef[i] is the index in pChanges->a[] of the
                         ** an expression for the i-th column of the table.
                         ** aXRef[i]==-1 if the i-th column is not changed. */
  u8 *aToOpen;           /* 1 for tables and indices to be opened */
  u8 chngPk;             /* PRIMARY KEY changed in a WITHOUT ROWID table */
  u8 chngRowid;          /* Rowid changed in a normal table */
  u8 chngKey;            /* Either chngPk or chngRowid */
  Expr *pRowidExpr = 0;  /* Expression defining the new record number */
  AuthContext sContext;  /* The authorization context */
  NameContext sNC;       /* The name-context to resolve expressions in */
  int iDb;               /* Database containing the table being updated */
  int eOnePass;          /* ONEPASS_XXX value from where.c */
  int hasFK;             /* True if foreign key processing is required */
  int labelBreak;        /* Jump here to break out of UPDATE loop */
  int labelContinue;     /* Jump here to continue next step of UPDATE loop */
  int flags;             /* Flags for sqlite3WhereBegin() */

#ifndef SQLITE_OMIT_TRIGGER
  int isView;            /* True when updating a view (INSTEAD OF trigger) */
  Trigger *pTrigger;     /* List of triggers on pTab, if required */
  int tmask;             /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */
#endif
  int newmask;           /* Mask of NEW.* columns accessed by BEFORE triggers */
  int iEph = 0;          /* Ephemeral table holding all primary key values */
  int nKey = 0;          /* Number of elements in regKey for WITHOUT ROWID */
  int aiCurOnePass[2];   /* The write cursors opened by WHERE_ONEPASS */
  int addrOpen = 0;      /* Address of OP_OpenEphemeral */
  int iPk = 0;           /* First of nPk cells holding PRIMARY KEY value */
  i16 nPk = 0;           /* Number of components of the PRIMARY KEY */
  int bReplace = 0;      /* True if REPLACE conflict resolution might happen */

  /* Register Allocations */
  int regRowCount = 0;   /* A count of rows changed */
  int regOldRowid = 0;   /* The old rowid */
  int regNewRowid = 0;   /* The new rowid */
  int regNew = 0;        /* Content of the NEW.* table in triggers */
  int regOld = 0;        /* Content of OLD.* table in triggers */

    }else{
      reg = 0;
      for(i=0; i<pIdx->nKeyCol; i++){
        i16 iIdxCol = pIdx->aiColumn[i];
        if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){
          reg = ++pParse->nMem;
          pParse->nMem += pIdx->nColumn;
          if( (onError==OE_Replace)
           || (onError==OE_Default && pIdx->onError==OE_Replace) 
          ){
            bReplace = 1;
          }
          break;
        }
      }
    }
    if( reg==0 ) aToOpen[j+1] = 0;
    aRegIdx[j] = reg;
  }
  if( bReplace ){
    /* If REPLACE conflict resolution might be invoked, open cursors on all 
    ** indexes in case they are needed to delete records.  */
    memset(aToOpen, 1, nIdx+1);
  }

  /* Begin generating code. */
  v = sqlite3GetVdbe(pParse);
  if( v==0 ) goto update_cleanup;
  if( pParse->nested==0 ) sqlite3VdbeCountChanges(v);
  sqlite3BeginWriteOperation(pParse, 1, iDb);

  if( IsVirtual(pTab) ){
    updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef,
                       pWhere, onError);
    goto update_cleanup;
  }
#endif

  /* Initialize the count of updated rows */




  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){




    regRowCount = ++pParse->nMem;


    sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);


  }



  if( HasRowid(pTab) ){
    sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid);
  }else{




    assert( pPk!=0 );
    nPk = pPk->nKeyCol;
    iPk = pParse->nMem+1;
    pParse->nMem += nPk;
    regKey = ++pParse->nMem;
    iEph = pParse->nTab++;

    sqlite3VdbeAddOp2(v, OP_Null, 0, iPk);
    addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
    sqlite3VdbeSetP4KeyInfo(pParse, pPk);
  }

  /* Begin the database scan. 
  **
  ** Do not consider a single-pass strategy for a multi-row update if
  ** there are any triggers or foreign keys to process, or rows may
  ** be deleted as a result of REPLACE conflict handling. Any of these
  ** things might disturb a cursor being used to scan through the table
  ** or index, causing a single-pass approach to malfunction.  */
  flags = WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE;
  if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
    flags |= WHERE_ONEPASS_MULTIROW;
  }
  pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);

  if( pWInfo==0 ) goto update_cleanup;

  /* A one-pass strategy that might update more than one row may not
  ** be used if any column of the index used for the scan is being
  ** updated. Otherwise, if there is an index on "b", statements like
  ** the following could create an infinite loop:
  **
  **   UPDATE t1 SET b=b+1 WHERE b>?
  **
  ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
  ** strategy that uses an index for which one or more columns are being
  ** updated.  */
  eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
  if( eOnePass==ONEPASS_MULTI ){
    int iCur = aiCurOnePass[1];
    if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
      eOnePass = ONEPASS_OFF;
    }
    assert( iCur!=iDataCur || !HasRowid(pTab) );
  }
  
  if( HasRowid(pTab) ){
    /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF
    ** mode, write the rowid into the FIFO. In either of the one-pass modes,
    ** leave it in register regOldRowid.  */
    sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid);
    if( eOnePass==ONEPASS_OFF ){
      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid);
    }
  }else{
    /* Read the PK of the current row into an array of registers. In
    ** ONEPASS_OFF mode, serialize the array into a record and store it in
    ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change
    ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table 
    ** is not required) and leave the PK fields in the array of registers.  */
    for(i=0; i<nPk; i++){
      assert( pPk->aiColumn[i]>=0 );
      sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i);

    }
    if( eOnePass ){
      sqlite3VdbeChangeToNoop(v, addrOpen);
      nKey = nPk;
      regKey = iPk;
    }else{
      sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey,
                        sqlite3IndexAffinityStr(db, pPk), nPk);
      sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk);
    }

  }





  if( eOnePass!=ONEPASS_MULTI ){
    sqlite3WhereEnd(pWInfo);
  }

  labelBreak = sqlite3VdbeMakeLabel(v);
  if( !isView ){
    int addrOnce = 0;

    /* Open every index that needs updating. */




    if( eOnePass!=ONEPASS_OFF ){
      if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;



      if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;

    }




    if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
      addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
    }
    sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen,
                               0, 0);
    if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
  }

  /* Top of the update loop */
  if( eOnePass!=ONEPASS_OFF ){
    if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
      assert( pPk );
      sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey);
      VdbeCoverageNeverTaken(v);
    }
    if( eOnePass==ONEPASS_SINGLE ){
      labelContinue = labelBreak;
    }else{
      labelContinue = sqlite3VdbeMakeLabel(v);
    }
    sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
    VdbeCoverageIf(v, pPk==0);
    VdbeCoverageIf(v, pPk!=0);
  }else if( pPk ){
    labelContinue = sqlite3VdbeMakeLabel(v);
    sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
    addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);

        sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i);
      }
    }
  }

  if( !isView ){
    int addr1 = 0;        /* Address of jump instruction */


    /* Do constraint checks. */
    assert( regOldRowid>0 );
    sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur,
        regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace,
        aXRef);

    ** pre-update hook. If the caller invokes preupdate_new(), the returned
    ** value is copied from memory cell (regNewRowid+1+iCol), where iCol
    ** is the column index supplied by the user.
    */
    assert( regNew==regNewRowid+1 );
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
    sqlite3VdbeAddOp3(v, OP_Delete, iDataCur,
        OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP),
        regNewRowid
    );
    if( eOnePass==ONEPASS_MULTI ){
      assert( hasFK==0 && chngKey==0 );
      sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION);
    }
    if( !pParse->nested ){
      sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
    }
#else
    if( hasFK || chngKey ){
      sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0);
    }
#endif
    if( bReplace || chngKey ){
      sqlite3VdbeJumpHere(v, addr1);
    }

    if( hasFK ){
      sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey);
    }
  
    /* Insert the new index entries and the new record. */
    sqlite3CompleteInsertion(
        pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx, 
        OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0), 
        0, 0
    );

    /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to
    ** handle rows (possibly in other tables) that refer via a foreign key
    ** to the row just updated. */ 
    if( hasFK ){
      sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey);
    }

  sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, 
      TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue);

  /* Repeat the above with the next record to be updated, until
  ** all record selected by the WHERE clause have been updated.
  */
  if( eOnePass==ONEPASS_SINGLE ){
    /* Nothing to do at end-of-loop for a single-pass */
  }else if( eOnePass==ONEPASS_MULTI ){
    sqlite3VdbeResolveLabel(v, labelContinue);
    sqlite3WhereEnd(pWInfo);
  }else if( pPk ){
    sqlite3VdbeResolveLabel(v, labelContinue);
    sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v);
  }else{
    sqlite3VdbeGoto(v, labelContinue);
  }
  sqlite3VdbeResolveLabel(v, labelBreak);

  if( pData->flags & MEM_Zero ){
    x.nZero = pData->u.nZero;
  }else{
    x.nZero = 0;
  }
  x.pKey = 0;
  rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
      (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), seekResult
  );
  pC->deferredMoveto = 0;
  pC->cacheStatus = CACHE_STALE;

  /* Invoke the update-hook if required. */
  if( rc ) goto abort_due_to_error;
  if( db->xUpdateCallback && op ){

    rc = sqlite3VdbeSorterWrite(pC, pIn2);
  }else{
    x.nKey = pIn2->n;
    x.pKey = pIn2->z;
    x.aMem = aMem + pOp->p3;
    x.nMem = (u16)pOp->p4.i;
    rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
         (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), 
        ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
        );
    assert( pC->deferredMoveto==0 );
    pC->cacheStatus = CACHE_STALE;
  }
  if( rc) goto abort_due_to_error;
  break;

            assert( x>=0 );
          }
          x = sqlite3ColumnOfIndex(pIdx, x);
          if( x>=0 ){
            pOp->p2 = x;
            pOp->p1 = pLevel->iIdxCur;
          }
          assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 
              || pWInfo->eOnePass );
        }else if( pOp->opcode==OP_Rowid ){
          pOp->p1 = pLevel->iIdxCur;
          pOp->opcode = OP_IdxRowid;
        }
      }
    }
  }

do_extract_hints_test 1.6 {
  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) LEFT JOIN t3 ON (d=f);
} {
  t2 {EQ(r[2],c0)} t3 {EQ(r[6],c1)}
}

if 0 {
  do_extract_hints_test 1.7 {
    SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND d=e) LEFT JOIN t3 ON (d=f);
  } {
    t2 {EQ(r[2],c0)} t3 {AND(EQ(r[6],c0),EQ(r[7],c1))}
  }
}

#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
  CREATE TABLE x1(x, y);
  CREATE TABLE x2(a, b);

# 2017 January 9
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix update2

db func repeat [list string repeat]

#-------------------------------------------------------------------------
# 1.1.* A one-pass UPDATE that does balance() operations on the IPK index
#       that it is scanning.
#
# 1.2.* Same again, but with a WITHOUT ROWID table.
#
set nrow [expr 10]
do_execsql_test 1.1.0 {
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
  CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
  WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow )
  INSERT INTO t1(b) SELECT char((i % 26) + 65) FROM s;
  INSERT INTO t2 SELECT * FROM t1;
}

do_execsql_test 1.1.1 {
  UPDATE t1 SET b = repeat(b, 100)
}

do_execsql_test 1.1.2 {
  SELECT * FROM t1;
} [db eval { SELECT a, repeat(b, 100) FROM t2 }]

do_execsql_test 1.2.0 {
  DROP TABLE t1;
  CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID;
  WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow )
  INSERT INTO t1(a, b) SELECT i+1, char((i % 26) + 65) FROM s;
}

#explain_i { UPDATE t1 SET b = repeat(b, 100) }
do_execsql_test 1.2.1 {
  UPDATE t1 SET b = repeat(b, 100)
}

do_execsql_test 1.2.2 {
  SELECT * FROM t1;
} [db eval { SELECT a, repeat(b, 100) FROM t2 }]


#-------------------------------------------------------------------------
# A one-pass UPDATE that does balance() operations on the IPK index
# that it is scanning.
#
do_execsql_test 2.1 {
  CREATE TABLE t3(a PRIMARY KEY, b, c);
  CREATE INDEX t3i ON t3(b);
} {}
do_execsql_test 2.2 { UPDATE t3 SET c=1 WHERE b=?      } {}
do_execsql_test 2.3 { UPDATE t3 SET c=1 WHERE rowid=?  } {}

#-------------------------------------------------------------------------
#
do_execsql_test 3.0 {
  CREATE TABLE t4(a PRIMARY KEY, b, c) WITHOUT ROWID;
  CREATE INDEX t4c ON t4(c);
  INSERT INTO t4 VALUES(1, 2, 3);
  INSERT INTO t4 VALUES(2, 3, 4);
}

do_execsql_test 3.1 {
  UPDATE t4 SET c=c+2 WHERE c>2;
  SELECT a, c FROM t4 ORDER BY a;
} {1 5 2 6}

#-------------------------------------------------------------------------
#
foreach {tn sql} {
  1 { 
    CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c);
    CREATE TABLE c1(a INTEGER PRIMARY KEY, b, c, d)
  }
  2 { 
    CREATE TABLE b1(a INT PRIMARY KEY, b, c) WITHOUT ROWID;
    CREATE TABLE c1(a INT PRIMARY KEY, b, c, d) WITHOUT ROWID;
  }
} {
  execsql { DROP TABLE IF EXISTS b1; DROP TABLE IF EXISTS c1; }
  execsql $sql

  do_execsql_test 4.$tn.0 {
    CREATE UNIQUE INDEX b1c ON b1(c);
    INSERT INTO b1 VALUES(1, 'a', 1);
    INSERT INTO b1 VALUES(2, 'b', 15);
    INSERT INTO b1 VALUES(3, 'c', 3);
    INSERT INTO b1 VALUES(4, 'd', 4);
    INSERT INTO b1 VALUES(5, 'e', 5);
    INSERT INTO b1 VALUES(6, 'f', 6);
    INSERT INTO b1 VALUES(7, 'g', 7);
  }

  do_execsql_test 4.$tn.1 {
    UPDATE OR REPLACE b1 SET c=c+10 WHERE a BETWEEN 4 AND 7;
    SELECT * FROM b1 ORDER BY a;
  } {
    1 a 1
    3 c 3
    4 d 14
    5 e 15
    6 f 16
    7 g 17
  }

  do_execsql_test 4.$tn.2 {
    CREATE INDEX c1d ON c1(d, b);
    CREATE UNIQUE INDEX c1c ON c1(c, b);

    INSERT INTO c1 VALUES(1, 'a', 1,  1);
    INSERT INTO c1 VALUES(2, 'a', 15, 2);
    INSERT INTO c1 VALUES(3, 'a', 3,  3);
    INSERT INTO c1 VALUES(4, 'a', 4,  4);
    INSERT INTO c1 VALUES(5, 'a', 5,  5);
    INSERT INTO c1 VALUES(6, 'a', 6,  6);
    INSERT INTO c1 VALUES(7, 'a', 7,  7);
  }

  do_execsql_test 4.$tn.3 {
    UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7;
    SELECT * FROM c1 ORDER BY a;
  } {
    1 a 1 1
    3 a 3 3
    4 a 14 4
    5 a 15 5
    6 a 16 6
    7 a 17 7
  }

  do_execsql_test 4.$tn.4 { PRAGMA integrity_check } ok

  do_execsql_test 4.$tn.5 {
    DROP INDEX c1d;
    DROP INDEX c1c;
    DELETE FROM c1;

    INSERT INTO c1 VALUES(1, 'a', 1,  1);
    INSERT INTO c1 VALUES(2, 'a', 15, 2);
    INSERT INTO c1 VALUES(3, 'a', 3,  3);
    INSERT INTO c1 VALUES(4, 'a', 4,  4);
    INSERT INTO c1 VALUES(5, 'a', 5,  5);
    INSERT INTO c1 VALUES(6, 'a', 6,  6);
    INSERT INTO c1 VALUES(7, 'a', 7,  7);

    CREATE INDEX c1d ON c1(d);
    CREATE UNIQUE INDEX c1c ON c1(c);
  }

  do_execsql_test 4.$tn.6 {
    UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7;
    SELECT * FROM c1 ORDER BY a;
  } {
    1 a 1 1
    3 a 3 3
    4 a 14 4
    5 a 15 5
    6 a 16 6
    7 a 17 7
  }
}

finish_test