/* ** 2014 August 30 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* */ #include #include #include #include #include "sqlite3.h" #include "sqlite3ota.h" /* ** The ota_state table is used to save the state of a partially applied ** update so that it can be resumed later. The table consists of integer ** keys mapped to values as follows: ** ** OTA_STATE_STAGE: ** May be set to integer values 1, 2 or 3. As follows: ** 0: Nothing at all has been done. ** 1: the *-ota file is currently under construction. ** 2: the *-ota file has been constructed, but not yet moved ** to the *-wal path. ** 3: the checkpoint is underway. ** ** OTA_STATE_TBL: ** Only valid if STAGE==1. The target database name of the table ** currently being written. ** ** OTA_STATE_IDX: ** Only valid if STAGE==1. The target database name of the index ** currently being written, or NULL if the main table is currently being ** updated. ** ** OTA_STATE_ROW: ** Only valid if STAGE==1. Number of rows already processed for the current ** table/index. ** ** OTA_STATE_PROGRESS: ** Total number of sqlite3ota_step() calls made so far as part of this ** ota update. ** ** OTA_STATE_CKPT: ** Valid if STAGE==3. The blob to pass to sqlite3ckpt_start() to resume ** the incremental checkpoint. ** */ #define OTA_STATE_STAGE 1 #define OTA_STATE_TBL 2 #define OTA_STATE_IDX 3 #define OTA_STATE_ROW 4 #define OTA_STATE_PROGRESS 5 #define OTA_STATE_CKPT 6 #define OTA_STAGE_OAL 1 #define OTA_STAGE_COPY 2 #define OTA_STAGE_CKPT 3 #define OTA_STAGE_DONE 4 #define OTA_CREATE_STATE "CREATE TABLE IF NOT EXISTS ota.ota_state" \ "(k INTEGER PRIMARY KEY, v)" typedef struct OtaState OtaState; typedef struct OtaObjIter OtaObjIter; /* ** A structure to store values read from the ota_state table in memory. */ struct OtaState { int eStage; char *zTbl; char *zIdx; unsigned char *pCkptState; int nCkptState; int nRow; sqlite3_int64 nProgress; }; /* ** An iterator of this type is used to iterate through all objects in ** the target database that require updating. For each such table, the ** iterator visits, in order: ** ** * the table itself, ** * each index of the table (zero or more points to visit), and ** * a special "cleanup table" point. */ struct OtaObjIter { sqlite3_stmt *pTblIter; /* Iterate through tables */ sqlite3_stmt *pIdxIter; /* Index iterator */ int nTblCol; /* Size of azTblCol[] array */ char **azTblCol; /* Array of quoted column names */ unsigned char *abTblPk; /* Array of flags - true for PK columns */ /* Output variables. zTbl==0 implies EOF. */ int bCleanup; /* True in "cleanup" state */ const char *zTbl; /* Name of target db table */ const char *zIdx; /* Name of target db index (or null) */ int iVisit; /* Number of points visited, incl. current */ /* Statements created by otaObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ sqlite3_stmt *pSelect; /* Source data */ sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ /* Last UPDATE used (for PK b-tree updates only), or NULL. */ char *zMask; /* Copy of update mask used with pUpdate */ sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ }; /* ** OTA handle. */ struct sqlite3ota { int eStage; /* Value of OTA_STATE_STAGE field */ sqlite3 *db; /* "main" -> target db, "ota" -> ota db */ char *zTarget; /* Path to target db */ char *zOta; /* Path to ota db */ int rc; /* Value returned by last ota_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ int nStep; /* Rows processed for current object */ int nProgress; /* Rows processed for all objects */ OtaObjIter objiter; /* Iterator for skipping through tbl/idx */ sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ }; /* ** Prepare the SQL statement in buffer zSql against database handle db. ** If successful, set *ppStmt to point to the new statement and return ** SQLITE_OK. ** ** Otherwise, if an error does occur, set *ppStmt to NULL and return ** an SQLite error code. Additionally, set output variable *pzErrmsg to ** point to a buffer containing an error message. It is the responsibility ** of the caller to (eventually) free this buffer using sqlite3_free(). */ static int prepareAndCollectError( sqlite3 *db, sqlite3_stmt **ppStmt, char **pzErrmsg, const char *zSql ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); *ppStmt = 0; } return rc; } /* ** Reset the SQL statement passed as the first argument. Return a copy ** of the value returned by sqlite3_reset(). ** ** If an error has occurred, then set *pzErrmsg to point to a buffer ** containing an error message. It is the responsibility of the caller ** to eventually free this buffer using sqlite3_free(). */ static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ int rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK ){ *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); } return rc; } /* ** Unless it is NULL, argument zSql points to a buffer allocated using ** sqlite3_malloc containing an SQL statement. This function prepares the SQL ** statement against database db and frees the buffer. If statement ** compilation is successful, *ppStmt is set to point to the new statement ** handle and SQLITE_OK is returned. ** ** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code ** returned. In this case, *pzErrmsg may also be set to point to an error ** message. It is the responsibility of the caller to free this error message ** buffer using sqlite3_free(). ** ** If argument zSql is NULL, this function assumes that an OOM has occurred. ** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. */ static int prepareFreeAndCollectError( sqlite3 *db, sqlite3_stmt **ppStmt, char **pzErrmsg, char *zSql ){ int rc; assert( *pzErrmsg==0 ); if( zSql==0 ){ rc = SQLITE_NOMEM; *ppStmt = 0; }else{ rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); sqlite3_free(zSql); } return rc; } /* ** Free the OtaObjIter.azTblCol[] and OtaObjIter.abTblPk[] arrays allocated ** by an earlier call to otaObjIterGetCols(). */ static void otaObjIterFreeCols(OtaObjIter *pIter){ int i; for(i=0; inTblCol; i++){ sqlite3_free(pIter->azTblCol[i]); } sqlite3_free(pIter->azTblCol); sqlite3_free(pIter->abTblPk); pIter->azTblCol = 0; pIter->abTblPk = 0; pIter->nTblCol = 0; sqlite3_free(pIter->zMask); pIter->zMask = 0; } /* ** Finalize all statements and free all allocations that are specific to ** the current object (table/index pair). */ static void otaObjIterClearStatements(OtaObjIter *pIter){ sqlite3_finalize(pIter->pSelect); sqlite3_finalize(pIter->pInsert); sqlite3_finalize(pIter->pDelete); sqlite3_finalize(pIter->pUpdate); pIter->pSelect = 0; pIter->pInsert = 0; pIter->pDelete = 0; pIter->pUpdate = 0; pIter->nCol = 0; } /* ** Clean up any resources allocated as part of the iterator object passed ** as the only argument. */ static void otaObjIterFinalize(OtaObjIter *pIter){ otaObjIterClearStatements(pIter); sqlite3_finalize(pIter->pTblIter); sqlite3_finalize(pIter->pIdxIter); otaObjIterFreeCols(pIter); memset(pIter, 0, sizeof(OtaObjIter)); } /* ** Advance the iterator to the next position. ** ** If no error occurs, SQLITE_OK is returned and the iterator is left ** pointing to the next entry. Otherwise, an error code and message is ** left in the OTA handle passed as the first argument. A copy of the ** error code is returned. */ static int otaObjIterNext(sqlite3ota *p, OtaObjIter *pIter){ int rc = p->rc; if( rc==SQLITE_OK ){ /* Free any SQLite statements used while processing the previous object */ otaObjIterClearStatements(pIter); if( pIter->bCleanup ){ otaObjIterFreeCols(pIter); pIter->bCleanup = 0; rc = sqlite3_step(pIter->pTblIter); if( rc!=SQLITE_ROW ){ rc = sqlite3_reset(pIter->pTblIter); pIter->zTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); rc = SQLITE_OK; } }else{ if( pIter->zIdx==0 ){ sqlite3_bind_text(pIter->pIdxIter, 1, pIter->zTbl, -1, SQLITE_STATIC); } rc = sqlite3_step(pIter->pIdxIter); if( rc!=SQLITE_ROW ){ rc = sqlite3_reset(pIter->pIdxIter); pIter->bCleanup = 1; pIter->zIdx = 0; }else{ pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); rc = SQLITE_OK; } } } if( rc!=SQLITE_OK ){ otaObjIterFinalize(pIter); p->rc = rc; } pIter->iVisit++; return rc; } /* ** Initialize the iterator structure passed as the second argument. ** ** If no error occurs, SQLITE_OK is returned and the iterator is left ** pointing to the first entry. Otherwise, an error code and message is ** left in the OTA handle passed as the first argument. A copy of the ** error code is returned. */ static int otaObjIterFirst(sqlite3ota *p, OtaObjIter *pIter){ int rc; memset(pIter, 0, sizeof(OtaObjIter)); rc = prepareAndCollectError(p->db, &pIter->pTblIter, &p->zErrmsg, "SELECT substr(name, 6) FROM ota.sqlite_master " "WHERE type='table' AND name LIKE 'data_%'" ); if( rc==SQLITE_OK ){ rc = prepareAndCollectError(p->db, &pIter->pIdxIter, &p->zErrmsg, "SELECT name FROM main.sqlite_master " "WHERE type='index' AND tbl_name = ?" ); } pIter->bCleanup = 1; p->rc = rc; return otaObjIterNext(p, pIter); } /* ** Allocate a buffer and populate it with the double-quoted version of the ** string in the argument buffer, suitable for use as an SQL identifier. ** For example: ** ** [quick `brown` fox] -> [`quick ``brown`` fox`] ** ** Assuming the allocation is successful, a pointer to the new buffer is ** returned. It is the responsibility of the caller to free it using ** sqlite3_free() at some point in the future. Or, if the allocation fails, ** a NULL pointer is returned. */ static char *otaQuoteName(const char *zName){ int nName = strlen(zName); char *zRet = sqlite3_malloc(nName * 2 + 2 + 1); if( zRet ){ int i; char *p = zRet; *p++ = '`'; for(i=0; irc==SQLITE_OK ){ char *zSql = sqlite3_vmprintf(zFmt, ap); if( zSql==0 ){ p->rc = SQLITE_NOMEM; }else{ p->rc = sqlite3_exec(p->db, zSql, 0, 0, &p->zErrmsg); sqlite3_free(zSql); } } va_end(ap); return p->rc; } /* ** If they are not already populated, populate the pIter->azTblCol[], ** pIter->abTblPk[] and pIter->nTblCol variables according to the table ** that the iterator currently points to. ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. If ** an error does occur, an error code and error message are also left in ** the OTA handle. */ static int otaObjIterGetCols(sqlite3ota *p, OtaObjIter *pIter){ if( pIter->azTblCol==0 ){ sqlite3_stmt *pStmt; char *zSql; int nCol = 0; int bSeenPk = 0; int rc2; /* sqlite3_finalize() return value */ zSql = sqlite3_mprintf("PRAGMA main.table_info(%Q)", pIter->zTbl); p->rc = prepareFreeAndCollectError(p->db, &pStmt, &p->zErrmsg, zSql); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ if( (nCol % 8)==0 ){ unsigned char *abNew; int nByte = sizeof(char*) * (nCol+8); char **azNew = (char**)sqlite3_realloc(pIter->azTblCol, nByte); abNew = (unsigned char*)sqlite3_realloc(pIter->abTblPk, nCol+8); if( azNew ) pIter->azTblCol = azNew; if( abNew ) pIter->abTblPk = abNew; if( azNew==0 || abNew==0 ) p->rc = SQLITE_NOMEM; } if( p->rc==SQLITE_OK ){ const char *zName = (const char*)sqlite3_column_text(pStmt, 1); pIter->abTblPk[nCol] = sqlite3_column_int(pStmt, 5); if( pIter->abTblPk[nCol] ) bSeenPk = 1; pIter->azTblCol[nCol] = otaQuoteName(zName); if( pIter->azTblCol[nCol]==0 ) p->rc = SQLITE_NOMEM; nCol++; } } pIter->nTblCol = nCol; rc2 = sqlite3_finalize(pStmt); if( p->rc==SQLITE_OK ) p->rc = rc2; if( p->rc==SQLITE_OK && bSeenPk==0 ){ p->zErrmsg = sqlite3_mprintf("table %s has no PRIMARY KEY", pIter->zTbl); p->rc = SQLITE_ERROR; } } return p->rc; } /* ** This function constructs and returns a pointer to a nul-terminated ** string containing some SQL clause or list based on one or more of the ** column names currently stored in the pIter->azTblCol[] array. ** ** If an OOM error is encountered, NULL is returned and an error code ** left in the OTA handle passed as the first argument. Otherwise, a pointer ** to the allocated string buffer is returned. It is the responsibility ** of the caller to eventually free this buffer using sqlite3_free(). ** ** The number of column names to include in the returned string is passed ** as the third argument. ** ** If arguments aiCol and azCollate are both NULL, then the returned string ** contains the first nCol column names as a comma-separated list. For ** example: ** ** "a", "b", "c" ** ** If argument aiCol is not NULL, it must point to an array containing nCol ** entries - the index of each column name to include in the comma-separated ** list. For example, if aiCol[] contains {2, 0, 1), then the returned ** string is changed to: ** ** "c", "a", "b" ** ** If azCollate is not NULL, it must also point to an array containing nCol ** entries - collation sequence names to associated with each element of ** the comma separated list. For example, ef azCollate[] contains ** {"BINARY", "NOCASE", "REVERSE"}, then the retuned string is: ** ** "c" COLLATE "BINARY", "a" COLLATE "NOCASE", "b" COLLATE "REVERSE" ** */ static char *otaObjIterGetCollist( sqlite3ota *p, /* OTA object */ OtaObjIter *pIter, /* Object iterator for column names */ int nCol, /* Number of column names */ int *aiCol, /* Array of nCol column indexes */ const char **azCollate /* Array of nCol collation sequence names */ ){ char *zList = 0; if( p->rc==SQLITE_OK ){ const char *zSep = ""; int i; for(i=0; iazTblCol[iCol]); if( zList && azCollate ){ zList = sqlite3_mprintf("%z COLLATE %Q", zList, azCollate[i]); } zSep = ", "; if( zList==0 ){ p->rc = SQLITE_NOMEM; break; } } } return zList; } static char *otaObjIterGetOldlist( sqlite3ota *p, OtaObjIter *pIter, const char *zObj ){ char *zList = 0; if( p->rc==SQLITE_OK ){ const char *zS = ""; int i; for(i=0; inTblCol; i++){ zList = sqlite3_mprintf("%z%s%s.%s", zList, zS, zObj, pIter->azTblCol[i]); zS = ", "; if( zList==0 ){ p->rc = SQLITE_NOMEM; break; } } } return zList; } static char *otaObjIterGetWhere( sqlite3ota *p, OtaObjIter *pIter ){ char *zList = 0; if( p->rc==SQLITE_OK ){ const char *zSep = ""; int i; for(i=0; inTblCol; i++){ if( pIter->abTblPk[i] ){ const char *zCol = pIter->azTblCol[i]; zList = sqlite3_mprintf("%z%s%s=?%d", zList, zSep, zCol, i+1); zSep = " AND "; if( zList==0 ){ p->rc = SQLITE_NOMEM; break; } } } } return zList; } /* ** The SELECT statement iterating through the keys for the current object ** (p->objiter.pSelect) currently points to a valid row. However, there ** is something wrong with the ota_control value in the ota_control value ** stored in the (p->nCol+1)'th column. Set the error code and error message ** of the OTA handle to something reflecting this. */ static void otaBadControlError(sqlite3ota *p){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("Invalid ota_control value"); } static char *otaObjIterGetSetlist( sqlite3ota *p, OtaObjIter *pIter, const char *zMask ){ char *zList = 0; if( p->rc==SQLITE_OK ){ int i; if( strlen(zMask)!=pIter->nTblCol ){ otaBadControlError(p); }else{ const char *zSep = ""; for(i=0; inTblCol; i++){ if( zMask[i]=='x' ){ zList = sqlite3_mprintf("%z%s%s=?%d", zList, zSep, pIter->azTblCol[i], i+1 ); if( zList==0 ){ p->rc = SQLITE_NOMEM; break; } zSep = ", "; } } } } return zList; } static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ char *zRet = 0; if( p->rc==SQLITE_OK ){ int nByte = nBind*2 + 1; zRet = sqlite3_malloc(nByte); if( zRet==0 ){ p->rc = SQLITE_NOMEM; }else{ int i; for(i=0; ibCleanup==0 ); if( pIter->pSelect==0 && otaObjIterGetCols(p, pIter)==SQLITE_OK ){ char *zCollist = 0; /* List of indexed columns */ char **pz = &p->zErrmsg; const char *zIdx = pIter->zIdx; char *zLimit = 0; if( nOffset ){ zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); if( !zLimit ) p->rc = SQLITE_NOMEM; } if( zIdx ){ int *aiCol; /* Column map */ const char **azColl; /* Collation sequences */ /* Create the index writers */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_index_writer( p->db, 0, zIdx, &pIter->pInsert, &azColl, &aiCol, &pIter->nCol ); } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_index_writer( p->db, 1, zIdx, &pIter->pDelete, &azColl, &aiCol, &pIter->nCol ); } /* Create the SELECT statement to read keys in sorted order */ zCollist = otaObjIterGetCollist(p, pIter, pIter->nCol, aiCol, azColl); if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( "SELECT %s, ota_control FROM ota.'data_%q' " "WHERE typeof(ota_control)='integer' AND ota_control!=1 " "UNION ALL " "SELECT %s, ota_control FROM ota.'ota_tmp_%q' " "ORDER BY %s%s", zCollist, pIter->zTbl, zCollist, pIter->zTbl, zCollist, zLimit ) ); } }else{ char *zBindings = otaObjIterGetBindlist(p, pIter->nTblCol); char *zWhere = otaObjIterGetWhere(p, pIter); char *zOldlist = otaObjIterGetOldlist(p, pIter, "old"); char *zNewlist = otaObjIterGetOldlist(p, pIter, "new"); zCollist = otaObjIterGetCollist(p, pIter, pIter->nTblCol, 0, 0); pIter->nCol = pIter->nTblCol; /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pSelect, pz, sqlite3_mprintf( "SELECT %s, ota_control FROM ota.'data_%q'%s", zCollist, pIter->zTbl, zLimit) ); } /* Create the INSERT statement to write to the target PK b-tree */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pInsert, pz, sqlite3_mprintf( "INSERT INTO main.%Q(%s) VALUES(%s)", pIter->zTbl, zCollist, zBindings ) ); } /* Create the DELETE statement to write to the target PK b-tree */ if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->db, &pIter->pDelete, pz, sqlite3_mprintf( "DELETE FROM main.%Q WHERE %s", pIter->zTbl, zWhere ) ); } if( p->rc==SQLITE_OK ){ otaMPrintfExec(p, "CREATE TABLE IF NOT EXISTS ota.'ota_tmp_%q' AS " "SELECT * FROM ota.'data_%q' WHERE 0;" "CREATE TEMP TRIGGER ota_delete_%q BEFORE DELETE ON main.%Q " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(2, %s);" "END;" "CREATE TEMP TRIGGER ota_update1_%q BEFORE UPDATE ON main.%Q " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(2, %s);" "END;" "CREATE TEMP TRIGGER ota_update2_%q AFTER UPDATE ON main.%Q " "BEGIN " " INSERT INTO 'ota_tmp_%q'(ota_control, %s) VALUES(3, %s);" "END;" , pIter->zTbl, pIter->zTbl, pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOldlist, pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zOldlist, pIter->zTbl, pIter->zTbl, pIter->zTbl, zCollist, zNewlist ); } /* Allocate space required for the zMask field. */ if( p->rc==SQLITE_OK ){ int nMask = pIter->nTblCol+1; pIter->zMask = (char*)sqlite3_malloc(nMask); if( pIter->zMask==0 ){ p->rc = SQLITE_NOMEM; }else{ memset(pIter->zMask, 0, nMask); } } sqlite3_free(zWhere); sqlite3_free(zOldlist); sqlite3_free(zNewlist); sqlite3_free(zBindings); } sqlite3_free(zCollist); sqlite3_free(zLimit); } return p->rc; } #define OTA_INSERT 1 #define OTA_DELETE 2 #define OTA_IDX_DELETE 3 #define OTA_IDX_INSERT 4 #define OTA_UPDATE 5 static int otaGetUpdateStmt( sqlite3ota *p, OtaObjIter *pIter, const char *zMask, sqlite3_stmt **ppStmt ){ if( pIter->pUpdate && strcmp(zMask, pIter->zMask)==0 ){ *ppStmt = pIter->pUpdate; }else{ char *zWhere = otaObjIterGetWhere(p, pIter); char *zSet = otaObjIterGetSetlist(p, pIter, zMask); char *zUpdate = 0; sqlite3_finalize(pIter->pUpdate); pIter->pUpdate = 0; if( p->rc==SQLITE_OK ){ zUpdate = sqlite3_mprintf("UPDATE %Q SET %s WHERE %s", pIter->zTbl, zSet, zWhere ); p->rc = prepareFreeAndCollectError( p->db, &pIter->pUpdate, &p->zErrmsg, zUpdate ); *ppStmt = pIter->pUpdate; } if( p->rc==SQLITE_OK ){ memcpy(pIter->zMask, zMask, pIter->nTblCol); } sqlite3_free(zWhere); sqlite3_free(zSet); } return p->rc; } /* ** Open the database handle and attach the OTA database as "ota". If an ** error occurs, leave an error code and message in the OTA handle. */ static void otaOpenDatabase(sqlite3ota *p){ assert( p->rc==SQLITE_OK ); sqlite3_close(p->db); p->db = 0; p->rc = sqlite3_open(p->zTarget, &p->db); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); } otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); } /* ** This routine is a copy of the sqlite3FileSuffix3() routine from the core. ** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. ** ** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than ** three characters, then shorten the suffix on z[] to be the last three ** characters of the original suffix. ** ** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always ** do the suffix shortening regardless of URI parameter. ** ** Examples: ** ** test.db-journal => test.nal ** test.db-wal => test.wal ** test.db-shm => test.shm ** test.db-mj7f3319fa => test.9fa */ static void otaFileSuffix3(const char *zBase, char *z){ #ifdef SQLITE_ENABLE_8_3_NAMES #if SQLITE_ENABLE_8_3_NAMES<2 if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) #endif { int i, sz; sz = sqlite3Strlen30(z); for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); } #endif } /* ** The OTA handle is currently in OTA_STAGE_OAL state, with a SHARED lock ** on the database file. This proc moves the *-oal file to the *-wal path, ** then reopens the database file (this time in vanilla, non-oal, WAL mode). ** If an error occurs, leave an error code and error message in the ota ** handle. */ static void otaMoveOalFile(sqlite3ota *p){ const char *zBase = sqlite3_db_filename(p->db, "main"); char *zWal = sqlite3_mprintf("%s-wal", zBase); char *zOal = sqlite3_mprintf("%s-oal", zBase); assert( p->eStage==OTA_STAGE_OAL ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); if( zWal==0 || zOal==0 ){ p->rc = SQLITE_NOMEM; }else{ /* Move the *-oal file to *-wal. At this point connection p->db is ** holding a SHARED lock on the target database file (because it is ** in WAL mode). So no other connection may be writing the db. */ otaFileSuffix3(zBase, zWal); otaFileSuffix3(zBase, zOal); rename(zOal, zWal); /* Re-open the databases. */ otaObjIterFinalize(&p->objiter); otaOpenDatabase(p); p->eStage = OTA_STAGE_CKPT; } sqlite3_free(zWal); sqlite3_free(zOal); } /* ** The SELECT statement iterating through the keys for the current object ** (p->objiter.pSelect) currently points to a valid row. This function ** determines the type of operation requested by this row and returns ** one of the following values to indicate the result: ** ** * OTA_INSERT ** * OTA_DELETE ** * OTA_IDX_DELETE ** * OTA_UPDATE ** ** If OTA_UPDATE is returned, then output variable *pzMask is set to ** point to the text value indicating the columns to update. ** ** If the ota_control field contains an invalid value, an error code and ** message are left in the OTA handle and zero returned. */ static int otaStepType(sqlite3ota *p, const char **pzMask){ int iCol = p->objiter.nCol; /* Index of ota_control column */ int res = 0; /* Return value */ switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){ case SQLITE_INTEGER: { int iVal = sqlite3_column_int(p->objiter.pSelect, iCol); if( iVal==0 ){ res = OTA_INSERT; }else if( iVal==1 ){ res = OTA_DELETE; }else if( iVal==2 ){ res = OTA_IDX_DELETE; }else if( iVal==3 ){ res = OTA_IDX_INSERT; } break; } case SQLITE_TEXT: *pzMask = (const char*)sqlite3_column_text(p->objiter.pSelect, iCol); res = OTA_UPDATE; break; default: break; } if( res==0 ){ otaBadControlError(p); } return res; } /* ** This function does the work for an sqlite3ota_step() call. ** ** The object-iterator (p->objiter) currently points to a valid object, ** and the input cursor (p->objiter.pSelect) currently points to a valid ** input row. Perform whatever processing is required and return. ** ** If no error occurs, SQLITE_OK is returned. Otherwise, an error code ** and message is left in the OTA handle and a copy of the error code ** returned. */ static int otaStep(sqlite3ota *p){ OtaObjIter *pIter = &p->objiter; const char *zMask = 0; int i; int eType = otaStepType(p, &zMask); if( eType ){ assert( eType!=OTA_UPDATE || pIter->zIdx==0 ); if( pIter->zIdx==0 && eType==OTA_IDX_DELETE ){ otaBadControlError(p); } else if( eType==OTA_INSERT || eType==OTA_DELETE || eType==OTA_IDX_DELETE || eType==OTA_IDX_INSERT ){ sqlite3_stmt *pWriter; assert( eType!=OTA_UPDATE ); assert( eType!=OTA_DELETE || pIter->zIdx==0 ); if( eType==OTA_IDX_DELETE || eType==OTA_DELETE ){ pWriter = pIter->pDelete; }else{ pWriter = pIter->pInsert; } for(i=0; inCol; i++){ sqlite3_value *pVal; if( eType==SQLITE_DELETE && pIter->zIdx==0 && pIter->abTblPk[i]==0 ){ continue; } pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pWriter, i+1, pVal); } sqlite3_step(pWriter); p->rc = resetAndCollectError(pWriter, &p->zErrmsg); }else if( eType==OTA_UPDATE ){ sqlite3_stmt *pUpdate = 0; otaGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ for(i=0; inCol; i++){ sqlite3_value *pVal = sqlite3_column_value(pIter->pSelect, i); sqlite3_bind_value(pUpdate, i+1, pVal); } sqlite3_step(pUpdate); p->rc = resetAndCollectError(pUpdate, &p->zErrmsg); } }else{ /* no-op */ assert( eType==OTA_DELETE && pIter->zIdx ); } } return p->rc; } /* ** Increment the schema cookie of the main database opened by p->db. */ static void otaIncrSchemaCookie(sqlite3ota *p){ int iCookie = 1000000; sqlite3_stmt *pStmt; assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); p->rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, "PRAGMA schema_version" ); if( p->rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ iCookie = sqlite3_column_int(pStmt, 0); } p->rc = sqlite3_finalize(pStmt); } if( p->rc==SQLITE_OK ){ otaMPrintfExec(p, "PRAGMA schema_version = %d", iCookie+1); } } /* ** Step the OTA object. */ int sqlite3ota_step(sqlite3ota *p){ if( p ){ switch( p->eStage ){ case OTA_STAGE_OAL: { OtaObjIter *pIter = &p->objiter; while( p && p->rc==SQLITE_OK && pIter->zTbl ){ if( pIter->bCleanup ){ /* Clean up the ota_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ otaMPrintfExec(p, "DELETE FROM ota.'ota_tmp_%q'", pIter->zTbl); }else{ otaObjIterPrepareAll(p, pIter, 0); /* Advance to the next row to process. */ if( p->rc==SQLITE_OK ){ int rc = sqlite3_step(pIter->pSelect); if( rc==SQLITE_ROW ){ p->nProgress++; p->nStep++; return otaStep(p); } p->rc = sqlite3_reset(pIter->pSelect); p->nStep = 0; } } otaObjIterNext(p, pIter); } if( p->rc==SQLITE_OK && pIter->zTbl==0 ){ p->nProgress++; otaIncrSchemaCookie(p); if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); } if( p->rc==SQLITE_OK ){ otaMoveOalFile(p); } } break; } case OTA_STAGE_CKPT: { if( p->rc==SQLITE_OK && p->pCkpt==0 ){ p->rc = sqlite3_ckpt_open(p->db, 0, 0, &p->pCkpt); } if( p->rc==SQLITE_OK ){ if( SQLITE_OK!=sqlite3_ckpt_step(p->pCkpt) ){ p->rc = sqlite3_ckpt_close(p->pCkpt, 0, 0); p->pCkpt = 0; if( p->rc==SQLITE_OK ){ p->eStage = OTA_STAGE_DONE; p->rc = SQLITE_DONE; } } p->nProgress++; } break; } default: break; } return p->rc; }else{ return SQLITE_NOMEM; } } static void otaSaveTransactionState(sqlite3ota *p){ sqlite3_stmt *pInsert; int rc; assert( (p->rc==SQLITE_OK || p->rc==SQLITE_DONE) && p->zErrmsg==0 ); rc = prepareFreeAndCollectError(p->db, &pInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT OR REPLACE INTO ota.ota_state(k, v) VALUES " "(%d, %d), " "(%d, %Q), " "(%d, %Q), " "(%d, %d), " "(%d, %lld), " "(%d, ?) ", OTA_STATE_STAGE, p->eStage, OTA_STATE_TBL, p->objiter.zTbl, OTA_STATE_IDX, p->objiter.zIdx, OTA_STATE_ROW, p->nStep, OTA_STATE_PROGRESS, p->nProgress, OTA_STATE_CKPT ) ); assert( pInsert==0 || rc==SQLITE_OK ); if( rc==SQLITE_OK ){ if( p->pCkpt ){ unsigned char *pCkptState = 0; int nCkptState = 0; rc = sqlite3_ckpt_close(p->pCkpt, &pCkptState, &nCkptState); p->pCkpt = 0; sqlite3_bind_blob(pInsert, 1, pCkptState, nCkptState, SQLITE_TRANSIENT); sqlite3_free(pCkptState); } } if( rc==SQLITE_OK ){ sqlite3_step(pInsert); rc = sqlite3_finalize(pInsert); }else{ sqlite3_finalize(pInsert); } if( rc!=SQLITE_OK ){ p->rc = rc; } } static char *otaStrndup(char *zStr, int nStr, int *pRc){ char *zRet = 0; assert( *pRc==SQLITE_OK ); if( zStr ){ int nCopy = nStr; if( nCopy<0 ) nCopy = strlen(zStr) + 1; zRet = (char*)sqlite3_malloc(nCopy); if( zRet ){ memcpy(zRet, zStr, nCopy); }else{ *pRc = SQLITE_NOMEM; } } return zRet; } static void otaFreeState(OtaState *p){ if( p ){ sqlite3_free(p->zTbl); sqlite3_free(p->zIdx); sqlite3_free(p->pCkptState); sqlite3_free(p); } } /* ** Allocate an OtaState object and load the contents of the ota_state ** table into it. Return a pointer to the new object. It is the ** responsibility of the caller to eventually free the object using ** sqlite3_free(). ** ** If an error occurs, leave an error code and message in the ota handle ** and return NULL. */ static OtaState *otaLoadState(sqlite3ota *p){ const char *zSelect = "SELECT k, v FROM ota.ota_state"; OtaState *pRet = 0; sqlite3_stmt *pStmt; int rc; int rc2; assert( p->rc==SQLITE_OK ); pRet = (OtaState*)sqlite3_malloc(sizeof(OtaState)); if( pRet==0 ){ rc = SQLITE_NOMEM; }else{ memset(pRet, 0, sizeof(OtaState)); rc = prepareAndCollectError(p->db, &pStmt, &p->zErrmsg, zSelect); } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ switch( sqlite3_column_int(pStmt, 0) ){ case OTA_STATE_STAGE: pRet->eStage = sqlite3_column_int(pStmt, 1); if( pRet->eStage!=OTA_STAGE_OAL && pRet->eStage!=OTA_STAGE_COPY && pRet->eStage!=OTA_STAGE_CKPT ){ p->rc = SQLITE_CORRUPT; } break; case OTA_STATE_TBL: pRet->zTbl = otaStrndup((char*)sqlite3_column_text(pStmt, 1), -1, &rc); break; case OTA_STATE_IDX: pRet->zIdx = otaStrndup((char*)sqlite3_column_text(pStmt, 1), -1, &rc); break; case OTA_STATE_ROW: pRet->nRow = sqlite3_column_int(pStmt, 1); break; case OTA_STATE_PROGRESS: pRet->nProgress = sqlite3_column_int64(pStmt, 1); break; case OTA_STATE_CKPT: pRet->nCkptState = sqlite3_column_bytes(pStmt, 1); pRet->pCkptState = (unsigned char*)otaStrndup( (char*)sqlite3_column_blob(pStmt, 1), pRet->nCkptState, &rc ); break; default: rc = SQLITE_CORRUPT; break; } } rc2 = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ) rc = rc2; p->rc = rc; return pRet; } static int otaStrCompare(const char *z1, const char *z2){ if( z1==0 && z2==0 ) return 0; if( z1==0 || z2==0 ) return 1; return (sqlite3_stricmp(z1, z2)!=0); } static void otaLoadTransactionState(sqlite3ota *p, OtaState *pState){ assert( p->rc==SQLITE_OK ); if( pState->zTbl ){ OtaObjIter *pIter = &p->objiter; int rc = SQLITE_OK; while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup || otaStrCompare(pIter->zTbl, pState->zTbl) || otaStrCompare(pIter->zIdx, pState->zIdx) )){ rc = otaObjIterNext(p, &p->objiter); } if( rc==SQLITE_OK && !p->objiter.zTbl ){ rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("ota_state mismatch error"); } if( rc==SQLITE_OK ){ p->nStep = pState->nRow; rc = otaObjIterPrepareAll(p, &p->objiter, p->nStep); } p->rc = rc; } } /* ** If there is a "*-oal" file in the file-system corresponding to the ** target database in the file-system, delete it. If an error occurs, ** leave an error code and error message in the ota handle. */ static void otaDeleteOalFile(sqlite3ota *p){ char *zOal = sqlite3_mprintf("%s-oal", p->zTarget); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); unlink(zOal); sqlite3_free(zOal); } /* ** Open and return a new OTA handle. */ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ sqlite3ota *p; int nTarget = strlen(zTarget); int nOta = strlen(zOta); p = (sqlite3ota*)sqlite3_malloc(sizeof(sqlite3ota)+nTarget+1+nOta+1); if( p ){ OtaState *pState = 0; /* Open the target database */ memset(p, 0, sizeof(sqlite3ota)); p->zTarget = (char*)&p[1]; memcpy(p->zTarget, zTarget, nTarget+1); p->zOta = &p->zTarget[nTarget+1]; memcpy(p->zOta, zOta, nOta+1); otaOpenDatabase(p); /* If it has not already been created, create the ota_state table */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->db, OTA_CREATE_STATE, 0, 0, &p->zErrmsg); } if( p->rc==SQLITE_OK ){ pState = otaLoadState(p); assert( pState || p->rc!=SQLITE_OK ); if( p->rc==SQLITE_OK ){ if( pState->eStage==0 ){ otaDeleteOalFile(p); p->eStage = 1; }else{ p->eStage = pState->eStage; } p->nProgress = pState->nProgress; } } assert( p->rc!=SQLITE_OK || p->eStage!=0 ); if( p->rc==SQLITE_OK ){ if( p->eStage==OTA_STAGE_OAL ){ const char *zScript = "PRAGMA journal_mode=off;" "PRAGMA pager_ota_mode=1;" "PRAGMA ota_mode=1;" "BEGIN IMMEDIATE;" ; p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ p->rc = otaObjIterFirst(p, &p->objiter); } if( p->rc==SQLITE_OK ){ otaLoadTransactionState(p, pState); } }else if( p->eStage==OTA_STAGE_CKPT ){ p->rc = sqlite3_ckpt_open( p->db, pState->pCkptState, pState->nCkptState, &p->pCkpt ); if( p->rc==SQLITE_MISMATCH ){ p->eStage = OTA_STAGE_DONE; p->rc = SQLITE_DONE; } }else if( p->eStage==OTA_STAGE_DONE ){ p->rc = SQLITE_DONE; } } otaFreeState(pState); } return p; } /* ** Close the OTA handle. */ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ int rc; if( p ){ /* If the update has not been fully applied, save the state in ** the ota db. If successful, this call also commits the open ** transaction on the ota db. */ assert( p->rc!=SQLITE_ROW ); if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ assert( p->zErrmsg==0 ); otaSaveTransactionState(p); } /* Close any open statement handles. */ otaObjIterFinalize(&p->objiter); /* Commit the transaction to the *-oal file. */ if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_OAL ){ p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); } if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ p->rc = sqlite3_exec(p->db, "PRAGMA pager_ota_mode=2", 0, 0, &p->zErrmsg); } /* Close the open database handle */ if( p->pCkpt ) sqlite3_ckpt_close(p->pCkpt, 0, 0); sqlite3_close(p->db); rc = p->rc; *pzErrmsg = p->zErrmsg; sqlite3_free(p); }else{ rc = SQLITE_NOMEM; *pzErrmsg = 0; } return rc; } /* ** Return the total number of key-value operations (inserts, deletes or ** updates) that have been performed on the target database since the ** current OTA update was started. */ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){ return pOta->nProgress; } /**************************************************************************/ #ifdef SQLITE_TEST #include /* From main.c (apparently...) */ extern const char *sqlite3ErrName(int); static int test_sqlite3ota_cmd( ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ int ret = TCL_OK; sqlite3ota *pOta = (sqlite3ota*)clientData; const char *azMethod[] = { "step", "close", 0 }; int iMethod; if( objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "METHOD"); return TCL_ERROR; } if( Tcl_GetIndexFromObj(interp, objv[1], azMethod, "method", 0, &iMethod) ){ return TCL_ERROR; } switch( iMethod ){ case 0: /* step */ { int rc = sqlite3ota_step(pOta); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); break; } case 1: /* close */ { char *zErrmsg = 0; int rc; Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); rc = sqlite3ota_close(pOta, &zErrmsg); if( rc==SQLITE_OK || rc==SQLITE_DONE ){ Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); assert( zErrmsg==0 ); }else{ Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); if( zErrmsg ){ Tcl_AppendResult(interp, " - ", zErrmsg, 0); sqlite3_free(zErrmsg); } ret = TCL_ERROR; } break; } default: /* seems unlikely */ assert( !"cannot happen" ); break; } return ret; } /* ** Tclcmd: sqlite3ota CMD */ static int test_sqlite3ota( ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ sqlite3ota *pOta = 0; const char *zCmd; const char *zTarget; const char *zOta; if( objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB OTA-DB"); return TCL_ERROR; } zCmd = Tcl_GetString(objv[1]); zTarget = Tcl_GetString(objv[2]); zOta = Tcl_GetString(objv[3]); pOta = sqlite3ota_open(zTarget, zOta); Tcl_CreateObjCommand(interp, zCmd, test_sqlite3ota_cmd, (ClientData)pOta, 0); Tcl_SetObjResult(interp, objv[1]); return TCL_OK; } int SqliteOta_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "sqlite3ota", test_sqlite3ota, 0, 0); return TCL_OK; } #endif /* ifdef SQLITE_TEST */