Index: lsm-test/lsmtest_func.c ================================================================== --- lsm-test/lsmtest_func.c +++ lsm-test/lsmtest_func.c @@ -5,20 +5,20 @@ int do_work(int nArg, char **azArg){ struct Option { const char *zName; } aOpt [] = { { "-nmerge" }, - { "-npage" }, + { "-nkb" }, { 0 } }; lsm_db *pDb; int rc; int i; const char *zDb; int nMerge = 1; - int nWork = (1<<30); + int nKB = (1<<30); if( nArg==0 ) goto usage; zDb = azArg[nArg-1]; for(i=0; i<(nArg-1); i++){ int iSel; @@ -31,11 +31,11 @@ nMerge = atoi(azArg[i]); break; case 1: i++; if( i==(nArg-1) ) goto usage; - nWork = atoi(azArg[i]); + nKB = atoi(azArg[i]); break; } } rc = lsm_new(0, &pDb); @@ -49,11 +49,11 @@ int n = -1; lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n); n = n*2; lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n); - rc = lsm_work(pDb, nMerge, nWork, 0); + rc = lsm_work(pDb, nMerge, nKB, 0); if( rc!=LSM_OK ){ testPrintError("lsm_work(): rc=%d\n", rc); } } } Index: lsm-test/lsmtest_main.c ================================================================== --- lsm-test/lsmtest_main.c +++ lsm-test/lsmtest_main.c @@ -565,11 +565,11 @@ " -write $write (default value 10000)\n" " -pause $pause (default value 0)\n" " -fetch $fetch (default value 0)\n" " -keysize $keysize (default value 12)\n" " -valsize $valsize (default value 100)\n" -" -system $system (default value \"lsm\"\n" +" -system $system (default value \"lsm\")\n" "\n" ); } int do_speed_test2(int nArg, char **azArg){ @@ -644,11 +644,11 @@ printf("#"); for(i=0; i=0 ){ printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]); - }else{ + }else if( aOpt[i].eVal==-1 ){ printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem); } } } printf("\n"); @@ -1331,11 +1331,11 @@ }else{ pClose = pInput = fopen(azArg[0], "r"); } zDb = azArg[1]; pEnv = tdb_lsm_env(); - rc = pEnv->xOpen(pEnv, zDb, &pOut); + rc = pEnv->xOpen(pEnv, zDb, 0, &pOut); if( rc!=LSM_OK ) return rc; while( feof(pInput)==0 ){ char zLine[80]; fgets(zLine, sizeof(zLine)-1, pInput); @@ -1428,11 +1428,11 @@ #include static void lsmtest_rusage_report(void){ int res; struct rusage r; - memset(&r, sizeof(r), 0); + memset(&r, 0, sizeof(r)); res = getrusage(RUSAGE_SELF, &r); assert( res==0 ); printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", @@ -1456,11 +1456,11 @@ {"insert", do_insert}, {"replay", do_replay}, {"speed", do_speed_tests}, - {"speed2", do_speed_test2}, + {"speed2", do_speed_test2}, {"show", st_do_show}, {"work", st_do_work}, {"test", do_test}, {0, 0} }; Index: lsm-test/lsmtest_tdb3.c ================================================================== --- lsm-test/lsmtest_tdb3.c +++ lsm-test/lsmtest_tdb3.c @@ -13,12 +13,12 @@ typedef struct LsmDb LsmDb; typedef struct LsmWorker LsmWorker; typedef struct LsmFile LsmFile; -#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024*1024) -#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024*1024) +#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) +#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) #ifdef LSM_MUTEX_PTHREADS #include #define LSMTEST_THREAD_CKPT 1 @@ -154,10 +154,11 @@ } static int testEnvOpen( lsm_env *pEnv, /* Environment for current LsmDb */ const char *zFile, /* Name of file to open */ + int flags, lsm_file **ppFile /* OUT: New file handle object */ ){ lsm_env *pRealEnv = tdb_lsm_env(); LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx; int rc; /* Return Code */ @@ -167,11 +168,11 @@ nFile = strlen(zFile); pRet = (LsmFile *)testMalloc(sizeof(LsmFile)); pRet->pDb = pDb; pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4)); - rc = pRealEnv->xOpen(pRealEnv, zFile, &pRet->pReal); + rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal); if( rc!=LSM_OK ){ testFree(pRet); pRet = 0; } @@ -369,10 +370,20 @@ if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ return LSM_BUSY; } return pRealEnv->xLock(p->pReal, iLock, eType); } + +static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ + LsmFile *p = (LsmFile *)pFile; + lsm_env *pRealEnv = tdb_lsm_env(); + + if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ + return LSM_BUSY; + } + return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType); +} static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){ LsmFile *p = (LsmFile *)pFile; lsm_env *pRealEnv = tdb_lsm_env(); return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp); @@ -402,11 +413,11 @@ for(iFile=0; iFile<2; iFile++){ lsm_file *pFile = 0; int i; - pEnv->xOpen(pEnv, zFile, &pFile); + pEnv->xOpen(pEnv, zFile, 0, &pFile); for(i=0; iaFile[iFile].nSector; i++){ u8 *aOld = pDb->aFile[iFile].aSector[i].aOld; if( aOld ){ int iOpt = testPrngValue(iSeed++) % 3; switch( iOpt ){ @@ -522,17 +533,17 @@ return rc; } static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ int nSleep = 0; - int nByte; + int nKB; int rc; do { - nByte = 0; - rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nByte); - if( rc!=LSM_OK || nBytenMtMaxCkpt ) break; + nKB = 0; + rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); + if( rc!=LSM_OK || nKBnMtMaxCkpt ) break; usleep(5000); nSleep += 5; }while( 1 ); #if 0 @@ -547,14 +558,14 @@ int nLimit = -1; int nSleep = 0; rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); do { - int bOld, nNew, rc; - rc = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &bOld, &nNew); + int nOld, nNew, rc; + rc = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); if( rc!=LSM_OK ) return rc; - if( bOld==0 || nNew<(nLimit/2) ) break; + if( nOld==0 || nNew<(nLimit/2) ) break; usleep(5000); nSleep += 5; }while( 1 ); #if 0 @@ -777,11 +788,10 @@ { "page_size", 0, LSM_CONFIG_PAGE_SIZE }, { "block_size", 0, LSM_CONFIG_BLOCK_SIZE }, { "safety", 0, LSM_CONFIG_SAFETY }, { "autowork", 0, LSM_CONFIG_AUTOWORK }, { "autocheckpoint", 0, LSM_CONFIG_AUTOCHECKPOINT }, - { "log_size", 0, LSM_CONFIG_LOG_SIZE }, { "mmap", 0, LSM_CONFIG_MMAP }, { "use_log", 0, LSM_CONFIG_USE_LOG }, { "automerge", 0, LSM_CONFIG_AUTOMERGE }, { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, @@ -828,14 +838,14 @@ z++; zStart = z; while( *z>='0' && *z<='9' ) z++; if( *z=='k' || *z=='K' ){ - iMul = 1024; + iMul = 1; z++; }else if( *z=='M' || *z=='M' ){ - iMul = 1024 * 1024; + iMul = 1024; z++; } nParam = z-zStart; if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; memcpy(zParam, zStart, nParam); @@ -853,14 +863,14 @@ break; case TEST_MT_MODE: if( pLsm ) nThread = iVal; break; case TEST_MT_MIN_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal; + if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal*1024; break; case TEST_MT_MAX_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal; + if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal*1024; break; #ifdef HAVE_ZLIB case TEST_COMPRESSION: testConfigureCompression(db); break; @@ -965,10 +975,11 @@ pDb->env.xUnmap = testEnvUnmap; pDb->env.xFileid = testEnvFileid; pDb->env.xClose = testEnvClose; pDb->env.xUnlink = testEnvUnlink; pDb->env.xLock = testEnvLock; + pDb->env.xTestLock = testEnvTestLock; pDb->env.xShmBarrier = testEnvShmBarrier; pDb->env.xShmMap = testEnvShmMap; pDb->env.xShmUnmap = testEnvShmUnmap; pDb->env.xSleep = testEnvSleep; @@ -1005,23 +1016,23 @@ int test_lsm_small_open( const char *zFile, int bClear, TestDb **ppDb ){ - const char *zCfg = "page_size=256 block_size=65536"; + const char *zCfg = "page_size=256 block_size=64"; return testLsmOpen(zCfg, zFile, bClear, ppDb); } int test_lsm_lomem_open( const char *zFilename, int bClear, TestDb **ppDb ){ - /* "max_freelist=4 autocheckpoint=32768 " */ + /* "max_freelist=4 autocheckpoint=32" */ const char *zCfg = - "page_size=256 block_size=65536 autoflush=16384 " - "autocheckpoint=32768 " + "page_size=256 block_size=64 autoflush=16 " + "autocheckpoint=32" "mmap=0 " ; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } @@ -1029,12 +1040,12 @@ const char *zFilename, int bClear, TestDb **ppDb ){ const char *zCfg = - "page_size=256 block_size=65536 autoflush=16384 " - "autocheckpoint=32768 compression=1 mmap=0 " + "page_size=256 block_size=64 autoflush=16 " + "autocheckpoint=32 compression=1 mmap=0 " ; return testLsmOpen(zCfg, zFilename, bClear, ppDb); } lsm_db *tdb_lsm(TestDb *pDb){ @@ -1149,13 +1160,13 @@ /* Do some work. If an error occurs, exit. */ pthread_mutex_unlock(&p->worker_mutex); if( p->eType==LSMTEST_THREAD_CKPT ){ - int nByte = 0; - rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nByte); - if( rc==LSM_OK && nByte>=p->pDb->nMtMinCkpt ){ + int nKB = 0; + rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB); + if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){ rc = lsm_checkpoint(pWorker, 0); } }else{ int nWrite; do { Index: lsm-test/lsmtest_util.c ================================================================== --- lsm-test/lsmtest_util.c +++ lsm-test/lsmtest_util.c @@ -1,13 +1,10 @@ -#include "lsmtest.h" - #include #include #include #include - /* ** Global variables used within this module. */ static struct TestutilGlobal { @@ -14,13 +11,13 @@ char **argv; int argc; } g = {0, 0}; static struct TestutilRnd { - u32 aRand1[2048]; /* Bits 0..10 */ - u32 aRand2[2048]; /* Bits 11..21 */ - u32 aRand3[1024]; /* Bits 22..31 */ + unsigned int aRand1[2048]; /* Bits 0..10 */ + unsigned int aRand2[2048]; /* Bits 11..21 */ + unsigned int aRand3[1024]; /* Bits 22..31 */ } r; /************************************************************************* ** The following block is a copy of the implementation of SQLite function ** sqlite3_randomness. This version has two important differences: @@ -70,11 +67,11 @@ 0xAE, 0x13, 0x06, 0xF5, 0x1C, 0x4E, 0x52, 0xF7 } }; /* Generate and return single random byte */ -static u8 randomByte(void){ +static unsigned char randomByte(void){ unsigned char t; sqlite3Prng.i++; t = sqlite3Prng.s[sqlite3Prng.i]; sqlite3Prng.j += t; sqlite3Prng.s[sqlite3Prng.i] = sqlite3Prng.s[sqlite3Prng.j]; @@ -96,33 +93,32 @@ ** End of code copied from SQLite. *************************************************************************/ int testPrngInit(void){ - sqlite3_initialize(); randomBlob(sizeof(r.aRand1), (unsigned char *)r.aRand1); randomBlob(sizeof(r.aRand2), (unsigned char *)r.aRand2); randomBlob(sizeof(r.aRand3), (unsigned char *)r.aRand3); - return LSM_OK; + return 0; } -u32 testPrngValue(u32 iVal){ +unsigned int testPrngValue(unsigned int iVal){ return r.aRand1[iVal & 0x000007FF] ^ r.aRand2[(iVal>>11) & 0x000007FF] ^ r.aRand3[(iVal>>22) & 0x000003FF] ; } -void testPrngArray(u32 iVal, u32 *aOut, int nOut){ +void testPrngArray(unsigned int iVal, unsigned int *aOut, int nOut){ int i; for(i=0; izName; - pEntry=(struct Entry *)&((u8 *)pEntry)[sz] + pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] ){ if( zPrev ){ testPrintError("%s, ", zPrev); } zPrev = pEntry->zName; } testPrintError("or %s\n", zPrev); @@ -175,11 +171,11 @@ int iOut = -1; int nOut = 0; for(pEntry=(struct Entry *)aData; pEntry->zName; - pEntry=(struct Entry *)&((u8 *)pEntry)[sz] + pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] ){ int nName = strlen(pEntry->zName); if( nArg<=nName && memcmp(pEntry->zName, zArg, nArg)==0 ){ iOut = i; if( nName==nArg ){ ADDED lsm-test/sqltest.c Index: lsm-test/sqltest.c ================================================================== --- /dev/null +++ lsm-test/sqltest.c @@ -0,0 +1,524 @@ +/* +** 2013 March 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code for a program that links against SQLite +** versions 3 and 4. It contains a few simple performance test routines +** that can be run against either database system. +*/ + +#include "sqlite4.h" +#include "sqlite3.h" +#include "lsm.h" + +#include +#include +#include +#include + +#define SQLITE3_DB_FILE "test.db3" +#define SQLITE4_DB_FILE "test.db4" + +#include "lsmtest_util.c" + +/* +** Unlink database zDb and its supporting files (wal, shm, journal, and log). +** This function works with both lsm and sqlite3 databases. +*/ +static int unlink_db(const char *zDb){ + int i; + const char *azExt[] = { "", "-shm", "-wal", "-journal", "-log", 0 }; + + for(i=0; azExt[i]; i++){ + char *zFile = sqlite4_mprintf(0, "%s%s", zDb, azExt[i]); + unlink(zFile); + sqlite4_free(0, zFile); + } + + return 0; +} + +static char *create_schema_sql(int nIdx){ + char *zSchema; + int i; + + zSchema = sqlite4_mprintf(0, "CREATE TABLE t1(k PRIMARY KEY,"); + for(i=0; i=nMin && nByte<=nMax ); + if( nByte>sizeof(aBlob) ) nByte = sizeof(aBlob); + testPrngArray(iSeed, (unsigned int *)aBlob, (nByte+3)/4); + + sqlite4_result_blob(ctx, aBlob, nByte, SQLITE4_TRANSIENT, 0); +} +static void install_rblob_function4(sqlite4 *db){ + testPrngInit(); + sqlite4_create_function(db, "rblob", 3, SQLITE4_UTF8, 0, rblobFunc4, 0, 0); +} + +/* sqlite3 implementation */ +static void rblobFunc3(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + unsigned char aBlob[1000]; + + int iSeed = sqlite3_value_int(apArg[0]); + int nMin = sqlite3_value_int(apArg[1]); + int nMax = sqlite3_value_int(apArg[2]); + int nByte; + + nByte = testPrngValue(iSeed + 1000000) & 0x7FFFFFFF; + nByte = (nByte % (nMax+1-nMin)) + nMin; + assert( nByte>=nMin && nByte<=nMax ); + if( nByte>sizeof(aBlob) ) nByte = sizeof(aBlob); + testPrngArray(iSeed, (unsigned int *)aBlob, (nByte+3)/4); + + sqlite3_result_blob(ctx, aBlob, nByte, SQLITE_TRANSIENT); +} +static void install_rblob_function3(sqlite3 *db){ + testPrngInit(); + sqlite3_create_function(db, "rblob", 3, SQLITE_UTF8, 0, rblobFunc3, 0, 0); +} +/* +** End of rblob() implementations. +*************************************************************************/ + +/************************************************************************* +** Integer query functions for sqlite3 and src4. +*/ +static int integer_query4(sqlite4 *db, const char *zSql){ + int iRet; + sqlite4_stmt *pStmt; + + EXPLODE( sqlite4_prepare(db, zSql, -1, &pStmt, 0) ); + EXPLODE( SQLITE_ROW!=sqlite4_step(pStmt) ); + iRet = sqlite4_column_int(pStmt, 0); + EXPLODE( sqlite4_finalize(pStmt) ); + + return iRet; +} +static int integer_query3(sqlite3 *db, const char *zSql){ + int iRet; + sqlite3_stmt *pStmt; + + EXPLODE( sqlite3_prepare(db, zSql, -1, &pStmt, 0) ); + EXPLODE( SQLITE_ROW!=sqlite3_step(pStmt) ); + iRet = sqlite3_column_int(pStmt, 0); + EXPLODE( sqlite3_finalize(pStmt) ); + + return iRet; +} +/* +** End of integer query implementations. +*************************************************************************/ + +static int do_insert1_test4( + int nRow, /* Number of rows to insert in total */ + int nRowPerTrans, /* Number of rows per transaction */ + int nIdx, /* Number of aux indexes (aside from PK) */ + int iSync /* PRAGMA synchronous value (0, 1 or 2) */ +){ + char *zCreateTbl; /* Create table statement */ + char *zInsert; /* INSERT statement */ + sqlite4_stmt *pInsert; /* Compiled INSERT statement */ + sqlite4 *db = 0; /* Database handle */ + int i; /* Counter to count nRow rows */ + int nMs; /* Test time in ms */ + + lsm_db *pLsm; + + unlink_db(SQLITE4_DB_FILE); + EXPLODE( sqlite4_open(0, SQLITE4_DB_FILE, &db) ); + sqlite4_kvstore_control(db, "main", SQLITE4_KVCTRL_LSM_HANDLE, &pLsm); + i = iSync; + lsm_config(pLsm, LSM_CONFIG_SAFETY, &i); + assert( i==iSync ); + + install_rblob_function4(db); + + zCreateTbl = create_schema_sql(nIdx); + zInsert = create_insert_sql(nIdx); + + /* Create the db schema and prepare the INSERT statement */ + EXPLODE( sqlite4_exec(db, zCreateTbl, 0, 0, 0) ); + EXPLODE( sqlite4_prepare(db, zInsert, -1, &pInsert, 0) ); + + /* Run the test */ + testTimeInit(); + for(i=0; iaArg[iSel].nMax ){ + fprintf(stderr, "option %s out of range (%d..%d)\n", + aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax + ); + return -1; + } + + switch( iSel ){ + case 0: iDb = iVal; break; + case 1: nRow = iVal; break; + case 2: nRowPerTrans = iVal; break; + case 3: nIdx = iVal; break; + case 4: iSync = iVal; break; + } + } + + printf("insert1: db=%d rows=%d rowspertrans=%d indexes=%d sync=%d ... ", + iDb, nRow, nRowPerTrans, nIdx, iSync + ); + fflush(stdout); + if( iDb==3 ){ + do_insert1_test3(nRow, nRowPerTrans, nIdx, iSync); + }else{ + do_insert1_test4(nRow, nRowPerTrans, nIdx, iSync); + } + + return 0; +} + +static int do_select1_test4( + int nRow, /* Number of rows to read in total */ + int nRowPerTrans, /* Number of rows per transaction */ + int iIdx +){ + int nMs = 0; + sqlite4_stmt *pSelect = 0; + char *zSelect; + sqlite4 *db; + int i; + int nTblRow; + + EXPLODE( sqlite4_open(0, SQLITE4_DB_FILE, &db) ); + install_rblob_function4(db); + + nTblRow = integer_query4(db, "SELECT count(*) FROM t1"); + + /* Create the db schema and prepare the INSERT statement */ + zSelect = create_select_sql(iIdx); + EXPLODE( sqlite4_prepare(db, zSelect, -1, &pSelect, 0) ); + + testTimeInit(); + for(i=0; iaArg[iSel].nMax ){ + fprintf(stderr, "option %s out of range (%d..%d)\n", + aArg[iSel].zArg, aArg[iSel].nMin, aArg[iSel].nMax + ); + return -1; + } + + switch( iSel ){ + case 0: iDb = iVal; break; + case 1: nRow = iVal; break; + case 2: nRowPerTrans = iVal; break; + case 3: iIdx = iVal; break; + } + } + + printf("select1: db=%d rows=%d rowspertrans=%d index=%d ... ", + iDb, nRow, nRowPerTrans, iIdx + ); + fflush(stdout); + if( iDb==3 ){ + do_select1_test3(nRow, nRowPerTrans, iIdx); + }else{ + do_select1_test4(nRow, nRowPerTrans, iIdx); + } + + return 0; +} + +int main(int argc, char **argv){ + struct SqltestArg { + const char *zPrg; + int (*xPrg)(int, char **); + } aArg[] = { + {"select", do_select1}, + {"insert", do_insert1}, + {0, 0} + }; + int iSel; + int rc; + + if( argc<2 ){ + fprintf(stderr, "Usage: %s sub-program...\n", argv[0]); + return -1; + } + + rc = testArgSelectX(aArg, "sub-program", sizeof(aArg[0]), argv[1], &iSel); + if( rc!=0 ) return -1; + + aArg[iSel].xPrg(argc-2, argv+2); + return 0; +} Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -42,16 +42,16 @@ ################################################################################ # FIXME: Required options for now. # OPTS += -DLSM_MUTEX_NONE -OPTS += -DSQLITE4_DEBUG=1 -DLSM_DEBUG=1 +#OPTS += -DSQLITE4_DEBUG=1 -DLSM_DEBUG=1 OPTS += -DHAVE_GMTIME_R OPTS += -DHAVE_LOCALTIME_R OPTS += -DHAVE_MALLOC_USABLE_SIZE OPTS += -DHAVE_USLEEP -OPTS += -DSQLITE4_MEMDEBUG=1 +#OPTS += -DSQLITE4_MEMDEBUG=1 #OPTS += -DSQLITE4_NO_SYNC=1 -DLSM_NO_SYNC=1 OPTS += -DSQLITE4_OMIT_ANALYZE OPTS += -DSQLITE4_OMIT_AUTOMATIC_INDEX OPTS += -DSQLITE4_OMIT_BTREECOUNT OPTS += -DSQLITE4_OMIT_VIRTUALTABLE=1 @@ -68,18 +68,18 @@ LIBOBJ+= vdbe.o parse.o \ alter.o analyze.o attach.o auth.o \ build.o \ - callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ - fts5.o fts5func.o \ + callback.o complete.o ctime.o date.o delete.o env.o expr.o \ + fault.o fkey.o fts5.o fts5func.o \ func.o global.o hash.o \ icu.o insert.o kv.o kvlsm.o kvmem.o legacy.o \ lsm_ckpt.o lsm_file.o lsm_log.o lsm_main.o lsm_mem.o lsm_mutex.o \ lsm_shared.o lsm_str.o lsm_sorted.o lsm_tree.o \ lsm_unix.o lsm_varint.o \ - main.o malloc.o math.o mem0.o mem1.o mem2.o mem3.o mem5.o \ + main.o malloc.o math.o mem.o mem0.o mem1.o mem2.o mem3.o mem5.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ opcodes.o os.o \ pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o select.o status.o \ tokenize.o trigger.o \ @@ -99,10 +99,11 @@ $(TOP)/src/callback.c \ $(TOP)/src/complete.c \ $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ $(TOP)/src/delete.c \ + $(TOP)/src/env.c \ $(TOP)/src/expr.c \ $(TOP)/src/fault.c \ $(TOP)/src/fkey.c \ $(TOP)/src/fts5.c \ $(TOP)/src/fts5func.c \ @@ -132,10 +133,11 @@ $(TOP)/src/lsm_unix.c \ $(TOP)/src/lsm_varint.c \ $(TOP)/src/main.c \ $(TOP)/src/malloc.c \ $(TOP)/src/math.c \ + $(TOP)/src/mem.c \ $(TOP)/src/mem0.c \ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ $(TOP)/src/mem3.c \ $(TOP)/src/mem5.c \ @@ -487,11 +489,11 @@ TESTFIXTURE_PREREQ += libsqlite4.a testfixture$(EXE): $(TESTFIXTURE_PREREQ) $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ - -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite4.a + -o testfixture$(EXE) $(LIBTCL) libsqlite4.a $(THREADLIB) amalgamation-testfixture$(EXE): sqlite4.c $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite4.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) @@ -529,10 +531,15 @@ $(TCCX) -O2 sqlite4.o $(TOP)/test/threadtest3.c \ -o threadtest3$(EXE) $(THREADLIB) threadtest: threadtest3$(EXE) ./threadtest3$(EXE) + +SQLSRC = $(TOP)/lsm-test/sqltest.c $(TOP)/lsm-test/lsmtest_util.c +sqltest$(EXE): $(SQLSRC) libsqlite4.a + $(TCCX) $(TOP)/lsm-test/sqltest.c \ + -o sqltest$(EXE) -lsqlite3 libsqlite4.a $(THREADLIB) TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO) $(TEST_EXTENSION): $(TOP)/test/test_loadext.c $(MKSHLIB) $(TOP)/test/test_loadext.c -o $(TEST_EXTENSION) Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -77,11 +77,11 @@ assert( len>0 ); } while( token!=TK_LP && token!=TK_USING ); zRet = sqlite4MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, zTableName, tname.z+tname.n); - sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, zRet); } } /* @@ -140,11 +140,11 @@ sqlite4DbFree(db, zParent); } } zResult = sqlite4MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), - sqlite4_result_text(context, zResult, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zResult, -1, SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, zOutput); sqlite4DbFree(db, zResult); } #endif @@ -218,11 +218,11 @@ /* Variable tname now contains the token that is the old table-name ** in the CREATE TRIGGER statement. */ zRet = sqlite4MPrintf(db, "%.*s\"%w\"%s", ((u8*)tname.z) - zSql, zSql, zTableName, tname.z+tname.n); - sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zRet, -1, SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, zRet); } } #endif /* !SQLITE4_OMIT_TRIGGER */ Index: src/attach.c ================================================================== --- src/attach.c +++ src/attach.c @@ -241,10 +241,12 @@ goto detach_error; } sqlite4KVStoreClose(pDb->pKV); pDb->pKV = 0; + sqlite4SchemaClear(db->pEnv, pDb->pSchema); + sqlite4DbFree(db, pDb->pSchema); pDb->pSchema = 0; sqlite4ResetInternalSchema(db, -1); return; detach_error: Index: src/callback.c ================================================================== --- src/callback.c +++ src/callback.c @@ -30,11 +30,11 @@ } #ifndef SQLITE4_OMIT_UTF16 if( db->xCollNeeded16 ){ char const *zExternal; sqlite4_value *pTmp = sqlite4ValueNew(db); - sqlite4ValueSetStr(pTmp, -1, zName, SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4ValueSetStr(pTmp, -1, zName, SQLITE4_UTF8, SQLITE4_STATIC, 0); zExternal = sqlite4ValueText(pTmp, SQLITE4_UTF16NATIVE); if( zExternal ){ db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); } sqlite4ValueFree(pTmp); Index: src/complete.c ================================================================== --- src/complete.c +++ src/complete.c @@ -267,11 +267,11 @@ #ifndef SQLITE4_OMIT_AUTOINIT rc = sqlite4_initialize(0); if( rc ) return rc; #endif pVal = sqlite4ValueNew(0); - sqlite4ValueSetStr(pVal, -1, zSql, SQLITE4_UTF16NATIVE, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, -1, zSql, SQLITE4_UTF16NATIVE, SQLITE4_STATIC, 0); zSql8 = sqlite4ValueText(pVal, SQLITE4_UTF8); if( zSql8 ){ rc = sqlite4_complete(zSql8); }else{ rc = SQLITE4_NOMEM; Index: src/date.c ================================================================== --- src/date.c +++ src/date.c @@ -812,11 +812,11 @@ if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeYMD_HMS(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%04d-%02d-%02d %02d:%02d:%02d", x.Y, x.M, x.D, x.h, x.m, (int)(x.s)); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } /* ** time( TIMESTRING, MOD, MOD, ...) @@ -831,11 +831,11 @@ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeHMS(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%02d:%02d:%02d", x.h, x.m, (int)x.s); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } /* ** date( TIMESTRING, MOD, MOD, ...) @@ -850,11 +850,11 @@ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ char zBuf[100]; computeYMD(&x); sqlite4_snprintf(zBuf,sizeof(zBuf), "%04d-%02d-%02d", x.Y, x.M, x.D); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) @@ -998,11 +998,11 @@ } } } z[j] = 0; sqlite4_result_text(context, z, -1, - z==zBuf ? SQLITE4_TRANSIENT : SQLITE4_DYNAMIC); + z==zBuf ? SQLITE4_TRANSIENT : SQLITE4_DYNAMIC, 0); } /* ** current_time() ** @@ -1085,11 +1085,11 @@ if( pTm ) memcpy(&sNow, pTm, sizeof(sNow)); sqlite4_mutex_leave(sqlite4MutexAlloc(SQLITE4_MUTEX_STATIC_MASTER)); #endif if( pTm ){ strftime(zBuf, 20, zFormat, &sNow); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } } #endif /* Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -116,34 +116,35 @@ sqlite4Select(pParse, pDup, &dest); sqlite4SelectDelete(db, pDup); } #endif /* !defined(SQLITE4_OMIT_VIEW) && !defined(SQLITE4_OMIT_TRIGGER) */ -#if defined(SQLITE4_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE4_OMIT_SUBQUERY) +#if defined(SQLITE4_ENABLE_UPDATE_DELETE_LIMIT) \ + && !defined(SQLITE4_OMIT_SUBQUERY) /* ** Generate an expression tree to implement the WHERE, ORDER BY, ** and LIMIT/OFFSET portion of DELETE and UPDATE statements. ** ** DELETE FROM table_wxyz WHERE a<5 ORDER BY a LIMIT 1; ** \__________________________/ ** pLimitWhere (pInClause) */ Expr *sqlite4LimitWhere( - Parse *pParse, /* The parser context */ - SrcList *pSrc, /* the FROM clause -- which tables to scan */ - Expr *pWhere, /* The WHERE clause. May be null */ - ExprList *pOrderBy, /* The ORDER BY clause. May be null */ - Expr *pLimit, /* The LIMIT clause. May be null */ - Expr *pOffset, /* The OFFSET clause. May be null */ - char *zStmtType /* Either DELETE or UPDATE. For error messages. */ + Parse *pParse, /* The parser context */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* The ORDER BY clause. May be null */ + Expr *pLimit, /* The LIMIT clause. May be null */ + Expr *pOffset, /* The OFFSET clause. May be null */ + char *zStmtType /* Either DELETE or UPDATE. For error messages. */ ){ - Expr *pWhereRowid = NULL; /* WHERE rowid .. */ - Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ - Expr *pSelectRowid = NULL; /* SELECT rowid ... */ - ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ - SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ - Select *pSelect = NULL; /* Complete SELECT tree */ + Expr *pWhereRowid = NULL; /* WHERE rowid .. */ + Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ + Expr *pSelectRowid = NULL;/* SELECT rowid ... */ + ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + SrcList *pSelectSrc = NULL;/* SELECT rowid FROM x ... (dup of pSrc) */ + Select *pSelect = NULL; /* Complete SELECT tree */ /* Check that there isn't an ORDER BY without a LIMIT clause. */ if( pOrderBy && (pLimit == 0) ) { sqlite4ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); @@ -207,11 +208,12 @@ sqlite4ExprListDelete(pParse->db, pOrderBy); sqlite4ExprDelete(pParse->db, pLimit); sqlite4ExprDelete(pParse->db, pOffset); return 0; } -#endif /* defined(SQLITE4_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE4_OMIT_SUBQUERY) */ +#endif /* defined(SQLITE4_ENABLE_UPDATE_DELETE_LIMIT) */ + /* && !defined(SQLITE4_OMIT_SUBQUERY) */ /* ** Generate code for a DELETE FROM statement. ** ** DELETE FROM table_wxyz WHERE a<5 AND b NOT NULL; @@ -263,11 +265,12 @@ assert( !IsView(pTab) || pTrigger ); assert( !IsView(pTab) || pTab->pIndex==0 ); /* Invoke the authorization callback */ rcauth = sqlite4AuthCheck(pParse, SQLITE4_DELETE, pTab->zName, 0, zDb); - assert( rcauth==SQLITE4_OK || rcauth==SQLITE4_DENY || rcauth==SQLITE4_IGNORE ); + assert( rcauth==SQLITE4_OK || rcauth==SQLITE4_DENY + || rcauth==SQLITE4_IGNORE ); if( rcauth==SQLITE4_DENY ){ goto delete_from_cleanup; } /* Assign a cursor number to the table or view this statement is ADDED src/env.c Index: src/env.c ================================================================== --- /dev/null +++ src/env.c @@ -0,0 +1,442 @@ +/* +** 2013 January 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to help implement the sqlite4_env object. +*/ +#include "sqliteInt.h" + + +/* +** Default factory objects +*/ +static KVFactory memFactory = { + 0, + "temp", + sqlite4KVStoreOpenMem, + 1 +}; +KVFactory sqlite4BuiltinFactory = { + &memFactory, + "main", + sqlite4KVStoreOpenLsm, + 1 +}; + +/* +** The following singleton contains the global configuration for +** the SQLite library. +*/ +struct sqlite4_env sqlite4DefaultEnv = { + sizeof(sqlite4_env), /* nByte */ + 1, /* iVersion */ + SQLITE4_DEFAULT_MEMSTATUS, /* bMemstat */ + 1, /* bCoreMutex */ + SQLITE4_THREADSAFE==1, /* bFullMutex */ + 0x7ffffffe, /* mxStrlen */ + 128, /* szLookaside */ + 500, /* nLookaside */ + &sqlite4MMSystem, /* pMM */ + {0,0,0,0,0,0,0,0,0}, /* m */ + {0,0,0,0,0,0,0,0,0,0}, /* mutex */ + (void*)0, /* pHeap */ + 0, /* nHeap */ + 0, 0, /* mnHeap, mxHeap */ + 0, /* mxParserStack */ + &sqlite4BuiltinFactory, /* pFactory */ + sqlite4OsRandomness, /* xRandomness */ + sqlite4OsCurrentTime, /* xCurrentTime */ + /* All the rest should always be initialized to zero */ + 0, /* isInit */ + 0, /* pFactoryMutex */ + 0, /* pPrngMutex */ + 0, 0, /* prngX, prngY */ + 0, /* xLog */ + 0, /* pLogArg */ + 0, /* bLocaltimeFault */ + 0, /* pMemMutex */ + {0,0,0,0}, /* nowValue[] */ + {0,0,0,0}, /* mxValue[] */ + {0,} /* hashGlobalFunc */ +}; + +/* +** Return the default environment +*/ +sqlite4_env *sqlite4_env_default(void){ return &sqlite4DefaultEnv; } + +/* +** Initialize SQLite. +** +** This routine must be called to initialize the run-time environment +** As long as you do not compile with SQLITE4_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite4_open(). +** +** This routine is a no-op except on its very first call for a given +** sqlite4_env object, or for the first call after a call to sqlite4_shutdown. +** +** This routine is not threadsafe. It should be called from a single +** thread to initialized the library in a multi-threaded system. Other +** threads should avoid using the sqlite4_env object until after it has +** completely initialized. +*/ +int sqlite4_initialize(sqlite4_env *pEnv){ + MUTEX_LOGIC( sqlite4_mutex *pMaster; ) /* The main static mutex */ + int rc; /* Result code */ + + if( pEnv==0 ) pEnv = &sqlite4DefaultEnv; + + /* If SQLite is already completely initialized, then this call + ** to sqlite4_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( pEnv->isInit ) return SQLITE4_OK; + + /* Initialize the mutex subsystem + */ + rc = sqlite4MutexInit(pEnv); + if( rc ){ + sqlite4MallocEnd(pEnv); + return rc; + } + + /* Initialize the memory allocation subsystem + */ + rc = sqlite4MallocInit(pEnv); + if( rc ) return rc; + + /* Create required mutexes + */ + if( pEnv->bCoreMutex ){ + pEnv->pMemMutex = sqlite4MutexAlloc(pEnv, SQLITE4_MUTEX_FAST); + pEnv->pPrngMutex = sqlite4MutexAlloc(pEnv, SQLITE4_MUTEX_FAST); + pEnv->pFactoryMutex = sqlite4MutexAlloc(pEnv, SQLITE4_MUTEX_FAST); + if( pEnv->pMemMutex==0 + || pEnv->pPrngMutex==0 + || pEnv->pFactoryMutex==0 + ){ + rc = SQLITE4_NOMEM; + } + }else{ + pEnv->pMemMutex = 0; + pEnv->pPrngMutex = 0; + } + pEnv->isInit = 1; + + sqlite4OsInit(pEnv); + + /* Register global functions */ + if( rc==SQLITE4_OK ){ + sqlite4RegisterGlobalFunctions(pEnv); + } + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE4_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if ( rc==SQLITE4_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite4IsNaN(y) ); + } +#endif +#endif + + return rc; +} + +/* +** Undo the effects of sqlite4_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +int sqlite4_shutdown(sqlite4_env *pEnv){ + if( pEnv==0 ) pEnv = &sqlite4DefaultEnv; + if( pEnv->isInit ){ + KVFactory *pMkr; + sqlite4_mutex_free(pEnv->pFactoryMutex); + sqlite4_mutex_free(pEnv->pPrngMutex); + sqlite4_mutex_free(pEnv->pMemMutex); + pEnv->pMemMutex = 0; + while( (pMkr = pEnv->pFactory)!=0 && pMkr->isPerm==0 ){ + KVFactory *pNext = pMkr->pNext; + sqlite4_free(pEnv, pMkr); + pMkr = pNext; + } + sqlite4MutexEnd(pEnv); + sqlite4MallocEnd(pEnv); + pEnv->isInit = 0; + } + return SQLITE4_OK; +} + +/* +** Return the size of an sqlite4_env object +*/ +int sqlite4_env_size(void){ return sizeof(sqlite4_env); } + +/* +** This API allows applications to modify the configuration described by +** an sqlite4_env object. +*/ +int sqlite4_env_config(sqlite4_env *pEnv, int op, ...){ + va_list ap; + int rc = SQLITE4_OK; + + if( pEnv==0 ) pEnv = sqlite4_env_default(); + + va_start(ap, op); + switch( op ){ + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_INIT, template); + ** + ** Turn bulk memory into a new sqlite4_env object. The template is + ** a prior sqlite4_env that is used as a template in initializing the + ** new sqlite4_env object. The size of the bulk memory must be at + ** least as many bytes as returned from sqlite4_env_size(). + */ + case SQLITE4_ENVCONFIG_INIT: { + /* Disable all mutexing */ + sqlite4_env *pTemplate = va_arg(ap, sqlite4_env*); + int n = pTemplate->nByte; + if( n>sizeof(sqlite4_env) ) n = sizeof(sqlite4_env); + memcpy(pEnv, pTemplate, n); + pEnv->pFactory = &sqlite4BuiltinFactory; + pEnv->isInit = 0; + break; + } + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE4_THREADSAFE) && SQLITE4_THREADSAFE>0 + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_SINGLETHREAD); + ** + ** Configure this environment for a single-threaded application. + */ + case SQLITE4_ENVCONFIG_SINGLETHREAD: { + /* Disable all mutexing */ + if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } + pEnv->bCoreMutex = 0; + pEnv->bFullMutex = 0; + break; + } + + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_MULTITHREAD); + ** + ** Configure this environment for a multi-threaded application where + ** the same database connection is never used by more than a single + ** thread at a time. + */ + case SQLITE4_ENVCONFIG_MULTITHREAD: { + /* Disable mutexing of database connections */ + /* Enable mutexing of core data structures */ + if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } + pEnv->bCoreMutex = 1; + pEnv->bFullMutex = 0; + break; + } + + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_SERIALIZED); + ** + ** Configure this environment for an unrestricted multi-threaded + ** application where any thread can do whatever it wants with any + ** database connection at any time. + */ + case SQLITE4_ENVCONFIG_SERIALIZED: { + /* Enable all mutexing */ + if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } + pEnv->bCoreMutex = 1; + pEnv->bFullMutex = 1; + break; + } + + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_MUTEXT, sqlite4_mutex_methods*) + ** + ** Configure this environment to use the mutex routines specified by the + ** argument. + */ + case SQLITE4_ENVCONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } + pEnv->mutex = *va_arg(ap, sqlite4_mutex_methods*); + break; + } + + /* + ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_GETMUTEX, sqlite4_mutex_methods*) + ** + ** Copy the mutex routines in use by this environment into the structure + ** given in the argument. + */ + case SQLITE4_ENVCONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite4_mutex_methods*) = pEnv->mutex; + break; + } +#endif + + + /* + ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_MALLOC, sqlite4_mem_methods*) + ** + ** Set the memory allocation routines to be used by this environment. + */ + case SQLITE4_ENVCONFIG_MALLOC: { + /* Specify an alternative malloc implementation */ + if( pEnv->isInit ) return SQLITE4_MISUSE; + pEnv->m = *va_arg(ap, sqlite4_mem_methods*); + break; + } + + /* + ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_GETMALLOC, sqlite4_mem_methods*) + ** + ** Copy the memory allocation routines in use by this environment + ** into the structure given in the argument. + */ + case SQLITE4_ENVCONFIG_GETMALLOC: { + /* Retrieve the current malloc() implementation */ + if( pEnv->m.xMalloc==0 ) sqlite4MemSetDefault(pEnv); + *va_arg(ap, sqlite4_mem_methods*) = pEnv->m; + break; + } + + /* sqlite4_env_config(p, SQLITE4_ENVCONFIG_MEMSTAT, int onoff); + ** + ** Enable or disable collection of memory usage statistics according to + ** the onoff parameter. + */ + case SQLITE4_ENVCONFIG_MEMSTATUS: { + /* Enable or disable the malloc status collection */ + pEnv->bMemstat = va_arg(ap, int); + break; + } + + /* + ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_LOOKASIDE, size, count); + ** + ** Set the default lookaside memory settings for all subsequent + ** database connections constructed in this environment. The size + ** parameter is the size of each lookaside memory buffer and the + ** count parameter is the number of lookaside buffers. Set both + ** to zero to disable lookaside memory. + */ + case SQLITE4_ENVCONFIG_LOOKASIDE: { + pEnv->szLookaside = va_arg(ap, int); + pEnv->nLookaside = va_arg(ap, int); + break; + } + + /* + ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_LOG, xOutput, pArg); + ** + ** Set the log function that is called in response to sqlite4_log() + ** calls. + */ + case SQLITE4_ENVCONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** pEnv->xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + pEnv->xLog = va_arg(ap, LOGFUNC_t); + pEnv->pLogArg = va_arg(ap, void*); + break; + } + + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_PUSH, zName,xFactory); + ** + ** Push a new KVStore factory onto the factory stack. The new factory + ** takes priority over prior factories. + */ + case SQLITE4_ENVCONFIG_KVSTORE_PUSH: { + const char *zName = va_arg(ap, const char*); + int nName = sqlite4Strlen30(zName); + KVFactory *pMkr = sqlite4_malloc(pEnv, sizeof(*pMkr)+nName+1); + char *z; + if( pMkr==0 ) return SQLITE4_NOMEM; + z = (char*)&pMkr[1]; + memcpy(z, zName, nName+1); + memset(pMkr, 0, sizeof(*pMkr)); + pMkr->zName = z; + pMkr->xFactory = va_arg(ap, sqlite4_kvfactory); + sqlite4_mutex_enter(pEnv->pFactoryMutex); + pMkr->pNext = pEnv->pFactory; + pEnv->pFactory = pMkr; + sqlite4_mutex_leave(pEnv->pFactoryMutex); + break; + } + + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_POP, zName, &pxFact); + ** + ** Remove a KVStore factory from the stack. + */ + /* + ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_GET, zName, &pxFact); + ** + ** Get the current factory pointer with the given name but leave the + ** factory on the stack. + */ + case SQLITE4_ENVCONFIG_KVSTORE_POP: + case SQLITE4_ENVCONFIG_KVSTORE_GET: { + typedef int (**PxFact)(sqlite4_env*,KVStore**,const char*,unsigned); + const char *zName = va_arg(ap, const char*); + KVFactory *pMkr, **ppPrev; + PxFact pxFact; + + pxFact = va_arg(ap,PxFact); + *pxFact = 0; + sqlite4_mutex_enter(pEnv->pFactoryMutex); + ppPrev = &pEnv->pFactory; + pMkr = *ppPrev; + while( pMkr && strcmp(zName, pMkr->zName)!=0 ){ + ppPrev = &pMkr->pNext; + pMkr = *ppPrev; + } + if( pMkr ){ + *pxFact = pMkr->xFactory; + if( op==SQLITE4_ENVCONFIG_KVSTORE_POP && pMkr->isPerm==0 ){ + *ppPrev = pMkr->pNext; + sqlite4_free(pEnv, pMkr); + } + } + sqlite4_mutex_leave(pEnv->pFactoryMutex); + break; + } + + + default: { + rc = SQLITE4_ERROR; + break; + } + } + va_end(ap); + return rc; +} Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -364,12 +364,11 @@ ** is responsible for making sure the node eventually gets freed. ** ** If dequote is true, then the token (if it exists) is dequoted. ** If dequote is false, no dequoting is performance. The deQuote ** parameter is ignored if pToken is NULL or if the token does not -** appear to be quoted. If the quotes were of the form "..." (double-quotes) -** then the EP_DblQuoted flag is set on the expression node. +** appear to be quoted. ** ** Special case: If op==TK_INTEGER and pToken points to a string that ** can be translated into a 32-bit integer, then the token is not ** stored in u.zToken. Instead, the integer values is written ** into u.iValue and the EP_IntValue flag is set. No extra storage @@ -405,13 +404,12 @@ pNew->u.zToken = (char*)&pNew[1]; assert( pToken->z!=0 || pToken->n==0 ); if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; if( dequote && nExtra>=3 - && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ + && ((c = pToken->z[0])=='\'' || c=='"' || c=='[') ){ sqlite4Dequote(pNew->u.zToken); - if( c=='"' ) pNew->flags |= EP_DblQuoted; } } } #if SQLITE4_MAX_EXPR_DEPTH>0 pNew->nHeight = 1; Index: src/fts5.c ================================================================== --- src/fts5.c +++ src/fts5.c @@ -3394,11 +3394,11 @@ } goto fts5_parse_expr_out; } fts5PrintExpr(db, azCol, pExpr, &zRet); - sqlite4_result_text(pCtx, zRet, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(pCtx, zRet, -1, SQLITE4_TRANSIENT, 0); fts5ExpressionFree(db, pExpr); sqlite4_free(sqlite4_db_env(db), zRet); fts5_parse_expr_out: if( p ) pTok->xDestroy(p); @@ -3421,6 +3421,5 @@ ); if( rc!=SQLITE4_OK ) return rc; #endif return sqlite4InitFts5Func(db); } - Index: src/fts5func.c ================================================================== --- src/fts5func.c +++ src/fts5func.c @@ -47,11 +47,11 @@ double *aAvgdl; /* Average document size of each field */ int nPhrase; /* Number of phrases in query */ double *aIdf; /* IDF weights for each phrase in query */ }; -static void fts5RankFreeCtx(void *pCtx){ +static void fts5RankFreeCtx(void *pNotUsed, void *pCtx){ if( pCtx ){ Fts5RankCtx *p = (Fts5RankCtx *)pCtx; sqlite4DbFree(p->db, p); } } @@ -134,11 +134,11 @@ int nByte; /* Number of bytes of data to allocate */ sqlite4_mi_phrase_count(pCtx, &nPhrase); nByte = sizeof(Fts5RankCtx) + (nPhrase+nField) * sizeof(double); p = (Fts5RankCtx *)sqlite4DbMallocZero(db, nByte); - sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx); + sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx, 0); p = sqlite4_get_auxdata(pCtx, 0); if( !p ){ rc = SQLITE4_NOMEM; }else{ @@ -268,11 +268,11 @@ if( rc==SQLITE4_OK ){ if( bExplain ){ zExplain = sqlite4MAppendf( db, zExplain, "%soverall rank=%.2f", zExplain, rank ); - sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT, 0); }else{ sqlite4_result_double(pCtx, rank); } }else{ sqlite4_result_error_code(pCtx, rc); @@ -544,10 +544,43 @@ mask = mask >> nShift; pSnip->iOff = iOff; pSnip->hlmask = mask; } + +/* +** Parameter aSnip points to an array of nSnip Snippet objects, where nSnip +** is less than or equal to 4. This function sorts the array in place in +** ascending order of Snippet.iCol and Snippet.iOff. +*/ +static void fts5SnippetSort(Snippet *aSnip, int nSnip){ + Snippet aTmp[4]; + int i; + + assert( nSnip<=4 && nSnip>=1 ); + + for(i=0; iiCol>=0 && (iBest<0 + || pTry->iColiCol==aSnip[iBest].iCol && pTry->iOff=0 ); + memcpy(&aTmp[i], &aSnip[iBest], sizeof(Snippet)); + aSnip[iBest].iCol = -1; + } + + memcpy(aSnip, aTmp, sizeof(Snippet)*nSnip); +} + static void fts5Snippet(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){ Snippet aSnip[4]; int nSnip; int iCol = -1; @@ -581,10 +614,11 @@ for(i=0; rc==SQLITE4_OK && ilen ){ p2 = len-p1; if( p2<0 ) p2 = 0; } - sqlite4_result_blob(context, (char*)&z[p1], (int)p2, SQLITE4_TRANSIENT); + sqlite4_result_blob(context, (char*)&z[p1], (int)p2, SQLITE4_TRANSIENT, 0); } } /* ** Implementation of the round() function @@ -334,11 +334,11 @@ z1 = contextMalloc(context, ((i64)n)+1); if( z1 ){ for(i=0; i>4)&0xf]; *(z++) = hexdigits[c&0xf]; } *z = 0; - sqlite4_result_text(context, zHex, n*2, SQLITE4_DYNAMIC); - } -} - -/* -** The zeroblob(N) function returns a zero-filled blob of size N bytes. -*/ -static void zeroblobFunc( - sqlite4_context *context, - int argc, - sqlite4_value **argv -){ - i64 n; - sqlite4 *db = sqlite4_context_db_handle(context); - assert( argc==1 ); - UNUSED_PARAMETER(argc); - n = sqlite4_value_int64(argv[0]); - testcase( n==db->aLimit[SQLITE4_LIMIT_LENGTH] ); - testcase( n==db->aLimit[SQLITE4_LIMIT_LENGTH]+1 ); - if( n>db->aLimit[SQLITE4_LIMIT_LENGTH] ){ - sqlite4_result_error_toobig(context); - }else{ - sqlite4_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ + sqlite4_result_text(context, zHex, n*2, SQLITE4_DYNAMIC, 0); } } /* ** The replace() function. Three arguments are all strings: call @@ -1045,11 +1007,11 @@ assert( j+nStr-i+1==nOut ); memcpy(&zOut[j], &zStr[i], nStr-i); j += nStr - i; assert( j<=nOut ); zOut[j] = 0; - sqlite4_result_text(context, (char*)zOut, j, SQLITE4_DYNAMIC); + sqlite4_result_text(context, (char*)zOut, j, SQLITE4_DYNAMIC, 0); } /* ** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. ** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. @@ -1129,11 +1091,11 @@ } if( zCharSet ){ sqlite4_free(sqlite4_context_env(context), azChar); } } - sqlite4_result_text(context, (char*)zIn, nIn, SQLITE4_TRANSIENT); + sqlite4_result_text(context, (char*)zIn, nIn, SQLITE4_TRANSIENT, 0); } /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE4_SOUNDEX compile-time option is used @@ -1184,15 +1146,15 @@ } while( j<4 ){ zResult[j++] = '0'; } zResult[j] = 0; - sqlite4_result_text(context, zResult, 4, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zResult, 4, SQLITE4_TRANSIENT, 0); }else{ /* IMP: R-64894-50321 The string "?000" is returned if the argument ** is NULL or contains no ASCII alphabetic characters. */ - sqlite4_result_text(context, "?000", 4, SQLITE4_STATIC); + sqlite4_result_text(context, "?000", 4, SQLITE4_STATIC, 0); } } #endif /* SQLITE4_SOUNDEX */ #if 0 /*ndef SQLITE4_OMIT_LOAD_EXTENSION*/ @@ -1305,19 +1267,10 @@ CountCtx *p; p = sqlite4_aggregate_context(context, sizeof(*p)); if( (argc==0 || SQLITE4_NULL!=sqlite4_value_type(argv[0])) && p ){ p->n++; } - -#ifndef SQLITE4_OMIT_DEPRECATED - /* The sqlite4_aggregate_count() function is deprecated. But just to make - ** sure it still operates correctly, verify that its count agrees with our - ** internal count when using count(*) and when the total count can be - ** expressed as a 32-bit integer. */ - assert( argc==1 || p==0 || p->n>0x7fffffff - || p->n==sqlite4_aggregate_count(context) ); -#endif } static void countFinalize(sqlite4_context *context){ CountCtx *p; p = sqlite4_aggregate_context(context, 0); sqlite4_result_int64(context, p ? p->n : 0); @@ -1416,11 +1369,11 @@ sqlite4_result_error_toobig(context); }else if( pAccum->mallocFailed ){ sqlite4_result_error_nomem(context); }else{ sqlite4_result_text(context, sqlite4StrAccumFinish(pAccum), -1, - SQLITE4_DYNAMIC); + SQLITE4_DYNAMIC, 0); } } } /* @@ -1559,15 +1512,13 @@ #ifndef SQLITE4_OMIT_COMPILEOPTION_DIAGS FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), #endif /* SQLITE4_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), - FUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), FUNCTION(changes, 0, 0, 0, changes ), FUNCTION(total_changes, 0, 0, 0, total_changes ), FUNCTION(replace, 3, 0, 0, replaceFunc ), - FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), #ifdef SQLITE4_SOUNDEX FUNCTION(soundex, 1, 0, 0, soundexFunc ), #endif #if 0 /*ndef SQLITE4_OMIT_LOAD_EXTENSION*/ FUNCTION(load_extension, 1, 0, 0, loadExt ), Index: src/global.c ================================================================== --- src/global.c +++ src/global.c @@ -127,67 +127,10 @@ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ }; #endif -/* -** Default factory objects -*/ -static KVFactory memFactory = { - 0, - "temp", - sqlite4KVStoreOpenMem, - 1 -}; -KVFactory sqlite4BuiltinFactory = { - &memFactory, - "main", - sqlite4KVStoreOpenLsm, - 1 -}; - -/* -** The following singleton contains the global configuration for -** the SQLite library. -*/ -struct sqlite4_env sqlite4DefaultEnv = { - sizeof(sqlite4_env), /* nByte */ - 1, /* iVersion */ - SQLITE4_DEFAULT_MEMSTATUS, /* bMemstat */ - 1, /* bCoreMutex */ - SQLITE4_THREADSAFE==1, /* bFullMutex */ - 0x7ffffffe, /* mxStrlen */ - 128, /* szLookaside */ - 500, /* nLookaside */ - {0,0,0,0,0,0,0,0,0}, /* m */ - {0,0,0,0,0,0,0,0,0,0}, /* mutex */ - (void*)0, /* pHeap */ - 0, /* nHeap */ - 0, 0, /* mnHeap, mxHeap */ - 0, /* mxParserStack */ - &sqlite4BuiltinFactory, /* pFactory */ - sqlite4OsRandomness, /* xRandomness */ - sqlite4OsCurrentTime, /* xCurrentTime */ - /* All the rest should always be initialized to zero */ - 0, /* isInit */ - 0, /* pFactoryMutex */ - 0, /* pPrngMutex */ - 0, 0, /* prngX, prngY */ - 0, /* xLog */ - 0, /* pLogArg */ - 0, /* bLocaltimeFault */ - 0, /* pMemMutex */ - {0,0,0,0}, /* nowValue[] */ - {0,0,0,0}, /* mxValue[] */ - {0,} /* hashGlobalFunc */ -}; - -/* -** Return the default environment -*/ -sqlite4_env *sqlite4_env_default(void){ return &sqlite4DefaultEnv; } - /* ** Constant tokens for values 0 and 1. */ const Token sqlite4IntTokens[] = { Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -1317,13 +1317,10 @@ int idx = pPk->aiColumn[i]; sqlite4VdbeAddOp2(v, OP_SCopy, regContent+idx, regTmp+i+pIdx->nColumn); } } sqlite4VdbeAddOp3(v, OP_MakeIdxKey, iIdx, regTmp, regKey); - if( pIdx==pPk && (pPk->fIndex & IDX_IntPK)!=0 ){ - sqlite4VdbeChangeP5(v, OPFLAG_LASTROWID); - } VdbeComment((v, "key for %s", pIdx->zName)); /* If Index.onError==OE_None, then pIdx is not a UNIQUE or PRIMARY KEY ** index. In this case there is no need to test the index for uniqueness ** - all that is required is to populate the regKey register. Jump @@ -1802,11 +1799,11 @@ addr1 = sqlite4VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } sqlite4VdbeAddOp2(v, OP_RowData, iSrc, regData); sqlite4VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); - sqlite4VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + sqlite4VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_APPEND); sqlite4VdbeChangeP4(v, -1, pDest->zName, 0); sqlite4VdbeAddOp2(v, OP_Next, iSrc, addr1); for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; Index: src/kvlsm.c ================================================================== --- src/kvlsm.c +++ src/kvlsm.c @@ -461,11 +461,11 @@ pNew->base.pStoreVfunc = &kvlsmMethods; pNew->base.pEnv = pEnv; rc = lsm_new(0, &pNew->pDb); if( rc==SQLITE4_OK ){ int i; - int bMmap = 0; + int bMmap = 1; lsm_config(pNew->pDb, LSM_CONFIG_MMAP, &bMmap); for(i=0; i=0 ); + assert( pSnap->iCmpId==pDb->compress.iId + || pSnap->iCmpId==LSM_COMPRESSION_EMPTY + ); ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); + ckptSetValue(&ckpt, CKPT_HDR_CMPID, pDb->compress.iId, &rc); ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); @@ -759,23 +771,24 @@ ** Initialize the shared-memory header with an empty snapshot. This function ** is called when no valid snapshot can be found in the database header. */ static void ckptLoadEmpty(lsm_db *pDb){ u32 aCkpt[] = { - 0, /* CKPT_HDR_ID_MSW */ - 10, /* CKPT_HDR_ID_LSW */ - 0, /* CKPT_HDR_NCKPT */ - 0, /* CKPT_HDR_NBLOCK */ - 0, /* CKPT_HDR_BLKSZ */ - 0, /* CKPT_HDR_NLEVEL */ - 0, /* CKPT_HDR_PGSZ */ - 0, /* CKPT_HDR_OVFL */ - 0, /* CKPT_HDR_NWRITE */ - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ - 0,0,0,0, 0,0,0,0, /* The append list */ - 0, /* The free block list */ - 0, 0 /* Space for checksum values */ + 0, /* CKPT_HDR_ID_MSW */ + 10, /* CKPT_HDR_ID_LSW */ + 0, /* CKPT_HDR_NCKPT */ + LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ + 0, /* CKPT_HDR_NBLOCK */ + 0, /* CKPT_HDR_BLKSZ */ + 0, /* CKPT_HDR_NLEVEL */ + 0, /* CKPT_HDR_PGSZ */ + 0, /* CKPT_HDR_OVFL */ + 0, /* CKPT_HDR_NWRITE */ + 0, 0, 1234, 5678, /* The log pointer and initial checksum */ + 0,0,0,0, 0,0,0,0, /* The append list */ + 0, /* The free block list */ + 0, 0 /* Space for checksum values */ }; u32 nCkpt = array_size(aCkpt); ShmHeader *pShm = pDb->pShmhdr; aCkpt[CKPT_HDR_NCKPT] = nCkpt; @@ -874,11 +887,23 @@ } } lsmShmBarrier(pDb); } - return LSM_PROTOCOL; + return LSM_PROTOCOL_BKPT; +} + +int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId){ + int rc; + + assert( db->pClient==0 && db->pWorker==0 ); + rc = lsmCheckpointLoad(db, 0); + if( rc==LSM_OK ){ + *piCmpId = db->aSnapshot[CKPT_HDR_CMPID]; + } + + return rc; } int lsmCheckpointLoadOk(lsm_db *pDb, int iSnap){ u32 *aShm; assert( iSnap==1 || iSnap==2 ); @@ -901,11 +926,11 @@ int nInt2; /* Must be holding the WORKER lock to do this. Or DMS2. */ assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) + || lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) ); /* Check that the two snapshots match. If not, repair them. */ nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; @@ -913,16 +938,20 @@ if( ckptChecksumOk(pShm->aSnap1) ){ memcpy(pShm->aSnap2, pShm->aSnap1, sizeof(u32)*nInt1); }else if( ckptChecksumOk(pShm->aSnap2) ){ memcpy(pShm->aSnap1, pShm->aSnap2, sizeof(u32)*nInt2); }else{ - return LSM_PROTOCOL; + return LSM_PROTOCOL_BKPT; } } rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; + + if( rc==LSM_OK ){ + rc = lsmCheckCompressionId(pDb, pDb->pWorker->iCmpId); + } #if 0 assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); #endif return rc; @@ -948,10 +977,11 @@ pNew->iId = lsmCheckpointId(aCkpt, 0); pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); + pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; /* Make a copy of the append-list */ for(i=0; iaiAppend[i] = ckptRead64(a); @@ -1097,11 +1127,11 @@ if( aCopy ){ memcpy(aCopy, aData, nCkpt*sizeof(u32)); ckptChangeEndianness(aCopy, nCkpt); if( ckptChecksumOk(aCopy) ){ if( piId ) *piId = lsmCheckpointId(aCopy, 0); - if( piLog ) *piLog = lsmCheckpointLogOffset(aCopy); + if( piLog ) *piLog = (lsmCheckpointLogOffset(aCopy) >> 1); if( pnWrite ) *pnWrite = aCopy[CKPT_HDR_NWRITE]; } lsmFree(pDb->pEnv, aCopy); } } @@ -1157,13 +1187,12 @@ void lsmCheckpointLogoffset( u32 *aCkpt, DbLog *pLog ){ - u32 iOffMSB = aCkpt[CKPT_HDR_LO_MSW]; - u32 iOffLSB = aCkpt[CKPT_HDR_LO_LSW]; - pLog->aRegion[2].iStart = (((i64)iOffMSB) << 32) + ((i64)iOffLSB); + pLog->aRegion[2].iStart = (lsmCheckpointLogOffset(aCkpt) >> 1); + pLog->cksum0 = aCkpt[CKPT_HDR_LO_CKSUM1]; pLog->cksum1 = aCkpt[CKPT_HDR_LO_CKSUM2]; pLog->iSnapshotId = lsmCheckpointId(aCkpt, 0); } @@ -1184,20 +1213,27 @@ memcpy(pDb->pShmhdr->aSnap1, pDb->aSnapshot, nCkpt*sizeof(u32)); memcpy(pDb->pShmhdr->aSnap2, pDb->aSnapshot, nCkpt*sizeof(u32)); } -int lsmCheckpointSize(lsm_db *db, int *pnByte){ +/* +** Set the output variable to the number of KB of data written into the +** database file since the most recent checkpoint. +*/ +int lsmCheckpointSize(lsm_db *db, int *pnKB){ ShmHeader *pShm = db->pShmhdr; int rc = LSM_OK; u32 nSynced; + /* Set nSynced to the number of pages that had been written when the + ** database was last checkpointed. */ rc = lsmCheckpointSynced(db, 0, 0, &nSynced); + if( rc==LSM_OK ){ u32 nPgsz = db->pShmhdr->aSnap1[CKPT_HDR_PGSZ]; u32 nWrite = db->pShmhdr->aSnap1[CKPT_HDR_NWRITE]; - *pnByte = (int)((nWrite - nSynced) * nPgsz); + *pnKB = (int)(( ((i64)(nWrite - nSynced) * nPgsz) + 1023) / 1024); } return rc; } Index: src/lsm_file.c ================================================================== --- src/lsm_file.c +++ src/lsm_file.c @@ -219,10 +219,12 @@ ** as the file grows), the Page.aData pointers are updated by iterating ** through the contents of this list. ** ** In non-mmap() mode, this list is an LRU list of cached pages with ** nRef==0. +** +** apHash, nHash: */ struct FileSystem { lsm_db *pDb; /* Database handle that owns this object */ lsm_env *pEnv; /* Environment pointer */ char *zDb; /* Database file name */ @@ -351,12 +353,12 @@ ** lsmEnvClose() ** lsmEnvTruncate() ** lsmEnvUnlink() ** lsmEnvRemap() */ -int lsmEnvOpen(lsm_env *pEnv, const char *zFile, lsm_file **ppNew){ - return pEnv->xOpen(pEnv, zFile, ppNew); +int lsmEnvOpen(lsm_env *pEnv, const char *zFile, int flags, lsm_file **ppNew){ + return pEnv->xOpen(pEnv, zFile, flags, ppNew); } static int lsmEnvRead( lsm_env *pEnv, lsm_file *pFile, lsm_i64 iOff, @@ -421,10 +423,20 @@ int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock){ if( pFile==0 ) return LSM_OK; return pEnv->xLock(pFile, iLock, eLock); } + +int lsmEnvTestLock( + lsm_env *pEnv, + lsm_file *pFile, + int iLock, + int nLock, + int eLock +){ + return pEnv->xTestLock(pFile, iLock, nLock, eLock); +} int lsmEnvShmMap( lsm_env *pEnv, lsm_file *pFile, int iChunk, @@ -486,11 +498,11 @@ if( pFS->fdLog==0 ) return LSM_OK; return lsmEnvTruncate(pFS->pEnv, pFS->fdLog, nByte); } /* -** Truncate the log file to nByte bytes in size. +** Truncate the db file to nByte bytes in size. */ int lsmFsTruncateDb(FileSystem *pFS, i64 nByte){ if( pFS->fdDb==0 ) return LSM_OK; return lsmEnvTruncate(pFS->pEnv, pFS->fdDb, nByte); } @@ -527,16 +539,20 @@ ** This is a helper function for lsmFsOpen(). It opens a single file on ** disk (either the database or log file). */ static lsm_file *fsOpenFile( FileSystem *pFS, /* File system object */ + int bReadonly, /* True to open this file read-only */ int bLog, /* True for log, false for db */ int *pRc /* IN/OUT: Error code */ ){ lsm_file *pFile = 0; if( *pRc==LSM_OK ){ - *pRc = lsmEnvOpen(pFS->pEnv, (bLog ? pFS->zLog : pFS->zDb), &pFile); + int flags = (bReadonly ? LSM_OPEN_READONLY : 0); + const char *zPath = (bLog ? pFS->zLog : pFS->zDb); + + *pRc = lsmEnvOpen(pFS->pEnv, zPath, flags, &pFile); } return pFile; } @@ -791,21 +807,51 @@ ** ** lsmFsWriteLog ** lsmFsSyncLog ** lsmFsReadLog */ -int lsmFsOpenLog(FileSystem *pFS){ +int lsmFsOpenLog(lsm_db *db, int *pbOpen){ int rc = LSM_OK; - if( 0==pFS->fdLog ){ pFS->fdLog = fsOpenFile(pFS, 1, &rc); } + FileSystem *pFS = db->pFS; + + if( 0==pFS->fdLog ){ + pFS->fdLog = fsOpenFile(pFS, db->bReadonly, 1, &rc); + + if( rc==LSM_IOERR_NOENT && db->bReadonly ){ + rc = LSM_OK; + } + } + + if( pbOpen ) *pbOpen = (pFS->fdLog!=0); return rc; } + +void lsmFsCloseLog(lsm_db *db){ + FileSystem *pFS = db->pFS; + if( pFS->fdLog ){ + lsmEnvClose(pFS->pEnv, pFS->fdLog); + pFS->fdLog = 0; + } +} /* ** Open a connection to a database stored within the file-system (the ** "system of files"). +** +** If parameter bReadonly is true, then open a read-only file-descriptor +** on the database file. It is possible that bReadonly will be false even +** if the user requested that pDb be opened read-only. This is because the +** file-descriptor may later on be recycled by a read-write connection. +** If the db file can be opened for read-write access, it always is. Parameter +** bReadonly is only ever true if it has already been determined that the +** db can only be opened for read-only access. */ -int lsmFsOpen(lsm_db *pDb, const char *zDb){ +int lsmFsOpen( + lsm_db *pDb, /* Database connection to open fd for */ + const char *zDb, /* Full path to database file */ + int bReadonly /* True to open db file read-only */ +){ FileSystem *pFS; int rc = LSM_OK; int nDb = strlen(zDb); int nByte; @@ -821,13 +867,11 @@ pFS->nPagesize = LSM_DFLT_PAGE_SIZE; pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; pFS->nMetasize = 4 * 1024; pFS->pDb = pDb; pFS->pEnv = pDb->pEnv; - if( pDb->compress.xCompress ){ - pFS->pCompress = &pDb->compress; - }else{ + if( !pDb->compress.xCompress ){ pFS->mmapmgr.eUseMmap = pDb->eMmap; pFS->mmapmgr.nMapsz = 1*1024*1024; pFS->mmapmgr.nMapsz = 4*1024; } @@ -850,11 +894,11 @@ pFS->fdDb = pLsmFile->pFile; memset(pLsmFile, 0, sizeof(LsmFile)); }else{ pFS->pLsmFile = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmFile), &rc); if( rc==LSM_OK ){ - pFS->fdDb = fsOpenFile(pFS, 0, &rc); + pFS->fdDb = fsOpenFile(pFS, bReadonly, 0, &rc); } } if( rc!=LSM_OK ){ lsmFsClose(pFS); @@ -865,10 +909,62 @@ } pDb->pFS = pFS; return rc; } + +/* +** Configure the file-system object according to the current values of +** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. +*/ +int lsmFsConfigure(lsm_db *db){ + FileSystem *pFS = db->pFS; + if( pFS ){ + lsm_env *pEnv = pFS->pEnv; + Page *pPg; + + assert( pFS->nOut==0 ); + assert( pFS->pWaiting==0 ); + + /* Reset any compression/decompression buffers already allocated */ + lsmFree(pEnv, pFS->aIBuffer); + lsmFree(pEnv, pFS->aOBuffer); + pFS->nBuffer = 0; + + /* Unmap the file, if it is currently mapped */ + if( pFS->pMap ){ + lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); + pFS->bUseMmap = 0; + } + + /* Free all allocate page structures */ + pPg = pFS->pLruFirst; + while( pPg ){ + Page *pNext = pPg->pLruNext; + if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); + lsmFree(pEnv, pPg); + pPg = pNext; + } + + /* Zero pointers that point to deleted page objects */ + pFS->nCacheAlloc = 0; + pFS->pLruFirst = 0; + pFS->pLruLast = 0; + pFS->pFree = 0; + + /* Configure the FileSystem object */ + if( db->compress.xCompress ){ + pFS->pCompress = &db->compress; + pFS->bUseMmap = 0; + }else{ + pFS->pCompress = 0; + pFS->bUseMmap = db->bMmap; + } + } + + return LSM_OK; +} /* ** Close and destroy a FileSystem object. */ void lsmFsClose(FileSystem *pFS){ @@ -1547,10 +1643,12 @@ if( rc!=LSM_OK ){ p->pHashNext = pFS->pFree; pFS->pFree = p; p = 0; } + + assert( (p->flags & PAGE_FREE)==0 ); }else{ /* Search the hash-table for the page */ iHash = fsHashKey(pFS->nHash, iReal); for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ Index: src/lsm_log.c ================================================================== --- src/lsm_log.c +++ src/lsm_log.c @@ -203,10 +203,13 @@ #define LSM_LOG_DELETE_CKSUM 0x09 /* Require a checksum every 32KB. */ #define LSM_CKSUM_MAXDATA (32*1024) +/* Do not wrap a log file smaller than this in bytes. */ +#define LSM_MIN_LOGWRAP (128*1024) + /* ** szSector: ** Commit records must be aligned to end on szSector boundaries. If ** the safety-mode is set to NORMAL or OFF, this value is 1. Otherwise, ** if the safety-mode is set to FULL, it is the size of the file-system @@ -299,12 +302,18 @@ ** If possible, reclaim log file space. Log file space is reclaimed after ** a snapshot that points to the same data in the database file is synced ** into the db header. */ static int logReclaimSpace(lsm_db *pDb){ - int rc = LSM_OK; + int rc; int iMeta; + int bRotrans; /* True if there exists some ro-trans */ + + /* Test if there exists some other connection with a read-only transaction + ** open. If there does, then log file space may not be reclaimed. */ + rc = lsmDetectRoTrans(pDb, &bRotrans); + if( rc!=LSM_OK || bRotrans ) return rc; iMeta = (int)pDb->pShmhdr->iMetaPage; if( iMeta==1 || iMeta==2 ){ DbLog *pLog = &pDb->treehdr.log; i64 iSyncedId; @@ -355,11 +364,11 @@ if( pDb->bUseLog==0 ) return LSM_OK; /* If the log file has not yet been opened, open it now. Also allocate ** the LogWriter structure, if it has not already been allocated. */ - rc = lsmFsOpenLog(pDb->pFS); + rc = lsmFsOpenLog(pDb, 0); if( pDb->pLogWriter==0 ){ pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); if( pNew ){ lsmStringInit(&pNew->buf, pDb->pEnv); rc = lsmStringExtend(&pNew->buf, 2); @@ -401,13 +410,12 @@ } /* There are now three scenarios: ** ** 1) Regions 0 and 1 are both zero bytes in size and region 2 begins - ** at a file offset greater than N, where N is the value configured - ** by LSM_CONFIG_LOG_SIZE. In this case, wrap around to the start - ** and write data into the start of the log file. + ** at a file offset greater than LSM_MIN_LOGWRAP. In this case, wrap + ** around to the start and write data into the start of the log file. ** ** 2) Region 1 is zero bytes in size and region 2 occurs earlier in the ** file than region 0. In this case, append data to region 2, but ** remember to jump over region 1 if required. ** @@ -419,11 +427,11 @@ assert( aReg[1].iEnd==0 || aReg[1].iEnd>aReg[1].iStart ); pNew->cksum0 = pDb->treehdr.log.cksum0; pNew->cksum1 = pDb->treehdr.log.cksum1; - if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=pDb->nLogSz ){ + if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=LSM_MIN_LOGWRAP ){ /* Case 1. Wrap around to the start of the file. Write an LSM_LOG_JUMP ** into the log file in this case. Pad it out to 8 bytes using a PAD2 ** record so that the checksums can be updated immediately. */ u8 aJump[] = { LSM_LOG_PAD2, 0x04, 0x00, 0x00, 0x00, 0x00, LSM_LOG_JUMP, 0x00 @@ -954,12 +962,13 @@ int rc = LSM_OK; /* Return code */ int nCommit = 0; /* Number of transactions to recover */ int iPass; int nJump = 0; /* Number of LSM_LOG_JUMP records in pass 0 */ DbLog *pLog; + int bOpen; - rc = lsmFsOpenLog(pDb->pFS); + rc = lsmFsOpenLog(pDb, &bOpen); if( rc!=LSM_OK ) return rc; rc = lsmTreeInit(pDb); if( rc!=LSM_OK ) return rc; @@ -972,124 +981,126 @@ /* The outer for() loop runs at most twice. The first iteration is to ** count the number of committed transactions in the log. The second ** iterates through those transactions and updates the in-memory tree ** structure with their contents. */ - for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ - int bEof = 0; - - while( rc==LSM_OK && !bEof ){ - u8 eType = 0; - logReaderByte(&reader, &eType, &rc); - - switch( eType ){ - case LSM_LOG_PAD1: - break; - - case LSM_LOG_PAD2: { - int nPad; - logReaderVarint(&reader, &buf1, &nPad, &rc); - logReaderBlob(&reader, &buf1, nPad, 0, &rc); - break; - } - - case LSM_LOG_WRITE: - case LSM_LOG_WRITE_CKSUM: { - int nKey; - int nVal; - u8 *aVal; - logReaderVarint(&reader, &buf1, &nKey, &rc); - logReaderVarint(&reader, &buf2, &nVal, &rc); - - if( eType==LSM_LOG_WRITE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey+nVal); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, 0, &rc); - logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); - if( iPass==1 && rc==LSM_OK ){ - rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - } - break; - } - - case LSM_LOG_DELETE: - case LSM_LOG_DELETE_CKSUM: { - int nKey; u8 *aKey; - logReaderVarint(&reader, &buf1, &nKey, &rc); - - if( eType==LSM_LOG_DELETE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); - if( iPass==1 && rc==LSM_OK ){ - rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); - } - break; - } - - case LSM_LOG_COMMIT: - logReaderCksum(&reader, &buf1, &bEof, &rc); - if( bEof==0 ){ - nCommit++; - assert( nCommit>0 || iPass==1 ); - if( nCommit==0 ) bEof = 1; - } - break; - - case LSM_LOG_JUMP: { - int iOff = 0; - logReaderVarint(&reader, &buf1, &iOff, &rc); - if( rc==LSM_OK ){ - if( iPass==1 ){ - if( pLog->aRegion[2].iStart==0 ){ - assert( pLog->aRegion[1].iStart==0 ); - pLog->aRegion[1].iEnd = reader.iOff; - }else{ - assert( pLog->aRegion[0].iStart==0 ); - pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[0].iEnd = reader.iOff - reader.buf.n+reader.iBuf; - } - pLog->aRegion[2].iStart = iOff; - }else{ - if( (nJump++)==2 ){ - bEof = 1; - } - } - - reader.iOff = iOff; - reader.buf.n = reader.iBuf; - } - break; - } - - default: - /* Including LSM_LOG_EOF */ - bEof = 1; - break; - } - } - - if( rc==LSM_OK && iPass==0 ){ - if( nCommit==0 ){ - if( pLog->aRegion[2].iStart==0 ){ - iPass = 1; - }else{ - pLog->aRegion[2].iStart = 0; - iPass = -1; - lsmCheckpointZeroLogoffset(pDb); - } - } - logReaderInit(pDb, pLog, 0, &reader); - nCommit = nCommit * -1; + if( bOpen ){ + for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ + int bEof = 0; + + while( rc==LSM_OK && !bEof ){ + u8 eType = 0; + logReaderByte(&reader, &eType, &rc); + + switch( eType ){ + case LSM_LOG_PAD1: + break; + + case LSM_LOG_PAD2: { + int nPad; + logReaderVarint(&reader, &buf1, &nPad, &rc); + logReaderBlob(&reader, &buf1, nPad, 0, &rc); + break; + } + + case LSM_LOG_WRITE: + case LSM_LOG_WRITE_CKSUM: { + int nKey; + int nVal; + u8 *aVal; + logReaderVarint(&reader, &buf1, &nKey, &rc); + logReaderVarint(&reader, &buf2, &nVal, &rc); + + if( eType==LSM_LOG_WRITE_CKSUM ){ + logReaderCksum(&reader, &buf1, &bEof, &rc); + }else{ + bEof = logRequireCksum(&reader, nKey+nVal); + } + if( bEof ) break; + + logReaderBlob(&reader, &buf1, nKey, 0, &rc); + logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); + if( iPass==1 && rc==LSM_OK ){ + rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); + } + break; + } + + case LSM_LOG_DELETE: + case LSM_LOG_DELETE_CKSUM: { + int nKey; u8 *aKey; + logReaderVarint(&reader, &buf1, &nKey, &rc); + + if( eType==LSM_LOG_DELETE_CKSUM ){ + logReaderCksum(&reader, &buf1, &bEof, &rc); + }else{ + bEof = logRequireCksum(&reader, nKey); + } + if( bEof ) break; + + logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); + if( iPass==1 && rc==LSM_OK ){ + rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); + } + break; + } + + case LSM_LOG_COMMIT: + logReaderCksum(&reader, &buf1, &bEof, &rc); + if( bEof==0 ){ + nCommit++; + assert( nCommit>0 || iPass==1 ); + if( nCommit==0 ) bEof = 1; + } + break; + + case LSM_LOG_JUMP: { + int iOff = 0; + logReaderVarint(&reader, &buf1, &iOff, &rc); + if( rc==LSM_OK ){ + if( iPass==1 ){ + if( pLog->aRegion[2].iStart==0 ){ + assert( pLog->aRegion[1].iStart==0 ); + pLog->aRegion[1].iEnd = reader.iOff; + }else{ + assert( pLog->aRegion[0].iStart==0 ); + pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; + pLog->aRegion[0].iEnd = reader.iOff-reader.buf.n+reader.iBuf; + } + pLog->aRegion[2].iStart = iOff; + }else{ + if( (nJump++)==2 ){ + bEof = 1; + } + } + + reader.iOff = iOff; + reader.buf.n = reader.iBuf; + } + break; + } + + default: + /* Including LSM_LOG_EOF */ + bEof = 1; + break; + } + } + + if( rc==LSM_OK && iPass==0 ){ + if( nCommit==0 ){ + if( pLog->aRegion[2].iStart==0 ){ + iPass = 1; + }else{ + pLog->aRegion[2].iStart = 0; + iPass = -1; + lsmCheckpointZeroLogoffset(pDb); + } + } + logReaderInit(pDb, pLog, 0, &reader); + nCommit = nCommit * -1; + } } } /* Initialize DbLog object */ if( rc==LSM_OK ){ @@ -1101,10 +1112,14 @@ if( rc==LSM_OK ){ rc = lsmFinishRecovery(pDb); }else{ lsmFinishRecovery(pDb); } + + if( pDb->bRoTrans ){ + lsmFsCloseLog(pDb); + } lsmStringClear(&buf1); lsmStringClear(&buf2); lsmStringClear(&reader.buf); return rc; Index: src/lsm_main.c ================================================================== --- src/lsm_main.c +++ src/lsm_main.c @@ -37,13 +37,14 @@ /* If there is at least one cursor or a write transaction open, the database ** handle must be holding a pointer to a client snapshot. And the reverse ** - if there are no open cursors and no write transactions then there must ** not be a client snapshot. */ - assert( (pDb->pCsr!=0 || pDb->nTransOpen>0)==(pDb->iReader>=0) ); + + assert( (pDb->pCsr!=0||pDb->nTransOpen>0)==(pDb->iReader>=0||pDb->bRoTrans) ); - assert( pDb->iReader<0 || pDb->pClient!=0 ); + assert( (pDb->iReader<0 && pDb->bRoTrans==0) || pDb->pClient!=0 ); assert( pDb->nTransOpen>=0 ); } #else # define assert_db_state(x) @@ -85,20 +86,21 @@ pDb->nTreeLimit = LSM_DFLT_AUTOFLUSH; pDb->nAutockpt = LSM_DFLT_AUTOCHECKPOINT; pDb->bAutowork = LSM_DFLT_AUTOWORK; pDb->eSafety = LSM_DFLT_SAFETY; pDb->xCmp = xCmp; - pDb->nLogSz = LSM_DFLT_LOG_SIZE; pDb->nDfltPgsz = LSM_DFLT_PAGE_SIZE; pDb->nDfltBlksz = LSM_DFLT_BLOCK_SIZE; pDb->nMerge = LSM_DFLT_AUTOMERGE; pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; pDb->bUseLog = LSM_DFLT_USE_LOG; pDb->iReader = -1; + pDb->iRwclient = -1; pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; pDb->eMmap = LSM_DFLT_MMAP; pDb->xLog = xLog; + pDb->compress.iId = LSM_COMPRESSION_NONE; return LSM_OK; } lsm_env *lsm_get_env(lsm_db *pDb){ assert( pDb->pEnv ); @@ -138,10 +140,29 @@ zAlloc = 0; } *pzAbs = zAlloc; return rc; } + +/* +** Check that the bits in the db->mLock mask are consistent with the +** value stored in db->iRwclient. An assert shall fail otherwise. +*/ +static void assertRwclientLockValue(lsm_db *db){ +#ifndef NDEBUG + u64 msk; /* Mask of mLock bits for RWCLIENT locks */ + u64 rwclient = 0; /* Bit corresponding to db->iRwclient */ + + if( db->iRwclient>=0 ){ + rwclient = ((u64)1 << (LSM_LOCK_RWCLIENT(db->iRwclient)-1)); + } + msk = ((u64)1 << (LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT)-1)) - 1; + msk -= (((u64)1 << (LSM_LOCK_RWCLIENT(0)-1)) - 1); + + assert( (db->mLock & msk)==rwclient ); +#endif +} /* ** Open a new connection to database zFilename. */ int lsm_open(lsm_db *pDb, const char *zFilename){ @@ -160,43 +181,60 @@ ** path is required to ensure that the correct files are operated ** on even if the application changes the cwd. */ rc = getFullpathname(pDb->pEnv, zFilename, &zFull); assert( rc==LSM_OK || zFull==0 ); - /* Connect to the database */ + /* Connect to the database. */ if( rc==LSM_OK ){ rc = lsmDbDatabaseConnect(pDb, zFull); } - /* Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so are - ** guaranteed not to change during the lifetime of this connection. */ - if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ - lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); - lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); + if( pDb->bReadonly==0 ){ + /* Configure the file-system connection with the page-size and block-size + ** of this database. Even if the database file is zero bytes in size + ** on disk, these values have been set in shared-memory by now, and so + ** are guaranteed not to change during the lifetime of this connection. + */ + if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ + lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); + lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); + } } lsmFree(pDb->pEnv, zFull); + assertRwclientLockValue(pDb); } + assert( pDb->bReadonly==0 || pDb->bReadonly==1 ); + assert( rc!=LSM_OK || (pDb->pShmhdr==0)==(pDb->bReadonly==1) ); + return rc; } - int lsm_close(lsm_db *pDb){ int rc = LSM_OK; if( pDb ){ assert_db_state(pDb); if( pDb->pCsr || pDb->nTransOpen ){ rc = LSM_MISUSE_BKPT; }else{ + lsmMCursorFreeCache(pDb); lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; + + assertRwclientLockValue(pDb); + lsmDbDatabaseRelease(pDb); lsmLogClose(pDb); lsmFsClose(pDb->pFS); + assert( pDb->mLock==0 ); + + /* Invoke any destructors registered for the compression or + ** compression factory callbacks. */ + if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); + if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); + lsmFree(pDb->pEnv, pDb->rollback.aArray); lsmFree(pDb->pEnv, pDb->aTrans); lsmFree(pDb->pEnv, pDb->apShm); lsmFree(pDb->pEnv, pDb); } @@ -209,15 +247,18 @@ va_list ap; va_start(ap, eParam); switch( eParam ){ case LSM_CONFIG_AUTOFLUSH: { + /* This parameter is read and written in KB. But all internal + ** processing is done in bytes. */ int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - pDb->nTreeLimit = *piVal; + int iVal = *piVal; + if( iVal>=0 && iVal<=(1024*1024) ){ + pDb->nTreeLimit = iVal*1024; } - *piVal = pDb->nTreeLimit; + *piVal = (pDb->nTreeLimit / 1024); break; } case LSM_CONFIG_AUTOWORK: { int *piVal = va_arg(ap, int *); @@ -227,24 +268,18 @@ *piVal = pDb->bAutowork; break; } case LSM_CONFIG_AUTOCHECKPOINT: { + /* This parameter is read and written in KB. But all internal processing + ** (including the lsm_db.nAutockpt variable) is done in bytes. */ int *piVal = va_arg(ap, int *); if( *piVal>=0 ){ - pDb->nAutockpt = *piVal; - } - *piVal = pDb->nAutockpt; - break; - } - - case LSM_CONFIG_LOG_SIZE: { - int *piVal = va_arg(ap, int *); - if( *piVal>0 ){ - pDb->nLogSz = *piVal; - } - *piVal = pDb->nLogSz; + int iVal = *piVal; + pDb->nAutockpt = (i64)iVal * 1024; + } + *piVal = (int)(pDb->nAutockpt / 1024); break; } case LSM_CONFIG_PAGE_SIZE: { int *piVal = va_arg(ap, int *); @@ -262,21 +297,24 @@ } break; } case LSM_CONFIG_BLOCK_SIZE: { + /* This parameter is read and written in KB. But all internal + ** processing is done in bytes. */ int *piVal = va_arg(ap, int *); if( pDb->pDatabase ){ /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the page-size according to the - ** FileSystem object. */ - *piVal = lsmFsBlockSize(pDb->pFS); - }else{ - if( *piVal>=65536 && ((*piVal-1) & *piVal)==0 ){ - pDb->nDfltBlksz = *piVal; - }else{ - *piVal = pDb->nDfltBlksz; + ** Set the output variable to the block-size in KB according to the + ** FileSystem object. */ + *piVal = lsmFsBlockSize(pDb->pFS) / 1024; + }else{ + int iVal = *piVal; + if( iVal>=64 && iVal<=65536 && ((iVal-1) & iVal)==0 ){ + pDb->nDfltBlksz = iVal * 1024; + }else{ + *piVal = pDb->nDfltBlksz / 1024; } } break; } @@ -289,12 +327,13 @@ break; } case LSM_CONFIG_MMAP: { int *piVal = va_arg(ap, int *); - if( pDb->pDatabase==0 && (*piVal>=0 && *piVal<=2) ){ - pDb->eMmap = *piVal; + if( pDb->iReader<0 && *piVal>=0 && *piVal<=1 ){ + pDb->bMmap = *piVal; + rc = lsmFsConfigure(pDb); } *piVal = pDb->eMmap; break; } @@ -333,19 +372,49 @@ }else{ pDb->bMultiProc = *piVal = (*piVal!=0); } break; } + + case LSM_CONFIG_READONLY: { + int *piVal = va_arg(ap, int *); + /* If lsm_open() has been called, this is a read-only parameter. */ + if( pDb->pDatabase==0 && *piVal>=0 ){ + pDb->bReadonly = *piVal = (*piVal!=0); + } + *piVal = pDb->bReadonly; + break; + } case LSM_CONFIG_SET_COMPRESSION: { lsm_compress *p = va_arg(ap, lsm_compress *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this call is against the rules. */ + if( pDb->iReader>=0 && pDb->bInFactory==0 ){ + /* May not change compression schemes with an open transaction */ rc = LSM_MISUSE_BKPT; }else{ - memcpy(&pDb->compress, p, sizeof(lsm_compress)); + if( pDb->compress.xFree ){ + /* Invoke any destructor belonging to the current compression. */ + pDb->compress.xFree(pDb->compress.pCtx); + } + if( p->xBound==0 ){ + memset(&pDb->compress, 0, sizeof(lsm_compress)); + pDb->compress.iId = LSM_COMPRESSION_NONE; + }else{ + memcpy(&pDb->compress, p, sizeof(lsm_compress)); + } + rc = lsmFsConfigure(pDb); + } + break; + } + + case LSM_CONFIG_SET_COMPRESSION_FACTORY: { + lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); + if( pDb->factory.xFree ){ + /* Invoke any destructor belonging to the current factory. */ + pDb->factory.xFree(pDb->factory.pCtx); } + memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); break; } case LSM_CONFIG_GET_COMPRESSION: { lsm_compress *p = va_arg(ap, lsm_compress *); @@ -431,11 +500,10 @@ int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ Snapshot *pWorker; /* Worker snapshot */ int bUnlock = 0; LsmString s; - int i; int rc; /* Obtain the worker snapshot */ rc = infoGetWorker(pDb, &pWorker, &bUnlock); if( rc!=LSM_OK ) return rc; @@ -451,14 +519,11 @@ /* Release the snapshot and return */ infoFreeWorker(pDb, bUnlock); return rc; } -static int infoFreelistSize(lsm_db *pDb, int *pnFree, int *pnWaiting){ -} - -static int infoTreeSize(lsm_db *db, int *pnOld, int *pnNew){ +static int infoTreeSize(lsm_db *db, int *pnOldKB, int *pnNewKB){ ShmHeader *pShm = db->pShmhdr; TreeHeader *p = &pShm->hdr1; /* The following code suffers from two race conditions, as it accesses and ** trusts the contents of shared memory without verifying checksums: @@ -477,19 +542,19 @@ ** ** Given the context in which this function is called (as a result of an ** lsm_info(LSM_INFO_TREE_SIZE) request), neither of these are considered to ** be problems. */ - *pnNew = (int)p->root.nByte; + *pnNewKB = ((int)p->root.nByte + 1023) / 1024; if( p->iOldShmid ){ if( p->iOldLog==lsmCheckpointLogOffset(pShm->aSnap1) ){ - *pnOld = 0; + *pnOldKB = 0; }else{ - *pnOld = (int)p->oldroot.nByte; + *pnOldKB = ((int)p->oldroot.nByte + 1023) / 1024; } }else{ - *pnOld = 0; + *pnOldKB = 0; } return LSM_OK; } @@ -556,21 +621,31 @@ rc = lsmInfoFreelist(pDb, pzVal); break; } case LSM_INFO_CHECKPOINT_SIZE: { - int *pnByte = va_arg(ap, int *); - rc = lsmCheckpointSize(pDb, pnByte); + int *pnKB = va_arg(ap, int *); + rc = lsmCheckpointSize(pDb, pnKB); break; } case LSM_INFO_TREE_SIZE: { int *pnOld = va_arg(ap, int *); int *pnNew = va_arg(ap, int *); rc = infoTreeSize(pDb, pnOld, pnNew); break; } + + case LSM_INFO_COMPRESSION_ID: { + unsigned int *piOut = va_arg(ap, unsigned int *); + if( pDb->pClient ){ + *piOut = pDb->pClient->iCmpId; + }else{ + rc = lsmInfoCompressionId(pDb, piOut); + } + break; + } default: rc = LSM_MISUSE; break; } @@ -679,25 +754,33 @@ ** ** If there are currently no other open cursor handles, and no open write ** transaction, open a read transaction here. */ int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr){ - int rc; /* Return code */ + int rc = LSM_OK; /* Return code */ MultiCursor *pCsr = 0; /* New cursor object */ /* Open a read transaction if one is not already open. */ assert_db_state(pDb); - rc = lsmBeginReadTrans(pDb); + + if( pDb->pShmhdr==0 ){ + assert( pDb->bReadonly ); + rc = lsmBeginRoTrans(pDb); + }else if( pDb->iReader<0 ){ + rc = lsmBeginReadTrans(pDb); + } /* Allocate the multi-cursor. */ - if( rc==LSM_OK ) rc = lsmMCursorNew(pDb, &pCsr); + if( rc==LSM_OK ){ + rc = lsmMCursorNew(pDb, &pCsr); + } /* If an error has occured, set the output to NULL and delete any partially ** allocated cursor. If this means there are no open cursors, release the ** client snapshot. */ if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr); + lsmMCursorClose(pCsr, 0); dbReleaseClientSnapshot(pDb); } assert_db_state(pDb); *ppCsr = (lsm_cursor *)pCsr; @@ -709,11 +792,11 @@ */ int lsm_csr_close(lsm_cursor *p){ if( p ){ lsm_db *pDb = lsmMCursorDb((MultiCursor *)p); assert_db_state(pDb); - lsmMCursorClose((MultiCursor *)p); + lsmMCursorClose((MultiCursor *)p, 1); dbReleaseClientSnapshot(pDb); assert_db_state(pDb); } return LSM_OK; } @@ -787,17 +870,17 @@ lsmStringClear(&s); } } int lsm_begin(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; + int rc; assert_db_state( pDb ); + rc = (pDb->bReadonly ? LSM_READONLY : LSM_OK); /* A value less than zero means open one more transaction. */ if( iLevel<0 ) iLevel = pDb->nTransOpen + 1; - if( iLevel>pDb->nTransOpen ){ int i; /* Extend the pDb->aTrans[] array if required. */ if( rc==LSM_OK && pDb->nTransAllocnTransOpen - 1); if( iLevelnTransOpen ){ if( iLevel==0 ){ - int bAutowork = 0; - /* Commit the transaction to disk. */ if( rc==LSM_OK ) rc = lsmLogCommit(pDb); if( rc==LSM_OK && pDb->eSafety==LSM_SAFETY_FULL ){ rc = lsmFsSyncLog(pDb->pFS); } Index: src/lsm_shared.c ================================================================== --- src/lsm_shared.c +++ src/lsm_shared.c @@ -47,10 +47,11 @@ int nName; /* strlen(zName) */ int nDbRef; /* Number of associated lsm_db handles */ Database *pDbNext; /* Next Database structure in global list */ /* Protected by the local mutex (pClientMutex) */ + int bReadonly; /* True if Database.pFile is read-only */ int bMultiProc; /* True if running in multi-process mode */ lsm_file *pFile; /* Used for locks/shm in multi-proc mode */ LsmFile *pLsmFile; /* List of deferred closes */ lsm_mutex *pClientMutex; /* Protects the apShmChunk[] and pConn */ int nShmChunk; /* Number of entries in apShmChunk[] array */ @@ -209,11 +210,11 @@ */ static int dbTruncateFile(lsm_db *pDb){ int rc; assert( pDb->pWorker==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) ); + assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) ); rc = lsmCheckpointLoadWorker(pDb); if( rc==LSM_OK ){ DbTruncateCtx ctx; @@ -239,52 +240,92 @@ } static void doDbDisconnect(lsm_db *pDb){ int rc; - /* Block for an exclusive lock on DMS1. This lock serializes all calls - ** to doDbConnect() and doDbDisconnect() across all processes. */ - rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); - if( rc==LSM_OK ){ - - /* Try an exclusive lock on DMS2. If successful, this is the last - ** connection to the database. In this case flush the contents of the - ** in-memory tree to disk and write a checkpoint. */ - rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - /* Flush the in-memory tree, if required. If there is data to flush, - ** this will create a new client snapshot in Database.pClient. The - ** checkpoint (serialization) of this snapshot may be written to disk - ** by the following block. - ** - ** There is no need to mess around with WRITER locks or anything at - ** this point. The lock on DMS2 guarantees that pDb has exclusive - ** access to the db at this point. - */ - rc = lsmTreeLoadHeader(pDb, 0); - if( rc==LSM_OK && (lsmTreeHasOld(pDb) || lsmTreeSize(pDb)>0) ){ - rc = lsmFlushTreeToDisk(pDb); - } - - /* Write a checkpoint to disk. */ - if( rc==LSM_OK ){ - rc = lsmCheckpointWrite(pDb, 1, 0); - } - - /* If the checkpoint was written successfully, delete the log file - ** and, if possible, truncate the database file. */ - if( rc==LSM_OK ){ - Database *p = pDb->pDatabase; - dbTruncateFile(pDb); - lsmFsCloseAndDeleteLog(pDb->pFS); - if( p->pFile && p->bMultiProc ) lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); - } - } - } - - lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); - lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + if( pDb->bReadonly ){ + lsmShmLock(pDb, LSM_LOCK_DMS3, LSM_LOCK_UNLOCK, 0); + }else{ + /* Block for an exclusive lock on DMS1. This lock serializes all calls + ** to doDbConnect() and doDbDisconnect() across all processes. */ + rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); + if( rc==LSM_OK ){ + + /* Try an exclusive lock on DMS2. If successful, this is the last + ** connection to the database. In this case flush the contents of the + ** in-memory tree to disk and write a checkpoint. */ + rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 1, LSM_LOCK_EXCL); + if( rc==LSM_OK ){ + rc = lsmShmTestLock(pDb, LSM_LOCK_CHECKPOINTER, 1, LSM_LOCK_EXCL); + } + if( rc==LSM_OK ){ + int bReadonly = 0; /* True if there exist read-only conns. */ + + /* Flush the in-memory tree, if required. If there is data to flush, + ** this will create a new client snapshot in Database.pClient. The + ** checkpoint (serialization) of this snapshot may be written to disk + ** by the following block. + ** + ** There is no need to take a WRITER lock here. That there are no + ** other locks on DMS2 guarantees that there are no other read-write + ** connections at this time (and the lock on DMS1 guarantees that + ** no new ones may appear). + */ + rc = lsmTreeLoadHeader(pDb, 0); + if( rc==LSM_OK && (lsmTreeHasOld(pDb) || lsmTreeSize(pDb)>0) ){ + rc = lsmFlushTreeToDisk(pDb); + } + + /* Now check if there are any read-only connections. If there are, + ** then do not truncate the db file or unlink the shared-memory + ** region. */ + if( rc==LSM_OK ){ + rc = lsmShmTestLock(pDb, LSM_LOCK_DMS3, 1, LSM_LOCK_EXCL); + if( rc==LSM_BUSY ){ + bReadonly = 1; + rc = LSM_OK; + } + } + + /* Write a checkpoint to disk. */ + if( rc==LSM_OK ){ + rc = lsmCheckpointWrite(pDb, (bReadonly==0), 0); + } + + /* If the checkpoint was written successfully, delete the log file + ** and, if possible, truncate the database file. */ + if( rc==LSM_OK ){ + int bRotrans = 0; + Database *p = pDb->pDatabase; + + /* The log file may only be deleted if there are no clients + ** read-only clients running rotrans transactions. */ + rc = lsmDetectRoTrans(pDb, &bRotrans); + if( rc==LSM_OK && bRotrans==0 ){ + lsmFsCloseAndDeleteLog(pDb->pFS); + } + + /* The database may only be truncated if there exist no read-only + ** clients - either connected or running rotrans transactions. */ + if( bReadonly==0 && bRotrans==0 ){ + dbTruncateFile(pDb); + if( p->pFile && p->bMultiProc ){ + lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); + } + } + } + } + } + + if( pDb->iRwclient>=0 ){ + lsmShmLock(pDb, LSM_LOCK_RWCLIENT(pDb->iRwclient), LSM_LOCK_UNLOCK, 0); + pDb->iRwclient = -1; + } + + lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); + lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + } pDb->pShmhdr = 0; } static int doDbConnect(lsm_db *pDb){ const int nUsMax = 100000; /* Max value for nUs */ @@ -291,10 +332,11 @@ int nUs = 1000; /* us to wait between DMS1 attempts */ int rc; /* Obtain a pointer to the shared-memory header */ assert( pDb->pShmhdr==0 ); + assert( pDb->bReadonly==0 ); rc = lsmShmCacheChunks(pDb, 1); if( rc!=LSM_OK ) return rc; pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; /* Block for an exclusive lock on DMS1. This lock serializes all calls @@ -309,20 +351,26 @@ if( rc!=LSM_OK ){ pDb->pShmhdr = 0; return rc; } - /* Try an exclusive lock on DMS2. If successful, this is the first and - ** only connection to the database. In this case initialize the + /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first + ** and only connection to the database. In this case initialize the ** shared-memory and run log file recovery. */ - rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL, 0); + assert( LSM_LOCK_DMS3==1+LSM_LOCK_DMS2 ); + rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 2, LSM_LOCK_EXCL); if( rc==LSM_OK ){ memset(pDb->pShmhdr, 0, sizeof(ShmHeader)); rc = lsmCheckpointRecover(pDb); if( rc==LSM_OK ){ rc = lsmLogRecover(pDb); } + if( rc==LSM_OK ){ + ShmHeader *pShm = pDb->pShmhdr; + pShm->aReader[0].iLsmId = lsmCheckpointId(pShm->aSnap1, 0); + pShm->aReader[0].iTreeId = pDb->treehdr.iUsedShmid; + } }else if( rc==LSM_BUSY ){ rc = LSM_OK; } /* Take a shared lock on DMS2. In multi-process mode this lock "cannot" @@ -336,16 +384,39 @@ */ if( rc==LSM_OK ){ rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_SHARED, 0); } - /* If anything went wrong, unlock DMS2. Unlock DMS1 in any case. */ + /* If anything went wrong, unlock DMS2. Otherwise, try to take an exclusive + ** lock on one of the LSM_LOCK_RWCLIENT() locks. Unlock DMS1 in any case. */ if( rc!=LSM_OK ){ - lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); pDb->pShmhdr = 0; + }else{ + int i; + for(i=0; iiRwclient = i; + if( rc2!=LSM_BUSY ){ + rc = rc2; + break; + } + } } lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + + return rc; +} + +static int dbOpenSharedFd(lsm_env *pEnv, Database *p, int bRoOk){ + int rc; + + rc = lsmEnvOpen(pEnv, p->zName, 0, &p->pFile); + if( rc==LSM_IOERR && bRoOk ){ + rc = lsmEnvOpen(pEnv, p->zName, LSM_OPEN_READONLY, &p->pFile); + p->bReadonly = 1; + } + return rc; } /* ** Return a reference to the shared Database handle for the database @@ -396,13 +467,16 @@ /* If nothing has gone wrong so far, open the shared fd. And if that ** succeeds and this connection requested single-process mode, ** attempt to take the exclusive lock on DMS2. */ if( rc==LSM_OK ){ - rc = lsmEnvOpen(pDb->pEnv, p->zName, &p->pFile); + int bReadonly = (pDb->bReadonly && pDb->bMultiProc); + rc = dbOpenSharedFd(pDb->pEnv, p, bReadonly); } + if( rc==LSM_OK && p->bMultiProc==0 ){ + assert( p->bReadonly==0 ); rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS2, LSM_LOCK_EXCL); } if( rc==LSM_OK ){ p->pDbNext = gShared.pDatabase; @@ -427,14 +501,24 @@ } pDb->pDatabase = p; if( rc==LSM_OK ){ assert( p ); - rc = lsmFsOpen(pDb, zName); + rc = lsmFsOpen(pDb, zName, p->bReadonly); } - if( rc==LSM_OK ){ - rc = doDbConnect(pDb); + + /* If the db handle is read-write, then connect to the system now. Run + ** recovery as necessary. Or, if this is a read-only database handle, + ** defer attempting to connect to the system until a read-transaction + ** is opened. */ + if( pDb->bReadonly==0 ){ + if( rc==LSM_OK ){ + rc = doDbConnect(pDb); + } + if( rc==LSM_OK ){ + rc = lsmFsConfigure(pDb); + } } return rc; } @@ -721,12 +805,27 @@ iInUse, iSynced, (pDb->iReader>=0 ? pDb->pClient->iId : 0) ); } #endif - /* Query the free block list for a suitable block */ - if( rc==LSM_OK ) rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); + + /* Unless there exists a read-only transaction (which prevents us from + ** recycling any blocks regardless, query the free block list for a + ** suitable block to reuse. + ** + ** It might seem more natural to check for a read-only transaction at + ** the start of this function. However, it is better do wait until after + ** the call to lsmCheckpointSynced() to do so. + */ + if( rc==LSM_OK ){ + int bRotrans; + rc = lsmDetectRoTrans(pDb, &bRotrans); + + if( rc==LSM_OK && bRotrans==0 ){ + rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); + } + } if( iBefore>0 && (iRet<=0 || iRet>=iBefore) ){ iRet = 0; }else if( rc==LSM_OK ){ @@ -783,11 +882,10 @@ ** block may be reused immediately. Whereas a freed block can not be reused ** until (at least) after the next checkpoint. */ int lsmBlockRefree(lsm_db *pDb, int iBlk){ int rc = LSM_OK; /* Return code */ - Snapshot *p = pDb->pWorker; #ifdef LSM_LOG_FREELIST lsmLogMessage(pDb, LSM_OK, "lsmBlockRefree(): Refree block %d", iBlk); #endif @@ -889,53 +987,160 @@ lsmFree(pEnv, p->freelist.aEntry); lsmFree(pEnv, p->redirect.a); lsmFree(pEnv, p); } } + +/* +** Attempt to populate one of the read-lock slots to contain lock values +** iLsm/iShm. Or, if such a slot exists already, this function is a no-op. +** +** It is not an error if no slot can be populated because the write-lock +** cannot be obtained. If any other error occurs, return an LSM error code. +** Otherwise, LSM_OK. +** +** This function is called at various points to try to ensure that there +** always exists at least one read-lock slot that can be used by a read-only +** client. And so that, in the usual case, there is an "exact match" available +** whenever a read transaction is opened by any client. At present this +** function is called when: +** +** * A write transaction that called lsmTreeDiscardOld() is committed, and +** * Whenever the working snapshot is updated (i.e. lsmFinishWork()). +*/ +static int dbSetReadLock(lsm_db *db, i64 iLsm, u32 iShm){ + int rc = LSM_OK; + ShmHeader *pShm = db->pShmhdr; + int i; + + /* Check if there is already a slot containing the required values. */ + for(i=0; iaReader[i]; + if( p->iLsmId==iLsm && p->iTreeId==iShm ) return LSM_OK; + } + + /* Iterate through all read-lock slots, attempting to take a write-lock + ** on each of them. If a write-lock succeeds, populate the locked slot + ** with the required values and break out of the loop. */ + for(i=0; rc==LSM_OK && iaReader[i]; + p->iLsmId = iLsm; + p->iTreeId = iShm; + lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); + break; + } + } + + return rc; +} + +/* +** Release the read-lock currently held by connection db. +*/ +int dbReleaseReadlock(lsm_db *db){ + int rc = LSM_OK; + if( db->iReader>=0 ){ + rc = lsmShmLock(db, LSM_LOCK_READER(db->iReader), LSM_LOCK_UNLOCK, 0); + db->iReader = -1; + } + db->bRoTrans = 0; + return rc; +} + /* ** Argument bFlush is true if the contents of the in-memory tree has just ** been flushed to disk. The significance of this is that once the snapshot ** created to hold the updated state of the database is synced to disk, log ** file space can be recycled. */ void lsmFinishWork(lsm_db *pDb, int bFlush, int *pRc){ - assert( *pRc!=0 || pDb->pWorker ); + int rc = *pRc; + assert( rc!=0 || pDb->pWorker ); if( pDb->pWorker ){ /* If no error has occurred, serialize the worker snapshot and write ** it to shared memory. */ - if( *pRc==LSM_OK ){ - *pRc = lsmSaveWorker(pDb, bFlush); + if( rc==LSM_OK ){ + rc = lsmSaveWorker(pDb, bFlush); + } + + /* Assuming no error has occurred, update a read lock slot with the + ** new snapshot id (see comments above function dbSetReadLock()). */ + if( rc==LSM_OK ){ + if( pDb->iReader<0 ){ + rc = lsmTreeLoadHeader(pDb, 0); + } + if( rc==LSM_OK ){ + rc = dbSetReadLock(pDb, pDb->pWorker->iId, pDb->treehdr.iUsedShmid); + } } + + /* Free the snapshot object. */ lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); pDb->pWorker = 0; } lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK, 0); + *pRc = rc; } - /* ** Called when recovery is finished. */ int lsmFinishRecovery(lsm_db *pDb){ lsmTreeEndTransaction(pDb, 1); return LSM_OK; } + +/* +** Check if the currently configured compression functions +** (LSM_CONFIG_SET_COMPRESSION) are compatible with a database that has its +** compression id set to iReq. Compression routines are compatible if iReq +** is zero (indicating the database is empty), or if it is equal to the +** compression id of the configured compression routines. +** +** If the check shows that the current compression are incompatible and there +** is a compression factory registered, give it a chance to install new +** compression routines. +** +** If, after any registered factory is invoked, the compression functions +** are still incompatible, return LSM_MISMATCH. Otherwise, LSM_OK. +*/ +int lsmCheckCompressionId(lsm_db *pDb, u32 iReq){ + if( iReq!=LSM_COMPRESSION_EMPTY && pDb->compress.iId!=iReq ){ + if( pDb->factory.xFactory ){ + pDb->bInFactory = 1; + pDb->factory.xFactory(pDb->factory.pCtx, pDb, iReq); + pDb->bInFactory = 0; + } + if( pDb->compress.iId!=iReq ){ + /* Incompatible */ + return LSM_MISMATCH; + } + } + /* Compatible */ + return LSM_OK; +} /* ** Begin a read transaction. This function is a no-op if the connection ** passed as the only argument already has an open read transaction. */ int lsmBeginReadTrans(lsm_db *pDb){ const int MAX_READLOCK_ATTEMPTS = 10; + const int nMaxAttempt = (pDb->bRoTrans ? 1 : MAX_READLOCK_ATTEMPTS); + int rc = LSM_OK; /* Return code */ int iAttempt = 0; assert( pDb->pWorker==0 ); - while( rc==LSM_OK && pDb->iReader<0 && (iAttempt++)iReader<0 && (iAttempt++)pCsr==0 && pDb->nTransOpen==0 ); /* Load the in-memory tree header. */ @@ -944,10 +1149,11 @@ /* Load the database snapshot */ if( rc==LSM_OK ){ if( lsmCheckpointClientCacheOk(pDb)==0 ){ lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; + lsmMCursorFreeCache(pDb); rc = lsmCheckpointLoad(pDb, &iSnap); }else{ iSnap = 1; } } @@ -973,14 +1179,21 @@ if( pDb->pClient==0 ){ rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); } assert( (rc==LSM_OK)==(pDb->pClient!=0) ); assert( pDb->iReader>=0 ); + + /* Check that the client has the right compression hooks loaded. + ** If not, set rc to LSM_MISMATCH. */ + if( rc==LSM_OK ){ + rc = lsmCheckCompressionId(pDb, pDb->pClient->iCmpId); + } }else{ - rc = lsmReleaseReadlock(pDb); + rc = dbReleaseReadlock(pDb); } } + if( rc==LSM_BUSY ){ rc = LSM_OK; } } #if 0 @@ -998,13 +1211,101 @@ if( rc==LSM_OK ){ rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); } if( rc!=LSM_OK ){ - lsmReleaseReadlock(pDb); + dbReleaseReadlock(pDb); } if( pDb->pClient==0 && rc==LSM_OK ) rc = LSM_BUSY; + return rc; +} + +/* +** This function is used by a read-write connection to determine if there +** are currently one or more read-only transactions open on the database +** (in this context a read-only transaction is one opened by a read-only +** connection on a non-live database). +** +** If no error occurs, LSM_OK is returned and *pbExists is set to true if +** some other connection has a read-only transaction open, or false +** otherwise. If an error occurs an LSM error code is returned and the final +** value of *pbExist is undefined. +*/ +int lsmDetectRoTrans(lsm_db *db, int *pbExist){ + int rc; + + /* Only a read-write connection may use this function. */ + assert( db->bReadonly==0 ); + + rc = lsmShmTestLock(db, LSM_LOCK_ROTRANS, 1, LSM_LOCK_EXCL); + if( rc==LSM_BUSY ){ + *pbExist = 1; + rc = LSM_OK; + }else{ + *pbExist = 0; + } + + return rc; +} + +/* +** db is a read-only database handle in the disconnected state. This function +** attempts to open a read-transaction on the database. This may involve +** connecting to the database system (opening shared memory etc.). +*/ +int lsmBeginRoTrans(lsm_db *db){ + int rc = LSM_OK; + + assert( db->bReadonly && db->pShmhdr==0 ); + assert( db->iReader<0 ); + + if( db->bRoTrans==0 ){ + + /* Attempt a shared-lock on DMS1. */ + rc = lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_SHARED, 0); + if( rc!=LSM_OK ) return rc; + + rc = lsmShmTestLock( + db, LSM_LOCK_RWCLIENT(0), LSM_LOCK_NREADER, LSM_LOCK_SHARED + ); + if( rc==LSM_OK ){ + /* System is not live. Take a SHARED lock on the ROTRANS byte and + ** release DMS1. Locking ROTRANS tells all read-write clients that they + ** may not recycle any disk space from within the database or log files, + ** as a read-only client may be using it. */ + rc = lsmShmLock(db, LSM_LOCK_ROTRANS, LSM_LOCK_SHARED, 0); + lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + + if( rc==LSM_OK ){ + db->bRoTrans = 1; + rc = lsmShmCacheChunks(db, 1); + if( rc==LSM_OK ){ + db->pShmhdr = (ShmHeader *)db->apShm[0]; + memset(db->pShmhdr, 0, sizeof(ShmHeader)); + rc = lsmCheckpointRecover(db); + if( rc==LSM_OK ){ + rc = lsmLogRecover(db); + } + } + } + }else if( rc==LSM_BUSY ){ + /* System is live! */ + rc = lsmShmLock(db, LSM_LOCK_DMS3, LSM_LOCK_SHARED, 0); + lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); + if( rc==LSM_OK ){ + rc = lsmShmCacheChunks(db, 1); + if( rc==LSM_OK ){ + db->pShmhdr = (ShmHeader *)db->apShm[0]; + } + } + } + + if( rc==LSM_OK ){ + rc = lsmBeginReadTrans(db); + } + } + return rc; } /* ** Close the currently open read transaction. @@ -1016,38 +1317,40 @@ ** transactions have been closed. Finally pClient should be non-NULL ** only iff pDb->iReader>=0. */ assert( pDb->pWorker==0 ); assert( pDb->pCsr==0 && pDb->nTransOpen==0 ); -#if 0 - if( pClient ){ - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - } -#endif - -#if 0 -if( pDb->pClient && pDb->iReader>=0 ){ - fprintf(stderr, - "finished reading %p: snapshot:%d\n", (void *)pDb, (int)pDb->pClient->iId - ); -} -#endif - if( pDb->iReader>=0 ) lsmReleaseReadlock(pDb); + if( pDb->bRoTrans ){ + int i; + for(i=0; inShm; i++){ + lsmFree(pDb->pEnv, pDb->apShm[i]); + } + lsmFree(pDb->pEnv, pDb->apShm); + pDb->apShm = 0; + pDb->nShm = 0; + pDb->pShmhdr = 0; + + lsmShmLock(pDb, LSM_LOCK_ROTRANS, LSM_LOCK_UNLOCK, 0); + } + dbReleaseReadlock(pDb); } /* ** Open a write transaction. */ int lsmBeginWriteTrans(lsm_db *pDb){ - int rc; /* Return code */ + int rc = LSM_OK; /* Return code */ ShmHeader *pShm = pDb->pShmhdr; /* Shared memory header */ assert( pDb->nTransOpen==0 ); + assert( pDb->bDiscardOld==0 ); + assert( pDb->bReadonly==0 ); /* If there is no read-transaction open, open one now. */ - rc = lsmBeginReadTrans(pDb); + if( pDb->iReader<0 ){ + rc = lsmBeginReadTrans(pDb); + } /* Attempt to take the WRITER lock */ if( rc==LSM_OK ){ rc = lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL, 0); } @@ -1075,10 +1378,11 @@ TreeHeader *p = &pDb->treehdr; pShm->bWriter = 1; p->root.iTransId++; if( lsmTreeHasOld(pDb) && p->iOldLog==pDb->pClient->iLogOff ){ lsmTreeDiscardOld(pDb); + pDb->bDiscardOld = 1; } }else{ lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); if( pDb->pCsr==0 ) lsmFinishReadTrans(pDb); } @@ -1107,14 +1411,20 @@ bFlush = 1; lsmTreeMakeOld(pDb); } lsmTreeEndTransaction(pDb, bCommit); - if( rc==LSM_OK && bFlush && pDb->bAutowork ){ - rc = lsmSortedAutoWork(pDb, 1); + if( rc==LSM_OK ){ + if( bFlush && pDb->bAutowork ){ + rc = lsmSortedAutoWork(pDb, 1); + }else if( bCommit && pDb->bDiscardOld ){ + rc = dbSetReadLock(pDb, pDb->pClient->iId, pDb->treehdr.iUsedShmid); + } } + pDb->bDiscardOld = 0; lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); + if( bFlush && pDb->bAutowork==0 && pDb->xWork ){ pDb->xWork(pDb, pDb->pWorkCtx); } return rc; } @@ -1147,10 +1457,16 @@ ShmHeader *pShm = db->pShmhdr; int i; assert( db->iReader<0 ); assert( shm_sequence_ge(iShmMax, iShmMin) ); + + /* This is a no-op if the read-only transaction flag is set. */ + if( db->bRoTrans ){ + db->iReader = 0; + return LSM_OK; + } /* Search for an exact match. */ for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; if( p->iLsmId==iLsm && p->iTreeId==iShmMax ){ @@ -1290,22 +1606,10 @@ return LSM_OK; } return isInUse(db, iLsmId, 0, pbInUse); } -/* -** Release the read-lock currently held by connection db. -*/ -int lsmReleaseReadlock(lsm_db *db){ - int rc = LSM_OK; - if( db->iReader>=0 ){ - rc = lsmShmLock(db, LSM_LOCK_READER(db->iReader), LSM_LOCK_UNLOCK, 0); - db->iReader = -1; - } - return rc; -} - /* ** This function may only be called after a successful call to ** lsmDbDatabaseConnect(). It returns true if the connection is in ** multi-process mode, or false otherwise. */ @@ -1327,11 +1631,10 @@ */ int lsmShmCacheChunks(lsm_db *db, int nChunk){ int rc = LSM_OK; if( nChunk>db->nShm ){ static const int NINCR = 16; - void *pRet = 0; Database *p = db->pDatabase; lsm_env *pEnv = db->pEnv; int nAlloc; int i; @@ -1346,50 +1649,59 @@ apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*nAlloc); if( !apShm ) return LSM_NOMEM_BKPT; db->apShm = apShm; } - /* Enter the client mutex */ - lsmMutexEnter(pEnv, p->pClientMutex); - - /* Extend the Database objects apShmChunk[] array if necessary. Using the - ** same pattern as for the lsm_db.apShm[] array above. */ - nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); - if( !apShm ){ - rc = LSM_NOMEM_BKPT; - break; - } - p->apShmChunk = apShm; - } - - for(i=db->nShm; rc==LSM_OK && i=p->nShmChunk ){ - void *pChunk = 0; - if( p->bMultiProc==0 ){ - /* Single process mode */ - pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - }else{ - /* Multi-process mode */ - rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); - } - if( rc==LSM_OK ){ - p->apShmChunk[i] = pChunk; - p->nShmChunk++; - } - } - if( rc==LSM_OK ){ - db->apShm[i] = p->apShmChunk[i]; - db->nShm++; - } - } - - /* Release the client mutex */ - lsmMutexLeave(pEnv, p->pClientMutex); + if( db->bRoTrans ){ + for(i=db->nShm; rc==LSM_OK && iapShm[i] = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); + db->nShm++; + } + + }else{ + + /* Enter the client mutex */ + lsmMutexEnter(pEnv, p->pClientMutex); + + /* Extend the Database objects apShmChunk[] array if necessary. Using the + ** same pattern as for the lsm_db.apShm[] array above. */ + nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; + while( nChunk>=nAlloc ){ + void **apShm; + nAlloc += NINCR; + apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); + if( !apShm ){ + rc = LSM_NOMEM_BKPT; + break; + } + p->apShmChunk = apShm; + } + + for(i=db->nShm; rc==LSM_OK && i=p->nShmChunk ){ + void *pChunk = 0; + if( p->bMultiProc==0 ){ + /* Single process mode */ + pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); + }else{ + /* Multi-process mode */ + rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); + } + if( rc==LSM_OK ){ + p->apShmChunk[i] = pChunk; + p->nShmChunk++; + } + } + if( rc==LSM_OK ){ + db->apShm[i] = p->apShmChunk[i]; + db->nShm++; + } + } + + /* Release the client mutex */ + lsmMutexLeave(pEnv, p->pClientMutex); + } } return rc; } @@ -1398,10 +1710,53 @@ if( p->bMultiProc ){ rc = lsmEnvLock(pEnv, p->pFile, iLock, eOp); } return rc; } + +/* +** Test if it would be possible for connection db to obtain a lock of type +** eType on the nLock locks starting at iLock. If so, return LSM_OK. If it +** would not be possible to obtain the lock due to a lock held by another +** connection, return LSM_BUSY. If an IO or other error occurs (i.e. in the +** lsm_env.xTestLock function), return some other LSM error code. +** +** Note that this function never actually locks the database - it merely +** queries the system to see if there exists a lock that would prevent +** it from doing so. +*/ +int lsmShmTestLock( + lsm_db *db, + int iLock, + int nLock, + int eOp +){ + int rc = LSM_OK; + lsm_db *pIter; + Database *p = db->pDatabase; + int i; + u64 mask = 0; + + for(i=iLock; i<(iLock+nLock); i++){ + mask |= ((u64)1 << (iLock-1)); + if( eOp==LSM_LOCK_EXCL ) mask |= ((u64)1 << (iLock+32-1)); + } + + lsmMutexEnter(db->pEnv, p->pClientMutex); + for(pIter=p->pConn; pIter; pIter=pIter->pNext){ + if( pIter!=db && (pIter->mLock & mask) ) break; + } + + if( pIter ){ + rc = LSM_BUSY; + }else if( p->bMultiProc ){ + rc = lsmEnvTestLock(db->pEnv, p->pFile, iLock, nLock, eOp); + } + + lsmMutexLeave(db->pEnv, p->pClientMutex); + return rc; +} /* ** Attempt to obtain the lock identified by the iLock and bExcl parameters. ** If successful, return LSM_OK. If the lock cannot be obtained because ** there exists some other conflicting lock, return LSM_BUSY. If some other @@ -1415,17 +1770,18 @@ int iLock, int eOp, /* One of LSM_LOCK_UNLOCK, SHARED or EXCL */ int bBlock /* True for a blocking lock */ ){ lsm_db *pIter; - const u32 me = (1 << (iLock-1)); - const u32 ms = (1 << (iLock+16-1)); + const u64 me = ((u64)1 << (iLock-1)); + const u64 ms = ((u64)1 << (iLock+32-1)); int rc = LSM_OK; Database *p = db->pDatabase; - assert( iLock>=1 && iLock<=LSM_LOCK_READER(LSM_LOCK_NREADER-1) ); - assert( iLock<=16 ); + assert( eOp!=LSM_LOCK_EXCL || db->bReadonly==0 ); + assert( iLock>=1 && iLock<=LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1) ); + assert( LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1)<=32 ); assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); /* Check for a no-op. Proceed only if this is not one of those. */ if( (eOp==LSM_LOCK_UNLOCK && (db->mLock & (me|ms))!=0) || (eOp==LSM_LOCK_SHARED && (db->mLock & (me|ms))!=ms) @@ -1464,22 +1820,26 @@ rc = LSM_BUSY; }else{ if( nShared==0 ){ rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_SHARED); } - db->mLock |= ms; - db->mLock &= ~me; + if( rc==LSM_OK ){ + db->mLock |= ms; + db->mLock &= ~me; + } } break; default: assert( eOp==LSM_LOCK_EXCL ); if( nExcl || nShared ){ rc = LSM_BUSY; }else{ rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_EXCL); - db->mLock |= (me|ms); + if( rc==LSM_OK ){ + db->mLock |= (me|ms); + } } break; } lsmMutexLeave(db->pEnv, p->pClientMutex); @@ -1489,12 +1849,12 @@ } #ifdef LSM_DEBUG int shmLockType(lsm_db *db, int iLock){ - const u32 me = (1 << (iLock-1)); - const u32 ms = (1 << (iLock+16-1)); + const u64 me = ((u64)1 << (iLock-1)); + const u64 ms = ((u64)1 << (iLock+32-1)); if( db->mLock & me ) return LSM_LOCK_EXCL; if( db->mLock & ms ) return LSM_LOCK_SHARED; return LSM_LOCK_UNLOCK; } @@ -1582,26 +1942,26 @@ void lsmShmBarrier(lsm_db *db){ lsmEnvShmBarrier(db->pEnv); } -int lsm_checkpoint(lsm_db *pDb, int *pnByte){ +int lsm_checkpoint(lsm_db *pDb, int *pnKB){ int rc; /* Return code */ u32 nWrite = 0; /* Number of pages checkpointed */ /* Attempt the checkpoint. If successful, nWrite is set to the number of ** pages written between this and the previous checkpoint. */ rc = lsmCheckpointWrite(pDb, 0, &nWrite); - /* If required, calculate the output variable (bytes of data checkpointed). + /* If required, calculate the output variable (KB of data checkpointed). ** Set it to zero if an error occured. */ - if( pnByte ){ - int nByte = 0; + if( pnKB ){ + int nKB = 0; if( rc==LSM_OK && nWrite ){ - nByte = (int)nWrite * lsmFsPageSize(pDb->pFS); + nKB = (((i64)nWrite * lsmFsPageSize(pDb->pFS)) + 1023) / 1024; } - *pnByte = nByte; + *pnKB = nKB; } return rc; } Index: src/lsm_sorted.c ================================================================== --- src/lsm_sorted.c +++ src/lsm_sorted.c @@ -2243,11 +2243,27 @@ pCsr->apTreeCsr[0] = 0; pCsr->apTreeCsr[1] = 0; pCsr->pBtCsr = 0; } -void lsmMCursorClose(MultiCursor *pCsr){ +void lsmMCursorFreeCache(lsm_db *pDb){ + MultiCursor *p; + MultiCursor *pNext; + for(p=pDb->pCsrCache; p; p=pNext){ + pNext = p->pNext; + lsmMCursorClose(p, 0); + } + pDb->pCsrCache = 0; +} + +/* +** Close the cursor passed as the first argument. +** +** If the bCache parameter is true, then shift the cursor to the pCsrCache +** list for possible reuse instead of actually deleting it. +*/ +void lsmMCursorClose(MultiCursor *pCsr, int bCache){ if( pCsr ){ lsm_db *pDb = pCsr->pDb; MultiCursor **pp; /* Iterator variable */ /* The cursor may or may not be currently part of the linked list @@ -2257,19 +2273,39 @@ *pp = pCsr->pNext; break; } } - /* Free the allocation used to cache the current key, if any. */ - sortedBlobFree(&pCsr->key); - sortedBlobFree(&pCsr->val); - - /* Free the component cursors */ - mcursorFreeComponents(pCsr); - - /* Free the cursor structure itself */ - lsmFree(pDb->pEnv, pCsr); + if( bCache ){ + int i; /* Used to iterate through segment-pointers */ + + /* Release any page references held by this cursor. */ + assert( !pCsr->pBtCsr ); + for(i=0; inPtr; i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + lsmFsPageRelease(pPtr->pPg); + pPtr->pPg = 0; + } + + /* Reset the tree cursors */ + lsmTreeCursorReset(pCsr->apTreeCsr[0]); + lsmTreeCursorReset(pCsr->apTreeCsr[1]); + + /* Add the cursor to the pCsrCache list */ + pCsr->pNext = pDb->pCsrCache; + pDb->pCsrCache = pCsr; + }else{ + /* Free the allocation used to cache the current key, if any. */ + sortedBlobFree(&pCsr->key); + sortedBlobFree(&pCsr->val); + + /* Free the component cursors */ + mcursorFreeComponents(pCsr); + + /* Free the cursor structure itself */ + lsmFree(pDb->pEnv, pCsr); + } } } #define TREE_NONE 0 #define TREE_OLD 1 @@ -2424,23 +2460,48 @@ return rc; } /* ** Allocate and return a new database cursor. +** +** This method should only be called to allocate user cursors. As it may +** recycle a cursor from lsm_db.pCsrCache. */ int lsmMCursorNew( lsm_db *pDb, /* Database handle */ MultiCursor **ppCsr /* OUT: Allocated cursor */ ){ MultiCursor *pCsr = 0; int rc = LSM_OK; - pCsr = multiCursorNew(pDb, &rc); - if( rc==LSM_OK ) rc = multiCursorInit(pCsr, pDb->pClient); + if( pDb->pCsrCache ){ + int bOld; /* True if there is an old in-memory tree */ + + /* Remove a cursor from the pCsrCache list and add it to the open list. */ + pCsr = pDb->pCsrCache; + pDb->pCsrCache = pCsr->pNext; + pCsr->pNext = pDb->pCsr; + pDb->pCsr = pCsr; + + /* The cursor can almost be used as is, except that the old in-memory + ** tree cursor may be present and not required, or required and not + ** present. Fix this if required. */ + bOld = (lsmTreeHasOld(pDb) && pDb->treehdr.iOldLog!=pDb->pClient->iLogOff); + if( !bOld && pCsr->apTreeCsr[1] ){ + lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); + pCsr->apTreeCsr[1] = 0; + }else if( bOld && !pCsr->apTreeCsr[1] ){ + rc = lsmTreeCursorNew(pDb, 1, &pCsr->apTreeCsr[1]); + } + + }else{ + pCsr = multiCursorNew(pDb, &rc); + if( rc==LSM_OK ) rc = multiCursorInit(pCsr, pDb->pClient); + } if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr); + lsmMCursorClose(pCsr, 0); pCsr = 0; } assert( (rc==LSM_OK)==(pCsr!=0) ); *ppCsr = pCsr; return rc; @@ -2555,11 +2616,11 @@ rc = multiCursorAdvance(pCsr, !bReverse); } } } - lsmMCursorClose(pCsr); + lsmMCursorClose(pCsr, 0); if( pSnap!=pDb->pWorker ){ lsmFreeSnapshot(pDb->pEnv, pSnap); } return rc; @@ -2598,11 +2659,11 @@ *pnVal = nVal; } } } - lsmMCursorClose(pCsr); + lsmMCursorClose(pCsr, 0); } return rc; } @@ -4033,11 +4094,11 @@ } pMerge->iOutputOff = -1; } - lsmMCursorClose(pCsr); + lsmMCursorClose(pCsr, 0); /* Persist and release the output page. */ if( rc==LSM_OK ) rc = mergeWorkerPersistAndRelease(pMW); if( rc==LSM_OK ) rc = mergeWorkerBtreeIndirect(pMW); if( rc==LSM_OK ) rc = mergeWorkerFinishHierarchy(pMW); @@ -4287,11 +4348,11 @@ multiCursorIgnoreDelete(pCsr); } } if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr); + lsmMCursorClose(pCsr, 0); }else{ Pgno iLeftPtr = 0; Merge merge; /* Merge object used to create new level */ MergeWorker mergeworker; /* MergeWorker object for the same purpose */ @@ -4740,16 +4801,29 @@ if( p->redirect.a==0 ){ int nByte = sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS; p->redirect.a = lsmMallocZeroRc(pDb->pEnv, nByte, &rc); } if( rc==LSM_OK ){ - memmove(&p->redirect.a[1], &p->redirect.a[0], - sizeof(struct RedirectEntry) * p->redirect.n - ); - p->redirect.a[0].iFrom = iFrom; - p->redirect.a[0].iTo = iTo; - p->redirect.n++; + + /* Check if the block just moved was already redirected. */ + int i; + for(i=0; iredirect.n; i++){ + if( p->redirect.a[i].iTo==iFrom ) break; + } + + if( i==p->redirect.n ){ + /* Block iFrom was not already redirected. Add a new array entry. */ + memmove(&p->redirect.a[1], &p->redirect.a[0], + sizeof(struct RedirectEntry) * p->redirect.n + ); + p->redirect.a[0].iFrom = iFrom; + p->redirect.a[0].iTo = iTo; + p->redirect.n++; + }else{ + /* Block iFrom was already redirected. Overwrite existing entry. */ + p->redirect.a[i].iTo = iTo; + } rc = lsmBlockFree(pDb, iFrom); *pnWrite = lsmFsBlockSize(pDb->pFS) / lsmFsPageSize(pDb->pFS); pLvl->lhs.pRedirect = &p->redirect; @@ -5198,21 +5272,23 @@ int rc = LSM_OK; /* Return code */ int nWrite = 0; /* Number of pages written */ assert( nMerge>=1 ); - if( nPage>0 ){ + if( nPage!=0 ){ int bCkpt = 0; do { int nThis = 0; + int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF); + bCkpt = 0; - rc = doLsmSingleWork(pDb, 0, nMerge, nPage-nWrite, &nThis, &bCkpt); + rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt); nWrite += nThis; if( rc==LSM_OK && bCkpt ){ rc = lsm_checkpoint(pDb, 0); } - }while( rc==LSM_OK && (nWritenTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT; - if( nMerge<=0 ) nMerge = pDb->nMerge; - return doLsmWork(pDb, nMerge, nPage, pnWrite); + + /* Convert from KB to pages */ + nPgsz = lsmFsPageSize(pDb->pFS); + if( nKB>=0 ){ + nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz; + }else{ + nPage = -1; + } + + rc = doLsmWork(pDb, nMerge, nPage, &nWrite); + + if( pnWrite ){ + /* Convert back from pages to KB */ + *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz); + } + return rc; } int lsm_flush(lsm_db *db){ int rc; @@ -5299,14 +5393,16 @@ nRemaining = nUnit * nDepth; #ifdef LSM_LOG_WORK lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", nUnit, nDepth, nRemaining); #endif + assert( nRemaining>=0 ); rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0); if( rc==LSM_BUSY ) rc = LSM_OK; if( bRestore && pDb->pCsr ){ + lsmMCursorFreeCache(pDb); lsmFreeSnapshot(pDb->pEnv, pDb->pClient); pDb->pClient = 0; rc = lsmCheckpointLoad(pDb, 0); if( rc==LSM_OK ){ rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot, &pDb->pClient); Index: src/lsm_tree.c ================================================================== --- src/lsm_tree.c +++ src/lsm_tree.c @@ -265,10 +265,17 @@ static void intArrayTruncate(IntArray *p, int nVal){ p->nArray = nVal; } /* End of IntArray methods. ***********************************************************************/ + +static int treeKeycmp(void *p1, int n1, void *p2, int n2){ + int res; + res = memcmp(p1, p2, LSM_MIN(n1, n2)); + if( res==0 ) res = (n1-n2); + return res; +} /* ** The pointer passed as the first argument points to an interior node, ** not a leaf. This function returns the offset of the iCell'th child ** sub-tree of the node. @@ -285,20 +292,23 @@ static int treeOffsetToChunk(u32 iOff){ assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); return (int)(iOff>>15); } +#define treeShmptrUnsafe(pDb, iPtr) \ +(&((u8*)((pDb)->apShm[(iPtr)>>15]))[(iPtr) & (LSM_SHM_CHUNK_SIZE-1)]) + /* ** Return a pointer to the mapped memory location associated with *-shm ** file offset iPtr. */ static void *treeShmptr(lsm_db *pDb, u32 iPtr){ assert( (iPtr>>15)nShm ); assert( pDb->apShm[iPtr>>15] ); - return iPtr?(&((u8*)(pDb->apShm[iPtr>>15]))[iPtr & (LSM_SHM_CHUNK_SIZE-1)]):0; + return iPtr ? treeShmptrUnsafe(pDb, iPtr) : 0; } static ShmChunk * treeShmChunk(lsm_db *pDb, int iChunk){ return (ShmChunk *)(pDb->apShm[iChunk]); } @@ -563,15 +573,20 @@ /* ** Return a pointer to the mapping of the TreeKey object that the cursor ** is pointing to. */ static TreeKey *csrGetKey(TreeCursor *pCsr, TreeBlob *pBlob, int *pRc){ - TreeKey *pRet = (TreeKey *)treeShmkey(pCsr->pDb, - pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]], - TKV_LOADVAL, pBlob, pRc - ); - assert( pRet==0 || assertFlagsOk(pRet->flags) ); + TreeKey *pRet; + lsm_db *pDb = pCsr->pDb; + u32 iPtr = pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]]; + + assert( iPtr ); + pRet = treeShmptrUnsafe(pDb, iPtr); + if( !(pRet->flags & LSM_CONTIGUOUS) ){ + pRet = treeShmkey(pDb, iPtr, TKV_LOADVAL, pBlob, pRc); + } + return pRet; } /* ** Save the current position of tree cursor pCsr. @@ -719,10 +734,11 @@ void *pVal, int nVal, /* Value data (or nVal<0 for delete) */ int *pRc ){ TreeKey *p; u32 iPtr; + u32 iEnd; int nRem; u8 *a; int n; /* Allocate space for the TreeKey structure itself */ @@ -752,10 +768,17 @@ } a = pVal; n = nRem = nVal; pVal = 0; } + + iEnd = iPtr + sizeof(TreeKey) + nKey + LSM_MAX(0, nVal); + if( (iPtr & ~(LSM_SHM_CHUNK_SIZE-1))!=(iEnd & ~(LSM_SHM_CHUNK_SIZE-1)) ){ + p->flags = 0; + }else{ + p->flags = LSM_CONTIGUOUS; + } if( *pRc ) return 0; #if 0 printf("store: %d %s\n", (int)iPtr, (char *)pKey); #endif @@ -1045,12 +1068,23 @@ return rc; } void lsmTreeMakeOld(lsm_db *pDb){ + + /* A write transaction must be open. Otherwise the code below that + ** assumes (pDb->pClient->iLogOff) is current may malfunction. + ** + ** Update: currently this assert fails due to lsm_flush(), which does + ** not set nTransOpen. + */ + assert( /* pDb->nTransOpen>0 && */ pDb->iReader>=0 ); + if( pDb->treehdr.iOldShmid==0 ){ - pDb->treehdr.iOldLog = pDb->treehdr.log.aRegion[2].iEnd; + pDb->treehdr.iOldLog = (pDb->treehdr.log.aRegion[2].iEnd << 1); + pDb->treehdr.iOldLog |= (~(pDb->pClient->iLogOff) & (i64)0x0001); + pDb->treehdr.oldcksum0 = pDb->treehdr.log.cksum0; pDb->treehdr.oldcksum1 = pDb->treehdr.log.cksum1; pDb->treehdr.iOldShmid = pDb->treehdr.iNextShmid-1; memcpy(&pDb->treehdr.oldroot, &pDb->treehdr.root, sizeof(TreeRoot)); @@ -1081,10 +1115,11 @@ */ int lsmTreeInit(lsm_db *pDb){ ShmChunk *pOne; int rc = LSM_OK; + memset(&pDb->treehdr, 0, sizeof(TreeHeader)); pDb->treehdr.root.iTransId = 1; pDb->treehdr.iFirst = 1; pDb->treehdr.nChunk = 2; pDb->treehdr.iWrite = LSM_SHM_CHUNK_SIZE + LSM_SHM_CHUNK_HDR; pDb->treehdr.iNextShmid = 2; @@ -1426,10 +1461,11 @@ assert( nVal>=0 || pVal==0 ); assert_tree_looks_ok(LSM_OK, pTree); assert( flags==LSM_INSERT || flags==LSM_POINT_DELETE || flags==LSM_START_DELETE || flags==LSM_END_DELETE ); + assert( (flags & LSM_CONTIGUOUS)==0 ); #if 0 dump_tree_contents(pDb, "before"); #endif if( p->iRoot ){ @@ -1483,11 +1519,12 @@ } /* Allocate and populate a new key-value pair structure */ pTreeKey = newTreeKey(pDb, &iTreeKey, pKey, nKey, pVal, nVal, &rc); if( rc!=LSM_OK ) return rc; - pTreeKey->flags = flags; + assert( pTreeKey->flags==0 || pTreeKey->flags==LSM_CONTIGUOUS ); + pTreeKey->flags |= flags; if( p->iRoot==0 ){ /* The tree is completely empty. Add a new root node and install ** (pKey/nKey) as the middle entry. Even though it is a leaf at the ** moment, use newTreeNode() to allocate the node (i.e. allocate enough @@ -1769,11 +1806,11 @@ int bDone = 0; TreeRoot *p = &db->treehdr.root; TreeBlob blob = {0, 0}; /* The range must be sensible - that (key1 < key2). */ - assert( db->xCmp(pKey1, nKey1, pKey2, nKey2)<0 ); + assert( treeKeycmp(pKey1, nKey1, pKey2, nKey2)<0 ); assert( assert_delete_ranges_match(db) ); #if 0 static int nCall = 0; printf("\n"); @@ -1801,11 +1838,11 @@ ** tree now contains no keys in the range being deleted. In this case ** break out of the loop. */ bDone = 1; if( lsmTreeCursorValid(&csr) ){ lsmTreeCursorKey(&csr, 0, &pDel, &nDel); - if( db->xCmp(pDel, nDel, pKey2, nKey2)<0 ) bDone = 0; + if( treeKeycmp(pDel, nDel, pKey2, nKey2)<0 ) bDone = 0; } if( bDone==0 ){ if( csr.iNode==(p->nHeight-1) ){ /* The element to delete already lies on a leaf node */ @@ -1918,11 +1955,11 @@ int cmp = 0; int rc = LSM_OK; assert( pCsr->iNode>=0 ); p = csrGetKey(pCsr, &pCsr->blob, &rc); if( p ){ - cmp = pCsr->pDb->xCmp(TKV_KEY(p), p->nKey, pKey, nKey); + cmp = treeKeycmp(TKV_KEY(p), p->nKey, pKey, nKey); } return cmp; } #endif @@ -1943,12 +1980,10 @@ */ int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes){ int rc = LSM_OK; /* Return code */ lsm_db *pDb = pCsr->pDb; TreeRoot *pRoot = pCsr->pRoot; - int (*xCmp)(void *, int, void *, int) = pDb->xCmp; - u32 iNodePtr; /* Location of current node in search */ /* Discard any saved position data */ treeCursorRestore(pCsr, 0); @@ -1963,41 +1998,49 @@ int res = 0; /* Result of comparison function */ int iNode = -1; while( iNodePtr ){ TreeNode *pNode; /* Node at location iNodePtr */ int iTest; /* Index of second key to test (0 or 2) */ + u32 iTreeKey; TreeKey *pTreeKey; /* Key to compare against */ - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); + pNode = (TreeNode *)treeShmptrUnsafe(pDb, iNodePtr); iNode++; pCsr->apTreeNode[iNode] = pNode; /* Compare (pKey/nKey) with the key in the middle slot of B-tree node ** pNode. The middle slot is never empty. If the comparison is a match, ** then the search is finished. Break out of the loop. */ - pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TKV_LOADKEY, &b, &rc); - if( rc!=LSM_OK ) break; - res = xCmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); + pTreeKey = treeShmptrUnsafe(pDb, pNode->aiKeyPtr[1]); + if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ + pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TKV_LOADKEY, &b, &rc); + if( rc!=LSM_OK ) break; + } + res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); if( res==0 ){ pCsr->aiCell[iNode] = 1; break; } /* Based on the results of the previous comparison, compare (pKey/nKey) ** to either the left or right key of the B-tree node, if such a key ** exists. */ iTest = (res>0 ? 0 : 2); - pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[iTest], TKV_LOADKEY, &b, &rc); - if( rc ) break; - if( pTreeKey==0 ){ - iTest = 1; - }else{ - res = xCmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); + iTreeKey = pNode->aiKeyPtr[iTest]; + if( iTreeKey ){ + pTreeKey = treeShmptrUnsafe(pDb, iTreeKey); + if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ + pTreeKey = treeShmkey(pDb, iTreeKey, TKV_LOADKEY, &b, &rc); + if( rc ) break; + } + res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); if( res==0 ){ pCsr->aiCell[iNode] = iTest; break; } + }else{ + iTest = 1; } if( iNode<(pRoot->nHeight-1) ){ iNodePtr = getChildPtr(pNode, pRoot->iTransId, iTest + (res<0)); }else{ @@ -2078,11 +2121,11 @@ } #ifndef NDEBUG if( pCsr->iNode>=0 ){ TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc || pDb->xCmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)>=0 ); + assert( rc||treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)>=0 ); } tblobFree(pDb, &key1); #endif return rc; @@ -2146,11 +2189,11 @@ } #ifndef NDEBUG if( pCsr->iNode>=0 ){ TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc || pDb->xCmp(TKV_KEY(pK2), pK2->nKey, TKV_KEY(pK1), pK1->nKey)<0 ); + assert( rc || treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)<0 ); } tblobFree(pDb, &key1); #endif return rc; @@ -2200,15 +2243,15 @@ int lsmTreeCursorFlags(TreeCursor *pCsr){ int flags = 0; if( pCsr && pCsr->iNode>=0 ){ int rc = LSM_OK; - TreeKey *pKey = (TreeKey *)treeShmptr(pCsr->pDb, + TreeKey *pKey = (TreeKey *)treeShmptrUnsafe(pCsr->pDb, pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]] ); assert( rc==LSM_OK ); - flags = pKey->flags; + flags = (pKey->flags & ~LSM_CONTIGUOUS); } return flags; } int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey){ @@ -2355,11 +2398,11 @@ return LSM_OK; } lsmShmBarrier(pDb); } - return LSM_PROTOCOL; + return LSM_PROTOCOL_BKPT; } int lsmTreeLoadHeaderOk(lsm_db *pDb, int iRead){ TreeHeader *p = (iRead==1) ? &pDb->pShmhdr->hdr1 : &pDb->pShmhdr->hdr2; assert( iRead==1 || iRead==2 ); Index: src/lsm_unix.c ================================================================== --- src/lsm_unix.c +++ src/lsm_unix.c @@ -36,10 +36,15 @@ #include #include #include "lsmInt.h" +/* There is no fdatasync() call on Android */ +#ifdef __ANDROID__ +# define fdatasync(x) fsync(x) +#endif + /* ** An open file is an instance of the following object */ typedef struct PosixFile PosixFile; struct PosixFile { @@ -51,12 +56,10 @@ off_t nMap; /* Size of mapping at pMap in bytes */ int nShm; /* Number of entries in array apShm[] */ void **apShm; /* Array of 32K shared memory segments */ }; -static int lsm_ioerr(void){ return LSM_IOERR; } - static char *posixShmFile(PosixFile *p){ char *zShm; int nName = strlen(p->zName); zShm = (char *)lsmMalloc(p->pEnv, nName+4+1); if( zShm ){ @@ -66,28 +69,35 @@ return zShm; } static int lsmPosixOsOpen( lsm_env *pEnv, - const char *zFile, + const char *zFile, + int flags, lsm_file **ppFile ){ int rc = LSM_OK; PosixFile *p; p = lsm_malloc(pEnv, sizeof(PosixFile)); if( p==0 ){ rc = LSM_NOMEM; }else{ + int bReadonly = (flags & LSM_OPEN_READONLY); + int oflags = (bReadonly ? O_RDONLY : (O_RDWR|O_CREAT)); memset(p, 0, sizeof(PosixFile)); p->zName = zFile; p->pEnv = pEnv; - p->fd = open(zFile, O_RDWR|O_CREAT, 0644); + p->fd = open(zFile, oflags, 0644); if( p->fd<0 ){ lsm_free(pEnv, p); p = 0; - rc = lsm_ioerr(); + if( errno==ENOENT ){ + rc = lsmErrorBkpt(LSM_IOERR_NOENT); + }else{ + rc = LSM_IOERR_BKPT; + } } } *ppFile = (lsm_file *)p; return rc; @@ -103,29 +113,33 @@ PosixFile *p = (PosixFile *)pFile; off_t offset; offset = lseek(p->fd, (off_t)iOff, SEEK_SET); if( offset!=iOff ){ - rc = lsm_ioerr(); + rc = LSM_IOERR_BKPT; }else{ ssize_t prc = write(p->fd, pData, (size_t)nData); - if( prc<0 ) rc = lsm_ioerr(); + if( prc<0 ) rc = LSM_IOERR_BKPT; } return rc; } static int lsmPosixOsTruncate( lsm_file *pFile, /* File to write to */ lsm_i64 nSize /* Size to truncate file to */ ){ - int rc = LSM_OK; + PosixFile *p = (PosixFile *)pFile; + int rc = LSM_OK; /* Return code */ int prc; /* Posix Return Code */ - PosixFile *p = (PosixFile *)pFile; - - prc = ftruncate(p->fd, (off_t)nSize); - if( prc<0 ) rc = lsm_ioerr(); + struct stat sStat; /* Result of fstat() invocation */ + + prc = fstat(p->fd, &sStat); + if( prc==0 && sStat.st_size>nSize ){ + prc = ftruncate(p->fd, (off_t)nSize); + } + if( prc<0 ) rc = LSM_IOERR_BKPT; return rc; } static int lsmPosixOsRead( @@ -138,15 +152,15 @@ PosixFile *p = (PosixFile *)pFile; off_t offset; offset = lseek(p->fd, (off_t)iOff, SEEK_SET); if( offset!=iOff ){ - rc = lsm_ioerr(); + rc = LSM_IOERR_BKPT; }else{ ssize_t prc = read(p->fd, pData, (size_t)nData); if( prc<0 ){ - rc = lsm_ioerr(); + rc = LSM_IOERR_BKPT; }else if( prcpMap ){ prc = msync(p->pMap, p->nMap, MS_SYNC); } if( prc==0 ) prc = fdatasync(p->fd); - if( prc<0 ) rc = lsm_ioerr(); + if( prc<0 ) rc = LSM_IOERR_BKPT; #else (void)pFile; #endif return rc; @@ -239,22 +253,24 @@ munmap(p->pMap, p->nMap); *ppOut = p->pMap = 0; *pnOut = p->nMap = 0; } - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - iSz = buf.st_size; - if( iSzfd, iSz); - if( prc!=0 ) return LSM_IOERR_BKPT; - } - - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - p->nMap = iSz; + if( iMin>=0 ){ + memset(&buf, 0, sizeof(buf)); + prc = fstat(p->fd, &buf); + if( prc!=0 ) return LSM_IOERR_BKPT; + iSz = buf.st_size; + if( iSzfd, iSz); + if( prc!=0 ) return LSM_IOERR_BKPT; + } + + p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); + p->nMap = iSz; + } *ppOut = p->pMap; *pnOut = p->nMap; return LSM_OK; } @@ -340,11 +356,11 @@ assert( aType[LSM_LOCK_UNLOCK]==F_UNLCK ); assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); assert( eType>=0 && eType0 && iLock<=16 ); + assert( iLock>0 && iLock<=32 ); memset(&lock, 0, sizeof(lock)); lock.l_whence = SEEK_SET; lock.l_len = 1; lock.l_type = aType[eType]; @@ -353,13 +369,40 @@ if( fcntl(p->fd, F_SETLK, &lock) ){ int e = errno; if( e==EACCES || e==EAGAIN ){ rc = LSM_BUSY; }else{ - rc = LSM_IOERR; + rc = LSM_IOERR_BKPT; } } + + return rc; +} + +int lsmPosixOsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ + int rc = LSM_OK; + PosixFile *p = (PosixFile *)pFile; + static const short aType[3] = { 0, F_RDLCK, F_WRLCK }; + struct flock lock; + + assert( eType==LSM_LOCK_SHARED || eType==LSM_LOCK_EXCL ); + assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); + assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); + assert( eType>=0 && eType0 && iLock<=32 ); + + memset(&lock, 0, sizeof(lock)); + lock.l_whence = SEEK_SET; + lock.l_len = nLock; + lock.l_type = aType[eType]; + lock.l_start = (4096-iLock); + + if( fcntl(p->fd, F_GETLK, &lock) ){ + rc = LSM_IOERR_BKPT; + }else if( lock.l_type!=F_UNLCK ){ + rc = LSM_BUSY; + } return rc; } int lsmPosixOsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ @@ -407,11 +450,11 @@ if( p->apShm[iChunk]==0 ){ p->apShm[iChunk] = mmap(0, LSM_SHM_CHUNK_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, p->shmfd, iChunk*LSM_SHM_CHUNK_SIZE ); - if( p->apShm[iChunk]==0 ) return LSM_IOERR; + if( p->apShm[iChunk]==0 ) return LSM_IOERR_BKPT; } *ppShm = p->apShm[iChunk]; return LSM_OK; } @@ -450,13 +493,15 @@ lsm_free(p->pEnv, p); return LSM_OK; } static int lsmPosixOsSleep(lsm_env *pEnv, int us){ - if( usleep(us) ){ - return LSM_IOERR; - } +#if 0 + /* Apparently on Android usleep() returns void */ + if( usleep(us) ) return LSM_IOERR; +#endif + usleep(us); return LSM_OK; } /**************************************************************************** ** Memory allocation routines. @@ -713,10 +758,11 @@ lsmPosixOsUnmap, /* xUnmap */ lsmPosixOsFileid, /* xFileid */ lsmPosixOsClose, /* xClose */ lsmPosixOsUnlink, /* xUnlink */ lsmPosixOsLock, /* xLock */ + lsmPosixOsTestLock, /* xTestLock */ lsmPosixOsShmMap, /* xShmMap */ lsmPosixOsShmBarrier, /* xShmBarrier */ lsmPosixOsShmUnmap, /* xShmUnmap */ /***** memory allocation *********/ 0, /* pMemCtx */ Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -27,18 +27,11 @@ #endif /* ** Dummy function used as a unique symbol for SQLITE4_DYNAMIC */ -void sqlite4_dynamic(void *p){ (void)p; } - -#ifndef SQLITE4_AMALGAMATION -/* IMPLEMENTATION-OF: R-46656-45156 The sqlite4_version[] string constant -** contains the text of SQLITE4_VERSION macro. -*/ -const char sqlite4_version[] = SQLITE4_VERSION; -#endif +void sqlite4_dynamic(void *pArg,void *p){ (void)pArg; (void)p; } /* IMPLEMENTATION-OF: R-53536-42575 The sqlite4_libversion() function returns ** a pointer to the to the sqlite4_version[] string constant. */ const char *sqlite4_libversion(void){ return SQLITE4_VERSION; } @@ -69,378 +62,10 @@ ** are intended for debugging activity only. */ void (*sqlite4IoTrace)(const char*, ...) = 0; #endif -/* -** Initialize SQLite. -** -** This routine must be called to initialize the run-time environment -** As long as you do not compile with SQLITE4_OMIT_AUTOINIT -** this routine will be called automatically by key routines such as -** sqlite4_open(). -** -** This routine is a no-op except on its very first call for a given -** sqlite4_env object, or for the first call after a call to sqlite4_shutdown. -** -** This routine is not threadsafe. It should be called from a single -** thread to initialized the library in a multi-threaded system. Other -** threads should avoid using the sqlite4_env object until after it has -** completely initialized. -*/ -int sqlite4_initialize(sqlite4_env *pEnv){ - MUTEX_LOGIC( sqlite4_mutex *pMaster; ) /* The main static mutex */ - int rc; /* Result code */ - - if( pEnv==0 ) pEnv = &sqlite4DefaultEnv; - - /* If SQLite is already completely initialized, then this call - ** to sqlite4_initialize() should be a no-op. But the initialization - ** must be complete. So isInit must not be set until the very end - ** of this routine. - */ - if( pEnv->isInit ) return SQLITE4_OK; - - /* Initialize the mutex subsystem - */ - rc = sqlite4MutexInit(pEnv); - if( rc ){ - sqlite4MallocEnd(pEnv); - return rc; - } - - /* Initialize the memory allocation subsystem - */ - rc = sqlite4MallocInit(pEnv); - if( rc ) return rc; - - /* Create required mutexes - */ - if( pEnv->bCoreMutex ){ - pEnv->pMemMutex = sqlite4MutexAlloc(pEnv, SQLITE4_MUTEX_FAST); - pEnv->pPrngMutex = sqlite4MutexAlloc(pEnv, SQLITE4_MUTEX_FAST); - pEnv->pFactoryMutex = sqlite4MutexAlloc(pEnv, SQLITE4_MUTEX_FAST); - if( pEnv->pMemMutex==0 - || pEnv->pPrngMutex==0 - || pEnv->pFactoryMutex==0 - ){ - rc = SQLITE4_NOMEM; - } - }else{ - pEnv->pMemMutex = 0; - pEnv->pPrngMutex = 0; - } - pEnv->isInit = 1; - - sqlite4OsInit(pEnv); - - /* Register global functions */ - if( rc==SQLITE4_OK ){ - sqlite4RegisterGlobalFunctions(pEnv); - } - - /* The following is just a sanity check to make sure SQLite has - ** been compiled correctly. It is important to run this code, but - ** we don't want to run it too often and soak up CPU cycles for no - ** reason. So we run it once during initialization. - */ -#ifndef NDEBUG -#ifndef SQLITE4_OMIT_FLOATING_POINT - /* This section of code's only "output" is via assert() statements. */ - if ( rc==SQLITE4_OK ){ - u64 x = (((u64)1)<<63)-1; - double y; - assert(sizeof(x)==8); - assert(sizeof(x)==sizeof(y)); - memcpy(&y, &x, 8); - assert( sqlite4IsNaN(y) ); - } -#endif -#endif - - return rc; -} - -/* -** Undo the effects of sqlite4_initialize(). Must not be called while -** there are outstanding database connections or memory allocations or -** while any part of SQLite is otherwise in use in any thread. This -** routine is not threadsafe. But it is safe to invoke this routine -** on when SQLite is already shut down. If SQLite is already shut down -** when this routine is invoked, then this routine is a harmless no-op. -*/ -int sqlite4_shutdown(sqlite4_env *pEnv){ - if( pEnv==0 ) pEnv = &sqlite4DefaultEnv; - if( pEnv->isInit ){ - KVFactory *pMkr; - sqlite4_mutex_free(pEnv->pFactoryMutex); - sqlite4_mutex_free(pEnv->pPrngMutex); - sqlite4_mutex_free(pEnv->pMemMutex); - pEnv->pMemMutex = 0; - while( (pMkr = pEnv->pFactory)!=0 && pMkr->isPerm==0 ){ - KVFactory *pNext = pMkr->pNext; - sqlite4_free(pEnv, pMkr); - pMkr = pNext; - } - sqlite4MutexEnd(pEnv); - sqlite4MallocEnd(pEnv); - pEnv->isInit = 0; - } - return SQLITE4_OK; -} - -/* -** Return the size of an sqlite4_env object -*/ -int sqlite4_env_size(void){ return sizeof(sqlite4_env); } - -/* -** This API allows applications to modify the configuration described by -** an sqlite4_env object. -*/ -int sqlite4_env_config(sqlite4_env *pEnv, int op, ...){ - va_list ap; - int rc = SQLITE4_OK; - - if( pEnv==0 ) pEnv = sqlite4_env_default(); - - va_start(ap, op); - switch( op ){ - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_INIT, template); - ** - ** Turn bulk memory into a new sqlite4_env object. The template is - ** a prior sqlite4_env that is used as a template in initializing the - ** new sqlite4_env object. The size of the bulk memory must be at - ** least as many bytes as returned from sqlite4_env_size(). - */ - case SQLITE4_ENVCONFIG_INIT: { - /* Disable all mutexing */ - sqlite4_env *pTemplate = va_arg(ap, sqlite4_env*); - int n = pTemplate->nByte; - if( n>sizeof(sqlite4_env) ) n = sizeof(sqlite4_env); - memcpy(pEnv, pTemplate, n); - pEnv->pFactory = &sqlite4BuiltinFactory; - pEnv->isInit = 0; - break; - } - - /* Mutex configuration options are only available in a threadsafe - ** compile. - */ -#if defined(SQLITE4_THREADSAFE) && SQLITE4_THREADSAFE>0 - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_SINGLETHREAD); - ** - ** Configure this environment for a single-threaded application. - */ - case SQLITE4_ENVCONFIG_SINGLETHREAD: { - /* Disable all mutexing */ - if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } - pEnv->bCoreMutex = 0; - pEnv->bFullMutex = 0; - break; - } - - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_MULTITHREAD); - ** - ** Configure this environment for a multi-threaded application where - ** the same database connection is never used by more than a single - ** thread at a time. - */ - case SQLITE4_ENVCONFIG_MULTITHREAD: { - /* Disable mutexing of database connections */ - /* Enable mutexing of core data structures */ - if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } - pEnv->bCoreMutex = 1; - pEnv->bFullMutex = 0; - break; - } - - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_MULTITHREAD); - ** - ** Configure this environment for an unrestricted multi-threaded - ** application where any thread can do whatever it wants with any - ** database connection at any time. - */ - case SQLITE4_ENVCONFIG_SERIALIZED: { - /* Enable all mutexing */ - if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } - pEnv->bCoreMutex = 1; - pEnv->bFullMutex = 1; - break; - } - - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_MUTEXT, sqlite4_mutex_methods*) - ** - ** Configure this environment to use the mutex routines specified by the - ** argument. - */ - case SQLITE4_ENVCONFIG_MUTEX: { - /* Specify an alternative mutex implementation */ - if( pEnv->isInit ){ rc = SQLITE4_MISUSE; break; } - pEnv->mutex = *va_arg(ap, sqlite4_mutex_methods*); - break; - } - - /* - ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_GETMUTEX, sqlite4_mutex_methods*) - ** - ** Copy the mutex routines in use by this environment into the structure - ** given in the argument. - */ - case SQLITE4_ENVCONFIG_GETMUTEX: { - /* Retrieve the current mutex implementation */ - *va_arg(ap, sqlite4_mutex_methods*) = pEnv->mutex; - break; - } -#endif - - - /* - ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_MALLOC, sqlite4_mem_methods*) - ** - ** Set the memory allocation routines to be used by this environment. - */ - case SQLITE4_ENVCONFIG_MALLOC: { - /* Specify an alternative malloc implementation */ - if( pEnv->isInit ) return SQLITE4_MISUSE; - pEnv->m = *va_arg(ap, sqlite4_mem_methods*); - break; - } - - /* - ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_GETMALLOC, sqlite4_mem_methods*) - ** - ** Copy the memory allocation routines in use by this environment - ** into the structure given in the argument. - */ - case SQLITE4_ENVCONFIG_GETMALLOC: { - /* Retrieve the current malloc() implementation */ - if( pEnv->m.xMalloc==0 ) sqlite4MemSetDefault(pEnv); - *va_arg(ap, sqlite4_mem_methods*) = pEnv->m; - break; - } - - /* sqlite4_env_config(p, SQLITE4_ENVCONFIG_MEMSTAT, int onoff); - ** - ** Enable or disable collection of memory usage statistics according to - ** the onoff parameter. - */ - case SQLITE4_ENVCONFIG_MEMSTATUS: { - /* Enable or disable the malloc status collection */ - pEnv->bMemstat = va_arg(ap, int); - break; - } - - /* - ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_LOOKASIDE, size, count); - ** - ** Set the default lookaside memory settings for all subsequent - ** database connections constructed in this environment. The size - ** parameter is the size of each lookaside memory buffer and the - ** count parameter is the number of lookaside buffers. Set both - ** to zero to disable lookaside memory. - */ - case SQLITE4_ENVCONFIG_LOOKASIDE: { - pEnv->szLookaside = va_arg(ap, int); - pEnv->nLookaside = va_arg(ap, int); - break; - } - - /* - ** sqlite4_env_config(p, SQLITE4_ENVCONFIG_LOG, xOutput, pArg); - ** - ** Set the log function that is called in response to sqlite4_log() - ** calls. - */ - case SQLITE4_ENVCONFIG_LOG: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** pEnv->xLog = va_arg(ap, void(*)(void*,int,const char*)); - */ - typedef void(*LOGFUNC_t)(void*,int,const char*); - pEnv->xLog = va_arg(ap, LOGFUNC_t); - pEnv->pLogArg = va_arg(ap, void*); - break; - } - - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_PUSH, zName,xFactory); - ** - ** Push a new KVStore factory onto the factory stack. The new factory - ** takes priority over prior factories. - */ - case SQLITE4_ENVCONFIG_KVSTORE_PUSH: { - const char *zName = va_arg(ap, const char*); - int nName = sqlite4Strlen30(zName); - KVFactory *pMkr = sqlite4_malloc(pEnv, sizeof(*pMkr)+nName+1); - char *z; - if( pMkr==0 ) return SQLITE4_NOMEM; - z = (char*)&pMkr[1]; - memcpy(z, zName, nName+1); - memset(pMkr, 0, sizeof(*pMkr)); - pMkr->zName = z; - pMkr->xFactory = va_arg(ap, sqlite4_kvfactory); - sqlite4_mutex_enter(pEnv->pFactoryMutex); - pMkr->pNext = pEnv->pFactory; - pEnv->pFactory = pMkr; - sqlite4_mutex_leave(pEnv->pFactoryMutex); - break; - } - - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_POP, zName, &pxFact); - ** - ** Remove a KVStore factory from the stack. - */ - /* - ** sqlite4_env_config(pEnv, SQLITE4_ENVCONFIG_KVSTORE_GET, zName, &pxFact); - ** - ** Get the current factory pointer with the given name but leave the - ** factory on the stack. - */ - case SQLITE4_ENVCONFIG_KVSTORE_POP: - case SQLITE4_ENVCONFIG_KVSTORE_GET: { - typedef int (**PxFact)(sqlite4_env*,KVStore**,const char*,unsigned); - const char *zName = va_arg(ap, const char*); - KVFactory *pMkr, **ppPrev; - PxFact pxFact; - - pxFact = va_arg(ap,PxFact); - *pxFact = 0; - sqlite4_mutex_enter(pEnv->pFactoryMutex); - ppPrev = &pEnv->pFactory; - pMkr = *ppPrev; - while( pMkr && strcmp(zName, pMkr->zName)!=0 ){ - ppPrev = &pMkr->pNext; - pMkr = *ppPrev; - } - if( pMkr ){ - *pxFact = pMkr->xFactory; - if( op==SQLITE4_ENVCONFIG_KVSTORE_POP && pMkr->isPerm==0 ){ - *ppPrev = pMkr->pNext; - sqlite4_free(pEnv, pMkr); - } - } - sqlite4_mutex_leave(pEnv->pFactoryMutex); - break; - } - - - default: { - rc = SQLITE4_ERROR; - break; - } - } - va_end(ap); - return rc; -} - /* ** Set up the lookaside buffers for a database connection. ** Return SQLITE4_OK on success. ** If lookaside is already active, return SQLITE4_BUSY. ** @@ -664,17 +289,10 @@ } } return nIn; } -/* -** Return the ROWID of the most recent insert -*/ -sqlite4_int64 sqlite4_last_insert_rowid(sqlite4 *db){ - return db->lastRowid; -} - /* ** Return the number of changes in the most recent call to sqlite4_exec(). */ int sqlite4_changes(sqlite4 *db){ return db->nChange; @@ -1265,11 +883,11 @@ z = (void *)outOfMem; }else{ z = sqlite4_value_text16(db->pErr); if( z==0 ){ sqlite4ValueSetStr(db->pErr, -1, sqlite4ErrStr(db->errCode), - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); z = sqlite4_value_text16(db->pErr); } /* A malloc() may have failed within the call to sqlite4_value_text16() ** above. If this is the case, then the db->mallocFailed flag needs to ** be cleared before returning. Do this directly, instead of via @@ -1931,27 +1549,15 @@ sqlite4_mutex_leave(db->mutex); return SQLITE4_OK; } #endif /* SQLITE4_OMIT_UTF16 */ -#ifndef SQLITE4_OMIT_DEPRECATED -/* -** This function is now an anachronism. It used to be used to recover from a -** malloc() failure, but SQLite now does this automatically. -*/ -int sqlite4_global_recover(void){ - return SQLITE4_OK; -} -#endif - /* ** Test to see whether or not the database connection is in autocommit ** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on ** by default. Autocommit is disabled by a BEGIN statement and reenabled ** by the next COMMIT or ROLLBACK. -** -******* THIS IS AN EXPERIMENTAL API AND IS SUBJECT TO CHANGE ****** */ int sqlite4_get_autocommit(sqlite4 *db){ return (db->pSavepoint==0); } Index: src/math.c ================================================================== --- src/math.c +++ src/math.c @@ -213,10 +213,24 @@ if( r.approx==0 && A.m%B.m!=0 ) r.approx = 1; r.m = A.m/B.m; r.e = A.e - B.e; return r; } + +/* +** Test if A is infinite. +*/ +int sqlite4_num_isinf(sqlite4_num A){ + return A.e>SQLITE4_MX_EXP && A.m!=0; +} + +/* +** Test if A is NaN. +*/ +int sqlite4_num_isnan(sqlite4_num A){ + return A.e>SQLITE4_MX_EXP && A.m==0; +} /* ** Compare numbers A and B. Return: ** ** 1 if ASQLITE4_MX_EXP ){ if( B.m==0 ) return 0; return B.sign ? 3 : 1; } if( A.sign!=B.sign ){ + if ( A.m==0 && B.m==0 ) return 2; return A.sign ? 1 : 3; } adjustExponent(&A, &B); if( A.sign ){ sqlite4_num t = A; @@ -321,11 +336,11 @@ i = incr; }else{ i = 0; } if( nIn<=0 ) goto not_a_valid_number; - if( nIn>=incr*2 + if( nIn>=incr*3 && ((c=zIn[i])=='i' || c=='I') && ((c=zIn[i+incr])=='n' || c=='N') && ((c=zIn[i+incr*2])=='f' || c=='F') ){ r.e = SQLITE4_MX_EXP+1; @@ -333,11 +348,11 @@ return r; } while( i='0' && c<='9' ){ - if( c==0 && nDigit==0 ){ + if( c=='0' && nDigit==0 ){ if( seenRadix && r.e > -(SQLITE4_MX_EXP+1000) ) r.e--; continue; } nDigit++; if( nDigit<=18 ){ @@ -347,44 +362,61 @@ if( c!='0' ) r.approx = 1; if( !seenRadix ) r.e++; } }else if( c=='.' ){ seenRadix = 1; - }else{ + }else if( c=='e' || c=='E' ){ + int exp = 0; + int expsign = 0; + int nEDigit = 0; + if( zIn[i]=='-' ){ + expsign = 1; + i += incr; + }else if( zIn[i]=='+' ){ + i += incr; + } + if( i>=nIn ) goto not_a_valid_number; + while( i'9' ) goto not_a_valid_number; + if( c=='0' && nEDigit==0 ) continue; + nEDigit++; + if( nEDigit>3 ) goto not_a_valid_number; + exp = exp*10 + c - '0'; + } + if( expsign ) exp = -exp; + r.e += exp; break; - } - } - if( c=='e' || c=='E' ){ - int exp = 0; - int expsign = 0; - int nEDigit = 0; - if( zIn[i]=='-' ){ - expsign = 1; - i += incr; - }else if( zIn[i]=='+' ){ - i += incr; - } - if( i>=nIn ) goto not_a_valid_number; - while( i'9' ) break; - if( c=='0' && nEDigit==0 ) continue; - nEDigit++; - if( nEDigit>3 ) goto not_a_valid_number; - exp = exp*10 + c - '0'; - } - if( expsign ) exp = -exp; - r.e += exp; - } - if( c!=0 ) goto not_a_valid_number; + }else{ + goto not_a_valid_number; + } + } return r; not_a_valid_number: r.e = SQLITE4_MX_EXP+1; r.m = 0; return r; } + +/* +** Convert an sqlite4_int64 to a number and return that number. +*/ +sqlite4_num sqlite4_num_from_int64(sqlite4_int64 n){ + sqlite4_num r; + r.approx = 0; + r.e = 0; + r.sign = n < 0; + if( n>=0 ){ + r.m = n; + }else if( n!=SMALLEST_INT64 ){ + r.m = -n; + }else{ + r.m = 1+(u64)LARGEST_INT64; + } + return r; +} /* ** Convert an integer into text in the buffer supplied. The ** text is zero-terminated and right-justified in the buffer. ** A pointer to the first character of text is returned. @@ -467,12 +499,14 @@ removeTrailingZeros(zNum, &n); if( n>0 ){ zOut[0] = '.'; memcpy(zOut+1, zNum, n); nOut += n; + zOut[n+1] = 0; + }else{ + zOut[0] = 0; } - zOut[n+1] = 0; return nOut; } if( x.e<0 && x.e >= -n-5 ){ /* Values less than 1 and with no more than 5 subsequent zeros prior ** to the first significant digit. Ex: 0.0000012345 */ Index: src/mem.c ================================================================== --- src/mem.c +++ src/mem.c @@ -11,13 +11,11 @@ ************************************************************************* ** ** This file contains the implementation of the "sqlite4_mm" memory ** allocator object. */ -#include "sqlite4.h" -#include "mem.h" -#include +#include "sqliteInt.h" /************************************************************************* ** The SQLITE4_MM_SYSTEM memory allocator. This allocator uses the ** malloc/realloc/free from the system library. It also tries to use ** the memory allocation sizer from the system library if such a routine @@ -98,33 +96,33 @@ #endif /* __APPLE__ or not __APPLE__ */ /* ** Implementations of core routines */ -static void *mmSysMalloc(sqlite4_mm *pMM, sqlite4_int64 iSize){ +static void *mmSysMalloc(sqlite4_mm *pMM, sqlite4_size_t iSize){ #ifdef SQLITE4_MALLOCSIZE return SQLITE4_MALLOC(iSize); #else unsigned char *pRes = SQLITE4_MALLOC(iSize+8); if( pRes ){ - *(sqlite4_int64*)pRes = iSize; + *(sqlite4_size_t*)pRes = iSize; pRes += 8; } return pRes; #endif } -static void *mmSysRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ +static void *mmSysRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_size_t iSz){ #ifdef SQLITE4_MALLOCSIZE return SQLITE4_REALLOC(pOld, iSz); #else unsigned char *pRes; if( pOld==0 ) return mmSysMalloc(pMM, iSz); pRes = (unsigned char*)pOld; pRes -= 8; pRes = SQLITE4_REALLOC(pRes, iSz+8); if( pRes ){ - *(sqlite4_int64*)pRes = iSz; + *(sqlite4_size_t*)pRes = iSz; pRes += 8; } return pRes; #endif } @@ -137,19 +135,19 @@ pRes = (unsigned char *)pOld; pRes -= 8; SQLITE4_FREE(pRes); #endif } -static sqlite4_int64 mmSysMsize(sqlite4_mm *pNotUsed, void *pOld){ +static sqlite4_size_t mmSysMsize(sqlite4_mm *pNotUsed, void *pOld){ #ifdef SQLITE4_MALLOCSIZE return SQLITE4_MALLOCSIZE(pOld); #else unsigned char *pX; if( pOld==0 ) return 0; pX = (unsigned char *)pOld; pX -= 8; - return *(sqlite4_int64*)pX; + return *(sqlite4_size_t*)pX; #endif } static const sqlite4_mm_methods mmSysMethods = { /* iVersion */ 1, @@ -157,16 +155,21 @@ /* xRealloc */ mmSysRealloc, /* xFree */ mmSysFree, /* xMsize */ mmSysMsize, /* xMember */ 0, /* xBenign */ 0, + /* xStat */ 0, + /* xCtrl */ 0, /* xFinal */ 0 }; -static sqlite4_mm mmSystem = { +sqlite4_mm sqlite4MMSystem = { /* pMethods */ &mmSysMethods }; +/* The system memory allocator is the default. */ +sqlite4_mm *sqlite4_mm_default(void){ return &sqlite4MMSystem; } + /************************************************************************* ** The SQLITE4_MM_OVERFLOW memory allocator. ** ** This memory allocator has two child memory allocators, A and B. Always ** try to fulfill the request using A first, then overflow to B if the request @@ -177,26 +180,33 @@ int (*xMemberOfA)(sqlite4_mm*, const void*); sqlite4_mm *pA; /* Primary memory allocator */ sqlite4_mm *pB; /* Backup memory allocator in case pA fails */ }; -static void *mmOvflMalloc(sqlite4_mm *pMM, sqlite4_int64 iSz){ +static void *mmOvflMalloc(sqlite4_mm *pMM, sqlite4_size_t iSz){ const struct mmOvfl *pOvfl = (const struct mmOvfl*)pMM; void *pRes; pRes = pOvfl->pA->pMethods->xMalloc(pOvfl->pA, iSz); if( pRes==0 ){ pRes = pOvfl->pB->pMethods->xMalloc(pOvfl->pB, iSz); } return pRes; } -static void *mmOvflRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ +static void *mmOvflRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_size_t iSz){ const struct mmOvfl *pOvfl; - void *pRes; + void *pRes, *pAlt; if( pOld==0 ) return mmOvflMalloc(pMM, iSz); pOvfl = (const struct mmOvfl*)pMM; if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ pRes = pOvfl->pA->pMethods->xRealloc(pOvfl->pA, pOld, iSz); + if( pRes==0 && (pAlt = pOvfl->pB->pMethods->xMalloc(pOvfl->pB, iSz))!=0 ){ + sqlite4_size_t nOld = pOvfl->pA->pMethods->xMsize(pOvfl->pA, pOld); + assert( nOldpA->pMethods->xFree(pOvfl->pA, pOld); + pRes = pAlt; + } }else{ pRes = pOvfl->pB->pMethods->xRealloc(pOvfl->pB, pOld, iSz); } return pRes; } @@ -208,13 +218,13 @@ pOvfl->pA->pMethods->xFree(pOvfl->pA, pOld); }else{ pOvfl->pB->pMethods->xFree(pOvfl->pB, pOld); } } -static sqlite4_int64 mmOvflMsize(sqlite4_mm *pMM, void *pOld){ +static sqlite4_size_t mmOvflMsize(sqlite4_mm *pMM, void *pOld){ const struct mmOvfl *pOvfl; - sqlite4_int64 iSz; + sqlite4_size_t iSz; if( pOld==0 ) return 0; pOvfl = (const struct mmOvfl*)pMM; if( pOvfl->xMemberOfA(pOvfl->pA, pOld) ){ iSz = sqlite4_mm_msize(pOvfl->pA, pOld); }else{ @@ -253,10 +263,12 @@ /* xRealloc */ mmOvflRealloc, /* xFree */ mmOvflFree, /* xMsize */ mmOvflMsize, /* xMember */ mmOvflMember, /* xBenign */ mmOvflBenign, + /* xStat */ 0, + /* xCtrl */ 0, /* xFinal */ mmOvflFinal }; static sqlite4_mm *mmOvflNew(sqlite4_mm *pA, sqlite4_mm *pB){ struct mmOvfl *pOvfl; if( pA->pMethods->xMember==0 ) return 0; @@ -288,22 +300,31 @@ sqlite4_mm base; /* Base class. Must be first. */ const void *pSpace; /* Space to allocate */ const void *pLast; /* Last possible allocation */ struct mmOneszBlock *pFree; /* List of free blocks */ int sz; /* Size of each allocation */ + unsigned nFailSize; /* Failures due to size */ + unsigned nFailMem; /* Failures due to OOM */ + unsigned nSlot; /* Number of available slots */ + unsigned nUsed; /* Current number of slots in use */ + unsigned nUsedHw; /* Highwater mark for slots in use */ + sqlite4_size_t mxSize; /* Maximum request size */ }; /* A free block in the buffer */ struct mmOneszBlock { struct mmOneszBlock *pNext; /* Next on the freelist */ }; -static void *mmOneszMalloc(sqlite4_mm *pMM, sqlite4_int64 iSz){ +static void *mmOneszMalloc(sqlite4_mm *pMM, sqlite4_size_t iSz){ struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; void *pRes; - if( iSz>pOnesz->sz ) return 0; - if( pOnesz->pFree==0 ) return 0; + if( iSz>pOnesz->mxSize ) pOnesz->mxSize = iSz; + if( iSz>pOnesz->sz ){ pOnesz->nFailSize++; return 0; } + if( pOnesz->pFree==0 ){ pOnesz->nFailMem++; return 0; } + pOnesz->nUsed++; + if( pOnesz->nUsed>pOnesz->nUsedHw ) pOnesz->nUsedHw = pOnesz->nUsed; pRes = pOnesz->pFree; pOnesz->pFree = pOnesz->pFree->pNext; return pRes; } static void mmOneszFree(sqlite4_mm *pMM, void *pOld){ @@ -310,48 +331,103 @@ struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; if( pOld ){ struct mmOneszBlock *pBlock = (struct mmOneszBlock*)pOld; pBlock->pNext = pOnesz->pFree; pOnesz->pFree = pBlock; + pOnesz->nUsed--; } } -static void *mmOneszRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSz){ +static void *mmOneszRealloc(sqlite4_mm *pMM, void *pOld, sqlite4_size_t iSz){ struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; if( pOld==0 ) return mmOneszMalloc(pMM, iSz); if( iSz<=0 ){ mmOneszFree(pMM, pOld); return 0; } if( iSz>pOnesz->sz ) return 0; return pOld; } -static sqlite4_int64 mmOneszMsize(sqlite4_mm *pMM, void *pOld){ +static sqlite4_size_t mmOneszMsize(sqlite4_mm *pMM, void *pOld){ struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; return pOld ? pOnesz->sz : 0; } static int mmOneszMember(sqlite4_mm *pMM, const void *pOld){ struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; return pOld && pOld>=pOnesz->pSpace && pOld<=pOnesz->pLast; +} +static sqlite4_int64 mmOneszStat(sqlite4_mm *pMM, int eType, unsigned flgs){ + struct mmOnesz *pOnesz = (struct mmOnesz*)pMM; + sqlite4_int64 x = -1; + switch( eType ){ + case SQLITE4_MMSTAT_OUT: { + x = pOnesz->nUsed*pOnesz->sz; + break; + } + case SQLITE4_MMSTAT_OUT_HW: { + x = pOnesz->nUsedHw*pOnesz->sz; + if( flgs & SQLITE4_MMSTAT_RESET ) pOnesz->nUsedHw = pOnesz->nUsed; + break; + } + case SQLITE4_MMSTAT_UNITS: { + x = pOnesz->nUsed; + break; + } + case SQLITE4_MMSTAT_UNITS_HW: { + x = pOnesz->nUsedHw; + if( flgs & SQLITE4_MMSTAT_RESET ) pOnesz->nUsedHw = pOnesz->nUsed; + break; + } + case SQLITE4_MMSTAT_SIZE: { + x = pOnesz->mxSize; + if( flgs & SQLITE4_MMSTAT_RESET ) pOnesz->mxSize = 0; + break; + } + case SQLITE4_MMSTAT_SZFAULT: { + x = pOnesz->nFailSize; + if( flgs & SQLITE4_MMSTAT_RESET ) pOnesz->nFailSize = 0; + break; + } + case SQLITE4_MMSTAT_MEMFAULT: { + x = pOnesz->nFailMem; + if( flgs & SQLITE4_MMSTAT_RESET ) pOnesz->nFailMem = 0; + break; + } + case SQLITE4_MMSTAT_FAULT: { + x = pOnesz->nFailSize + pOnesz->nFailMem; + if( flgs & SQLITE4_MMSTAT_RESET ){ + pOnesz->nFailSize = 0; + pOnesz->nFailMem = 0; + } + break; + } + } + return x; } static const sqlite4_mm_methods mmOneszMethods = { /* iVersion */ 1, /* xMalloc */ mmOneszMalloc, /* xRealloc */ mmOneszRealloc, /* xFree */ mmOneszFree, /* xMsize */ mmOneszMsize, /* xMember */ mmOneszMember, /* xBenign */ 0, + /* xStat */ mmOneszStat, + /* xCtrl */ 0, /* xFinal */ 0 }; static sqlite4_mm *mmOneszNew(void *pSpace, int sz, int cnt){ struct mmOnesz *pOnesz; unsigned char *pMem; - if( szbase.pMethods = &mmOneszMethods; pOnesz->pSpace = (const void*)pMem; pOnesz->sz = sz; pOnesz->pLast = (const void*)(pMem + sz*(cnt-2)); pOnesz->pFree = 0; @@ -366,34 +442,49 @@ } /************************************************************************* ** Main interfaces. */ -void *sqlite4_mm_malloc(sqlite4_mm *pMM, sqlite4_int64 iSize){ - if( pMM==0 ) pMM = &mmSystem; +void *sqlite4_mm_malloc(sqlite4_mm *pMM, sqlite4_size_t iSize){ + if( pMM==0 ) pMM = &sqlite4MMSystem; return pMM->pMethods->xMalloc(pMM,iSize); } -void *sqlite4_mm_realloc(sqlite4_mm *pMM, void *pOld, sqlite4_int64 iSize){ - if( pMM==0 ) pMM = &mmSystem; +void *sqlite4_mm_realloc(sqlite4_mm *pMM, void *pOld, sqlite4_size_t iSize){ + if( pMM==0 ) pMM = &sqlite4MMSystem; return pMM->pMethods->xRealloc(pMM,pOld,iSize); } void sqlite4_mm_free(sqlite4_mm *pMM, void *pOld){ - if( pMM==0 ) pMM = &mmSystem; + if( pMM==0 ) pMM = &sqlite4MMSystem; pMM->pMethods->xFree(pMM,pOld); } -sqlite4_int64 sqlite4_mm_msize(sqlite4_mm *pMM, void *pOld){ - if( pMM==0 ) pMM = &mmSystem; +sqlite4_size_t sqlite4_mm_msize(sqlite4_mm *pMM, void *pOld){ + if( pMM==0 ) pMM = &sqlite4MMSystem; return pMM->pMethods->xMsize(pMM,pOld); } int sqlite4_mm_member(sqlite4_mm *pMM, const void *pOld){ return (pMM && pMM->pMethods->xMember!=0) ? pMM->pMethods->xMember(pMM,pOld) : -1; } -void sqlite4_mm_benign_failure(sqlite4_mm *pMM, int bEnable){ +void sqlite4_mm_benign_failures(sqlite4_mm *pMM, int bEnable){ if( pMM && pMM->pMethods->xBenign ){ pMM->pMethods->xBenign(pMM, bEnable); } +} +sqlite4_int64 sqlite4_mm_stat(sqlite4_mm *pMM, int eStatType, unsigned flags){ + if( pMM==0 ) return -1; + if( pMM->pMethods->xStat==0 ) return -1; + return pMM->pMethods->xStat(pMM, eStatType, flags); +} +int sqlite4_mm_control(sqlite4_mm *pMM, int eCtrlType, ...){ + int rc = SQLITE4_NOTFOUND; + if( pMM && pMM->pMethods->xCtrl ){ + va_list ap; + va_start(ap, eCtrlType); + rc = pMM->pMethods->xCtrl(pMM, eCtrlType, ap); + va_end(ap); + } + return rc; } void sqlite4_mm_destroy(sqlite4_mm *pMM){ if( pMM && pMM->pMethods->xFinal ) pMM->pMethods->xFinal(pMM); } @@ -406,11 +497,11 @@ sqlite4_mm *pMM; va_start(ap, eType); switch( eType ){ case SQLITE4_MM_SYSTEM: { - pMM = &mmSystem; + pMM = &sqlite4MMSystem; break; } case SQLITE4_MM_OVERFLOW: { sqlite4_mm *pA = va_arg(ap, sqlite4_mm*); sqlite4_mm *pB = va_arg(ap, sqlite4_mm*); Index: src/mem.h ================================================================== --- src/mem.h +++ src/mem.h @@ -7,14 +7,13 @@ ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* -** This header file defines the interface that the SQLite4 memory -** management logic. ** -** Some of this will eventually fold into sqliteInt.h. +** This file defines the sqlite4_mm "SQLite4 Memory Manager" object and +** its interfaces. */ /* ** object declarations @@ -30,36 +29,57 @@ struct sqlite4_mm { const sqlite4_mm_methods *pMethods; }; /* -** An instance of the following object defines a BESPOKE memory alloator +** Memory statistics reporting +*/ +typedef enum { + SQLITE4_MMSTAT_OUT = 1, /* Bytes of memory outstanding */ + SQLITE4_MMSTAT_UNITS = 2, /* Separate allocations outstanding */ + SQLITE4_MMSTAT_SIZE = 3, /* Size of the allocation */ + SQLITE4_MMSTAT_SZFAULT = 4, /* Number of faults due to size */ + SQLITE4_MMSTAT_MEMFAULT = 5, /* Number of faults due to out of space */ + SQLITE4_MMSTAT_FAULT = 6, /* Total number of faults */ +}; + +/* +** Bit flags for the 3rd parameter of xStat() +*/ +#define SQLITE4_MMSTAT_HIGHWATER 0x01 +#define SQLITE4_MMSTAT_RESET 0x02 +#define SQLITE4_MMSTAT_HWRESET 0x03 + +/* +** An instance of the following object defines the methods on +** a BESPOKE memory allocator. */ struct sqlite4_mm_methods { int iVersion; void *(*xMalloc)(sqlite4_mm*, sqlite4_int64); void *(*xRealloc)(sqlite4_mm*, void*, sqlite4_int64); void (*xFree)(sqlite4_mm*, void*); sqlite4_int64 (*xMsize)(sqlite4_mm*, void*); int (*xMember)(sqlite4_mm*, const void*); void (*xBenign)(sqlite4_mm*, int); + sqlite4_int64 (*xStat)(sqlite4_mm*, sqlite4_mm_stattype, unsigned flags); void (*xFinal)(sqlite4_mm*); }; /* ** Available memory management types: */ typedef enum { - SQLITE4_MM_SYSTEM, /* Use the system malloc() */ - SQLITE4_MM_ONESIZE, /* All allocations map to a fixed size */ - SQLITE4_MM_OVERFLOW, /* Two allocators. Use A first; failover to B */ - SQLITE4_MM_COMPACT, /* Like memsys3 from SQLite3 */ - SQLITE4_MM_ROBSON, /* Like memsys5 from SQLite3 */ - SQLITE4_MM_LINEAR, /* Allocate from a fixed buffer w/o free */ + SQLITE4_MM_SYSTEM = 1, /* Use the system malloc() */ + SQLITE4_MM_ONESIZE = 2, /* All allocations map to a fixed size */ + SQLITE4_MM_OVERFLOW = 3, /* Two allocators. Use A first; failover to B */ + SQLITE4_MM_COMPACT = 4, /* Like memsys3 from SQLite3 */ + SQLITE4_MM_ROBSON = 5, /* Like memsys5 from SQLite3 */ + SQLITE4_MM_LINEAR = 6, /* Allocate from a fixed buffer w/o free */ + SQLITE4_MM_BESPOKE = 7, /* Caller-defined implementation */ SQLITE4_MM_DEBUG, /* Debugging memory allocator */ - SQLITE4_MM_STATS, /* Keep memory statistics */ - SQLITE4_MM_BESPOKE /* Caller-defined implementation */ + SQLITE4_MM_STATS /* Keep memory statistics */ } sqlite4_mm_type; /* ** Allocate a new memory manager. Return NULL if unable. */ @@ -105,5 +125,8 @@ ** nested. In benign failure mode, OOM errors do not necessarily propagate ** back out to the application but can be dealt with internally. Memory ** allocations that occur in benign failure mode are considered "optional". */ void sqlite4_mm_benign_failures(sqlite4_mm*, int bEnable); + +/* +** Rest Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -75,10 +75,11 @@ { "vdbe_listing", SQLITE4_VdbeListing }, { "vdbe_trace", SQLITE4_VdbeTrace }, { "kv_trace", SQLITE4_KvTrace }, { "trace", SQLITE4_SqlTrace | SQLITE4_VdbeListing | SQLITE4_VdbeTrace | SQLITE4_KvTrace }, + { "vdbe_addoptrace", SQLITE4_VdbeAddopTrace }, #endif #ifndef SQLITE4_OMIT_CHECK { "ignore_check_constraints", SQLITE4_IgnoreChecks }, #endif /* The following is VERY experimental */ Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -340,26 +340,10 @@ if( cnt==0 ){ pNC = pNC->pNext; } } - /* - ** If X and Y are NULL (in other words if only the column name Z is - ** supplied) and the value of Z is enclosed in double-quotes, then - ** Z is a string literal if it doesn't match any column names. In that - ** case, we need to return right away and not make any changes to - ** pExpr. - ** - ** Because no reference was made to outer contexts, the pNC->nRef - ** fields are not changed in any context. - */ - if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ - pExpr->op = TK_STRING; - pExpr->pTab = 0; - return WRC_Prune; - } - /* ** cnt==0 means there was not match. cnt>1 means there were two or ** more matches. Either way, we have an error. */ if( cnt!=1 ){ Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -946,16 +946,16 @@ int ptab2 = pParse->nTab++; sqlite4VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); addr = 1 + sqlite4VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); codeOffset(v, p, addrContinue); sqlite4VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite4VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite4VdbeAddOp3(v, OP_Column, ptab2, 0, regRow); sqlite4VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ addr = 1 + sqlite4VdbeAddOp2(v, OP_Sort, iTab, addrBreak); codeOffset(v, p, addrContinue); - /* sqlite4VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); */ + sqlite4VdbeAddOp3(v, OP_Column, iTab, 0, regRow); } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { testcase( eDest==SRT_Table ); @@ -1280,11 +1280,11 @@ int nCol; /* Number of columns in the result set */ Expr *p; /* Expression for a single result column */ char *zName; /* Column name */ int nName; /* Size of name in zName[] */ - *pnCol = nCol = pEList->nExpr; + *pnCol = nCol = pEList ? pEList->nExpr : 0; aCol = *paCol = sqlite4DbMallocZero(db, sizeof(aCol[0])*nCol); if( aCol==0 ) return SQLITE4_NOMEM; for(i=0, pCol=aCol; ipEList->nExpr; assert( nOrderBy>=nExpr || db->mallocFailed ); - regPrev = sqlite4GetTempRange(pParse, nExpr+1); + regPrev = pParse->nMem + 1; + pParse->nMem += nExpr + 1; sqlite4VdbeAddOp2(v, OP_Integer, 0, regPrev); pKeyDup = sqlite4DbMallocZero(db, sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) ); if( pKeyDup ){ pKeyDup->aSortOrder = (u8*)&pKeyDup->aColl[nExpr]; @@ -2488,16 +2489,10 @@ sqlite4VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite4VdbeAddOp4(v, OP_Compare, destA.iMem, destB.iMem, nOrderBy, (char*)pKeyMerge, P4_KEYINFO_HANDOFF); sqlite4VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - /* Release temporary registers - */ - if( regPrev ){ - sqlite4ReleaseTempRange(pParse, regPrev, nOrderBy+1); - } - /* Jump to the this point in order to terminate the query. */ sqlite4VdbeResolveLabel(v, labelEnd); /* Set the number of output columns @@ -2701,10 +2696,16 @@ ** restriction (4). ** ** (18) If the sub-query is a compound select, then all terms of the ** ORDER by clause of the parent must be simple references to ** columns of the sub-query. +** +** Also, each component of the sub-query must return the same number +** of result columns. This is actually a requirement for any compound +** SELECT statement, but all the code here does is make sure that no +** such (illegal) sub-query is flattened. The caller will detect the +** syntax error and return a detailed message. ** ** (19) The subquery does not use LIMIT or the outer query does not ** have a WHERE clause. ** ** (20) If the sub-query is a compound select, then it must not use @@ -2841,10 +2842,11 @@ testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); assert( pSub->pSrc!=0 ); if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 || (pSub1->pPrior && pSub1->op!=TK_ALL) || pSub1->pSrc->nSrc<1 + || pSub->pEList->nExpr!=pSub1->pEList->nExpr ){ return 0; } testcase( pSub1->pSrc->nSrc>1 ); } Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -319,11 +319,11 @@ ){ assert( 0==argc ); assert( zShellStatic ); UNUSED_PARAMETER(argc); UNUSED_PARAMETER(argv); - sqlite4_result_text(context, zShellStatic, -1, SQLITE4_STATIC); + sqlite4_result_text(context, zShellStatic, -1, SQLITE4_STATIC, 0); } /* ** This routine reads a line of text from FILE in, stores @@ -1805,11 +1805,11 @@ if( z[j]=='"' ){ j++; if( z[j]==0 ) break; } z[k++] = z[j]; } z[k] = 0; } - sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC, 0); } sqlite4_step(pStmt); rc = sqlite4_reset(pStmt); free(zLine); if( rc!=SQLITE4_OK ){ @@ -2170,13 +2170,13 @@ sqlite4_free(0, zSql); if( rc ) return rc; nRow = nAlloc = 0; azResult = 0; if( nArg>1 ){ - sqlite4_bind_text(pStmt, 1, azArg[1], -1, SQLITE4_TRANSIENT); + sqlite4_bind_text(pStmt, 1, azArg[1], -1, SQLITE4_TRANSIENT, 0); }else{ - sqlite4_bind_text(pStmt, 1, "%", -1, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, 1, "%", -1, SQLITE4_STATIC, 0); } while( sqlite4_step(pStmt)==SQLITE4_ROW ){ if( nRow>=nAlloc ){ char **azNew; int n = nAlloc*2 + 10; Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -7,26 +7,17 @@ ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* -** This header file defines the interface that the SQLite library +** +** This header file defines the interface that the SQLite4 library ** presents to client programs. If a C-function, structure, datatype, ** or constant definition does not appear in this file, then it is ** not a published API of SQLite, is subject to change without ** notice, and should not be referenced by programs that use SQLite. ** -** Some of the definitions that are in this file are marked as -** "experimental". Experimental interfaces are normally new -** features recently added to SQLite. We do not anticipate changes -** to experimental interfaces but reserve the right to make minor changes -** if experience from use "in the wild" suggest such changes are prudent. -** -** The official C-language API documentation for SQLite is derived -** from comments in this file. This file is the authoritative source -** on how SQLite interfaces are suppose to operate. -** ** The name of this file under configuration management is "sqlite.h.in". ** The makefile makes some minor changes to this file (such as inserting ** the version number) and changes its name to "sqlite4.h" as ** part of the build process. */ @@ -47,35 +38,184 @@ */ #ifndef SQLITE4_EXTERN # define SQLITE4_EXTERN extern #endif -/* -** These no-op macros are used in front of interfaces to mark those -** interfaces as either deprecated or experimental. New applications -** should not use deprecated interfaces - they are support for backwards -** compatibility only. Application writers should be aware that -** experimental interfaces are subject to change in point releases. -** -** These macros used to resolve to various kinds of compiler magic that -** would generate warning messages when they were used. But that -** compiler magic ended up generating such a flurry of bug reports -** that we have taken it all out and gone back to using simple -** noop macros. -*/ -#define SQLITE4_DEPRECATED -#define SQLITE4_EXPERIMENTAL - /* ** Ensure these symbols were not defined by some previous header file. */ #ifdef SQLITE4_VERSION # undef SQLITE4_VERSION #endif #ifdef SQLITE4_VERSION_NUMBER # undef SQLITE4_VERSION_NUMBER #endif + +/* +** CAPIREF: 64-Bit Integer Types +** KEYWORDS: sqlite4_int64 sqlite4_uint64 +** +** Because there is no cross-platform way to specify 64-bit integer types +** SQLite includes typedefs for 64-bit signed and unsigned integers. +** +** ^The sqlite4_int64 and sqlite_int64 types can store integer values +** between -9223372036854775808 and +9223372036854775807 inclusive. ^The +** sqlite4_uint64 and sqlite_uint64 types can store integer values +** between 0 and +18446744073709551615 inclusive. +*/ +#ifdef SQLITE4_INT64_TYPE + typedef SQLITE4_INT64_TYPE sqlite4_int64_t; + typedef unsigned SQLITE4_INT64_TYPE sqlite4_uint64_t; +#elif defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 sqlite4_int64_t; + typedef unsigned __int64 sqlite4_uint64_t; +#else + typedef long long int sqlite4_int64_t; + typedef unsigned long long int sqlite4_uint64_t; +#endif +typedef sqlite4_int64_t sqlite4_int64; +typedef sqlite4_uint64_t sqlite4_uint64; + +/* +** CAPIREF: String length type +** +** A type for measuring the length of the string. Like size_t but +** does not require <stddef.h> +*/ +typedef int sqlite4_size_t; + +/* +** Available memory allocator object subtypes: +*/ +typedef enum { + SQLITE4_MM_SYSTEM = 1, /* Use the system malloc() */ + SQLITE4_MM_ONESIZE = 2, /* All allocations map to a fixed size */ + SQLITE4_MM_OVERFLOW = 3, /* Two allocators. Use A first; failover to B */ + SQLITE4_MM_COMPACT = 4, /* Like memsys3 from SQLite3 */ + SQLITE4_MM_ROBSON = 5, /* Like memsys5 from SQLite3 */ + SQLITE4_MM_LINEAR = 6, /* Allocate from a fixed buffer w/o free */ + SQLITE4_MM_BESPOKE = 7, /* Caller-defined implementation */ + SQLITE4_MM_DEBUG, /* Debugging memory allocator */ + SQLITE4_MM_STATS /* Keep memory statistics */ +} sqlite4_mm_type; + +/* +** Base class for the memory allocator object. +** +** Implementations may extend this with additional +** fields specific to its own needs. This needs to be public so that +** applications can supply their on customized memory allocators. +*/ +typedef struct sqlite4_mm sqlite4_mm; +typedef struct sqlite4_mm_methods sqlite4_mm_methods; +struct sqlite4_mm { + const struct sqlite4_mm_methods *pMethods; +}; +struct sqlite4_mm_methods { + int iVersion; + void *(*xMalloc)(sqlite4_mm*, sqlite4_size_t); + void *(*xRealloc)(sqlite4_mm*, void*, sqlite4_size_t); + void (*xFree)(sqlite4_mm*, void*); + sqlite4_size_t (*xMsize)(sqlite4_mm*, void*); + int (*xMember)(sqlite4_mm*, const void*); + void (*xBenign)(sqlite4_mm*, int); + sqlite4_int64 (*xStat)(sqlite4_mm*, unsigned eType, unsigned bFlags); + int (*xCtrl)(sqlite4_mm*, unsigned eType, va_list); + void (*xFinal)(sqlite4_mm*); +}; + +/* +** Return a pointer to the default memory allocator, which is basically +** a wrapper around system malloc()/realloc()/free(). +*/ +sqlite4_mm *sqlite4_mm_default(void); + + +/* +** Create a new memory allocator object. +*/ +sqlite4_mm *sqlite4_mm_new(sqlite4_mm_type, ...); + + +/* +** Allocate a new memory manager. Return NULL if unable. +*/ +sqlite4_mm *sqlite4_mm_new(sqlite4_mm_type, ...); + +/* +** Free the sqlite4_mm object. +** +** All outstanding memory for the allocator must be freed prior to +** invoking this interface, or else the behavior is undefined. +*/ +void sqlite4_mm_destroy(sqlite4_mm*); + +/* +** Core memory allocation routines: +*/ +void *sqlite4_mm_malloc(sqlite4_mm*, sqlite4_size_t); +void *sqlite4_mm_realloc(sqlite4_mm*, void*, sqlite4_size_t); +void sqlite4_mm_free(sqlite4_mm*, void*); + +/* +** Return the size of a memory allocation. +** +** All memory allocators in SQLite4 must be able to report their size. +** When using system malloc() on system that lack the malloc_usable_size() +** routine or its equivalent, then the sqlite4_mm object allocates 8 extra +** bytes for each memory allocation and stores the allocation size in those +** initial 8 bytes. +*/ +sqlite4_size_t sqlite4_mm_msize(sqlite4_mm*, void*); + +/* +** Check to see if pOld is a memory allocation from pMM. If it is, return +** 1. If not, return 0. If we cannot determine an answer, return -1. +** +** If pOld is not a valid memory allocation or is a memory allocation that +** has previously been freed, then the result of this routine is undefined. +*/ +int sqlite4_mm_member(sqlite4_mm *pMM, const void *pOld); + +/* +** Allowed values for the second parameter ("eType") to sqlite4_mm_type(). +*/ +#define SQLITE4_MMSTAT_OUT 1 +#define SQLITE4_MMSTAT_OUT_HW 2 +#define SQLITE4_MMSTAT_UNITS 3 +#define SQLITE4_MMSTAT_UNITS_HW 4 +#define SQLITE4_MMSTAT_SIZE 5 +#define SQLITE4_MMSTAT_SZFAULT 6 +#define SQLITE4_MMSTAT_MEMFAULT 7 +#define SQLITE4_MMSTAT_FAULT 8 + +/* +** Bits for the bit vector third parameter ("flags") to sqlite4_mm_type() +*/ +#define SQLITE4_MMSTAT_RESET 0x01 + +/* +** Return statistics or status information about a memory allocator. +** Not all memory allocators provide all stat values. Some memory +** allocators provides no states at all. If a particular stat for +** a memory allocator is unavailable, then -1 is returned. +*/ +sqlite4_int64 sqlite4_mm_stat(sqlite4_mm *pMM, int eType, unsigned flags); + +/* +** Send a control message into a memory allocator. +*/ +int sqlit4_mm_control(sqlite4_mm *pMM, int eType, ...); + +/* +** Enable or disable benign failure mode. Benign failure mode can be +** nested. In benign failure mode, OOM errors do not necessarily propagate +** back out to the application but can be dealt with internally. Memory +** allocations that occur in benign failure mode are considered "optional". +*/ +void sqlite4_mm_benign_failures(sqlite4_mm*, int bEnable); + /* ** CAPIREF: Run-time Environment Object ** ** An instance of the following object defines the run-time environment @@ -252,43 +392,10 @@ ** [sqlite4_busy_timeout()] to name but three) that are methods on an ** sqlite4 object. */ typedef struct sqlite4 sqlite4; -/* -** CAPIREF: 64-Bit Integer Types -** KEYWORDS: sqlite4_int64 sqlite4_uint64 -** -** Because there is no cross-platform way to specify 64-bit integer types -** SQLite includes typedefs for 64-bit signed and unsigned integers. -** -** ^The sqlite4_int64 and sqlite_int64 types can store integer values -** between -9223372036854775808 and +9223372036854775807 inclusive. ^The -** sqlite4_uint64 and sqlite_uint64 types can store integer values -** between 0 and +18446744073709551615 inclusive. -*/ -#ifdef SQLITE4_INT64_TYPE - typedef SQLITE4_INT64_TYPE sqlite4_int64_t; - typedef unsigned SQLITE4_INT64_TYPE sqlite4_uint64_t; -#elif defined(_MSC_VER) || defined(__BORLANDC__) - typedef __int64 sqlite4_int64_t; - typedef unsigned __int64 sqlite4_uint64_t; -#else - typedef long long int sqlite4_int64_t; - typedef unsigned long long int sqlite4_uint64_t; -#endif -typedef sqlite4_int64_t sqlite4_int64; -typedef sqlite4_uint64_t sqlite4_uint64; - -/* -** CAPIREF: String length type -** -** A type for measuring the length of the string. Like size_t but -** does not require <stddef.h> -*/ -typedef int sqlite4_size_t; - /* ** If compiling for a processor that lacks floating point support, ** substitute integer for floating-point. */ #ifdef SQLITE4_OMIT_FLOATING_POINT @@ -739,53 +846,10 @@ #define SQLITE4_DBCONFIG_LOOKASIDE 1001 /* void* int int */ #define SQLITE4_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ #define SQLITE4_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ -/* -** CAPIREF: Last Insert Rowid -** -** ^The sqlite4_last_insert_rowid(D) routine returns the primary key value -** of the the most recent successful [INSERT] from [database connection] D -** into a table where the primary key of the inserted row is a single -** integer. -** ^If there have been no successful [INSERT]s of rows with a single integer -** PRIMARY KEY value on database connection D, then this routine returns -** zero. -** -** ^(If an [INSERT] occurs within a trigger or within a [virtual table] -** method, then this routine will return the [rowid] of the inserted -** row as long as the trigger or virtual table method is running. -** But once the trigger or virtual table method ends, the value returned -** by this routine reverts to what it was before the trigger or virtual -** table method began.)^ -** -** ^An [INSERT] that fails due to a constraint violation is not a -** successful [INSERT] and does not change the value returned by this -** routine. ^Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, -** and INSERT OR ABORT make no changes to the return value of this -** routine when their insertion fails. ^(When INSERT OR REPLACE -** encounters a constraint violation, it does not fail. The -** INSERT continues to completion after deleting rows that caused -** the constraint problem so INSERT OR REPLACE will always change -** the return value of this interface.)^ -** -** ^For the purposes of this routine, an [INSERT] is considered to -** be successful even if it is subsequently rolled back. -** -** This function is accessible to SQL statements via the -** [last_insert_rowid() SQL function]. -** -** If a separate thread performs a new [INSERT] on the same -** database connection while the [sqlite4_last_insert_rowid()] -** function is running and thus changes the last insert [rowid], -** then the value returned by [sqlite4_last_insert_rowid()] is -** unpredictable and might not equal either the old or the new -** last insert [rowid]. -*/ -sqlite4_int64 sqlite4_last_insert_rowid(sqlite4*); - /* ** CAPIREF: Count The Number Of Rows Modified ** ** ^This function returns the number of database rows that were changed ** or inserted or deleted by the most recently completed SQL statement @@ -1313,11 +1377,11 @@ ** might provide greater resolution on the profiler callback. The ** sqlite4_profile() function is considered experimental and is ** subject to change in future versions of SQLite. */ void *sqlite4_trace(sqlite4*, void(*xTrace)(void*,const char*), void*); -SQLITE4_EXPERIMENTAL void *sqlite4_profile(sqlite4*, +void *sqlite4_profile(sqlite4*, void(*xProfile)(void*,const char*,sqlite4_uint64), void*); /* ** CAPIREF: Query Progress Callbacks ** @@ -1805,18 +1869,10 @@ ** information is in static, unmanaged space and does not need to be freed. ** ^If the fifth argument has the value [SQLITE4_TRANSIENT], then ** SQLite makes its own private copy of the data immediately, before ** the sqlite4_bind_*() routine returns. ** -** ^The sqlite4_bind_zeroblob() routine binds a BLOB of length N that -** is filled with zeroes. ^A zeroblob uses a fixed amount of memory -** (just an integer to hold its size) while it is being processed. -** Zeroblobs are intended to serve as placeholders for BLOBs whose -** content is later written using -** [sqlite4_blob_open | incremental BLOB I/O] routines. -** ^A negative value for the zeroblob results in a zero-length BLOB. -** ** ^If any of the sqlite4_bind_*() routines are called with a NULL pointer ** for the [prepared statement] or with a prepared statement for which ** [sqlite4_step()] has been called more recently than [sqlite4_reset()], ** then the call will return [SQLITE4_MISUSE]. If any sqlite4_bind_() ** routine is passed a [prepared statement] that has been finalized, the @@ -1831,19 +1887,21 @@ ** index is out of range. ^[SQLITE4_NOMEM] is returned if malloc() fails. ** ** See also: [sqlite4_bind_parameter_count()], ** [sqlite4_bind_parameter_name()], and [sqlite4_bind_parameter_index()]. */ -int sqlite4_bind_blob(sqlite4_stmt*, int, const void*, int n, void(*)(void*)); +int sqlite4_bind_blob(sqlite4_stmt*, int, const void*, int n, + void(*)(void*,void*),void*); int sqlite4_bind_double(sqlite4_stmt*, int, double); int sqlite4_bind_int(sqlite4_stmt*, int, int); int sqlite4_bind_int64(sqlite4_stmt*, int, sqlite4_int64); int sqlite4_bind_null(sqlite4_stmt*, int); -int sqlite4_bind_text(sqlite4_stmt*, int, const char*, int n, void(*)(void*)); -int sqlite4_bind_text16(sqlite4_stmt*, int, const void*, int, void(*)(void*)); +int sqlite4_bind_text(sqlite4_stmt*, int, const char*, int n, + void(*)(void*,void*),void*); +int sqlite4_bind_text16(sqlite4_stmt*, int, const void*, int, + void(*)(void*,void*),void*); int sqlite4_bind_value(sqlite4_stmt*, int, const sqlite4_value*); -int sqlite4_bind_zeroblob(sqlite4_stmt*, int, int n); /* ** CAPIREF: Number Of SQL Parameters ** ** ^This routine can be used to find the number of [SQL parameters] @@ -2148,11 +2206,12 @@ ** are pending, then the results are undefined. ** ** ^The sqlite4_column_type() routine returns the ** [SQLITE4_INTEGER | datatype code] for the initial data type ** of the result column. ^The returned value is one of [SQLITE4_INTEGER], -** [SQLITE4_FLOAT], [SQLITE4_TEXT], [SQLITE4_BLOB], or [SQLITE4_NULL]. The value +** [SQLITE4_FLOAT], [SQLITE4_TEXT], [SQLITE4_BLOB], or [SQLITE4_NULL]. +** The value ** returned by sqlite4_column_type() is only meaningful if no type ** conversions have occurred as described below. After a type conversion, ** the value returned by sqlite4_column_type() is undefined. Future ** versions of SQLite may change the behavior of sqlite4_column_type() ** following a type conversion. @@ -2477,27 +2536,10 @@ #define SQLITE4_UTF16BE 3 #define SQLITE4_UTF16 4 /* Use native byte order */ #define SQLITE4_ANY 5 /* sqlite4_create_function only */ #define SQLITE4_UTF16_ALIGNED 8 /* sqlite4_create_collation only */ -/* -** CAPIREF: Deprecated Functions -** DEPRECATED -** -** These functions are [deprecated]. In order to maintain -** backwards compatibility with older code, these functions continue -** to be supported. However, new applications should avoid -** the use of these functions. To help encourage people to avoid -** using these functions, we are not going to tell you what they do. -*/ -#ifndef SQLITE4_OMIT_DEPRECATED -SQLITE4_DEPRECATED int sqlite4_aggregate_count(sqlite4_context*); -SQLITE4_DEPRECATED int sqlite4_expired(sqlite4_stmt*); -SQLITE4_DEPRECATED int sqlite4_transfer_bindings(sqlite4_stmt*, sqlite4_stmt*); -SQLITE4_DEPRECATED int sqlite4_global_recover(void); -#endif - /* ** CAPIREF: Obtaining SQL Function Parameter Values ** ** The C-language implementation of SQL functions and aggregates uses ** this set of interface routines to access the parameter values on @@ -2662,11 +2704,12 @@ ** ** These routines must be called from the same thread in which ** the SQL function is running. */ void *sqlite4_get_auxdata(sqlite4_context*, int N); -void sqlite4_set_auxdata(sqlite4_context*, int N, void*, void (*)(void*)); +void sqlite4_set_auxdata(sqlite4_context*, int N, void*, + void (*)(void*,void*),void*); /* ** CAPIREF: Constants Defining Special Destructor Behavior ** @@ -2679,12 +2722,12 @@ ** the content before returning. ** ** The typedef is necessary to work around problems in certain ** C++ compilers. See ticket #2191. */ -typedef void (*sqlite4_destructor_type)(void*); -void sqlite4_dynamic(void*); +typedef void (*sqlite4_destructor_type)(void*,void*); +void sqlite4_dynamic(void*,void*); #define SQLITE4_STATIC ((sqlite4_destructor_type)0) #define SQLITE4_TRANSIENT ((sqlite4_destructor_type)-1) #define SQLITE4_DYNAMIC (sqlite4_dynamic) @@ -2703,14 +2746,10 @@ ** ^The sqlite4_result_blob() interface sets the result from ** an application-defined function to be the BLOB whose content is pointed ** to by the second parameter and which is N bytes long where N is the ** third parameter. ** -** ^The sqlite4_result_zeroblob() interfaces set the result of -** the application-defined function to be a BLOB containing all zero -** bytes and N bytes in size, where N is the value of the 2nd parameter. -** ** ^The sqlite4_result_double() interface sets the result from ** an application-defined function to be a floating point value specified ** by its 2nd argument. ** ** ^The sqlite4_result_error() and sqlite4_result_error16() functions @@ -2730,11 +2769,12 @@ ** routines make a private copy of the error message text before ** they return. Hence, the calling function can deallocate or ** modify the text after they return without harm. ** ^The sqlite4_result_error_code() function changes the error code ** returned by SQLite as a result of an error in a function. ^By default, -** the error code is SQLITE4_ERROR. ^A subsequent call to sqlite4_result_error() +** the error code is SQLITE4_ERROR. +** ^A subsequent call to sqlite4_result_error() ** or sqlite4_result_error16() resets the error code to SQLITE4_ERROR. ** ** ^The sqlite4_result_toobig() interface causes SQLite to throw an error ** indicating that a string or BLOB is too long to represent. ** @@ -2796,26 +2836,30 @@ ** ** If these routines are called from within the different thread ** than the one containing the application-defined function that received ** the [sqlite4_context] pointer, the results are undefined. */ -void sqlite4_result_blob(sqlite4_context*, const void*, int, void(*)(void*)); +void sqlite4_result_blob(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); void sqlite4_result_double(sqlite4_context*, double); void sqlite4_result_error(sqlite4_context*, const char*, int); void sqlite4_result_error16(sqlite4_context*, const void*, int); void sqlite4_result_error_toobig(sqlite4_context*); void sqlite4_result_error_nomem(sqlite4_context*); void sqlite4_result_error_code(sqlite4_context*, int); void sqlite4_result_int(sqlite4_context*, int); void sqlite4_result_int64(sqlite4_context*, sqlite4_int64); void sqlite4_result_null(sqlite4_context*); -void sqlite4_result_text(sqlite4_context*, const char*, int, void(*)(void*)); -void sqlite4_result_text16(sqlite4_context*, const void*, int, void(*)(void*)); -void sqlite4_result_text16le(sqlite4_context*, const void*, int,void(*)(void*)); -void sqlite4_result_text16be(sqlite4_context*, const void*, int,void(*)(void*)); +void sqlite4_result_text(sqlite4_context*, const char*, int, + void(*)(void*,void*),void*); +void sqlite4_result_text16(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); +void sqlite4_result_text16le(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); +void sqlite4_result_text16be(sqlite4_context*, const void*, int, + void(*)(void*,void*),void*); void sqlite4_result_value(sqlite4_context*, sqlite4_value*); -void sqlite4_result_zeroblob(sqlite4_context*, int n); /* ** CAPIREF: Define New Collating Sequences ** ** ^This function adds, removes, or modifies a [collation] associated @@ -3858,11 +3902,12 @@ ** [sqlite4_db_status()] to make sure that the call worked. ** The [sqlite4_db_status()] interface will return a non-zero error code ** if a discontinued or unsupported verb is invoked. ** **
-** [[SQLITE4_DBSTATUS_LOOKASIDE_USED]] ^(
SQLITE4_DBSTATUS_LOOKASIDE_USED
+** [[SQLITE4_DBSTATUS_LOOKASIDE_USED]] +** ^(
SQLITE4_DBSTATUS_LOOKASIDE_USED
**
This parameter returns the number of lookaside memory slots currently ** checked out.
)^ ** ** [[SQLITE4_DBSTATUS_LOOKASIDE_HIT]] ^(
SQLITE4_DBSTATUS_LOOKASIDE_HIT
**
This parameter returns the number malloc attempts that were @@ -3963,11 +4008,12 @@ ** These preprocessor macros define integer codes that name counter ** values associated with the [sqlite4_stmt_status()] interface. ** The meanings of the various counters are as follows: ** **
-** [[SQLITE4_STMTSTATUS_FULLSCAN_STEP]]
SQLITE4_STMTSTATUS_FULLSCAN_STEP
+** [[SQLITE4_STMTSTATUS_FULLSCAN_STEP]] +**
SQLITE4_STMTSTATUS_FULLSCAN_STEP
**
^This is the number of times that SQLite has stepped forward in ** a table as part of a full table scan. Large numbers for this counter ** may indicate opportunities for performance improvement through ** careful use of indices.
** @@ -3986,131 +4032,10 @@ */ #define SQLITE4_STMTSTATUS_FULLSCAN_STEP 1 #define SQLITE4_STMTSTATUS_SORT 2 #define SQLITE4_STMTSTATUS_AUTOINDEX 3 - -/* -** CAPIREF: Unlock Notification -** -** ^When running in shared-cache mode, a database operation may fail with -** an [SQLITE4_LOCKED] error if the required locks on the shared-cache or -** individual tables within the shared-cache cannot be obtained. See -** [SQLite Shared-Cache Mode] for a description of shared-cache locking. -** ^This API may be used to register a callback that SQLite will invoke -** when the connection currently holding the required lock relinquishes it. -** ^This API is only available if the library was compiled with the -** [SQLITE4_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. -** -** See Also: [Using the SQLite Unlock Notification Feature]. -** -** ^Shared-cache locks are released when a database connection concludes -** its current transaction, either by committing it or rolling it back. -** -** ^When a connection (known as the blocked connection) fails to obtain a -** shared-cache lock and SQLITE4_LOCKED is returned to the caller, the -** identity of the database connection (the blocking connection) that -** has locked the required resource is stored internally. ^After an -** application receives an SQLITE4_LOCKED error, it may call the -** sqlite4_unlock_notify() method with the blocked connection handle as -** the first argument to register for a callback that will be invoked -** when the blocking connections current transaction is concluded. ^The -** callback is invoked from within the [sqlite4_step] or [sqlite4_close] -** call that concludes the blocking connections transaction. -** -** ^(If sqlite4_unlock_notify() is called in a multi-threaded application, -** there is a chance that the blocking connection will have already -** concluded its transaction by the time sqlite4_unlock_notify() is invoked. -** If this happens, then the specified callback is invoked immediately, -** from within the call to sqlite4_unlock_notify().)^ -** -** ^If the blocked connection is attempting to obtain a write-lock on a -** shared-cache table, and more than one other connection currently holds -** a read-lock on the same table, then SQLite arbitrarily selects one of -** the other connections to use as the blocking connection. -** -** ^(There may be at most one unlock-notify callback registered by a -** blocked connection. If sqlite4_unlock_notify() is called when the -** blocked connection already has a registered unlock-notify callback, -** then the new callback replaces the old.)^ ^If sqlite4_unlock_notify() is -** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is canceled. ^The blocked connections -** unlock-notify callback may also be canceled by closing the blocked -** connection using [sqlite4_close()]. -** -** The unlock-notify callback is not reentrant. If an application invokes -** any sqlite4_xxx API functions from within an unlock-notify callback, a -** crash or deadlock may be the result. -** -** ^Unless deadlock is detected (see below), sqlite4_unlock_notify() always -** returns SQLITE4_OK. -** -** Callback Invocation Details -** -** When an unlock-notify callback is registered, the application provides a -** single void* pointer that is passed to the callback when it is invoked. -** However, the signature of the callback function allows SQLite to pass -** it an array of void* context pointers. The first argument passed to -** an unlock-notify callback is a pointer to an array of void* pointers, -** and the second is the number of entries in the array. -** -** When a blocking connections transaction is concluded, there may be -** more than one blocked connection that has registered for an unlock-notify -** callback. ^If two or more such blocked connections have specified the -** same callback function, then instead of invoking the callback function -** multiple times, it is invoked once with the set of void* context pointers -** specified by the blocked connections bundled together into an array. -** This gives the application an opportunity to prioritize any actions -** related to the set of unblocked database connections. -** -** Deadlock Detection -** -** Assuming that after registering for an unlock-notify callback a -** database waits for the callback to be issued before taking any further -** action (a reasonable assumption), then using this API may cause the -** application to deadlock. For example, if connection X is waiting for -** connection Y's transaction to be concluded, and similarly connection -** Y is waiting on connection X's transaction, then neither connection -** will proceed and the system may remain deadlocked indefinitely. -** -** To avoid this scenario, the sqlite4_unlock_notify() performs deadlock -** detection. ^If a given call to sqlite4_unlock_notify() would put the -** system in a deadlocked state, then SQLITE4_LOCKED is returned and no -** unlock-notify callback is registered. The system is said to be in -** a deadlocked state if connection A has registered for an unlock-notify -** callback on the conclusion of connection B's transaction, and connection -** B has itself registered for an unlock-notify callback when connection -** A's transaction is concluded. ^Indirect deadlock is also detected, so -** the system is also considered to be deadlocked if connection B has -** registered for an unlock-notify callback on the conclusion of connection -** C's transaction, where connection C is waiting on connection A. ^Any -** number of levels of indirection are allowed. -** -** The "DROP TABLE" Exception -** -** When a call to [sqlite4_step()] returns SQLITE4_LOCKED, it is almost -** always appropriate to call sqlite4_unlock_notify(). There is however, -** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement, -** SQLite checks if there are any currently executing SELECT statements -** that belong to the same connection. If there are, SQLITE4_LOCKED is -** returned. In this case there is no "blocking connection", so invoking -** sqlite4_unlock_notify() results in the unlock-notify callback being -** invoked immediately. If the application then re-attempts the "DROP TABLE" -** or "DROP INDEX" query, an infinite loop might be the result. -** -** One way around this problem is to check the extended error code returned -** by an sqlite4_step() call. ^(If there is a blocking connection, then the -** extended error code is set to SQLITE4_LOCKED_SHAREDCACHE. Otherwise, in -** the special "DROP TABLE/INDEX" case, the extended error code is just -** SQLITE4_LOCKED.)^ -*/ -int sqlite4_unlock_notify( - sqlite4 *pBlocked, /* Waiting connection */ - void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ - void *pNotifyArg /* Argument to pass to xNotify */ -); - /* ** CAPIREF: String Comparison ** ** ^The [sqlite4_strnicmp()] API allows applications and extensions to @@ -4204,11 +4129,12 @@ /* ** CAPIREF: Determine The Virtual Table Conflict Policy ** ** This function may only be called from within a call to the [xUpdate] method ** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The -** value returned is one of [SQLITE4_ROLLBACK], [SQLITE4_IGNORE], [SQLITE4_FAIL], +** value returned is one of [SQLITE4_ROLLBACK], [SQLITE4_IGNORE], +** [SQLITE4_FAIL], ** [SQLITE4_ABORT], or [SQLITE4_REPLACE], according to the [ON CONFLICT] mode ** of the SQL statement that triggered the call to the [xUpdate] method of the ** [virtual table]. */ int sqlite4_vtab_on_conflict(sqlite4 *); @@ -4325,11 +4251,11 @@ ** CAPIREF: Key-value storage object factory ** ** New key/value storage engines can be added to SQLite4 at run-time. ** In order to create a new KV storage engine, the application must ** supply a "factory" function that creates an instance of the -** sqlite4_kvstore object. This is typedef defines the signature +** sqlite4_kvstore object. This typedef defines the signature ** of that factory function. */ typedef int (*sqlite4_kvfactory)( sqlite4_env *pEnv, /* The environment to use */ sqlite4_kvstore **ppKVStore, /* OUT: New KV store returned here */ @@ -4345,11 +4271,11 @@ */ typedef struct sqlite4_num sqlite4_num; struct sqlite4_num { unsigned char sign; /* Sign of the overall value */ unsigned char approx; /* True if the value is approximate */ - unsigned short e; /* The exponent. */ + short e; /* The exponent. */ sqlite4_uint64 m; /* The significant */ }; /* ** CAPI4REF: Operations On SQLite Number Objects Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -166,15 +166,17 @@ ** pared it down to just these three.) ** ** If none of the above are defined, then set SQLITE4_SYSTEM_MALLOC as ** the default. */ -#if defined(SQLITE4_SYSTEM_MALLOC)+defined(SQLITE4_WIN32_MALLOC)+defined(SQLITE4_MEMDEBUG)>1 +#if defined(SQLITE4_SYSTEM_MALLOC)+defined(SQLITE4_WIN32_MALLOC)\ + +defined(SQLITE4_MEMDEBUG)>1 # error "At most one of the following compile-time configuration options\ is allows: SQLITE4_SYSTEM_MALLOC, SQLITE4_WIN32_MALLOC, SQLITE4_MEMDEBUG" #endif -#if defined(SQLITE4_SYSTEM_MALLOC)+defined(SQLITE4_WIN32_MALLOC)+defined(SQLITE4_MEMDEBUG)==0 +#if defined(SQLITE4_SYSTEM_MALLOC)+defined(SQLITE4_WIN32_MALLOC)\ + +defined(SQLITE4_MEMDEBUG)==0 # define SQLITE4_SYSTEM_MALLOC 1 #endif /* ** If SQLITE4_MALLOC_SOFT_LIMIT is not zero, then try to keep the @@ -195,11 +197,12 @@ ** if it is already defined or if it is unneeded because we are ** not doing a threadsafe build. Ticket #2681. ** ** See also ticket #2741. */ -#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE4_THREADSAFE +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__)\ + && SQLITE4_THREADSAFE # define _XOPEN_SOURCE 500 /* Needed to enable pthread recursive mutexes */ #endif /* ** The TCL headers are only needed when compiling the TCL bindings. @@ -462,11 +465,11 @@ ** These macros are designed to work correctly on both 32-bit and 64-bit ** compilers. */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -#define LARGEST_UINT64 (0xffffffff|(((i64)0xffffffff)<<32)) +#define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) /* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. */ @@ -762,23 +765,16 @@ /* ** Each database connection is an instance of the following structure. ** -** The sqlite.lastRowid records the last insert rowid generated by an -** insert statement. Inserts on views do not affect its value. Each -** trigger has its own context, so that lastRowid can be updated inside -** triggers as usual. The previous value will be restored once the trigger -** exits. Upon entering a before or instead of trigger, lastRowid is no -** longer (since after version 2.8.12) reset to -1. -** ** The sqlite.nChange does not count changes within triggers and keeps no ** context. It is reset at start of sqlite4_exec. ** The sqlite.lsChange represents the number of changes made by the last ** insert, update, or delete statement. It remains constant throughout the ** length of a statement and is then updated by OP_SetCounts. It keeps a -** context stack just like lastRowid so that the count of changes +** context stack so that the count of changes ** within a trigger is not seen outside the trigger. Changes to views do not ** affect the value of lsChange. ** The sqlite.csChange keeps track of the number of current changes (since ** the last statement) and is used to update sqlite_lsChange. ** @@ -801,11 +797,10 @@ u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ int nextPagesize; /* Pagesize after VACUUM if >0 */ int nTable; /* Number of tables in the database */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ - i64 lastRowid; /* ROWID of most recent insert (see above) */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite4_changes() */ int nTotalChange; /* Value returned by sqlite4_total_changes() */ sqlite4_mutex *mutex; /* Connection mutex */ int aLimit[SQLITE4_N_LIMIT]; /* Limits */ @@ -891,55 +886,51 @@ /* ** Possible values for the sqlite4.flags. */ #define SQLITE4_VdbeTrace 0x00000100 /* True to trace VDBE execution */ -#define SQLITE4_InternChanges 0x00000200 /* Uncommitted Hash table changes */ -#define SQLITE4_CountRows 0x00001000 /* Count rows changed by INSERT, */ - /* DELETE, or UPDATE and return */ - /* the count using a callback. */ -#define SQLITE4_SqlTrace 0x00004000 /* Debug print SQL as it executes */ -#define SQLITE4_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ -#define SQLITE4_WriteSchema 0x00010000 /* OK to update SQLITE4_MASTER */ -#define SQLITE4_KvTrace 0x00020000 /* Trace Key/value storage calls */ -#define SQLITE4_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ -#define SQLITE4_ReadUncommitted 0x0080000 /* For shared-cache mode */ -#define SQLITE4_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ -#define SQLITE4_RecoveryMode 0x00800000 /* Ignore schema errors */ +#define SQLITE4_SqlTrace 0x00000200 /* Debug print SQL as it executes */ +#define SQLITE4_VdbeListing 0x00000400 /* Debug listings of VDBE programs */ +#define SQLITE4_KvTrace 0x00000800 /* Trace Key/value storage calls */ +#define SQLITE4_VdbeAddopTrace 0x00001000 /* Trace sqlite4VdbeAddOp() calls */ +#define SQLITE4_InternChanges 0x00010000 /* Uncommitted Hash table changes */ +#define SQLITE4_WriteSchema 0x00020000 /* OK to update SQLITE4_MASTER */ +#define SQLITE4_IgnoreChecks 0x00040000 /* Dont enforce check constraints */ +#define SQLITE4_RecoveryMode 0x00080000 /* Ignore schema errors */ #define SQLITE4_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ #define SQLITE4_RecTriggers 0x02000000 /* Enable recursive triggers */ -#define SQLITE4_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ +#define SQLITE4_ForeignKeys 0x04000000 /* Enable foreign key constraints */ #define SQLITE4_AutoIndex 0x08000000 /* Enable automatic indexes */ #define SQLITE4_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ #define SQLITE4_EnableTrigger 0x40000000 /* True to enable triggers */ /* ** Bits of the sqlite4.flags field that are used by the ** sqlite4_test_control(SQLITE4_TESTCTRL_OPTIMIZATIONS,...) interface. ** These must be the low-order bits of the flags field. */ -#define SQLITE4_QueryFlattener 0x01 /* Disable query flattening */ -#define SQLITE4_ColumnCache 0x02 /* Disable the column cache */ -#define SQLITE4_IndexSort 0x04 /* Disable indexes for sorting */ -#define SQLITE4_IndexSearch 0x08 /* Disable indexes for searching */ -#define SQLITE4_IndexCover 0x10 /* Disable index covering table */ -#define SQLITE4_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ -#define SQLITE4_FactorOutConst 0x40 /* Disable factoring out constants */ -#define SQLITE4_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ -#define SQLITE4_DistinctOpt 0x80 /* DISTINCT using indexes */ -#define SQLITE4_OptMask 0xff /* Mask of all disablable opts */ +#define SQLITE4_QueryFlattener 0x01 /* Disable query flattening */ +#define SQLITE4_ColumnCache 0x02 /* Disable the column cache */ +#define SQLITE4_IndexSort 0x04 /* Disable indexes for sorting */ +#define SQLITE4_IndexSearch 0x08 /* Disable indexes for searching */ +#define SQLITE4_IndexCover 0x10 /* Disable index covering table */ +#define SQLITE4_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ +#define SQLITE4_FactorOutConst 0x40 /* Disable factoring out constants */ +#define SQLITE4_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ +#define SQLITE4_DistinctOpt 0x80 /* DISTINCT using indexes */ +#define SQLITE4_OptMask 0xff /* Mask of all disablable opts */ /* ** Possible values for the sqlite.magic field. ** The numbers are obtained at random and have no special meaning, other ** than being distinct from one another. */ -#define SQLITE4_MAGIC_OPEN 0xa029a697 /* Database is open */ -#define SQLITE4_MAGIC_CLOSED 0x9f3c2d33 /* Database is closed */ -#define SQLITE4_MAGIC_SICK 0x4b771290 /* Error and awaiting close */ -#define SQLITE4_MAGIC_BUSY 0xf03b7906 /* Database currently in use */ -#define SQLITE4_MAGIC_ERROR 0xb5357930 /* An SQLITE4_MISUSE error occurred */ +#define SQLITE4_MAGIC_OPEN 0x4d06c919 /* Database is open */ +#define SQLITE4_MAGIC_CLOSED 0x5f2246b4 /* Database is closed */ +#define SQLITE4_MAGIC_SICK 0xcaad9e61 /* Error and awaiting close */ +#define SQLITE4_MAGIC_BUSY 0xb07f8c8c /* Database currently in use */ +#define SQLITE4_MAGIC_ERROR 0x912e4c46 /* An SQLITE4_MISUSE error occurred */ /* ** This structure encapsulates a user-function destructor callback (as ** configured using create_function_v2()) and a reference counter. When ** create_function_v2() is called to create a function with a destructor, @@ -966,11 +957,11 @@ #define SQLITE4_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */ #define SQLITE4_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */ #define SQLITE4_FUNC_NEEDCOLL 0x08 /* sqlite4GetFuncCollSeq() might be called */ #define SQLITE4_FUNC_PRIVATE 0x10 /* Allowed for internal use only */ #define SQLITE4_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */ -#define SQLITE4_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */ +#define SQLITE4_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() func */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are ** used to create the initializers for the FuncDef structures. ** @@ -1124,13 +1115,13 @@ /* ** Additional bit values that can be ORed with an affinity without ** changing the affinity. */ -#define SQLITE4_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ -#define SQLITE4_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ -#define SQLITE4_NULLEQ 0x80 /* NULL=NULL */ +#define SQLITE4_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ +#define SQLITE4_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ +#define SQLITE4_NULLEQ 0x80 /* NULL=NULL */ /* ** An object of this type is created for each virtual table present in ** the database schema. ** @@ -1654,11 +1645,10 @@ #define EP_Agg 0x0002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x0004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x0008 /* Expression contains one or more errors */ #define EP_Distinct 0x0010 /* Aggregate function with DISTINCT keyword */ #define EP_VarSelect 0x0020 /* pSelect is correlated, not constant */ -#define EP_DblQuoted 0x0040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x0080 /* True for an infix function: LIKE, GLOB, etc */ #define EP_ExpCollate 0x0100 /* Collating sequence specified explicitly */ #define EP_FixedDest 0x0200 /* Result needed in a specific register */ #define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ @@ -2186,11 +2176,11 @@ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int iNewidxReg; /* First argument to OP_NewIdxid */ u8 nColCache; /* Number of entries in aColCache[] */ u8 iColCache; /* Next entry in aColCache[] to replace */ - ParseYColCache aColCache[SQLITE4_N_COLCACHE]; /* One for each column cache entry */ + ParseYColCache aColCache[SQLITE4_N_COLCACHE]; /* One per colcache entry */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ u8 isMultiWrite; /* True if statement may affect/insert multiple rows */ u8 mayAbort; /* True if statement may throw an ABORT exception */ int cookieGoto; /* Address of OP_Goto to cookie verifier subroutine */ @@ -2258,11 +2248,11 @@ /* ** Bitfield flags for P5 value in OP_Insert and OP_Delete */ #define OPFLAG_NCHANGE 0x01 /* Set to update db->nChange */ -#define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ +#define OPFLAG_PARTIALKEY 0x02 /* Not all values given to OP_MakeIdxKey */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_SEQCOUNT 0x10 /* Append sequence number to key */ #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ #define OPFLAG_APPENDBIAS 0x40 /* Bias inserts for appending */ @@ -2418,10 +2408,11 @@ int bCoreMutex; /* True to enable core mutexing */ int bFullMutex; /* True to enable full mutexing */ int mxStrlen; /* Maximum string length */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ + sqlite4_mm *pMM; /* Memory allocator for this environment */ sqlite4_mem_methods m; /* Low-level memory allocation interface */ sqlite4_mutex_methods mutex; /* Low-level mutex interface */ void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ @@ -2493,10 +2484,15 @@ if( (*(zIn++))>=0xc0 ){ \ while( (*zIn & 0xc0)==0x80 ){ zIn++; } \ } \ } +/* +** Default memory allocator +*/ +extern sqlite4_mm sqlite4MMSystem; + /* ** The SQLITE4_*_BKPT macros are substitutes for the error codes with ** the same name but without the _BKPT suffix. These macros invoke ** routines that report the line-number on which the error originated ** using sqlite4_log(). The routines also provide a convenient place @@ -2744,12 +2740,13 @@ Expr*,ExprList*,int,Expr*,Expr*); void sqlite4SelectDelete(sqlite4*, Select*); Table *sqlite4SrcListLookup(Parse*, SrcList*); int sqlite4IsReadOnly(Parse*, Table*, int); void sqlite4OpenTable(Parse*, int iCur, int iDb, Table*, int); -#if defined(SQLITE4_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE4_OMIT_SUBQUERY) -Expr *sqlite4LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *); +#if defined(SQLITE4_ENABLE_UPDATE_DELETE_LIMIT) \ + && !defined(SQLITE4_OMIT_SUBQUERY) +Expr *sqlite4LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); #endif void sqlite4DeleteFrom(Parse*, SrcList*, Expr*); void sqlite4Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite4WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite4WhereEnd(WhereInfo*); @@ -2925,12 +2922,15 @@ ** ** x = getVarint32( A, B ); ** x = putVarint32( A, B ); ** */ -#define getVarint32(A,B) (u8)((*(A)<(u8)0x80) ? ((B) = (u32)*(A)),1 : sqlite4GetVarint32((A), (u32 *)&(B))) -#define putVarint32(A,B) (u8)(((u32)(B)<(u32)0x80) ? (*(A) = (unsigned char)(B)),1 : sqlite4PutVarint32((A), (B))) +#define getVarint32(A,B) \ + (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite4GetVarint32((A),(u32 *)&(B))) +#define putVarint32(A,B) \ + (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1\ + :sqlite4PutVarint32((A),(B))) #define getVarint sqlite4GetVarint #define putVarint sqlite4PutVarint const char *sqlite4IndexAffinityStr(Vdbe *, Index *); @@ -2965,11 +2965,11 @@ u8 sqlite4GetBoolean(const char *z); const void *sqlite4ValueText(sqlite4_value*, u8); int sqlite4ValueBytes(sqlite4_value*, u8); void sqlite4ValueSetStr(sqlite4_value*, int, const void *,u8, - void(*)(void*)); + void(*)(void*,void*),void*); void sqlite4ValueFree(sqlite4_value*); sqlite4_value *sqlite4ValueNew(sqlite4 *); char *sqlite4Utf16to8(sqlite4 *, const void*, int, u8); #ifdef SQLITE4_ENABLE_STAT3 char *sqlite4Utf8to16(sqlite4 *, u8, char *, int, int *); Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -116,11 +116,10 @@ char *zProgress; /* The progress callback routine */ char *zAuth; /* The authorization callback routine */ int disableAuth; /* Disable the authorizer if it exists */ char *zNull; /* Text to substitute for an SQL NULL value */ SqlFunc *pFunc; /* List of SQL functions */ - Tcl_Obj *pUnlockNotify; /* Unlock notify script (if any) */ SqlCollate *pCollate; /* List of SQL collation functions */ int rc; /* Return code of most recent sqlite4_exec() */ Tcl_Obj *pCollateNeeded; /* Collation needed script */ SqlPreparedStmt *stmtList; /* List of prepared statements*/ SqlPreparedStmt *stmtLast; /* Last statement in the list */ @@ -314,37 +313,10 @@ Tcl_DStringFree(&str); Tcl_ResetResult(pDb->interp); } #endif -#if defined(SQLITE4_TEST) && defined(SQLITE4_ENABLE_UNLOCK_NOTIFY) -static void setTestUnlockNotifyVars(Tcl_Interp *interp, int iArg, int nArg){ - char zBuf[64]; - sprintf(zBuf, "%d", iArg); - Tcl_SetVar(interp, "sqlite_unlock_notify_arg", zBuf, TCL_GLOBAL_ONLY); - sprintf(zBuf, "%d", nArg); - Tcl_SetVar(interp, "sqlite_unlock_notify_argcount", zBuf, TCL_GLOBAL_ONLY); -} -#else -# define setTestUnlockNotifyVars(x,y,z) -#endif - -#ifdef SQLITE4_ENABLE_UNLOCK_NOTIFY -static void DbUnlockNotify(void **apArg, int nArg){ - int i; - for(i=0; iinterp, i, nArg); - assert( pDb->pUnlockNotify); - Tcl_EvalObjEx(pDb->interp, pDb->pUnlockNotify, flags); - Tcl_DecrRefCount(pDb->pUnlockNotify); - pDb->pUnlockNotify = 0; - } -} -#endif - static void tclCollateNeeded( void *pCtx, sqlite4 *db, int enc, const char *zName @@ -504,11 +476,11 @@ char c = zType[0]; if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ /* Only return a BLOB type if the Tcl variable is a bytearray and ** has no string representation. */ data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite4_result_blob(context, data, n, SQLITE4_TRANSIENT); + sqlite4_result_blob(context, data, n, SQLITE4_TRANSIENT, 0); }else if( c=='b' && strcmp(zType,"boolean")==0 ){ Tcl_GetIntFromObj(0, pVar, &n); sqlite4_result_int(context, n); }else if( c=='d' && strcmp(zType,"double")==0 ){ double r; @@ -519,11 +491,11 @@ Tcl_WideInt v; Tcl_GetWideIntFromObj(0, pVar, &v); sqlite4_result_int64(context, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite4_result_text(context, (char *)data, n, SQLITE4_TRANSIENT); + sqlite4_result_text(context, (char *)data, n, SQLITE4_TRANSIENT, 0); } } } #ifndef SQLITE4_OMIT_AUTHORIZATION @@ -846,11 +818,11 @@ (c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0) ){ /* Load a BLOB type if the Tcl variable is a bytearray and ** it has no string representation or the host ** parameter name begins with "@". */ data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite4_bind_blob(pStmt, i, data, n, SQLITE4_STATIC); + sqlite4_bind_blob(pStmt, i, data, n, SQLITE4_STATIC, 0); Tcl_IncrRefCount(pVar); pPreStmt->apParm[iParm++] = pVar; }else if( c=='b' && strcmp(zType,"boolean")==0 ){ Tcl_GetIntFromObj(interp, pVar, &n); sqlite4_bind_int(pStmt, i, n); @@ -863,11 +835,11 @@ Tcl_WideInt v; Tcl_GetWideIntFromObj(interp, pVar, &v); sqlite4_bind_int64(pStmt, i, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite4_bind_text(pStmt, i, (char *)data, n, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, i, (char *)data, n, SQLITE4_STATIC, 0); Tcl_IncrRefCount(pVar); pPreStmt->apParm[iParm++] = pVar; } }else{ sqlite4_bind_null(pStmt, i); @@ -1271,25 +1243,25 @@ static const char *DB_strs[] = { "authorizer", "cache", "changes", "close", "collate", "collation_needed", "complete", "copy", "enable_load_extension", "errorcode", "eval", "exists", - "function", "interrupt", "last_insert_rowid", + "function", "interrupt", "nullvalue", "onecolumn", "profile", "rekey", "status", "total_changes", - "trace", "transaction", "unlock_notify", + "trace", "transaction", "version", 0 }; enum DB_enum { DB_AUTHORIZER, DB_CACHE, DB_CHANGES, DB_CLOSE, DB_COLLATE, DB_COLLATION_NEEDED, DB_COMPLETE, DB_COPY, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_EVAL, DB_EXISTS, - DB_FUNCTION, DB_INTERRUPT, DB_LAST_INSERT_ROWID, + DB_FUNCTION, DB_INTERRUPT, DB_NULLVALUE, DB_ONECOLUMN, DB_PROFILE, DB_REKEY, DB_STATUS, DB_TOTAL_CHANGES, - DB_TRACE, DB_TRANSACTION, DB_UNLOCK_NOTIFY, + DB_TRACE, DB_TRANSACTION, DB_VERSION }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ if( objc<2 ){ @@ -1668,11 +1640,11 @@ if( (nNull>0 && strcmp(azCol[i], zNull)==0) || strlen30(azCol[i])==0 ){ sqlite4_bind_null(pStmt, i+1); }else{ - sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC); + sqlite4_bind_text(pStmt, i+1, azCol[i], -1, SQLITE4_STATIC, 0); } } sqlite4_step(pStmt); rc = sqlite4_reset(pStmt); free(zLine); @@ -1901,28 +1873,10 @@ } Tcl_SetObjResult(interp, dbTextToObj(pDb->zNull)); break; } - /* - ** $db last_insert_rowid - ** - ** Return an integer which is the ROWID for the most recent insert. - */ - case DB_LAST_INSERT_ROWID: { - Tcl_Obj *pResult; - Tcl_WideInt rowid; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 2, objv, ""); - return TCL_ERROR; - } - rowid = sqlite4_last_insert_rowid(pDb->db); - pResult = Tcl_GetObjResult(interp); - Tcl_SetWideIntObj(pResult, rowid); - break; - } - /* ** The DB_ONECOLUMN method is implemented together with DB_EXISTS. */ @@ -2134,46 +2088,10 @@ Tcl_NRAddCallback(interp, DbTransPostCmd, cd, 0, 0, 0); Tcl_NREvalObj(interp, pScript, 0); }else{ rc = DbTransPostCmd(&cd, interp, Tcl_EvalObjEx(interp, pScript, 0)); } - break; - } - - /* - ** $db unlock_notify ?script? - */ - case DB_UNLOCK_NOTIFY: { -#ifndef SQLITE4_ENABLE_UNLOCK_NOTIFY - Tcl_AppendResult(interp, "unlock_notify not available in this build", 0); - rc = TCL_ERROR; -#else - if( objc!=2 && objc!=3 ){ - Tcl_WrongNumArgs(interp, 2, objv, "?SCRIPT?"); - rc = TCL_ERROR; - }else{ - void (*xNotify)(void **, int) = 0; - void *pNotifyArg = 0; - - if( pDb->pUnlockNotify ){ - Tcl_DecrRefCount(pDb->pUnlockNotify); - pDb->pUnlockNotify = 0; - } - - if( objc==3 ){ - xNotify = DbUnlockNotify; - pNotifyArg = (void *)pDb; - pDb->pUnlockNotify = objv[2]; - Tcl_IncrRefCount(pDb->pUnlockNotify); - } - - if( sqlite4_unlock_notify(pDb->db, xNotify, pNotifyArg) ){ - Tcl_AppendResult(interp, sqlite4_errmsg(pDb->db), 0); - rc = TCL_ERROR; - } - } -#endif break; } /* $db version ** @@ -2801,11 +2719,11 @@ unsigned char digest[16]; char zBuf[33]; p = sqlite4_aggregate_context(context, sizeof(*p)); MD5Final(digest,p); MD5DigestToBase16(digest, zBuf); - sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT); + sqlite4_result_text(context, zBuf, -1, SQLITE4_TRANSIENT, 0); } int Md5_Register(sqlite4 *db){ int rc = sqlite4_create_function(db, "md5sum", -1, SQLITE4_UTF8, 0, 0, md5step, md5finalize); sqlite4_overload_function(db, "md5sum", -1); /* To exercise this API */ Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -225,11 +225,10 @@ } case '~': { *tokenType = TK_BITNOT; return 1; } - case '`': case '\'': case '"': { int delim = z[0]; testcase( delim=='`' ); testcase( delim=='\'' ); Index: src/utf.c ================================================================== --- src/utf.c +++ src/utf.c @@ -472,11 +472,11 @@ */ char *sqlite4Utf16to8(sqlite4 *db, const void *z, int nByte, u8 enc){ Mem m; memset(&m, 0, sizeof(m)); m.db = db; - sqlite4VdbeMemSetStr(&m, z, nByte, enc, SQLITE4_STATIC); + sqlite4VdbeMemSetStr(&m, z, nByte, enc, SQLITE4_STATIC, 0); sqlite4VdbeChangeEncoding(&m, SQLITE4_UTF8); if( db->mallocFailed ){ sqlite4VdbeMemRelease(&m); m.z = 0; } @@ -500,11 +500,11 @@ #ifdef SQLITE4_ENABLE_STAT3 char *sqlite4Utf8to16(sqlite4 *db, u8 enc, char *z, int n, int *pnOut){ Mem m; memset(&m, 0, sizeof(m)); m.db = db; - sqlite4VdbeMemSetStr(&m, z, n, SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4VdbeMemSetStr(&m, z, n, SQLITE4_UTF8, SQLITE4_STATIC, 0); if( sqlite4VdbeMemTranslate(&m, enc) ){ assert( db->mallocFailed ); return 0; } assert( m.z==m.zMalloc ); Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -130,13 +130,13 @@ char *z; va_list ap; va_start(ap, zFormat); z = sqlite4VMPrintf(db, zFormat, ap); va_end(ap); - sqlite4ValueSetStr(db->pErr, -1, z, SQLITE4_UTF8, SQLITE4_DYNAMIC); + sqlite4ValueSetStr(db->pErr, -1, z, SQLITE4_UTF8, SQLITE4_DYNAMIC, 0); }else{ - sqlite4ValueSetStr(db->pErr, 0, 0, SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4ValueSetStr(db->pErr, 0, 0, SQLITE4_UTF8, SQLITE4_STATIC, 0); } } } /* Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -351,13 +351,10 @@ if( z<32 || z>126 ) *zCsr++ = '.'; else *zCsr++ = z; } zCsr += sqlite4_snprintf(zCsr, 100, "]%s", encnames[pMem->enc]); - if( f & MEM_Zero ){ - zCsr += sqlite4_snprintf(zCsr, 100, "+%dz",pMem->u.nZero); - } *zCsr = '\0'; }else if( f & MEM_Str ){ int j, k; zBuf[0] = ' '; if( f & MEM_Dyn ){ @@ -527,11 +524,10 @@ Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ int iCompare = 0; /* Result of last OP_Compare operation */ int *aPermute = 0; /* Permutation of columns for OP_Compare */ - i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ int origPc; /* Program counter at start of opcode */ #endif /*** INSERT STACK UNION HERE ***/ @@ -796,11 +792,10 @@ VdbeFrame *pFrame = p->pFrame; p->pFrame = pFrame->pParent; p->nFrame--; sqlite4VdbeSetChanges(db, p->nChange); pc = sqlite4VdbeFrameRestore(pFrame); - lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ /* Instruction pc is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt ** instruction is set to OE_Ignore, then the sub-program is throwing ** an IGNORE exception. In this case jump to the address specified @@ -882,11 +877,12 @@ pOp->opcode = OP_String; pOp->p1 = sqlite4Strlen30(pOp->p4.z); #ifndef SQLITE4_OMIT_UTF16 if( encoding!=SQLITE4_UTF8 ){ - rc = sqlite4VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE4_UTF8, SQLITE4_STATIC); + rc = sqlite4VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE4_UTF8, + SQLITE4_STATIC, 0); if( rc==SQLITE4_TOOBIG ) goto too_big; if( SQLITE4_OK!=sqlite4VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->zMalloc==pOut->z ); assert( pOut->flags & MEM_Dyn ); pOut->zMalloc = 0; @@ -948,11 +944,11 @@ ** P4 points to a blob of data P1 bytes long. Store this ** blob in register P2. */ case OP_Blob: { /* out2-prerelease */ assert( pOp->p1 <= SQLITE4_MAX_LENGTH ); - sqlite4VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + sqlite4VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0, 0); pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -1143,11 +1139,10 @@ assert( pIn1!=pOut ); if( (pIn1->flags | pIn2->flags) & MEM_Null ){ sqlite4VdbeMemSetNull(pOut); break; } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; Stringify(pIn1, encoding); Stringify(pIn2, encoding); nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE4_LIMIT_LENGTH] ){ goto too_big; @@ -1384,13 +1379,11 @@ assert( pOp>aOp ); assert( pOp[-1].p4type==P4_COLLSEQ ); assert( pOp[-1].opcode==OP_CollSeq ); ctx.pColl = pOp[-1].p4.pColl; } - db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* If any auxiliary data functions have been called by this user function, ** immediately call the destructor for any non-static values. */ if( ctx.pVdbeFunc ){ @@ -1585,13 +1578,12 @@ memAboutToChange(p, pIn1); if( pIn1->flags & MEM_Null ) break; assert( MEM_Str==(MEM_Blob>>3) ); pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; applyAffinity(pIn1, SQLITE4_AFF_TEXT, encoding); - rc = ExpandBlob(pIn1); assert( pIn1->flags & MEM_Str || db->mallocFailed ); - pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob); UPDATE_MAX_BLOBSIZE(pIn1); break; } /* Opcode: ToBlob P1 * * * * @@ -2199,13 +2191,14 @@ ** ** If the OPFLAG_SEQCOUNT bit of P5 is set, then a sequence number ** (unique within the cursor) is appended to the record. The sole purpose ** of this is to ensure that the key blob is unique within the cursors table. ** -** If the OPFLAG_LASTROWID bit of P5 is set and the value of the first and -** only field of the key is an integer, then set the lastRowid field to the -** value of that integer. +** If the OPFLAG_PARTIALKEY bit of P5 is set, that means the value supplied +** for N is not the true number of values in the key, only the number that +** need to be encoded for this operation. This effects the encoding of +** final BLOBs. */ case OP_MakeIdxKey: { VdbeCursor *pC; KeyInfo *pKeyInfo; Mem *pData0; /* First in array of input registers */ @@ -2234,32 +2227,31 @@ aSeq[sizeof(aSeq)-nSeq] = (u8)(iSeq & 0x007F); iSeq = iSeq >> 7; }while( iSeq ); aSeq[sizeof(aSeq)-nSeq] |= 0x80; } - if( (pOp->p5 & OPFLAG_LASTROWID)!=0 && (pData0->flags & MEM_Int)!=0 ){ - lastRowid = pData0->u.i; - } memAboutToChange(p, pOut); nField = pKeyInfo->nField; if( pOp->p4type==P4_INT32 && pOp->p4.i ){ nField = pOp->p4.i; assert( nField<=pKeyInfo->nField ); } rc = sqlite4VdbeEncodeKey( - db, pData0, nField, pC->iRoot, pKeyInfo, &aRec, &nRec, nSeq + db, pData0, nField, nField+(pOp->p5 & OPFLAG_PARTIALKEY), + pC->iRoot, pKeyInfo, &aRec, &nRec, nSeq ); if( rc ){ sqlite4DbFree(db, aRec); }else{ if( nSeq ){ memcpy(&aRec[nRec], &aSeq[sizeof(aSeq)-nSeq], nSeq); } - rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec+nSeq, 0, SQLITE4_DYNAMIC); + rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec+nSeq, 0, + SQLITE4_DYNAMIC, 0); REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); } break; @@ -2331,25 +2323,24 @@ for(pMem=pData0; pMem<=pLast; pMem++){ assert( memIsValid(pMem) ); if( zAffinity ){ applyAffinity(pMem, *(zAffinity++), encoding); } - if( pMem->flags&MEM_Zero ){ - (void)ExpandBlob(pMem); - } } /* Compute the key (if this is a MakeKey opcode) */ if( pC ){ aRec = 0; rc = sqlite4VdbeEncodeKey(db, - pData0, pC->pKeyInfo->nField, pC->iRoot, pC->pKeyInfo, &aRec, &nRec, 0 + pData0, pC->pKeyInfo->nField, pC->pKeyInfo->nField, + pC->iRoot, pC->pKeyInfo, &aRec, &nRec, 0 ); if( rc ){ sqlite4DbFree(db, aRec); }else{ - rc = sqlite4VdbeMemSetStr(pKeyOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC); + rc = sqlite4VdbeMemSetStr(pKeyOut, (char *)aRec, nRec, 0, + SQLITE4_DYNAMIC, 0); REGISTER_TRACE(keyReg, pKeyOut); UPDATE_MAX_BLOBSIZE(pKeyOut); } } @@ -2361,11 +2352,11 @@ aRec = 0; rc = sqlite4VdbeEncodeData(db, pData0, nField, &aRec, &nRec); if( rc ){ sqlite4DbFree(db, aRec); }else{ - rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC); + rc = sqlite4VdbeMemSetStr(pOut, (char *)aRec, nRec, 0, SQLITE4_DYNAMIC,0); REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); } } break; @@ -2580,11 +2571,11 @@ break; } /* Opcode: VerifyCookie P1 P2 P3 * * ** -** cHECK THe value of global database parameter number 0 (the +** Check the value of global database parameter number 0 (the ** schema version) and make sure it is equal to P2 and that the ** generation counter on the local schema parse equals P3. ** ** P1 is the database number which is 0 for the main database file ** and 1 for the file holding temporary tables and some higher number @@ -2998,11 +2989,12 @@ ** free byte at the end of the database key (see below). */ op = pOp->opcode; nField = pOp->p4.i; pIn3 = &aMem[pOp->p3]; rc = sqlite4VdbeEncodeKey( - db, pIn3, nField, pC->iRoot, pC->pKeyInfo, &aProbe, &nProbe, 1 + db, pIn3, nField, nField+(pOp->p5 & OPFLAG_PARTIALKEY), + pC->iRoot, pC->pKeyInfo, &aProbe, &nProbe, 1 ); /* Opcode search-dir increment-key ** -------------------------------------- ** SeekLt -1 no @@ -3059,11 +3051,11 @@ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->isTable ); pKVCur = pC->pKVCur; - rc = sqlite4VdbeEncodeKey(db, aMem+pOp->p2, 1, pC->iRoot, 0, + rc = sqlite4VdbeEncodeKey(db, aMem+pOp->p2, 1, 1, pC->iRoot, 0, &aKey, &nKey, 0); if( rc==SQLITE4_OK ){ rc = sqlite4KVCursorSeek(pKVCur, aKey, nKey, 0); if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_CORRUPT_BKPT; } @@ -3136,12 +3128,14 @@ assert( pC!=0 ); pIn3 = &aMem[pOp->p3]; assert( pC->pKVCur!=0 ); assert( pC->isTable==0 || pOp->opcode==OP_NotExists ); if( pOp->p4.i>0 ){ - rc = sqlite4VdbeEncodeKey(db, pIn3, pOp->p4.i, pC->iRoot, - pC->pKeyInfo, &pProbe, &nProbe, 0); + rc = sqlite4VdbeEncodeKey( + db, pIn3, pOp->p4.i, pOp->p4.i + (pOp->p5 & OPFLAG_PARTIALKEY), + pC->iRoot, pC->pKeyInfo, &pProbe, &nProbe, 0 + ); pFree = pProbe; }else{ pProbe = (KVByteArray*)pIn3->z; nProbe = pIn3->n; pFree = 0; @@ -3436,11 +3430,10 @@ /* assert( pOp->opcode==OP_InsertInt ); */ iKey = pOp->p3; } if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey; if( pData->flags & MEM_Null ){ pData->z = 0; pData->n = 0; }else{ assert( pData->flags & (MEM_Blob|MEM_Str) ); @@ -3519,30 +3512,20 @@ if( (pIn3->flags & MEM_Blob) && pIn3->n==nKey && 0==memcmp(pIn3->z, aKey, nKey) ){ pc = pOp->p2-1; }else{ - sqlite4VdbeMemSetStr(pIn3, (const char*)aKey, nKey, 0, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pIn3, (const char*)aKey, nKey, 0, SQLITE4_TRANSIENT,0); } break; }; /* Opcode: SorterData P1 P2 * * * ** ** Write into register P2 the current sorter data for sorter cursor P1. */ -case OP_SorterData: { - VdbeCursor *pC; - pOut = &aMem[pOp->p2]; - pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); - pOp->opcode = OP_RowData; - pc--; - break; -} - /* Opcode: RowData P1 P2 * * * ** ** Write into register P2 the complete row data for cursor P1. ** There is no interpretation of the data. ** It is just copied onto the P2 register exactly as @@ -3559,10 +3542,11 @@ ** it is found in the database file. ** ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. */ +case OP_SorterData: case OP_RowKey: case OP_RowData: { VdbeCursor *pC; KVCursor *pCrsr; const KVByteArray *pData; @@ -3586,11 +3570,11 @@ rc = sqlite4KVCursorData(pCrsr, 0, -1, &pData, &nData); } if( rc==SQLITE4_OK && nData>db->aLimit[SQLITE4_LIMIT_LENGTH] ){ goto too_big; } - sqlite4VdbeMemSetStr(pOut, (const char*)pData, nData, 0, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pOut, (const char*)pData, nData, 0, SQLITE4_TRANSIENT,0); pOut->enc = SQLITE4_UTF8; /* In case the blob is ever cast to text */ UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -3648,11 +3632,10 @@ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; - pC->rowidIsValid = 0; break; } /* Opcode: Last P1 P2 * * * ** @@ -3789,11 +3772,10 @@ #endif }else if( rc==SQLITE4_NOTFOUND ){ pC->nullRow = 1; rc = SQLITE4_OK; } - pC->rowidIsValid = 0; break; } /* Opcode: SorterInsert P1 P2 P3 @@ -4107,11 +4089,12 @@ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p3]; if( (pIn1->flags & MEM_RowSet) && (aKey = sqlite4RowSetRead(pIn1->u.pRowSet, &nKey)) ){ - rc = sqlite4VdbeMemSetStr(pOut, (char const *)aKey, nKey, 0, SQLITE4_TRANSIENT); + rc = sqlite4VdbeMemSetStr(pOut, (char const *)aKey, nKey, 0, + SQLITE4_TRANSIENT, 0); sqlite4RowSetNext(pIn1->u.pRowSet); }else{ /* The RowSet is empty */ sqlite4VdbeMemSetNull(pIn1); pc = pOp->p2 - 1; @@ -4203,11 +4186,10 @@ assert( pc==pFrame->pc ); } p->nFrame++; pFrame->pParent = p->pFrame; - pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; p->nChange = 0; p->pFrame = pFrame; p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; p->nMem = pFrame->nChildMem; @@ -4828,14 +4810,10 @@ } db->vtabOnConflict = pOp->p5; rc = pModule->xUpdate(pVtab, nArg, apArg, &rowid); db->vtabOnConflict = vtabOnConflict; importVtabErrMsg(p, pVtab); - if( rc==SQLITE4_OK && pOp->p1 ){ - assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); - db->lastRowid = lastRowid = rowid; - } if( rc==SQLITE4_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ if( pOp->p5==OE_Ignore ){ rc = SQLITE4_OK; }else{ p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); @@ -5084,11 +5062,10 @@ /* This is the only way out of this procedure. We have to ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: - db->lastRowid = lastRowid; return rc; /* Jump to here if a string or blob larger than SQLITE4_MAX_LENGTH ** is encountered. */ Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -202,11 +202,11 @@ #endif void sqlite4VdbeResetStepResult(Vdbe*); void sqlite4VdbeRewind(Vdbe*); int sqlite4VdbeReset(Vdbe*); void sqlite4VdbeSetNumCols(Vdbe*,int); -int sqlite4VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); +int sqlite4VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*,void*)); void sqlite4VdbeCountChanges(Vdbe*); sqlite4 *sqlite4VdbeDb(Vdbe*); void sqlite4VdbeSetSql(Vdbe*, const char *z, int n); void sqlite4VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite4VdbeTakeOpArray(Vdbe*, int*, int*); Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -56,21 +56,19 @@ int iDb; /* Index of cursor database in db->aDb[] (or -1) */ int iRoot; /* Root page of the table */ int pseudoTableReg; /* Register holding pseudotable content. */ int nField; /* Number of fields in the header */ Bool zeroed; /* True if zeroed out and ready for reuse */ - Bool rowidIsValid; /* True if lastRowid is valid */ Bool atFirst; /* True if pointing to first entry */ Bool nullRow; /* True if pointing to a row with no data */ Bool isTable; /* True if a table requiring integer keys */ Bool isIndex; /* True if an index containing keys only - no data */ Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ sqlite4_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ const sqlite4_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred move-to */ - i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ Fts5Cursor *pFts; /* Fts5 cursor object (or NULL) */ /* Result of last sqlite4-Moveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ @@ -111,11 +109,10 @@ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ u16 nCursor; /* Number of entries in apCsr */ void *token; /* Copy of SubProgram.token */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - i64 lastRowid; /* Last insert rowid (sqlite4.lastRowid) */ int nChange; /* Statement changes (Vdbe.nChanges) */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) @@ -134,11 +131,10 @@ sqlite4 *db; /* The associated database connection */ char *z; /* String or BLOB value */ double r; /* Real value */ union { i64 i; /* Integer value used when MEM_Int is set in flags */ - int nZero; /* Used when bit MEM_Zero is set in flags */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; int n; /* Number of characters in string value, excluding '\0' */ @@ -147,11 +143,12 @@ u8 enc; /* SQLITE4_UTF8, SQLITE4_UTF16BE, SQLITE4_UTF16LE */ #ifdef SQLITE4_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ #endif - void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ + void (*xDel)(void*,void*); /* Function to delete Mem.z */ + void *pDelArg; /* First argument to xDel() */ char *zMalloc; /* Dynamic buffer allocated by sqlite4_malloc() */ }; /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. @@ -184,17 +181,16 @@ #define MEM_Term 0x0200 /* String rep is nul terminated */ #define MEM_Dyn 0x0400 /* Need to call sqliteFree() on Mem.z */ #define MEM_Static 0x0800 /* Mem.z points to a static string */ #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ -#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ /* ** Clear any existing type flags from a Mem and replace them with f */ #define MemSetTypeFlag(p, f) \ - ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) + ((p)->flags = ((p)->flags&~(MEM_TypeMask))|f) /* ** Return true if a memory cell is not marked as invalid. This macro ** is for use inside assert() statements only. */ @@ -215,11 +211,12 @@ struct VdbeFunc { FuncDef *pFunc; /* The definition of the function */ int nAux; /* Number of entries allocated for apAux[] */ struct AuxData { void *pAux; /* Aux data for the i-th argument */ - void (*xDelete)(void *); /* Destructor for the aux data */ + void (*xDelete)(void*,void*); /* Destructor for the aux data */ + void *pDeleteArg; /* First argument to xDelete */ } apAux[1]; /* One slot for each function argument */ }; /* ** The "context" argument for a installable function. A pointer to an @@ -377,15 +374,16 @@ ); int sqlite4VdbeEncodeKey( sqlite4 *db, /* The database connection */ Mem *aIn, /* Values to be encoded */ int nIn, /* Number of entries in aIn[] */ + int nInTotal, /* Number of values in complete key */ int iTabno, /* The table this key applies to */ KeyInfo *pKeyInfo, /* Collating sequence information */ u8 **pzOut, /* Write the resulting key here */ int *pnOut, /* Number of bytes in the key */ - int bIncr /* Make the key "incrementable" */ + int nExtra /* Append extra bytes on end of key */ ); int sqlite4VdbeEncodeIntKey(u8 *aBuf,sqlite4_int64 v); int sqlite4VdbeDecodeIntKey(const KVByteArray*, KVSize, sqlite4_int64*); int sqlite4VdbeShortKey(const u8 *, int, int); int sqlite4MemCompare(const Mem*, const Mem*, const CollSeq*); @@ -396,19 +394,19 @@ int sqlite4VdbeMemTooBig(Mem*); int sqlite4VdbeMemCopy(Mem*, const Mem*); void sqlite4VdbeMemShallowCopy(Mem*, const Mem*, int); void sqlite4VdbeMemMove(Mem*, Mem*); int sqlite4VdbeMemNulTerminate(Mem*); -int sqlite4VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); +int sqlite4VdbeMemSetStr(Mem*, const char*, int, u8, + void(*)(void*,void*),void*); void sqlite4VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE4_OMIT_FLOATING_POINT # define sqlite4VdbeMemSetDouble sqlite4VdbeMemSetInt64 #else void sqlite4VdbeMemSetDouble(Mem*, double); #endif void sqlite4VdbeMemSetNull(Mem*); -void sqlite4VdbeMemSetZeroBlob(Mem*,int); int sqlite4VdbeMemMakeWriteable(Mem*); int sqlite4VdbeMemStringify(Mem*, int); i64 sqlite4VdbeIntValue(Mem*); int sqlite4VdbeMemIntegerify(Mem*); double sqlite4VdbeRealValue(Mem*); @@ -453,9 +451,6 @@ void sqlite4VdbeMemPrettyPrint(Mem *pMem, char *zBuf); #endif int sqlite4VdbeMemHandleBom(Mem *pMem); -#define sqlite4VdbeMemExpandBlob(x) SQLITE4_OK -#define ExpandBlob(P) SQLITE4_OK - #endif /* !defined(_VDBEINT_H_) */ Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -14,25 +14,10 @@ ** VDBE. */ #include "sqliteInt.h" #include "vdbeInt.h" -#ifndef SQLITE4_OMIT_DEPRECATED -/* -** Return TRUE (non-zero) of the statement supplied as an argument needs -** to be recompiled. A statement needs to be recompiled whenever the -** execution environment changes in a way that would alter the program -** that sqlite4_prepare() generates. For example, if new functions or -** collating sequences are registered or if an authorizer function is -** added or changed. -*/ -int sqlite4_expired(sqlite4_stmt *pStmt){ - Vdbe *p = (Vdbe*)pStmt; - return p==0 || p->expired; -} -#endif - /* ** Check on a Vdbe to make sure it has not been finalized. Log ** an error and return true if it has been finalized (or is otherwise ** invalid). Return false if it is ok. */ @@ -137,11 +122,10 @@ ** structure. */ const void *sqlite4_value_blob(sqlite4_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ - (void)sqlite4VdbeMemExpandBlob(p); p->flags &= ~MEM_Str; p->flags |= MEM_Blob; return p->n ? p->z : 0; }else{ return sqlite4_value_text(pVal); @@ -187,49 +171,51 @@ ** The setStrOrError() funtion calls sqlite4VdbeMemSetStr() to store the ** result as a string or blob but if the string or blob is too large, it ** then sets the error code to SQLITE4_TOOBIG */ static void setResultStrOrError( - sqlite4_context *pCtx, /* Function context */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*) /* Destructor function */ + sqlite4_context *pCtx, /* Function context */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*,void*), /* Destructor function */ + void *pDelArg /* First argument to xDel() */ ){ if( xDel==SQLITE4_DYNAMIC ){ assert( sqlite4MemdebugHasType(z, MEMTYPE_HEAP) ); assert( sqlite4MemdebugNoType(z, ~MEMTYPE_HEAP) ); sqlite4MemdebugSetType((char*)z, MEMTYPE_DB | MEMTYPE_HEAP); } - if( sqlite4VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE4_TOOBIG ){ + if( sqlite4VdbeMemSetStr(&pCtx->s, z, n, enc, xDel,pDelArg)==SQLITE4_TOOBIG ){ sqlite4_result_error_toobig(pCtx); } } void sqlite4_result_blob( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( n>=0 ); assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, 0, xDel); + setResultStrOrError(pCtx, z, n, 0, xDel, pDelArg); } void sqlite4_result_double(sqlite4_context *pCtx, double rVal){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemSetDouble(&pCtx->s, rVal); } void sqlite4_result_error(sqlite4_context *pCtx, const char *z, int n){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_ERROR; - sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); } #ifndef SQLITE4_OMIT_UTF16 void sqlite4_result_error16(sqlite4_context *pCtx, const void *z, int n){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_ERROR; - sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF16NATIVE, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(&pCtx->s, z, n, SQLITE4_UTF16NATIVE,SQLITE4_TRANSIENT,0); } #endif void sqlite4_result_int(sqlite4_context *pCtx, int iVal){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemSetInt64(&pCtx->s, (i64)iVal); @@ -244,66 +230,66 @@ } void sqlite4_result_text( sqlite4_context *pCtx, const char *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF8, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF8, xDel, pDelArg); } #ifndef SQLITE4_OMIT_UTF16 void sqlite4_result_text16( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16NATIVE, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16NATIVE, xDel, pDelArg); } void sqlite4_result_text16be( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16BE, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16BE, xDel, pDelArg); } void sqlite4_result_text16le( sqlite4_context *pCtx, const void *z, int n, - void (*xDel)(void *) + void (*xDel)(void*,void*), + void *pDelArg ){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE4_UTF16LE, xDel); + setResultStrOrError(pCtx, z, n, SQLITE4_UTF16LE, xDel, pDelArg); } #endif /* SQLITE4_OMIT_UTF16 */ void sqlite4_result_value(sqlite4_context *pCtx, sqlite4_value *pValue){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); sqlite4VdbeMemCopy(&pCtx->s, pValue); } -void sqlite4_result_zeroblob(sqlite4_context *pCtx, int n){ - assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); - sqlite4VdbeMemSetZeroBlob(&pCtx->s, n); -} void sqlite4_result_error_code(sqlite4_context *pCtx, int errCode){ pCtx->isError = errCode; if( pCtx->s.flags & MEM_Null ){ sqlite4VdbeMemSetStr(&pCtx->s, sqlite4ErrStr(errCode), -1, - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); } } /* Force an SQLITE4_TOOBIG error. */ void sqlite4_result_error_toobig(sqlite4_context *pCtx){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); pCtx->isError = SQLITE4_TOOBIG; sqlite4VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); } /* An SQLITE4_NOMEM error. */ void sqlite4_result_error_nomem(sqlite4_context *pCtx){ assert( sqlite4_mutex_held(pCtx->s.db->mutex) ); @@ -589,11 +575,12 @@ */ void sqlite4_set_auxdata( sqlite4_context *pCtx, int iArg, void *pAux, - void (*xDelete)(void*) + void (*xDelete)(void*,void*), + void *pDeleteArg ){ struct AuxData *pAuxData; VdbeFunc *pVdbeFunc; if( iArg<0 ) goto failed; @@ -612,37 +599,22 @@ pVdbeFunc->pFunc = pCtx->pFunc; } pAuxData = &pVdbeFunc->apAux[iArg]; if( pAuxData->pAux && pAuxData->xDelete ){ - pAuxData->xDelete(pAuxData->pAux); + pAuxData->xDelete(pAuxData->pDeleteArg, pAuxData->pAux); } pAuxData->pAux = pAux; pAuxData->xDelete = xDelete; + pAuxData->pDeleteArg = pDeleteArg; return; failed: if( xDelete ){ - xDelete(pAux); - } -} - -#ifndef SQLITE4_OMIT_DEPRECATED -/* -** Return the number of times the Step function of a aggregate has been -** called. -** -** This function is deprecated. Do not use it for new code. It is -** provide only to avoid breaking legacy code. New aggregate function -** implementations should keep their own counts within their aggregate -** context. -*/ -int sqlite4_aggregate_count(sqlite4_context *p){ - assert( p && p->pMem && p->pFunc && p->pFunc->xStep ); - return p->pMem->n; -} -#endif + xDelete(pDeleteArg, pAux); + } +} /* ** Return the number of columns in the result set for the statement pStmt. */ int sqlite4_column_count(sqlite4_stmt *pStmt){ @@ -743,15 +715,10 @@ ** in the result set. */ const void *sqlite4_column_blob(sqlite4_stmt *pStmt, int i){ const void *val; val = sqlite4_value_blob( columnMem(pStmt,i) ); - /* Even though there is no encoding conversion, value_blob() might - ** need to call malloc() to expand the result of a zeroblob() - ** expression. - */ - columnMallocFailure(pStmt); return val; } int sqlite4_column_bytes(sqlite4_stmt *pStmt, int i){ int val = sqlite4_value_bytes( columnMem(pStmt,i) ); columnMallocFailure(pStmt); @@ -1006,35 +973,36 @@ /* ** Bind a text or BLOB value. */ static int bindText( - sqlite4_stmt *pStmt, /* The statement to bind against */ - int i, /* Index of the parameter to bind */ - const void *zData, /* Pointer to the data to be bound */ - int nData, /* Number of bytes of data to be bound */ - void (*xDel)(void*), /* Destructor for the data */ - u8 encoding /* Encoding for the data */ + sqlite4_stmt *pStmt, /* The statement to bind against */ + int i, /* Index of the parameter to bind */ + const void *zData, /* Pointer to the data to be bound */ + int nData, /* Number of bytes of data to be bound */ + void (*xDel)(void*,void*), /* Destructor for the data */ + void *pDelArg, /* First argument to xDel() */ + u8 encoding /* Encoding for the data */ ){ Vdbe *p = (Vdbe *)pStmt; Mem *pVar; int rc; rc = vdbeUnbind(p, i); if( rc==SQLITE4_OK ){ if( zData!=0 ){ pVar = &p->aVar[i-1]; - rc = sqlite4VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + rc = sqlite4VdbeMemSetStr(pVar, zData, nData, encoding, xDel, pDelArg); if( rc==SQLITE4_OK && encoding!=0 ){ rc = sqlite4VdbeChangeEncoding(pVar, ENC(p->db)); } sqlite4Error(p->db, rc, 0); rc = sqlite4ApiExit(p->db, rc); } sqlite4_mutex_leave(p->db->mutex); }else if( xDel!=SQLITE4_STATIC && xDel!=SQLITE4_TRANSIENT ){ - xDel((void*)zData); + xDel(pDelArg, (void*)zData); } return rc; } @@ -1044,13 +1012,14 @@ int sqlite4_bind_blob( sqlite4_stmt *pStmt, int i, const void *zData, int nData, - void (*xDel)(void*) + void (*xDel)(void*,void*), + void *pDelArg ){ - return bindText(pStmt, i, zData, nData, xDel, 0); + return bindText(pStmt, i, zData, nData, xDel, pDelArg, 0); } int sqlite4_bind_double(sqlite4_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; rc = vdbeUnbind(p, i); @@ -1085,23 +1054,25 @@ int sqlite4_bind_text( sqlite4_stmt *pStmt, int i, const char *zData, int nData, - void (*xDel)(void*) + void (*xDel)(void*,void*), + void *pDelArg ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE4_UTF8); + return bindText(pStmt, i, zData, nData, xDel, pDelArg, SQLITE4_UTF8); } #ifndef SQLITE4_OMIT_UTF16 int sqlite4_bind_text16( sqlite4_stmt *pStmt, int i, const void *zData, int nData, - void (*xDel)(void*) + void (*xDel)(void*,void*), + void *pDelArg ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE4_UTF16NATIVE); + return bindText(pStmt, i, zData, nData, xDel, pDelArg, SQLITE4_UTF16NATIVE); } #endif /* SQLITE4_OMIT_UTF16 */ int sqlite4_bind_value(sqlite4_stmt *pStmt, int i, const sqlite4_value *pValue){ int rc; switch( pValue->type ){ @@ -1112,37 +1083,24 @@ case SQLITE4_FLOAT: { rc = sqlite4_bind_double(pStmt, i, pValue->r); break; } case SQLITE4_BLOB: { - if( pValue->flags & MEM_Zero ){ - rc = sqlite4_bind_zeroblob(pStmt, i, pValue->u.nZero); - }else{ - rc = sqlite4_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE4_TRANSIENT); - } + rc = sqlite4_bind_blob(pStmt, i, pValue->z, pValue->n, + SQLITE4_TRANSIENT, 0); break; } case SQLITE4_TEXT: { - rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE4_TRANSIENT, + rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE4_TRANSIENT, 0, pValue->enc); break; } default: { rc = sqlite4_bind_null(pStmt, i); break; } } - return rc; -} -int sqlite4_bind_zeroblob(sqlite4_stmt *pStmt, int i, int n){ - int rc; - Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); - if( rc==SQLITE4_OK ){ - sqlite4VdbeMemSetZeroBlob(&p->aVar[i-1], n); - sqlite4_mutex_leave(p->db->mutex); - } return rc; } /* ** Return the number of wildcards that can be potentially bound to. @@ -1206,39 +1164,10 @@ } sqlite4_mutex_leave(pTo->db->mutex); return SQLITE4_OK; } -#ifndef SQLITE4_OMIT_DEPRECATED -/* -** Deprecated external interface. Internal/core SQLite code -** should call sqlite4TransferBindings. -** -** Is is misuse to call this routine with statements from different -** database connections. But as this is a deprecated interface, we -** will not bother to check for that condition. -** -** If the two statements contain a different number of bindings, then -** an SQLITE4_ERROR is returned. Nothing else can go wrong, so otherwise -** SQLITE4_OK is returned. -*/ -int sqlite4_transfer_bindings(sqlite4_stmt *pFromStmt, sqlite4_stmt *pToStmt){ - Vdbe *pFrom = (Vdbe*)pFromStmt; - Vdbe *pTo = (Vdbe*)pToStmt; - if( pFrom->nVar!=pTo->nVar ){ - return SQLITE4_ERROR; - } - if( pTo->expmask ){ - pTo->expired = 1; - } - if( pFrom->expmask ){ - pFrom->expired = 1; - } - return sqlite4TransferBindings(pFromStmt, pToStmt); -} -#endif - /* ** Return the sqlite4* database handle to which the prepared statement given ** in the argument belongs. This is the same database handle that was ** the first argument to the sqlite4_prepare() that was used to create ** the statement in the first place. Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -16,21 +16,10 @@ */ #include "sqliteInt.h" #include "vdbeInt.h" - -/* -** When debugging the code generator in a symbolic debugger, one can -** set the sqlite4VdbeAddopTrace to 1 and all opcodes will be printed -** as they are added to the instruction stream. -*/ -#ifdef SQLITE4_DEBUG -int sqlite4VdbeAddopTrace = 0; -#endif - - /* ** Create a new virtual database engine. */ Vdbe *sqlite4VdbeCreate(sqlite4 *db){ Vdbe *p; @@ -150,11 +139,13 @@ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; #ifdef SQLITE4_DEBUG pOp->zComment = 0; - if( sqlite4VdbeAddopTrace ) sqlite4VdbePrintOp(0, i, &p->aOp[i]); + if( p->db->flags & SQLITE4_VdbeAddopTrace ){ + sqlite4VdbePrintOp(0, i, &p->aOp[i]); + } #endif #ifdef VDBE_PROFILE pOp->cycles = 0; pOp->cnt = 0; #endif @@ -503,11 +494,11 @@ pOut->p4type = P4_NOTUSED; pOut->p4.p = 0; pOut->p5 = 0; #ifdef SQLITE4_DEBUG pOut->zComment = 0; - if( sqlite4VdbeAddopTrace ){ + if( p->db->flags & SQLITE4_VdbeAddopTrace ){ sqlite4VdbePrintOp(0, i+addr, &p->aOp[i+addr]); } #endif } p->nOp += nOp; @@ -1193,11 +1184,11 @@ return SQLITE4_ERROR; } pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; z = displayP4(pOp, pMem->z, 32); if( z!=pMem->z ){ - sqlite4VdbeMemSetStr(pMem, z, -1, SQLITE4_UTF8, 0); + sqlite4VdbeMemSetStr(pMem, z, -1, SQLITE4_UTF8, 0, 0); }else{ assert( pMem->z!=0 ); pMem->n = sqlite4Strlen30(pMem->z); pMem->enc = SQLITE4_UTF8; } @@ -1531,11 +1522,10 @@ v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; v->nMem = pFrame->nMem; v->apCsr = pFrame->apCsr; v->nCursor = pFrame->nCursor; - v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; return pFrame->pc; } /* @@ -1636,23 +1626,25 @@ int sqlite4VdbeSetColName( Vdbe *p, /* Vdbe being configured */ int idx, /* Index of column zName applies to */ int var, /* One of the COLNAME_* constants */ const char *zName, /* Pointer to buffer containing name */ - void (*xDel)(void*) /* Memory management strategy for zName */ + void (*xDel)(void*,void*) /* Memory management strategy for zName */ ){ int rc; Mem *pColName; assert( idxnResColumn ); assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE4_DYNAMIC ); return SQLITE4_NOMEM; } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResColumn]); - rc = sqlite4VdbeMemSetStr(pColName, zName, -1, SQLITE4_UTF8, xDel); + rc = sqlite4VdbeMemSetStr(pColName, zName, -1, SQLITE4_UTF8, xDel, 0); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; } /* @@ -1974,11 +1966,12 @@ sqlite4 *db = p->db; int rc = p->rc; if( p->zErrMsg ){ u8 mallocFailed = db->mallocFailed; sqlite4BeginBenignMalloc(db->pEnv); - sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, + SQLITE4_TRANSIENT, 0); sqlite4EndBenignMalloc(db->pEnv); db->mallocFailed = mallocFailed; db->errCode = rc; }else{ sqlite4Error(db, rc, 0); @@ -2021,11 +2014,12 @@ /* The expired flag was set on the VDBE before the first call ** to sqlite4_step(). For consistency (since sqlite4_step() was ** called), set the database error in this case as well. */ sqlite4Error(db, p->rc, 0); - sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE4_UTF8, + SQLITE4_TRANSIENT, 0); sqlite4DbFree(db, p->zErrMsg); p->zErrMsg = 0; } /* Reclaim all memory used by the VDBE @@ -2083,11 +2077,11 @@ int i; for(i=0; inAux; i++){ struct AuxData *pAux = &pVdbeFunc->apAux[i]; if( (i>31 || !(mask&(((u32)1)<pAux ){ if( pAux->xDelete ){ - pAux->xDelete(pAux->pAux); + pAux->xDelete(pAux->pDeleteArg, pAux->pAux); } pAux->pAux = 0; } } } @@ -2221,13 +2215,10 @@ if( flags&MEM_Real ){ return 7; } assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); n = pMem->n; - if( flags & MEM_Zero ){ - n += pMem->u.nZero; - } assert( n>=0 ); return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } /* @@ -2337,23 +2328,14 @@ return len; } /* String or blob */ if( serial_type>=12 ){ - assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) - == (int)sqlite4VdbeSerialTypeLen(serial_type) ); + assert( pMem->n == (int)sqlite4VdbeSerialTypeLen(serial_type) ); assert( pMem->n<=nBuf ); len = pMem->n; memcpy(buf, pMem->z, len); - if( pMem->flags & MEM_Zero ){ - len += pMem->u.nZero; - assert( nBuf>=0 ); - if( len > (u32)nBuf ){ - len = (u32)nBuf; - } - memset(&buf[pMem->n], 0, len-pMem->n); - } return len; } /* NULL or constants 0 or 1 */ return 0; Index: src/vdbecodec.c ================================================================== --- src/vdbecodec.c +++ src/vdbecodec.c @@ -129,31 +129,36 @@ if( n!=size ) return SQLITE4_CORRUPT; r = (double)x; if( e&1 ) r = -r; if( e&2 ){ e = -(e>>2); - while( e<=-10 ){ r /= 1.0e10; e += 10; } - while( e<0 ){ r /= 10.0; e++; } + if( e==0 ){ + r *= 1e+300*1e+300; + }else{ + while( e<=-10 ){ r /= 1.0e10; e += 10; } + while( e<0 ){ r /= 10.0; e++; } + } }else{ e = e>>2; while( e>=10 ){ r *= 1.0e10; e -= 10; } while( e>0 ){ r *= 10.0; e--; } } sqlite4VdbeMemSetDouble(pOut, r); }else if( cclass==0 ){ if( size==0 ){ - sqlite4VdbeMemSetStr(pOut, "", 0, SQLITE4_UTF8, SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pOut, "", 0, SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); }else if( p->a[ofst]>0x02 ){ sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, - SQLITE4_UTF8, SQLITE4_TRANSIENT); + SQLITE4_UTF8, SQLITE4_TRANSIENT, 0); }else{ static const u8 enc[] = {SQLITE4_UTF8,SQLITE4_UTF16LE,SQLITE4_UTF16BE }; sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst+1), size-1, - enc[p->a[ofst]], SQLITE4_TRANSIENT); + enc[p->a[ofst]], SQLITE4_TRANSIENT, 0); } }else{ - sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, 0,SQLITE4_TRANSIENT); + sqlite4VdbeMemSetStr(pOut, (char*)(p->a+ofst), size, 0, + SQLITE4_TRANSIENT, 0); } } testcase( i==iVal ); testcase( i==iVal+1 ); if( i<=iVal ){ @@ -503,14 +508,15 @@ /* ** Encode a single column of the key */ static int encodeOneKeyValue( - KeyEncoder *p, - Mem *pMem, - u8 sortOrder, - CollSeq *pColl + KeyEncoder *p, /* Key encoder context */ + Mem *pMem, /* Value to be encoded */ + u8 sortOrder, /* Sort order for this value */ + u8 isLastValue, /* True if this is the last value in the key */ + CollSeq *pColl /* Collating sequence for the value */ ){ int flags = pMem->flags; int i, e; int n; int iStart = p->nOut; @@ -608,12 +614,19 @@ p->aOut[p->nOut++] = 0x00; /* Release any memory allocated to hold the translated text */ if( pEnc==&sMem ) sqlite4VdbeMemRelease(&sMem); - }else - { + }else if( isLastValue ){ + /* A BLOB value that is the right-most value of a key */ + assert( flags & MEM_Blob ); + if( enlargeEncoderAllocation(p, pMem->n+1) ) return SQLITE4_NOMEM; + p->aOut[p->nOut++] = 0x26; + memcpy(p->aOut+p->nOut, pMem->z, pMem->n); + p->nOut += pMem->n; + }else{ + /* A BLOB value that is followed by other values */ const unsigned char *a; unsigned char s, t; assert( flags & MEM_Blob ); n = pMem->n; a = (u8*)pMem->z; @@ -678,10 +691,14 @@ case 0xDB: /* Text (descending index) */ case 0xDA: /* Blob (descending index) */ while( (0xFF!=*(p++)) ); break; + + case 0x26: /* Blob-final (ascending) */ + case 0xD9: /* Blob-final (descending) */ + return nKey; case 0x22: case 0xDD: /* Large positive number */ case 0x14: case 0xEB: /* Small negative number */ case 0x16: case 0xE9: /* Small positive number */ case 0x08: case 0xF7: { /* Large negative number */ @@ -724,15 +741,16 @@ */ int sqlite4VdbeEncodeKey( sqlite4 *db, /* The database connection */ Mem *aIn, /* Values to be encoded */ int nIn, /* Number of entries in aIn[] */ + int nInTotal, /* Number of values in a complete key */ int iTabno, /* The table this key applies to */ KeyInfo *pKeyInfo, /* Collating sequence and sort-order info */ u8 **paOut, /* Write the resulting key here */ int *pnOut, /* Number of bytes in the key */ - int nExtra /* See above */ + int nExtra /* extra bytes of space appended to the key */ ){ int i; int rc = SQLITE4_OK; KeyEncoder x; u8 *so; @@ -751,11 +769,12 @@ if( enlargeEncoderAllocation(&x, (nIn+1)*10) ) return SQLITE4_NOMEM; x.nOut = sqlite4PutVarint64(x.aOut, iTabno); aColl = pKeyInfo->aColl; so = pKeyInfo->aSortOrder; for(i=0; iz && preserve && pMem->zMalloc && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( pMem->flags&MEM_Dyn && pMem->xDel ){ assert( pMem->xDel!=SQLITE4_DYNAMIC ); - pMem->xDel((void *)(pMem->z)); + pMem->xDel(pMem->pDelArg, (void *)(pMem->z)); } pMem->z = pMem->zMalloc; if( pMem->z==0 ){ pMem->flags = MEM_Null; @@ -116,11 +116,10 @@ */ int sqlite4VdbeMemMakeWriteable(Mem *pMem){ int f; assert( pMem->db==0 || sqlite4_mutex_held(pMem->db->mutex) ); assert( (pMem->flags&MEM_RowSet)==0 ); - (void)ExpandBlob(pMem); f = pMem->flags; if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ if( sqlite4VdbeMemGrow(pMem, pMem->n + 2, 1) ){ return SQLITE4_NOMEM; } @@ -170,11 +169,10 @@ int rc = SQLITE4_OK; int fg = pMem->flags; const int nByte = 32; assert( pMem->db==0 || sqlite4_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); assert( !(fg&(MEM_Str|MEM_Blob)) ); assert( fg&(MEM_Int|MEM_Real) ); assert( (pMem->flags&MEM_RowSet)==0 ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -242,11 +240,11 @@ assert( (p->flags & MEM_Agg)==0 ); sqlite4VdbeMemRelease(p); }else if( p->flags&MEM_Dyn && p->xDel ){ assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE4_DYNAMIC ); - p->xDel((void *)p->z); + p->xDel(p->pDelArg, (void *)p->z); p->xDel = 0; }else if( p->flags&MEM_RowSet ){ sqlite4RowSetClear(p->u.pRowSet); }else if( p->flags&MEM_Frame ){ sqlite4VdbeMemSetNull(p); @@ -456,24 +454,10 @@ } MemSetTypeFlag(pMem, MEM_Null); pMem->type = SQLITE4_NULL; } -/* -** Delete any previous value and set the value to be a BLOB of length -** n containing all zeros. -*/ -void sqlite4VdbeMemSetZeroBlob(Mem *pMem, int n){ - sqlite4VdbeMemRelease(pMem); - pMem->flags = MEM_Blob|MEM_Zero; - pMem->type = SQLITE4_BLOB; - pMem->n = 0; - if( n<0 ) n = 0; - pMem->u.nZero = n; - pMem->enc = SQLITE4_UTF8; -} - /* ** Delete any previous value and set the value stored in *pMem to val, ** manifest type INTEGER. */ void sqlite4VdbeMemSetInt64(Mem *pMem, i64 val){ @@ -525,13 +509,10 @@ */ int sqlite4VdbeMemTooBig(Mem *p){ assert( p->db!=0 ); if( p->flags & (MEM_Str|MEM_Blob) ){ int n = p->n; - if( p->flags & MEM_Zero ){ - n += p->u.nZero; - } return n>p->db->aLimit[SQLITE4_LIMIT_LENGTH]; } return 0; } @@ -634,15 +615,16 @@ ** stored without allocating memory, then it is. If a memory allocation ** is required to store the string, then value of pMem is unchanged. In ** either case, SQLITE4_TOOBIG is returned. */ int sqlite4VdbeMemSetStr( - Mem *pMem, /* Memory cell to set to string value */ - const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ - u8 enc, /* Encoding of z. 0 for BLOBs */ - void (*xDel)(void*) /* Destructor function */ + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*,void*),/* Destructor function */ + void *pDelArg /* First argument to xDel() */ ){ int nByte = n; /* New value for pMem->n */ int iLimit; /* Maximum allowed string or blob size */ u16 flags = 0; /* New value for pMem->flags */ @@ -693,10 +675,11 @@ pMem->xDel = 0; }else{ sqlite4VdbeMemRelease(pMem); pMem->z = (char *)z; pMem->xDel = xDel; + pMem->pDelArg = pDelArg; flags |= ((xDel==SQLITE4_STATIC)?MEM_Static:MEM_Dyn); } pMem->n = nByte; pMem->flags = flags; @@ -854,11 +837,10 @@ if( pVal->flags&MEM_Null ){ return 0; } assert( (MEM_Blob>>3) == MEM_Str ); pVal->flags |= (pVal->flags & MEM_Blob)>>3; - (void)ExpandBlob(pVal); if( pVal->flags&MEM_Str ){ sqlite4VdbeChangeEncoding(pVal, enc & ~SQLITE4_UTF16_ALIGNED); if( (enc & SQLITE4_UTF16_ALIGNED)!=0 && 1==(1&SQLITE4_PTR_TO_INT(pVal->z)) ){ assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); if( sqlite4VdbeMemMakeWriteable(pVal)!=SQLITE4_OK ){ @@ -949,11 +931,11 @@ if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite4VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ zVal = sqlite4MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; - sqlite4ValueSetStr(pVal, -1, zVal, SQLITE4_UTF8, SQLITE4_DYNAMIC); + sqlite4ValueSetStr(pVal, -1, zVal, SQLITE4_UTF8, SQLITE4_DYNAMIC, 0); if( op==TK_FLOAT ) pVal->type = SQLITE4_FLOAT; } if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE4_AFF_NONE ){ sqlite4ValueApplyAffinity(pVal, SQLITE4_AFF_NUMERIC, SQLITE4_UTF8); }else{ @@ -990,11 +972,11 @@ if( !pVal ) goto no_mem; zVal = &pExpr->u.zToken[2]; nVal = sqlite4Strlen30(zVal)-1; assert( zVal[nVal]=='\'' ); sqlite4VdbeMemSetStr(pVal, sqlite4HexToBlob(db, zVal, nVal), nVal/2, - 0, SQLITE4_DYNAMIC); + 0, SQLITE4_DYNAMIC, 0); } #endif if( pVal ){ sqlite4VdbeMemStoreType(pVal); @@ -1012,17 +994,18 @@ /* ** Change the string value of an sqlite4_value object */ void sqlite4ValueSetStr( - sqlite4_value *v, /* Value to be set */ - int n, /* Length of string z */ - const void *z, /* Text of the new string */ - u8 enc, /* Encoding to use */ - void (*xDel)(void*) /* Destructor for the string */ + sqlite4_value *v, /* Value to be set */ + int n, /* Length of string z */ + const void *z, /* Text of the new string */ + u8 enc, /* Encoding to use */ + void (*xDel)(void*,void*), /* Destructor for the string */ + void *pDelArg /* First argument to xDel() */ ){ - if( v ) sqlite4VdbeMemSetStr((Mem *)v, z, n, enc, xDel); + if( v ) sqlite4VdbeMemSetStr((Mem *)v, z, n, enc, xDel, pDelArg); } /* ** Free an sqlite4_value object */ @@ -1037,13 +1020,9 @@ ** that it uses the encoding "enc" */ int sqlite4ValueBytes(sqlite4_value *pVal, u8 enc){ Mem *p = (Mem*)pVal; if( (p->flags & MEM_Blob)!=0 || sqlite4ValueText(pVal, enc) ){ - if( p->flags & MEM_Zero ){ - return p->n + p->u.nZero; - }else{ - return p->n; - } + return p->n; } return 0; } Index: src/vdbetrace.c ================================================================== --- src/vdbetrace.c +++ src/vdbetrace.c @@ -128,21 +128,19 @@ u8 enc = ENC(db); if( enc!=SQLITE4_UTF8 ){ Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; - sqlite4VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE4_STATIC); + sqlite4VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE4_STATIC, 0); sqlite4VdbeChangeEncoding(&utf8, SQLITE4_UTF8); sqlite4XPrintf(&out, "'%.*q'", utf8.n, utf8.z); sqlite4VdbeMemRelease(&utf8); }else #endif { sqlite4XPrintf(&out, "'%.*q'", pVar->n, pVar->z); } - }else if( pVar->flags & MEM_Zero ){ - sqlite4XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ assert( pVar->flags & MEM_Blob ); sqlite4StrAccumAppend(&out, "x'", 2); for(i=0; in; i++){ sqlite4XPrintf(&out, "%02x", pVar->z[i]&0xff); Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -4163,10 +4163,13 @@ testcase( op==OP_SeekGt ); testcase( op==OP_SeekGe ); testcase( op==OP_SeekLe ); testcase( op==OP_SeekLt ); sqlite4VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + if( (pIdx->nColumn + (pIdx==pPk ? 0 : pPk->nColumn))>nEq ){ + sqlite4VdbeChangeP5(v, OPFLAG_PARTIALKEY); + } /* Set variable op to the instruction required to determine if the ** cursor is passed the end of the range. If the range is unbounded, ** then set op to OP_Noop. Nothing to do in this case. */ assert( (endEq==0 || endEq==1) ); DELETED test/bindxfer.test Index: test/bindxfer.test ================================================================== --- test/bindxfer.test +++ /dev/null @@ -1,76 +0,0 @@ -# 2005 April 21 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this script testing the sqlite_transfer_bindings() API. -# -# $Id: bindxfer.test,v 1.9 2009/04/17 11:56:28 drh Exp $ -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -proc sqlite_step {stmt VALS COLS} { - upvar #0 $VALS vals - upvar #0 $COLS cols - set vals [list] - set cols [list] - - set rc [sqlite4_step $stmt] - for {set i 0} {$i < [sqlite4_column_count $stmt]} {incr i} { - lappend cols [sqlite4_column_name $stmt $i] - } - for {set i 0} {$i < [sqlite4_data_count $stmt]} {incr i} { - lappend vals [sqlite4_column_text $stmt $i] - } - - return $rc -} - -do_test bindxfer-1.1 { - set DB [sqlite4_connection_pointer db] - execsql {CREATE TABLE t1(a,b,c);} - set VM1 [sqlite4_prepare $DB {SELECT ?, ?, ?} -1 TAIL] - set TAIL -} {} -do_test bindxfer-1.2 { - sqlite4_bind_parameter_count $VM1 -} 3 -do_test bindxfer-1.3 { - set VM2 [sqlite4_prepare $DB {SELECT ?, ?, ?} -1 TAIL] - set TAIL -} {} -do_test bindxfer-1.4 { - sqlite4_bind_parameter_count $VM2 -} 3 -ifcapable deprecated { - do_test bindxfer-1.5 { - sqlite_bind $VM1 1 one normal - set sqlite_static_bind_value two - sqlite_bind $VM1 2 {} static - sqlite_bind $VM1 3 {} null - sqlite4_transfer_bindings $VM1 $VM2 - sqlite_step $VM1 VALUES COLNAMES - } SQLITE4_ROW - do_test bindxfer-1.6 { - set VALUES - } {{} {} {}} - do_test bindxfer-1.7 { - sqlite_step $VM2 VALUES COLNAMES - } SQLITE4_ROW - do_test bindxfer-1.8 { - set VALUES - } {one two {}} -} -catch {sqlite4_finalize $VM1} -catch {sqlite4_finalize $VM2} - - -finish_test Index: test/boundary4.test ================================================================== --- test/boundary4.test +++ test/boundary4.test @@ -212,11 +212,11 @@ UPDATE t1 SET oid=a, a=oid } } {} do_test boundary4-3.2 { db eval { - ALTER TABLE t1 ADD COLUMN z; UPDATE t1 SET z=zeroblob(600) + ALTER TABLE t1 ADD COLUMN z; UPDATE t1 SET z=randomblob(600) } } {} do_test boundary4-3.3 { db eval { SELECT oid, a, x FROM t1 ORDER BY +oid Index: test/ckpt1.test ================================================================== --- test/ckpt1.test +++ test/ckpt1.test @@ -74,11 +74,10 @@ INSERT INTO t1 SELECT randstr(100,100), randstr(100,100) FROM t1; -- 16K INSERT INTO t1 SELECT randstr(100,100), randstr(100,100) FROM t1; -- 32K INSERT INTO t1 SELECT randstr(100,100), randstr(100,100) FROM t1; -- 64K } do_test 3.2 { - sqlite4_lsm_flush db main sqlite4_lsm_work db main -nmerge 1 -npage 1000000 execsql { SELECT count(*) FROM t1 } } {65536} do_test 3.3 { db close @@ -85,12 +84,13 @@ sqlite4 db test.db execsql { SELECT count(*) FROM t1 } } {65536} do_test 3.4 { execsql { INSERT INTO t1 VALUES(randstr(100,100), randstr(100,100)) } - sqlite4_lsm_flush db main + db close + sqlite4 db test.db sqlite4_lsm_work db main -nmerge 1 -npage 1000000 execsql { SELECT count(*) FROM t1 } } {65537} finish_test Index: test/csr1.test ================================================================== --- test/csr1.test +++ test/csr1.test @@ -75,12 +75,12 @@ populate_db_2 do_execsql_test 3.1 { BEGIN; INSERT INTO t1 VALUES(10, randstr(910, 910)); } -do_test 3.2 { sqlite4_lsm_config db main autoflush } [expr 1*1024*1024] -do_test 3.3 { sqlite4_lsm_config db main autoflush 4096 } 4096 +do_test 3.2 { sqlite4_lsm_config db main autoflush } 1024 +do_test 3.3 { sqlite4_lsm_config db main autoflush 4 } 4 do_test 3.4 { set res [list] db eval { SELECT a, length(b) AS l FROM t1 } { lappend res $a $l Index: test/ctime.test ================================================================== --- test/ctime.test +++ test/ctime.test @@ -70,15 +70,10 @@ } {0 1} do_test ctime-1.4.2 { catchsql { SELECT sqlite_compileoption_used('THREADSAFE'); } -} {0 1} -do_test ctime-1.4.3 { - catchsql { - SELECT sqlite_compileoption_used("THREADSAFE"); - } } {0 1} do_test ctime-1.5 { set ans1 [ catchsql { SELECT sqlite_compileoption_used('THREADSAFE=0'); @@ -120,15 +115,10 @@ do_test ctime-2.1.2 { catchsql { SELECT sqlite_compileoption_used(NULL); } } {0 {{}}} -do_test ctime-2.1.3 { - catchsql { - SELECT sqlite_compileoption_used(""); - } -} {0 0} do_test ctime-2.1.4 { catchsql { SELECT sqlite_compileoption_used(''); } } {0 0} Index: test/enc4.test ================================================================== --- test/enc4.test +++ test/enc4.test @@ -52,15 +52,11 @@ } $enc set j 1 foreach init $inits { - do_test enc4-$i.$j.2 { - set S [sqlite4_prepare db "SELECT $init+?" -1 dummy] - sqlite4_expired $S - } {0} - + set S [sqlite4_prepare db "SELECT $init+?" -1 dummy] set k 1 foreach val $vals { for {set x 1} {$x<16} {incr x} { set part [expr $init + [string range $val 0 [expr $x-1]]] Index: test/in.test ================================================================== --- test/in.test +++ test/in.test @@ -271,15 +271,10 @@ do_test in-8.1 { execsql { SELECT b FROM t1 WHERE a IN ('hello','there') } -} {world} -do_test in-8.2 { - execsql { - SELECT b FROM t1 WHERE a IN ("hello",'there') - } } {world} # Test constructs of the form: expr IN tablename # do_test in-9.1 { Index: test/join.test ================================================================== --- test/join.test +++ test/join.test @@ -429,25 +429,25 @@ # A test for ticket #247. # do_test join-7.1 { execsql { CREATE TABLE t7 (x, y); - INSERT INTO t7 VALUES ("pa1", 1); - INSERT INTO t7 VALUES ("pa2", NULL); - INSERT INTO t7 VALUES ("pa3", NULL); - INSERT INTO t7 VALUES ("pa4", 2); - INSERT INTO t7 VALUES ("pa30", 131); - INSERT INTO t7 VALUES ("pa31", 130); - INSERT INTO t7 VALUES ("pa28", NULL); + INSERT INTO t7 VALUES ('pa1', 1); + INSERT INTO t7 VALUES ('pa2', NULL); + INSERT INTO t7 VALUES ('pa3', NULL); + INSERT INTO t7 VALUES ('pa4', 2); + INSERT INTO t7 VALUES ('pa30', 131); + INSERT INTO t7 VALUES ('pa31', 130); + INSERT INTO t7 VALUES ('pa28', NULL); CREATE TABLE t8 (a integer primary key, b); - INSERT INTO t8 VALUES (1, "pa1"); - INSERT INTO t8 VALUES (2, "pa4"); + INSERT INTO t8 VALUES (1, 'pa1'); + INSERT INTO t8 VALUES (2, 'pa4'); INSERT INTO t8 VALUES (3, NULL); INSERT INTO t8 VALUES (4, NULL); - INSERT INTO t8 VALUES (130, "pa31"); - INSERT INTO t8 VALUES (131, "pa30"); + INSERT INTO t8 VALUES (130, 'pa31'); + INSERT INTO t8 VALUES (131, 'pa30'); SELECT coalesce(t8.a,999) from t7 LEFT JOIN t8 on y=a; } } {1 999 999 2 131 130 999} DELETED test/lastinsert.test Index: test/lastinsert.test ================================================================== --- test/lastinsert.test +++ /dev/null @@ -1,366 +0,0 @@ -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Tests to make sure that value returned by last_insert_rowid() (LIRID) -# is updated properly, especially inside triggers -# -# Note 1: insert into table is now the only statement which changes LIRID -# Note 2: upon entry into before or instead of triggers, -# LIRID is unchanged (rather than -1) -# Note 3: LIRID is changed within the context of a trigger, -# but is restored once the trigger exits -# Note 4: LIRID is not changed by an insert into a view (since everything -# is done within instead of trigger context) -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -# ---------------------------------------------------------------------------- -# 1.x - basic tests (no triggers) - -# LIRID changed properly after an insert into a table -do_test lastinsert-1.1 { - catchsql { - create table t1 (k integer primary key); - insert into t1 values (1); - insert into t1 values (NULL); - insert into t1 values (NULL); - select last_insert_rowid(); - } -} {0 3} - -# LIRID unchanged after an update on a table -do_test lastinsert-1.2 { - catchsql { - update t1 set k=4 where k=2; - select last_insert_rowid(); - } -} {0 3} - -# LIRID unchanged after a delete from a table -do_test lastinsert-1.3 { - catchsql { - delete from t1 where k=4; - select last_insert_rowid(); - } -} {0 3} - -# LIRID unchanged after create table/view statements -do_test lastinsert-1.4.1 { - catchsql { - create table t2 (k integer primary key, val1, val2, val3); - select last_insert_rowid(); - } -} {0 3} -ifcapable view { -do_test lastinsert-1.4.2 { - catchsql { - create view v as select * from t1; - select last_insert_rowid(); - } -} {0 3} -} ;# ifcapable view - -# All remaining tests involve triggers. Skip them if triggers are not -# supported in this build. -# -ifcapable {!trigger} { - finish_test - return -} - -# ---------------------------------------------------------------------------- -# 2.x - tests with after insert trigger - -# LIRID changed properly after an insert into table containing an after trigger -do_test lastinsert-2.1 { - catchsql { - delete from t2; - create trigger r1 after insert on t1 for each row begin - insert into t2 values (NEW.k*2, last_insert_rowid(), NULL, NULL); - update t2 set k=k+10, val2=100+last_insert_rowid(); - update t2 set val3=1000+last_insert_rowid(); - end; - insert into t1 values (13); - select last_insert_rowid(); - } -} {0 13} - -# LIRID equals NEW.k upon entry into after insert trigger -do_test lastinsert-2.2 { - catchsql { - select val1 from t2; - } -} {0 13} - -# LIRID changed properly by insert within context of after insert trigger -do_test lastinsert-2.3 { - catchsql { - select val2 from t2; - } -} {0 126} - -# LIRID unchanged by update within context of after insert trigger -do_test lastinsert-2.4 { - catchsql { - select val3 from t2; - } -} {0 1026} - -# ---------------------------------------------------------------------------- -# 3.x - tests with after update trigger - -# LIRID not changed after an update onto a table containing an after trigger -do_test lastinsert-3.1 { - catchsql { - delete from t2; - drop trigger r1; - create trigger r1 after update on t1 for each row begin - insert into t2 values (NEW.k*2, last_insert_rowid(), NULL, NULL); - update t2 set k=k+10, val2=100+last_insert_rowid(); - update t2 set val3=1000+last_insert_rowid(); - end; - update t1 set k=14 where k=3; - select last_insert_rowid(); - } -} {0 13} - -# LIRID unchanged upon entry into after update trigger -do_test lastinsert-3.2 { - catchsql { - select val1 from t2; - } -} {0 13} - -# LIRID changed properly by insert within context of after update trigger -do_test lastinsert-3.3 { - catchsql { - select val2 from t2; - } -} {0 128} - -# LIRID unchanged by update within context of after update trigger -do_test lastinsert-3.4 { - catchsql { - select val3 from t2; - } -} {0 1028} - -# ---------------------------------------------------------------------------- -# 4.x - tests with instead of insert trigger -# These may not be run if either views or triggers were disabled at -# compile-time - -ifcapable {view && trigger} { -# LIRID not changed after an insert into view containing an instead of trigger -do_test lastinsert-4.1 { - catchsql { - delete from t2; - drop trigger r1; - create trigger r1 instead of insert on v for each row begin - insert into t2 values (NEW.k*2, last_insert_rowid(), NULL, NULL); - update t2 set k=k+10, val2=100+last_insert_rowid(); - update t2 set val3=1000+last_insert_rowid(); - end; - insert into v values (15); - select last_insert_rowid(); - } -} {0 13} - -# LIRID unchanged upon entry into instead of trigger -do_test lastinsert-4.2 { - catchsql { - select val1 from t2; - } -} {0 13} - -# LIRID changed properly by insert within context of instead of trigger -do_test lastinsert-4.3 { - catchsql { - select val2 from t2; - } -} {0 130} - -# LIRID unchanged by update within context of instead of trigger -do_test lastinsert-4.4 { - catchsql { - select val3 from t2; - } -} {0 1030} -} ;# ifcapable (view && trigger) - -# ---------------------------------------------------------------------------- -# 5.x - tests with before delete trigger - -# LIRID not changed after a delete on a table containing a before trigger -do_test lastinsert-5.1 { - catchsql { - drop trigger r1; -- This was not created if views are disabled. - } - catchsql { - delete from t2; - create trigger r1 before delete on t1 for each row begin - insert into t2 values (77, last_insert_rowid(), NULL, NULL); - update t2 set k=k+10, val2=100+last_insert_rowid(); - update t2 set val3=1000+last_insert_rowid(); - end; - delete from t1 where k=1; - select last_insert_rowid(); - } -} {0 13} - -# LIRID unchanged upon entry into delete trigger -do_test lastinsert-5.2 { - catchsql { - select val1 from t2; - } -} {0 13} - -# LIRID changed properly by insert within context of delete trigger -do_test lastinsert-5.3 { - catchsql { - select val2 from t2; - } -} {0 177} - -# LIRID unchanged by update within context of delete trigger -do_test lastinsert-5.4 { - catchsql { - select val3 from t2; - } -} {0 1077} - -# ---------------------------------------------------------------------------- -# 6.x - tests with instead of update trigger -# These tests may not run if either views or triggers are disabled. - -ifcapable {view && trigger} { -# LIRID not changed after an update on a view containing an instead of trigger -do_test lastinsert-6.1 { - catchsql { - delete from t2; - drop trigger r1; - create trigger r1 instead of update on v for each row begin - insert into t2 values (NEW.k*2, last_insert_rowid(), NULL, NULL); - update t2 set k=k+10, val2=100+last_insert_rowid(); - update t2 set val3=1000+last_insert_rowid(); - end; - update v set k=16 where k=14; - select last_insert_rowid(); - } -} {0 13} - -# LIRID unchanged upon entry into instead of trigger -do_test lastinsert-6.2 { - catchsql { - select val1 from t2; - } -} {0 13} - -# LIRID changed properly by insert within context of instead of trigger -do_test lastinsert-6.3 { - catchsql { - select val2 from t2; - } -} {0 132} - -# LIRID unchanged by update within context of instead of trigger -do_test lastinsert-6.4 { - catchsql { - select val3 from t2; - } -} {0 1032} -} ;# ifcapable (view && trigger) - -# ---------------------------------------------------------------------------- -# 7.x - complex tests with temporary tables and nested instead of triggers -# These do not run if views or triggers are disabled. - -ifcapable {trigger && view && tempdb} { -do_test lastinsert-7.1 { - catchsql { - drop table t1; drop table t2; drop trigger r1; - create temp table t1 (k integer primary key); - create temp table t2 (k integer primary key); - create temp view v1 as select * from t1; - create temp view v2 as select * from t2; - create temp table rid (k integer primary key, rin, rout); - insert into rid values (1, NULL, NULL); - insert into rid values (2, NULL, NULL); - create temp trigger r1 instead of insert on v1 for each row begin - update rid set rin=last_insert_rowid() where k=1; - insert into t1 values (100+NEW.k); - insert into v2 values (100+last_insert_rowid()); - update rid set rout=last_insert_rowid() where k=1; - end; - create temp trigger r2 instead of insert on v2 for each row begin - update rid set rin=last_insert_rowid() where k=2; - insert into t2 values (1000+NEW.k); - update rid set rout=last_insert_rowid() where k=2; - end; - insert into t1 values (77); - select last_insert_rowid(); - } -} {0 77} - -do_test lastinsert-7.2 { - catchsql { - insert into v1 values (5); - select last_insert_rowid(); - } -} {0 77} - -do_test lastinsert-7.3 { - catchsql { - select rin from rid where k=1; - } -} {0 77} - -do_test lastinsert-7.4 { - catchsql { - select rout from rid where k=1; - } -} {0 105} - -do_test lastinsert-7.5 { - catchsql { - select rin from rid where k=2; - } -} {0 105} - -do_test lastinsert-7.6 { - catchsql { - select rout from rid where k=2; - } -} {0 1205} - -do_test lastinsert-8.1 { - db close - sqlite4 db test.db - execsql { - CREATE TABLE t2(x INTEGER PRIMARY KEY, y); - CREATE TABLE t3(a, b); - CREATE TRIGGER after_t2 AFTER INSERT ON t2 BEGIN - INSERT INTO t3 VALUES(new.x, new.y); - END; - INSERT INTO t2 VALUES(5000000000, 1); - SELECT last_insert_rowid(); - } -} 5000000000 - -do_test lastinsert-9.1 { - db eval {INSERT INTO t2 VALUES(123456789012345,0)} - db last_insert_rowid -} {123456789012345} - - -} ;# ifcapable (view && trigger) - -finish_test Index: test/log3.test ================================================================== --- test/log3.test +++ test/log3.test @@ -34,11 +34,10 @@ do_test 1.9 { sqlite4_lsm_config db main safety } {2} #------------------------------------------------------------------------- reset_db do_test 2.0 { sqlite4_lsm_config db main safety 2 } {2} -do_test 2.1 { sqlite4_lsm_config db main log-size 1024 } {1024} do_execsql_test 2.2 { CREATE TABLE t1(a PRIMARY KEY, b); INSERT INTO t1 VALUES(randstr(50,50), randstr(50,50)); } {} @@ -54,11 +53,11 @@ INSERT INTO t1 VALUES(randstr(50,50), randstr(50,50)); COMMIT; } {} do_filesize_test 2.5 0 2048 -do_test 2.6 { sqlite4_lsm_flush db main } {} +do_test 2.6 { optimize_db } {} do_execsql_test 2.7 { INSERT INTO t1 VALUES(randstr(50,50), randstr(50,50)) } do_test 2.8 { sqlite4_lsm_checkpoint db main } {} do_test 2.9 { sqlite4_lsm_info db main log-structure } {0 0 0 0 2560 3072} for {set i 1} {$i <= 6} {incr i} { @@ -69,9 +68,9 @@ do_recover_test 2.10.$i.3 { SELECT count(*) FROM t1 } [expr 8 + $i] } do_test 2.11 { sqlite4_lsm_info db main log-structure -} {2560 3080 0 2216 3584 4608} +} {0 0 0 0 2560 6144} finish_test Index: test/lsm1.test ================================================================== --- test/lsm1.test +++ test/lsm1.test @@ -88,11 +88,10 @@ db write eee five db write fff six reopen db delete_range a bbb reopen -breakpoint db work 10 } {1} do_contents_test 2.2 { {bbb two} {ccc three} {ddd four} {eee five} {fff six} } @@ -117,12 +116,12 @@ do_test 3.2 { db write bx seven reopen db delete_range aaa bx reopen - db work 10 -} {2} + db work 2 10 +} {1} do_contents_test 3.3 { {aaa one} {bx seven} {ccc three} {ddd four} {eee five} {fff six} } ADDED test/lsm3.test Index: test/lsm3.test ================================================================== --- /dev/null +++ test/lsm3.test @@ -0,0 +1,89 @@ +# 2012 November 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that the LSM_CONFIG_MULTIPLE_PROCESSES parameter seems +# to work as documented. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix lsm3 +db close + +do_multiclient_test tn { + + # The [do_multiclient_test] command automatically opens a connection + # in each process (or three connections in this process). We don't want + # them in this case. + code1 { db close } + code2 { db2 close } + code3 { db3 close } + + if { $tn==1 } { + set locked {1 {database is locked}} + } else { + set locked {0 {}} + } + + # Open a single-process connection to the database from an external + # process (if $tn==1, otherwise open it from within the current + # process). + code2 { sqlite4 db2 file:test.db?lsm_multiple_processes=0 } + + # Try to open some other connections to the database file, both in multi + # and single process mode. If ($tn==1), then all such attempts fail. Or, + # if ($tn==2), they all succeed. + do_test $tn.1 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.2 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.3 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } $locked + do_test $tn.4 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } $locked + + # Now open a connection from an external process in multi-proc mode. + # Observe that further connections are allowed if they are from within + # the same process or if the LSM_CONFIG_MULTIPLE_PROCESSES parameter + # is set to true. + code2 { + db2 close + sqlite4 db2 file:test.db + } + + do_test $tn.5 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.6 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=0} msg] $msg + } $locked + do_test $tn.7 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } {0 {}} + do_test $tn.8 { + catch { db close } + list [catch {sqlite4 db file:test.db?lsm_multiple_processes=1} msg] $msg + } {0 {}} +} + + +finish_test ADDED test/lsm4.test Index: test/lsm4.test ================================================================== --- /dev/null +++ test/lsm4.test @@ -0,0 +1,132 @@ +# 2013 February 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the LSM library. More specifically, +# it focuses on testing the compression, compression-id and +# compression-factory functionality. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix lsm4 +db close + +# Compression scheme ids (defined in test_lsm.c): +# +set compression_id(encrypt) 43 +set compression_id(rle) 44 +set compression_id(noop) 45 + +proc db_fetch {db key} { + db csr_open csr + csr seek $key eq + set ret [csr value] + csr close + set ret +} + +do_test 1.1 { + lsm_open db test.db + db config {set_compression noop} + db write 1 abc + db write 2 def + db close +} {} + +do_test 1.2 { + lsm_open db test.db + db config {set_compression noop} + list [db_fetch db 1] [db_fetch db 2] +} {abc def} + +do_test 1.3 { + db close + lsm_open db test.db + db config {set_compression rle} + list [catch {db_fetch db 1} msg] $msg +} {1 {error in lsm_csr_open() - 50}} + +do_test 1.4 { + db close + lsm_open db test.db + list [catch {db_fetch db 1} msg] $msg +} {1 {error in lsm_csr_open() - 50}} + +do_test 1.5 { + db config {set_compression_factory true} + list [db_fetch db 1] [db_fetch db 2] +} {abc def} + +do_test 1.6 { db info compression_id } $compression_id(noop) +db close + +#------------------------------------------------------------------------- +# +forcedelete test.db + +do_test 2.1 { + lsm_open db test.db + db info compression_id +} {0} + +do_test 2.2 { + db write 1 abc + db write 2 abc + db info compression_id +} {0} + +do_test 2.3 { + lsm_open db2 test.db + db2 info compression_id +} {0} + +do_test 2.4 { + db close + db2 info compression_id +} {0} + +do_test 2.5 { + db2 close + lsm_open db test.db + db info compression_id +} {1} + +db close +forcedelete test.db + +do_test 2.6 { + lsm_open db test.db + db config {set_compression rle} + db write 3 three + db write 4 four + db close + + lsm_open db test.db + db info compression_id +} $compression_id(rle) + +do_test 2.7 { + db config {set_compression rle} + list [db_fetch db 3] [db_fetch db 4] +} {three four} + +#------------------------------------------------------------------------- +# +catch {db close} +forcedelete test.db + +do_test 3.1 { + lsm_open db test.db + db_fetch db abc +} {} + +finish_test + ADDED test/lsm5.test Index: test/lsm5.test ================================================================== --- /dev/null +++ test/lsm5.test @@ -0,0 +1,205 @@ +# 2013 February 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the LSM library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lsm_common.tcl +set testprefix lsm5 +db close + +# Create a new database with file name $file. +# +proc create_abc_db {file} { + forcedelete $file + lsm_open db $file {block_size 256} + db write a alpha + db write b bravo + db write c charlie + db close +} + +proc create_abc_log {file} { + forcedelete $file ${file}-2 + lsm_open db ${file}-2 + db write a alpha + db write b bravo + db write c charlie + file copy ${file}-2 $file + file copy ${file}-2-log $file-log + db close +} + +#------------------------------------------------------------------------- +# When the database system is shut down (i.e. when the last connection +# disconnects), an attempt is made to truncate the database file to the +# minimum number of blocks required. +# +# This test case checks that this process does not actually cause the +# database to grow. +# +do_test 1.1 { + lsm_open db test.db + db config {mmap 0} +} {0} +do_test 1.2 { + db write 1 one + db write 2 two + db close +} {} +do_test 1.3 { + expr [file size test.db] < (64*1024) +} 1 + +#------------------------------------------------------------------------- +# Test that if an attempt is made to open a read-write connection to a +# non-live database that the client does not have permission to write to is +# attempted an error is reported. In order to open a read-write connection +# to a database, the client requires: +# +# * read-write access to the db file, +# * read-write access to the log file, +# * for multi-process mode, read-write access to the shm file. +# +# In the above, "read-write access" includes the ability to create the db, +# log or shm file if it does not exist. +# +# These tests verify that the lsm_open() command returns LSM_IOERR. At some +# point in the future this will be improved. Likely when sqlite4 level tests +# for opening read-only databases are added. +# +foreach {tn filename setup} { + + 1 test.dir/test.db { + # Create a directory "test.dir". + forcedelete test.dir + file mkdir test.dir + + # Create a database within test.dir + create_abc_db test.dir/test.db + + # Now make the db and its directory read-only. + file attr test.dir/test.db -perm r--r--r-- + file attr test.dir -perm r-xr-xr-x + } + + 2 test.db { + # Create a database test.db and set its permissions to read-only + create_abc_db test.db + file attr test.db -perm r--r--r-- + } + + 3 test.dir/test.db { + # Create a directory "test.dir". + forcedelete test.dir + file mkdir test.dir + + # Create a database within test.dir + create_abc_db test.dir/test.db + + # Now make test.dir read-only. + file attr test.dir -perm r-xr-xr-x + } + +} { + do_test 2.$tn.1 { + eval $setup + set rc [catch {lsm_open db $filename} msg] + list $rc $msg + } {1 {error in lsm_open() - 10}} + + do_test 2.$tn.2 { + eval $setup + lsm_open db $filename {readonly 1} + set res [list [db_fetch db a] [db_fetch db b] [db_fetch db c]] + db close + set res + } {alpha bravo charlie} +} + +#------------------------------------------------------------------------- +# Try having a read-only connection connect to a non-live system where the +# log file contains content. In this scenario the read-only client must +# read the contents from the log file at the start of each read-transaction. +# +do_test 3.1 { + create_abc_log test.db + list [file size test.db] [file size test.db-log] +} {0 56} +do_test 3.2 { + lsm_open db $filename {readonly 1} + set res [list [db_fetch db a] [db_fetch db b] [db_fetch db c]] + db close + set res +} {alpha bravo charlie} +do_test 3.3 { + list [file size test.db] [file size test.db-log] +} {0 56} + +# Now make the same db live and check the read-only connection can still +# read it. +do_test 3.4 { file exists test.db-shm } 0 +do_test 3.5 { + lsm_open db_rw test.db + file exists test.db-shm +} 1 +do_test 3.6 { + lsm_open db test.db {readonly 1} + list [db_fetch db a] [db_fetch db b] [db_fetch db c] +} {alpha bravo charlie} + +# Close the read-write connection. This should cause a checkpoint and delete +# the log file, even though the system remains live. +do_test 3.7 { + db_rw close + list [file exists test.db-log] [file exists test.db-shm] +} {0 1} + +# Now close the read-only connection. The system is now non-live, but the +# *-shm remains in the file-system (the readonly connection cannot unlink it). +do_test 3.8 { + db close + list [file exists test.db-log] [file exists test.db-shm] +} {0 1} + +#------------------------------------------------------------------------- +# +do_test 4.1 { + create_abc_log test.db + list [file size test.db] [file size test.db-log] +} {0 56} + +do_test 4.2 { + lsm_open db test.db {readonly 1} + db csr_open T + list [db_fetch db a] [db_fetch db b] [db_fetch db c] +} {alpha bravo charlie} + +do_test 4.3 { + lsm_open db_rw test.db {block_size 64} + db_rw write b BRAVO + db_rw close + list [file size test.db] [file size test.db-log] +} {65536 74} + +do_test 4.4 { + list [db_fetch db a] [db_fetch db b] [db_fetch db c] +} {alpha bravo charlie} + +do_test 4.5 { + T close + list [db_fetch db a] [db_fetch db b] [db_fetch db c] +} {alpha BRAVO charlie} + +finish_test + ADDED test/lsm6.test Index: test/lsm6.test ================================================================== --- /dev/null +++ test/lsm6.test @@ -0,0 +1,50 @@ +# 2013 February 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the LSM library. Specifically, it +# checks that the in-memory tree is flushed to disk when the last connection +# is closed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lsm_common.tcl +set testprefix lsm6 +db close + +foreach {tn mp lf} { + 1 1 1 + 2 1 0 + 3 0 1 + 4 0 0 +} { + + do_test $tn.1 { + forcedelete test.db test.db-log + lsm_open db test.db [list multi_proc $mp use_log $lf] + for {set i 0} {$i < 1000} {incr i} { + db write $i [string repeat "$i." 1000] + } + expr {[file size test.db-log] > 0} + } $lf + + do_test $tn.2 { + db close + lsm_open db test.db + db_fetch db 999 + } [string repeat 999. 1000] + + db close +} + +finish_test + + ADDED test/lsm_common.tcl Index: test/lsm_common.tcl ================================================================== --- /dev/null +++ test/lsm_common.tcl @@ -0,0 +1,25 @@ +# 2013 Feb 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains common code used the various lsm tests in this +# directory. +# + + +proc db_fetch {db key} { + db csr_open csr + csr seek $key eq + set ret [csr value] + csr close + set ret +} + + ADDED test/num.test Index: test/num.test ================================================================== --- /dev/null +++ test/num.test @@ -0,0 +1,91 @@ +# 2001 September 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite_*_printf() interface. +# +# $Id: printf.test,v 1.31 2009/02/01 00:21:10 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test num-1.1.1 { + sqlite4_num_compare 20 20 +} {equal} +do_test num-1.1.2 { + sqlite4_num_compare 20 2e1 +} {equal} +do_test num-1.1.3 { + sqlite4_num_compare -00034 -3.4e1 +} {equal} +do_test num-1.1.4 { + sqlite4_num_compare -inf +inf +} {lesser} +do_test num-1.1.5 { + sqlite4_num_compare -inf 0 +} {lesser} +do_test num-1.1.6 { + sqlite4_num_compare inf 4 +} {greater} +do_test num-1.1.7 { + sqlite4_num_compare nan 7 +} {incomparable} +# Is +0 > -0? +#do_test num-equal-1.1.4 { +# sqlite4_num_compare +0 -0 +#} {equal} + +do_test num-2.1.1 { + sqlite4_num_to_text [sqlite4_num_from_text 37] +} {37} +do_test num-2.1.2 { + sqlite4_num_to_text [sqlite4_num_from_text 37 2] +} {37} +do_test num-2.1.4 { + sqlite4_num_compare [sqlite4_num_from_text 2.9e2X 5] 290 +} {equal} +do_test num-2.1.5 { + sqlite4_num_isnan [sqlite4_num_from_text inf 2] +} {true} +do_test num-2.1.6 { + sqlite4_num_isinf [sqlite4_num_from_text inf 3] +} {true} + +do_test num-3.1.1 { + sqlite4_num_to_text [sqlite4_num_add 5 7] +} {12} + +do_test num-4.1.1 { + sqlite4_num_to_text [sqlite4_num_sub 9 3] +} {6} +do_test num-4.1.2 { + sqlite4_num_to_text [sqlite4_num_sub 5 12] +} {-7} +do_test num-4.2.1 { + sqlite4_num_compare [sqlite4_num_sub 1 1] [sqlite4_num_sub -1 -1] +} {equal} + +do_test num-5.1.1 { + sqlite4_num_to_text [sqlite4_num_mul 9 8] +} {72} + +do_test num-6.1.1 { + sqlite4_num_to_text [sqlite4_num_div 6 5] +} {1.2} +do_test num-6.1.2 { + sqlite4_num_compare 2 [sqlite4_num_div 2 1] +} {equal} +do_test num-6.1.3 { + sqlite4_num_to_text [sqlite4_num_div 2 1] +} {2} +do_test num-6.1.4 { + sqlite4_num_to_text [sqlite4_num_div 22 10] +} {2.2} +finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -133,11 +133,11 @@ test_suite "src4" -prefix "" -description { } -files { simple.test simple2.test log3.test - lsm1.test lsm2.test + lsm1.test lsm2.test lsm3.test lsm4.test lsm5.test csr1.test ckpt1.test mc1.test fts5expr1.test fts5query1.test fts5rnd1.test fts5create.test fts5snippet.test @@ -147,11 +147,10 @@ autoindex1.test badutf.test between.test bigrow.test bind.test - bindxfer.test boundary1.test boundary4.test cast.test coalesce.test collate1.test collate2.test collate3.test collate4.test collate5.test Index: test/quote.test ================================================================== --- test/quote.test +++ test/quote.test @@ -32,13 +32,10 @@ catchsql {SELECT * FROM '@abc'} } {0 {5 hello}} do_test quote-1.2.2 { catchsql {SELECT * FROM [@abc]} ;# SqlServer compatibility } {0 {5 hello}} -do_test quote-1.2.3 { - catchsql {SELECT * FROM `@abc`} ;# MySQL compatibility -} {0 {5 hello}} do_test quote-1.3 { catchsql { SELECT '@abc'.'!pqr', '@abc'.'#xyz'+5 FROM '@abc' } } {0 {hello 10}} @@ -49,15 +46,10 @@ } {0 {!pqr 5}} do_test quote-1.3.2 { catchsql { SELECT "!pqr", "#xyz"+5 FROM '@abc' } -} {0 {hello 10}} -do_test quote-1.3.3 { - catchsql { - SELECT [!pqr], `#xyz`+5 FROM '@abc' - } } {0 {hello 10}} do_test quote-1.3.4 { set r [catch { execsql {SELECT '@abc'.'!pqr', '@abc'.'#xyz'+5 FROM '@abc'} } msg ] Index: test/select6.test ================================================================== --- test/select6.test +++ test/select6.test @@ -163,15 +163,15 @@ } } {x 3 y 2} do_test select6-3.2 { execsql { SELECT * FROM - (SELECT a.q, a.p, b.r + (SELECT a.q AS x, a.p, b.r FROM (SELECT count(*) as p , b as q FROM t2 GROUP BY q) AS a, (SELECT max(a) as r, b as s FROM t2 GROUP BY s) as b WHERE a.q=b.s ORDER BY a.q) - ORDER BY "a.q" + ORDER BY x } } {1 1 1 2 2 3 3 4 7 4 8 15 5 5 20} do_test select6-3.3 { execsql { SELECT a,b,a+b FROM (SELECT avg(x) as 'a', avg(y) as 'b' FROM t1) Index: test/simple.test ================================================================== --- test/simple.test +++ test/simple.test @@ -85,11 +85,11 @@ do_execsql_test 3.2 { SELECT * FROM sqlite_master } { table t1 t1 2 {CREATE TABLE t1(k PRIMARY KEY, v UNIQUE)} - index sqlite_autoindex_t1_2 t1 3 {} + index sqlite_t1_unique2 t1 3 {} } #explain { INSERT INTO t1 VALUES('one', '111') } #execsql { PRAGMA vdbe_trace = 1 } #execsql { PRAGMA kv_trace = 1 } @@ -125,11 +125,11 @@ do_execsql_test 5.3 { SELECT * FROM sqlite_master } { table t1 t1 3 {CREATE TABLE t1(k, v UNIQUE)} - index sqlite_autoindex_t1_1 t1 2 {} + index sqlite_t1_unique1 t1 2 {} index i1 t1 4 {CREATE INDEX i1 ON t1(v)} } do_execsql_test 5.3 { INSERT INTO t1 VALUES('one', '111') } {} do_execsql_test 5.4 { SELECT * FROM t1 } {one 111} @@ -1459,9 +1459,26 @@ db close sqlite4 db test.db execsql "INSERT INTO t1 VALUES('k$i', randstr(500,500))" } } {} -breakpoint db close + +#------------------------------------------------------------------------- +# Index on a blob. +# +reset_db + +do_execsql_test 75.1 { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(x'12345678', x'12345678'); +} + +do_execsql_test 75.2 { + SELECT count(*) FROM t1 WHERE a = x'12345678' +} 1 + +do_execsql_test 75.3 { + SELECT count(*) FROM t1 WHERE b = x'12345678' +} 1 finish_test Index: test/subquery.test ================================================================== --- test/subquery.test +++ test/subquery.test @@ -387,11 +387,11 @@ INSERT INTO t5 VALUES(1,11); INSERT INTO t5 VALUES(2,22); INSERT INTO t5 VALUES(3,33); INSERT INTO t5 VALUES(4,44); SELECT b FROM t5 WHERE a IN - (SELECT callcnt(y)+0 FROM t4 WHERE x="two") + (SELECT callcnt(y)+0 FROM t4 WHERE x='two') } } {22} do_test subquery-5.2 { # This is the key test. The subquery should have only run once. If # The double-quoted identifier "two" were causing the subquery to be Index: test/test_func.c ================================================================== --- test/test_func.c +++ test/test_func.c @@ -67,11 +67,11 @@ sqlite4_randomness(pEnv, n, zBuf); for(i=0; ipEnv, p); } /* @@ -258,11 +263,11 @@ sqlite4_result_error_nomem(pCtx); return; } pCounter->cnt = sqlite4_value_int(argv[0]); pCounter->pEnv = sqlite4_context_env(pCtx); - sqlite4_set_auxdata(pCtx, 0, pCounter, counterFree); + sqlite4_set_auxdata(pCtx, 0, pCounter, counterFree, 0); }else{ pCounter->cnt++; } sqlite4_result_int(pCtx, pCounter->cnt); } @@ -321,11 +326,11 @@ char *zErr; sqlite4_env *pEnv = sqlite4_context_env(pCtx); assert( pStmt==0 ); zErr = sqlite4_mprintf(pEnv, "sqlite4_prepare() error: %s", sqlite4_errmsg(db)); - sqlite4_result_text(pCtx, zErr, -1, SQLITE4_DYNAMIC); + sqlite4_result_text(pCtx, zErr, -1, SQLITE4_DYNAMIC, 0); sqlite4_result_error_code(pCtx, rc); } } @@ -374,11 +379,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text16be(pCtx, zOut, n/2, SQLITE4_DYNAMIC); + sqlite4_result_text16be(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); } } #endif /* @@ -401,11 +406,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text(pCtx, zOut, n/2, SQLITE4_DYNAMIC); + sqlite4_result_text(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); } } /* ** hex_to_utf16le(HEX) @@ -428,11 +433,11 @@ zOut = sqlite4_malloc(sqlite4_context_env(pCtx), n/2 ); if( zOut==0 ){ sqlite4_result_error_nomem(pCtx); }else{ testHexToBin(zIn, zOut); - sqlite4_result_text16le(pCtx, zOut, n/2, SQLITE4_DYNAMIC); + sqlite4_result_text16le(pCtx, zOut, n/2, SQLITE4_DYNAMIC, 0); } } #endif static int registerTestFunctions(sqlite4 *db){ Index: test/test_kv2.c ================================================================== --- test/test_kv2.c +++ test/test_kv2.c @@ -289,12 +289,25 @@ Tcl_WrongNumArgs(interp, 2, objv, ""); return TCL_ERROR; } if( kvwg.xFactory==0 ){ - sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_GET, "main", &kvwg.xFactory); - sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_PUSH, "main",newFileStorage); + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_GET,"main", &kvwg.xFactory); + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_PUSH,"main",newFileStorage); + } + return TCL_OK; +} + +static int kvwrap_uninstall_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 2, objv, ""); + return TCL_ERROR; + } + + if( kvwg.xFactory ){ + sqlite4_env_config(0, SQLITE4_ENVCONFIG_KVSTORE_POP,"main", &kvwg.xFactory); + kvwg.xFactory = 0; } return TCL_OK; } static int kvwrap_seek_cmd(Tcl_Interp *interp, int objc, Tcl_Obj **objv){ @@ -342,14 +355,15 @@ ){ struct Subcmd { const char *zCmd; int (*xCmd)(Tcl_Interp *, int, Tcl_Obj **); } aSub[] = { - { "install", kvwrap_install_cmd }, - { "step", kvwrap_step_cmd }, - { "seek", kvwrap_seek_cmd }, - { "reset", kvwrap_reset_cmd }, + { "install", kvwrap_install_cmd }, + { "step", kvwrap_step_cmd }, + { "seek", kvwrap_seek_cmd }, + { "reset", kvwrap_reset_cmd }, + { "uninstall", kvwrap_uninstall_cmd }, }; int iSub; int rc; rc = Tcl_GetIndexFromObjStruct( Index: test/test_lsm.c ================================================================== --- test/test_lsm.c +++ test/test_lsm.c @@ -7,11 +7,10 @@ ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* -** */ #include #include "lsm.h" #include "sqlite4.h" @@ -18,10 +17,291 @@ #include #include extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite4 **ppDb); extern const char *sqlite4TestErrorName(int); + +/************************************************************************* +*/ +#define ENCRYPTION_XOR_MASK 0x23b2bbb6 +static int testCompressEncBound(void *pCtx, int nSrc){ + return nSrc; +} +static int testCompressEncCompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + int i; + unsigned int *aIn = (unsigned int *)pOut; + unsigned int *aOut = (unsigned int *)pIn; + + assert( (nIn%4)==0 ); + for(i=0; i<(nIn/4); i++){ + aOut[i] = (aIn[i] ^ ENCRYPTION_XOR_MASK); + } + *pnOut = nIn; + + return LSM_OK; +} +static int testCompressEncUncompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + return testCompressEncCompress(pCtx, pOut, pnOut, pIn, nIn); +} +static void testCompressEncFree(void *pCtx){ + /* no-op */ +} +/* +** End of compression routines "encrypt". +*************************************************************************/ + +/************************************************************************* +*/ +static int testCompressRleBound(void *pCtx, int nSrc){ + return nSrc*2; +} +static int testCompressRleCompress( + void *pCtx, + char *pOut, int *pnOut, + const char *pIn, int nIn +){ + int iOut = 0; + int i; + char c; + int n; + + c = pIn[0]; + n = 1; + for(i=1; idb); ckfree((char *)p); } } + +static int testInfoLsm(Tcl_Interp *interp, lsm_db *db, Tcl_Obj *pObj){ + struct Lsminfo { + const char *zOpt; + int eOpt; + } aInfo[] = { + { "compression_id", LSM_INFO_COMPRESSION_ID }, + { 0, 0 } + }; + int rc; + int iOpt; + + rc = Tcl_GetIndexFromObjStruct( + interp, pObj, aInfo, sizeof(aInfo[0]), "option", 0, &iOpt + ); + if( rc==LSM_OK ){ + switch( aInfo[iOpt].eOpt ){ + case LSM_INFO_COMPRESSION_ID: { + unsigned int iCmpId = 0; + rc = lsm_info(db, LSM_INFO_COMPRESSION_ID, &iCmpId); + if( rc==LSM_OK ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj((Tcl_WideInt)iCmpId)); + }else{ + test_lsm_error(interp, "lsm_info", rc); + } + break; + } + } + } + + return rc; +} /* ** Usage: CSR sub-command ... */ static int test_lsm_cursor_cmd( @@ -519,10 +715,11 @@ /* 7 */ {"csr_open", 1, "CSR"}, /* 8 */ {"work", -1, "?NMERGE? NPAGE"}, /* 9 */ {"flush", 0, ""}, /* 10 */ {"config", 1, "LIST"}, /* 11 */ {"checkpoint", 0, ""}, + /* 12 */ {"info", 1, "OPTION"}, {0, 0, 0} }; int iCmd; int rc; TclLsm *p = (TclLsm *)clientData; @@ -621,11 +818,10 @@ case 8: assert( 0==strcmp(aCmd[8].zCmd, "work") ); { int nWork = 0; int nMerge = 1; int nWrite = 0; - int i; if( objc==3 ){ rc = Tcl_GetIntFromObj(interp, objv[2], &nWork); }else if( objc==4 ){ rc = Tcl_GetIntFromObj(interp, objv[2], &nMerge); @@ -647,17 +843,26 @@ rc = lsm_flush(p->db); return test_lsm_error(interp, "lsm_flush", rc); } case 10: assert( 0==strcmp(aCmd[10].zCmd, "config") ); { - return testConfigureLsm(interp, p->db, objv[2]); + Tcl_Obj **apObj; + int nObj; + if( TCL_OK==Tcl_ListObjGetElements(interp, objv[2], &nObj, &apObj) ){ + return testConfigureLsm(interp, p->db, nObj, apObj); + } + return TCL_ERROR; } case 11: assert( 0==strcmp(aCmd[11].zCmd, "checkpoint") ); { rc = lsm_checkpoint(p->db, 0); return test_lsm_error(interp, "lsm_checkpoint", rc); } + + case 12: assert( 0==strcmp(aCmd[12].zCmd, "info") ); { + return testInfoLsm(interp, p->db, objv[2]); + } default: assert( 0 ); } @@ -701,11 +906,16 @@ test_lsm_error(interp, "lsm_new", rc); return TCL_ERROR; } if( objc==4 ){ - rc = testConfigureLsm(interp, p->db, objv[3]); + Tcl_Obj **apObj; + int nObj; + rc = Tcl_ListObjGetElements(interp, objv[3], &nObj, &apObj); + if( rc==TCL_OK ){ + rc = testConfigureLsm(interp, p->db, nObj, apObj); + } if( rc!=TCL_OK ){ test_lsm_del((void *)p); return rc; } } @@ -729,11 +939,10 @@ const char *zName; Tcl_ObjCmdProc *xCmd; } aCmd[] = { { "sqlite4_lsm_work", test_sqlite4_lsm_work }, { "sqlite4_lsm_checkpoint", test_sqlite4_lsm_checkpoint }, - { "sqlite4_lsm_flush", test_sqlite4_lsm_flush }, { "sqlite4_lsm_info", test_sqlite4_lsm_info }, { "sqlite4_lsm_config", test_sqlite4_lsm_config }, { "lsm_open", test_lsm_open }, }; int i; Index: test/test_main.c ================================================================== --- test/test_main.c +++ test/test_main.c @@ -565,35 +565,10 @@ Tcl_AppendResult(interp, zStr, 0); return TCL_OK; } - -/* -** Usage: sqlite4_last_insert_rowid DB -** -** Returns the integer ROWID of the most recent insert. -*/ -static int test_last_rowid( - void *NotUsed, - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int argc, /* Number of arguments */ - char **argv /* Text of each argument */ -){ - sqlite4 *db; - char zBuf[30]; - - if( argc!=2 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB\"", 0); - return TCL_ERROR; - } - if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; - sprintf(zBuf, "%lld", sqlite4_last_insert_rowid(db)); - Tcl_AppendResult(interp, zBuf, 0); - return SQLITE4_OK; -} - /* ** Usage: sqlite4_key DB KEY ** ** Set the codec key. */ @@ -684,11 +659,11 @@ int i; for(i=0; imutex); pVal = sqlite4ValueNew(db); - sqlite4ValueSetStr(pVal, -1, "x_sqlite_exec", SQLITE4_UTF8, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, -1, "x_sqlite_exec", SQLITE4_UTF8, + SQLITE4_STATIC, 0); zUtf16 = sqlite4ValueText(pVal, SQLITE4_UTF16NATIVE); if( db->mallocFailed ){ rc = SQLITE4_NOMEM; }else{ rc = sqlite4_create_function16(db, zUtf16, @@ -1034,24 +1010,10 @@ sqlite4_result_int(context, p ? p->n : 0); } } } -#ifndef SQLITE4_OMIT_DEPRECATED -static void legacyCountStep( - sqlite4_context *context, - int argc, - sqlite4_value **argv -){ - /* no-op */ -} - -static void legacyCountFinalize(sqlite4_context *context){ - sqlite4_result_int(context, sqlite4_aggregate_count(context)); -} -#endif - /* ** Usage: sqlite4_create_aggregate DB ** ** Call the sqlite4_create_function API on the given database in order ** to create a function named "x_count". This function is similar @@ -1087,17 +1049,10 @@ t1CountStep,t1CountFinalize); if( rc==SQLITE4_OK ){ rc = sqlite4_create_function(db, "x_count", 1, SQLITE4_UTF8, 0, 0, t1CountStep,t1CountFinalize); } -#ifndef SQLITE4_OMIT_DEPRECATED - if( rc==SQLITE4_OK ){ - rc = sqlite4_create_function(db, "legacy_count", 0, SQLITE4_ANY, 0, 0, - legacyCountStep, legacyCountFinalize - ); - } -#endif if( sqlite4TestErrCode(interp, db, rc) ) return TCL_ERROR; Tcl_SetResult(interp, (char *)t1ErrorName(rc), 0); return TCL_OK; } @@ -1712,11 +1667,11 @@ sqlite4_result_int(context, sqlite4_value_int(argv[1])); }else if( sqlite4_stricmp(zArg0,"int64")==0 ){ sqlite4_result_int64(context, sqlite4_value_int64(argv[1])); }else if( sqlite4_stricmp(zArg0,"string")==0 ){ sqlite4_result_text(context, (char*)sqlite4_value_text(argv[1]), -1, - SQLITE4_TRANSIENT); + SQLITE4_TRANSIENT, 0); }else if( sqlite4_stricmp(zArg0,"double")==0 ){ sqlite4_result_double(context, sqlite4_value_double(argv[1])); }else if( sqlite4_stricmp(zArg0,"null")==0 ){ sqlite4_result_null(context); }else if( sqlite4_stricmp(zArg0,"value")==0 ){ @@ -1988,60 +1943,10 @@ } */ return TCL_OK; } -/* -** Usage: sqlite4_expired STMT -** -** Return TRUE if a recompilation of the statement is recommended. -*/ -static int test_expired( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ -#ifndef SQLITE4_OMIT_DEPRECATED - sqlite4_stmt *pStmt; - if( objc!=2 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " ", 0); - return TCL_ERROR; - } - if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; - Tcl_SetObjResult(interp, Tcl_NewBooleanObj(sqlite4_expired(pStmt))); -#endif - return TCL_OK; -} - -/* -** Usage: sqlite4_transfer_bindings FROMSTMT TOSTMT -** -** Transfer all bindings from FROMSTMT over to TOSTMT -*/ -static int test_transfer_bind( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ -#ifndef SQLITE4_OMIT_DEPRECATED - sqlite4_stmt *pStmt1, *pStmt2; - if( objc!=3 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " FROM-STMT TO-STMT", 0); - return TCL_ERROR; - } - if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt1)) return TCL_ERROR; - if( getStmtPointer(interp, Tcl_GetString(objv[2]), &pStmt2)) return TCL_ERROR; - Tcl_SetObjResult(interp, - Tcl_NewIntObj(sqlite4_transfer_bindings(pStmt1,pStmt2))); -#endif - return TCL_OK; -} - /* ** Usage: sqlite4_changes DB ** ** Return the number of changes made to the database by the last SQL ** execution. @@ -2098,18 +2003,18 @@ if( getStmtPointer(interp, argv[1], &pStmt) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &idx) ) return TCL_ERROR; if( strcmp(argv[4],"null")==0 ){ rc = sqlite4_bind_null(pStmt, idx); }else if( strcmp(argv[4],"static")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, -1, 0); + rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, -1, 0, 0); }else if( strcmp(argv[4],"static-nbytes")==0 ){ rc = sqlite4_bind_text(pStmt, idx, sqlite_static_bind_value, - sqlite_static_bind_nbyte, 0); + sqlite_static_bind_nbyte, 0, 0); }else if( strcmp(argv[4],"normal")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, argv[3], -1, SQLITE4_TRANSIENT); + rc = sqlite4_bind_text(pStmt, idx, argv[3], -1, SQLITE4_TRANSIENT, 0); }else if( strcmp(argv[4],"blob10")==0 ){ - rc = sqlite4_bind_text(pStmt, idx, "abc\000xyz\000pq", 10, SQLITE4_STATIC); + rc = sqlite4_bind_text(pStmt, idx, "abc\000xyz\000pq", 10,SQLITE4_STATIC,0); }else{ Tcl_AppendResult(interp, "4th argument should be " "\"null\" or \"static\" or \"normal\"", 0); return TCL_ERROR; } @@ -2185,15 +2090,15 @@ } sqlite4BeginBenignMalloc(pEnv); pVal = sqlite4ValueNew(0); if( pVal ){ - sqlite4ValueSetStr(pVal, nA, zA, encin, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, nA, zA, encin, SQLITE4_STATIC, 0); n = sqlite4_value_bytes(pVal); Tcl_ListObjAppendElement(i,pX, Tcl_NewStringObj((char*)sqlite4_value_text(pVal),n)); - sqlite4ValueSetStr(pVal, nB, zB, encin, SQLITE4_STATIC); + sqlite4ValueSetStr(pVal, nB, zB, encin, SQLITE4_STATIC, 0); n = sqlite4_value_bytes(pVal); Tcl_ListObjAppendElement(i,pX, Tcl_NewStringObj((char*)sqlite4_value_text(pVal),n)); sqlite4ValueFree(pVal); } @@ -2387,16 +2292,17 @@ Tcl_ListObjAppendElement(interp, pX, Tcl_NewStringObj("UTF-8", -1)); Tcl_ListObjAppendElement(interp, pX, Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); - sqlite4_result_text(pCtx, Tcl_GetStringResult(interp), -1, SQLITE4_TRANSIENT); + sqlite4_result_text(pCtx, Tcl_GetStringResult(interp), -1, + SQLITE4_TRANSIENT, 0); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); sqlite4_result_text16be(pCtx, sqlite4_value_text16be(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4ValueFree(pVal); } static void test_function_utf16le( sqlite4_context *pCtx, int nArg, @@ -2413,12 +2319,13 @@ Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC); - sqlite4_result_text(pCtx,(char*)sqlite4_value_text(pVal),-1,SQLITE4_TRANSIENT); + SQLITE4_UTF8, SQLITE4_STATIC, 0); + sqlite4_result_text(pCtx, (char*)sqlite4_value_text(pVal), -1, + SQLITE4_TRANSIENT, 0); sqlite4ValueFree(pVal); } static void test_function_utf16be( sqlite4_context *pCtx, int nArg, @@ -2435,17 +2342,17 @@ Tcl_NewStringObj((char*)sqlite4_value_text(argv[0]), -1)); Tcl_EvalObjEx(interp, pX, 0); Tcl_DecrRefCount(pX); pVal = sqlite4ValueNew(0); sqlite4ValueSetStr(pVal, -1, Tcl_GetStringResult(interp), - SQLITE4_UTF8, SQLITE4_STATIC); + SQLITE4_UTF8, SQLITE4_STATIC, 0); sqlite4_result_text16(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4_result_text16be(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4_result_text16le(pCtx, sqlite4_value_text16le(pVal), - -1, SQLITE4_TRANSIENT); + -1, SQLITE4_TRANSIENT, 0); sqlite4ValueFree(pVal); } #endif /* SQLITE4_OMIT_UTF16 */ static int test_function( void * clientData, @@ -2532,46 +2439,10 @@ char **argv /* Text of each argument */ ){ return TCL_OK; /* Do nothing */ } -/* -** Usage: sqlite4_bind_zeroblob STMT IDX N -** -** Test the sqlite4_bind_zeroblob interface. STMT is a prepared statement. -** IDX is the index of a wildcard in the prepared statement. This command -** binds a N-byte zero-filled BLOB to the wildcard. -*/ -static int test_bind_zeroblob( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite4_stmt *pStmt; - int idx; - int n; - int rc; - - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "STMT IDX N"); - return TCL_ERROR; - } - - if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; - if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; - if( Tcl_GetIntFromObj(interp, objv[3], &n) ) return TCL_ERROR; - - rc = sqlite4_bind_zeroblob(pStmt, idx, n); - if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; - if( rc!=SQLITE4_OK ){ - return TCL_ERROR; - } - - return TCL_OK; -} - /* ** Usage: sqlite4_bind_int STMT N VALUE ** ** Test the sqlite4_bind_int interface. STMT is a prepared statement. ** N is the index of a wildcard in the prepared statement. This command @@ -2786,11 +2657,11 @@ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; value = (char*)Tcl_GetByteArrayFromObj(objv[3], &bytes); if( Tcl_GetIntFromObj(interp, objv[4], &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_text(pStmt, idx, value, bytes, SQLITE4_TRANSIENT); + rc = sqlite4_bind_text(pStmt, idx, value, bytes, SQLITE4_TRANSIENT, 0); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ Tcl_AppendResult(interp, sqlite4TestErrorName(rc), 0); return TCL_ERROR; } @@ -2834,11 +2705,11 @@ if( getStmtPointer(interp, Tcl_GetString(oStmt), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, oN, &idx) ) return TCL_ERROR; value = (char*)Tcl_GetByteArrayFromObj(oString, 0); if( Tcl_GetIntFromObj(interp, oBytes, &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_text16(pStmt, idx, (void *)value, bytes, xDel); + rc = sqlite4_bind_text16(pStmt, idx, (void *)value, bytes, xDel, 0); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ Tcl_AppendResult(interp, sqlite4TestErrorName(rc), 0); return TCL_ERROR; } @@ -2881,11 +2752,11 @@ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; value = Tcl_GetString(objv[3]); if( Tcl_GetIntFromObj(interp, objv[4], &bytes) ) return TCL_ERROR; - rc = sqlite4_bind_blob(pStmt, idx, value, bytes, xDestructor); + rc = sqlite4_bind_blob(pStmt, idx, value, bytes, xDestructor, 0); if( sqlite4TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE4_OK ){ return TCL_ERROR; } @@ -3545,28 +3416,10 @@ if( Tcl_GetIntFromObj(interp, objv[2], &col) ) return TCL_ERROR; zRet = xFunc(pStmt, col); if( zRet ){ Tcl_SetResult(interp, (char *)zRet, 0); } - return TCL_OK; -} - -static int test_global_recover( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ -#ifndef SQLITE4_OMIT_DEPRECATED - int rc; - if( objc!=1 ){ - Tcl_WrongNumArgs(interp, 1, objv, ""); - return TCL_ERROR; - } - rc = sqlite4_global_recover(); - Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); -#endif return TCL_OK; } /* ** Usage: sqlite4_column_text STMT column @@ -4347,10 +4200,209 @@ return TCL_ERROR; } sqlite4_test_control(SQLITE4_TESTCTRL_OPTIMIZATIONS, db, mask); return TCL_OK; } + +#define NUM_FORMAT "sign:%d approx:%d e:%d m:%lld" + +/* Append a return value representing a sqlite4_num. +*/ +static void append_num_result( Tcl_Interp *interp, sqlite4_num A ){ + char buf[100]; + sprintf( buf, NUM_FORMAT, A.sign, A.approx, A.e, A.m ); + Tcl_AppendResult(interp, buf, 0); +} + +/* Convert a string either representing a sqlite4_num (listing its fields as +** returned by append_num_result) or that can be parsed as one. Invalid +** strings become NaN. +*/ +static sqlite4_num test_parse_num( char *arg ){ + sqlite4_num A; + int sign, approx, e; + if( sscanf( arg, NUM_FORMAT, &sign, &approx, &e, &A.m)==4 ){ + A.sign = sign; + A.approx = approx; + A.e = e; + return A; + } else { + return sqlite4_num_from_text(arg, -1, 0); + } +} + +/* Convert return values of sqlite4_num to strings that will be readable in +** the tests. +*/ +static char *describe_num_comparison( int code ){ + switch( code ){ + case 0: return "incomparable"; + case 1: return "lesser"; + case 2: return "equal"; + case 3: return "greater"; + default: return "error"; + } +} + +/* Compare two numbers A and B. Returns "incomparable", "lesser", "equal", +** "greater", or "error". +*/ +static int test_num_compare( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + sqlite4_num A, B; + int cmp; + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM NUM\"", 0); + return TCL_ERROR; + } + + A = test_parse_num( argv[1] ); + B = test_parse_num( argv[2] ); + cmp = sqlite4_num_compare(A, B); + Tcl_AppendResult( interp, describe_num_comparison( cmp ), 0); + return TCL_OK; +} + +/* Create a sqlite4_num from a string. The optional second argument specifies +** how many bytes may be read. +*/ +static int test_num_from_text( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + sqlite4_num A; + int len; + if( argc!=2 && argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " STRING\" or \"", argv[0], " STRING INTEGER\"", 0); + return TCL_ERROR; + } + + if( argc==3 ){ + if ( Tcl_GetInt(interp, argv[2], &len) ) return TCL_ERROR; + }else{ + len = -1; + } + + A = sqlite4_num_from_text( argv[1], len, 0 ); + append_num_result(interp, A); + return TCL_OK; +} + +static int test_num_to_text( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + char text[30]; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM\"", 0); + return TCL_ERROR; + } + sqlite4_num_to_text( test_parse_num( argv[1] ), text ); + Tcl_AppendResult( interp, text, 0 ); + return TCL_OK; +} + +static int test_num_binary_op( + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv, /* Text of each argument */ + sqlite4_num (*op) (sqlite4_num, sqlite4_num) +){ + sqlite4_num A, B, R; + if( argc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM NUM\"", 0); + return TCL_ERROR; + } + A = test_parse_num(argv[1]); + B = test_parse_num(argv[2]); + R = op(A, B); + append_num_result(interp, R); + return TCL_OK; +} + +static int test_num_add( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_add ); +} + +static int test_num_sub( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_sub ); +} + +static int test_num_mul( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_mul ); +} + +static int test_num_div( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_binary_op( interp, argc, argv, sqlite4_num_div ); +} + +static int test_num_predicate( + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv, /* Text of each argument */ + int (*pred) (sqlite4_num) +){ + sqlite4_num A; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NUM\"", 0); + return TCL_ERROR; + } + A = test_parse_num(argv[1]); + Tcl_AppendResult(interp, pred(A) ? "true" : "false", 0); + return TCL_OK; +} + +static int test_num_isinf( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_predicate( interp, argc, argv, sqlite4_num_isinf ); +} + +static int test_num_isnan( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + return test_num_predicate( interp, argc, argv, sqlite4_num_isnan ); +} /* ** Register commands with the TCL interpreter. */ int Sqlitetest1_Init(Tcl_Interp *interp){ @@ -4379,11 +4431,10 @@ { "sqlite4_mprintf_scaled", (Tcl_CmdProc*)sqlite4_mprintf_scaled }, { "sqlite4_mprintf_hexdouble", (Tcl_CmdProc*)sqlite4_mprintf_hexdouble}, { "sqlite4_mprintf_z_test", (Tcl_CmdProc*)test_mprintf_z }, { "sqlite4_mprintf_n_test", (Tcl_CmdProc*)test_mprintf_n }, { "sqlite4_snprintf_int", (Tcl_CmdProc*)test_snprintf_int }, - { "sqlite4_last_insert_rowid", (Tcl_CmdProc*)test_last_rowid }, { "sqlite4_exec_printf", (Tcl_CmdProc*)test_exec_printf }, { "sqlite4_exec_hex", (Tcl_CmdProc*)test_exec_hex }, { "sqlite4_exec", (Tcl_CmdProc*)test_exec }, { "sqlite4_exec_nr", (Tcl_CmdProc*)test_exec_nr }, { "sqlite4_close", (Tcl_CmdProc*)sqlite_test_close }, @@ -4400,20 +4451,28 @@ { "sqlite_delete_function", (Tcl_CmdProc*)delete_function }, { "sqlite_delete_collation", (Tcl_CmdProc*)delete_collation }, { "sqlite4_get_autocommit", (Tcl_CmdProc*)get_autocommit }, { "sqlite4_stack_used", (Tcl_CmdProc*)test_stack_used }, { "printf", (Tcl_CmdProc*)test_printf }, - { "sqlite4IoTrace", (Tcl_CmdProc*)test_io_trace }, + { "sqlite4IoTrace", (Tcl_CmdProc*)test_io_trace }, + { "sqlite4_num_compare", (Tcl_CmdProc*)test_num_compare }, + { "sqlite4_num_from_text", (Tcl_CmdProc*)test_num_from_text }, + { "sqlite4_num_to_text", (Tcl_CmdProc*)test_num_to_text }, + { "sqlite4_num_add", (Tcl_CmdProc*)test_num_add }, + { "sqlite4_num_sub", (Tcl_CmdProc*)test_num_sub }, + { "sqlite4_num_mul", (Tcl_CmdProc*)test_num_mul }, + { "sqlite4_num_div", (Tcl_CmdProc*)test_num_div }, + { "sqlite4_num_isinf", (Tcl_CmdProc*)test_num_isinf }, + { "sqlite4_num_isnan", (Tcl_CmdProc*)test_num_isnan }, }; static struct { char *zName; Tcl_ObjCmdProc *xProc; void *clientData; } aObjCmd[] = { { "sqlite4_connection_pointer", get_sqlite_pointer, 0 }, { "sqlite4_bind_int", test_bind_int, 0 }, - { "sqlite4_bind_zeroblob", test_bind_zeroblob, 0 }, { "sqlite4_bind_int64", test_bind_int64, 0 }, { "sqlite4_bind_double", test_bind_double, 0 }, { "sqlite4_bind_null", test_bind_null ,0 }, { "sqlite4_bind_text", test_bind_text ,0 }, { "sqlite4_bind_text16", test_bind_text16 ,0 }, @@ -4432,12 +4491,10 @@ { "sqlite4_prepare", test_prepare ,0 }, { "sqlite4_prepare_tkt3134", test_prepare_tkt3134, 0}, { "sqlite4_finalize", test_finalize ,0 }, { "sqlite4_stmt_status", test_stmt_status ,0 }, { "sqlite4_reset", test_reset ,0 }, - { "sqlite4_expired", test_expired ,0 }, - { "sqlite4_transfer_bindings", test_transfer_bind ,0 }, { "sqlite4_changes", test_changes ,0 }, { "sqlite4_step", test_step ,0 }, { "sqlite4_sql", test_sql ,0 }, { "sqlite4_next_stmt", test_next_stmt ,0 }, { "sqlite4_stmt_readonly", test_stmt_readonly ,0 }, @@ -4488,11 +4545,10 @@ {"sqlite4_column_table_name16", test_stmt_utf16, (void*)sqlite4_column_table_name16}, {"sqlite4_column_origin_name16", test_stmt_utf16, (void*)sqlite4_column_origin_name16}, #endif #endif { "sqlite4_create_collation", test_create_collation, 0 }, - { "sqlite4_global_recover", test_global_recover, 0 }, { "working_64bit_int", working_64bit_int, 0 }, { "sqlite4_create_function_v2", test_create_function_v2, 0 }, /* Functions from os.h */ #ifndef SQLITE4_OMIT_UTF16 @@ -4521,11 +4577,10 @@ extern int sqlite4_os_type; #endif #ifdef SQLITE4_DEBUG extern int sqlite4WhereTrace; extern int sqlite4OSTrace; - extern int sqlite4VdbeAddopTrace; #endif #ifdef SQLITE4_TEST extern char sqlite4_query_plan[]; static char *query_plan = sqlite4_query_plan; #ifdef SQLITE4_ENABLE_FTS3 @@ -4576,12 +4631,10 @@ #ifdef SQLITE4_TEST Tcl_LinkVar(interp, "sqlite_query_plan", (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); #endif #ifdef SQLITE4_DEBUG - Tcl_LinkVar(interp, "sqlite_addop_trace", - (char*)&sqlite4VdbeAddopTrace, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite_where_trace", (char*)&sqlite4WhereTrace, TCL_LINK_INT); #endif Tcl_LinkVar(interp, "sqlite_static_bind_value", (char*)&sqlite_static_bind_value, TCL_LINK_STRING); Index: test/test_utf.c ================================================================== --- test/test_utf.c +++ test/test_utf.c @@ -114,11 +114,13 @@ return SQLITE4_UTF16NATIVE; } return pEnc->enc; } -static void freeStr(void *pStr){ sqlite4_free(0, pStr); } +static void freeStr(void *pEnv, void *pStr){ + sqlite4_free((sqlite4_env*)pEnv, pStr); +} /* ** Usage: test_translate ?? ** */ @@ -132,11 +134,11 @@ u8 enc_to; sqlite4_value *pVal; char *z; int len; - void (*xDel)(void *p) = SQLITE4_STATIC; + void (*xDel)(void*,void*) = SQLITE4_STATIC; if( objc!=4 && objc!=5 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetStringFromObj(objv[0], 0), " ", 0 @@ -157,19 +159,19 @@ if( enc_from==SQLITE4_UTF8 ){ z = Tcl_GetString(objv[1]); if( objc==5 ){ z = sqlite4_mprintf(0, "%s", z); } - sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel); + sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel, 0); }else{ z = (char*)Tcl_GetByteArrayFromObj(objv[1], &len); if( objc==5 ){ char *zTmp = z; z = sqlite4_malloc(0, len); memcpy(z, zTmp, len); } - sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel); + sqlite4ValueSetStr(pVal, -1, z, enc_from, xDel, 0); } z = (char *)sqlite4ValueText(pVal, enc_to); len = sqlite4ValueBytes(pVal, enc_to) + (enc_to==SQLITE4_UTF8?1:2); Tcl_SetObjResult(interp, Tcl_NewByteArrayObj((u8*)z, len)); Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -760,10 +760,11 @@ puts "******************************************************************" } if {$::cmdlinearg(binarylog)} { vfslog finalize binarylog } + kvwrap uninstall if {[lindex [sqlite4_env_status SQLITE4_ENVSTATUS_MALLOC_COUNT 0] 1]>0 || [sqlite4_memory_used]>0} { puts "Unfreed memory: [sqlite4_memory_used] bytes in\ [lindex [sqlite4_env_status SQLITE4_ENVSTATUS_MALLOC_COUNT 0] 1] allocations" incr nErr @@ -1563,11 +1564,15 @@ # the next rowid allocated for each table with an IPK will be as expected # by SQLite 3 tests. # proc optimize_db {} { #catch { - sqlite4_lsm_flush db main + set af [sqlite4_lsm_config db main autoflush] + sqlite4_lsm_config db main autoflush 0 + db eval { BEGIN EXCLUSIVE; COMMIT; } + sqlite4_lsm_config db main autoflush $af + sqlite4_lsm_work db main -nmerge 1 -npage 100000 sqlite4_lsm_checkpoint db main #} return "" } Index: test/tkt-bd484a090c.test ================================================================== --- test/tkt-bd484a090c.test +++ test/tkt-bd484a090c.test @@ -22,10 +22,11 @@ } {0} do_test 1.2 { lindex [catchsql { SELECT datetime('now', 'utc') }] 0 } {0} +if 0 { sqlite4_test_control SQLITE4_TESTCTRL_LOCALTIME_FAULT 1 do_test 2.1 { catchsql { SELECT datetime('now', 'localtime') } } {1 {local time unavailable}} @@ -32,7 +33,8 @@ do_test 2.2 { catchsql { SELECT datetime('now', 'utc') } } {1 {local time unavailable}} sqlite4_test_control SQLITE4_TESTCTRL_LOCALTIME_FAULT 0 +} finish_test Index: test/tkt3442.test ================================================================== --- test/tkt3442.test +++ test/tkt3442.test @@ -47,13 +47,10 @@ # and verify that the query plan is the same. # ifcapable explain { do_test tkt3442-1.2 { EQP { SELECT node FROM listhash WHERE id='5000' LIMIT 1; } - } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} - do_test tkt3442-1.3 { - EQP { SELECT node FROM listhash WHERE id="5000" LIMIT 1; } } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} } # Some extra tests testing other permutations of 5000. Index: test/tkt35xx.test ================================================================== --- test/tkt35xx.test +++ test/tkt35xx.test @@ -31,21 +31,21 @@ do_test tkt35xx-1.1 { execsql { PRAGMA auto_vacuum = 0; CREATE TABLE t1(a,b,c); CREATE INDEX i1 ON t1(c); - INSERT INTO t1 VALUES(0, 0, zeroblob(676)); - INSERT INTO t1 VALUES(1, 1, zeroblob(676)); + INSERT INTO t1 VALUES(0, 0, randomblob(676)); + INSERT INTO t1 VALUES(1, 1, randomblob(676)); DELETE FROM t1; BEGIN; - INSERT INTO t1 VALUES(0, 0, zeroblob(676)); - INSERT INTO t1 VALUES(1, 1, zeroblob(676)); + INSERT INTO t1 VALUES(0, 0, randomblob(676)); + INSERT INTO t1 VALUES(1, 1, randomblob(676)); ROLLBACK; - INSERT INTO t1 VALUES(0, 0, zeroblob(676)); + INSERT INTO t1 VALUES(0, 0, randomblob(676)); } execsql { - INSERT INTO t1 VALUES(1, 1, zeroblob(676)); + INSERT INTO t1 VALUES(1, 1, randomblob(676)); } } {} # Trigger the problem using statement rollback. # Index: test/tkt3761.test ================================================================== --- test/tkt3761.test +++ test/tkt3761.test @@ -14,14 +14,19 @@ # # $Id: tkt3761.test,v 1.1 2009/03/31 02:54:40 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl + +proc zeroblob {n} { + string repeat [binary format c 0] $n +} do_test tkt3761-1.1 { db close sqlite4 db :memory: + db func zeroblob zeroblob db eval { PRAGMA auto_vacuum=INCREMENTAL; CREATE TABLE t1(x); INSERT INTO t1 VALUES(zeroblob(900)); INSERT INTO t1 VALUES(zeroblob(900)); Index: test/tkt3841.test ================================================================== --- test/tkt3841.test +++ test/tkt3841.test @@ -25,18 +25,18 @@ do_test tkt3841.1 { execsql { CREATE TABLE table2 (key TEXT, x TEXT); CREATE TABLE list (key TEXT, value TEXT); - INSERT INTO table2 VALUES ("a", "alist"); - INSERT INTO table2 VALUES ("b", "blist"); - INSERT INTO list VALUES ("a", 1); - INSERT INTO list VALUES ("a", 2); - INSERT INTO list VALUES ("a", 3); - INSERT INTO list VALUES ("b", 4); - INSERT INTO list VALUES ("b", 5); - INSERT INTO list VALUES ("b", 6); + INSERT INTO table2 VALUES ('a', 'alist'); + INSERT INTO table2 VALUES ('b', 'blist'); + INSERT INTO list VALUES ('a', 1); + INSERT INTO list VALUES ('a', 2); + INSERT INTO list VALUES ('a', 3); + INSERT INTO list VALUES ('b', 4); + INSERT INTO list VALUES ('b', 5); + INSERT INTO list VALUES ('b', 6); SELECT table2.x, (SELECT group_concat(list.value) FROM list Index: test/tkt3918.test ================================================================== --- test/tkt3918.test +++ test/tkt3918.test @@ -22,12 +22,12 @@ } } {} do_test tkt3918.2 { execsql { INSERT INTO t1 VALUES(1, randstr(1000,1000)); - INSERT INTO t1 VALUES(2, zeroblob(248*1020 + 100)); - INSERT INTO t1 VALUES(3, zeroblob(2*1020 + 100)); + INSERT INTO t1 VALUES(2, randomblob(248*1020 + 100)); + INSERT INTO t1 VALUES(3, randomblob(2*1020 + 100)); } } {} # This set of statements sets up the free list so that the # first free-list trunk page contains only a single leaf. Index: test/where8.test ================================================================== --- test/where8.test +++ test/where8.test @@ -328,11 +328,11 @@ INSERT INTO t3 VALUES(1339.360726, 2847564.823, 'over'); INSERT INTO t3 VALUES('villages', 'their', 'have'); INSERT INTO t3 VALUES('unalike', 'remarkably', 'in'); INSERT INTO t3 VALUES('and', 8979323846, 'and'); INSERT INTO t3 VALUES(NULL, 1415926535, 'an'); - INSERT INTO t3 VALUES(271.2019091, 8628034825, 0.4811174502); + INSERT INTO t3 VALUES(271.25, 8628034825, 0.4811174502); INSERT INTO t3 VALUES('all', 3421170679, 'the'); INSERT INTO t3 VALUES('Not', 'and', 1415926535); INSERT INTO t3 VALUES('of', 'other', 'light'); INSERT INTO t3 VALUES(NULL, 'towering', 'Not'); INSERT INTO t3 VALUES(346.0348610, NULL, 'other'); DELETED test/zeroblob.test Index: test/zeroblob.test ================================================================== --- test/zeroblob.test +++ /dev/null @@ -1,259 +0,0 @@ -# 2007 May 02 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing of the zero-filled blob functionality -# including the sqlite4_bind_zeroblob(), sqlite4_result_zeroblob(), -# and the built-in zeroblob() SQL function. -# -# $Id: zeroblob.test,v 1.14 2009/07/14 02:33:02 drh Exp $ - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -ifcapable !incrblob { - finish_test - return -} - -# When zeroblob() is used for the last field of a column, then the -# content of the zeroblob is never instantiated on the VDBE stack. -# But it does get inserted into the database correctly. -# -db eval {PRAGMA cache_size=10} -sqlite4_memory_highwater 1 -unset -nocomplain memused -set memused [sqlite4_memory_used] -do_test zeroblob-1.1 { - execsql { - CREATE TABLE t1(a,b,c,d); - } - set ::sqlite4_max_blobsize 0 - execsql { - INSERT INTO t1 VALUES(2,3,4,zeroblob(1000000)); - } - set ::sqlite4_max_blobsize -} {10} -do_test zeroblob-1.1.1 { - expr {[sqlite4_memory_highwater]<$::memused+25000} -} {1} -do_test zeroblob-1.2 { - execsql { - SELECT length(d) FROM t1 - } -} {1000000} - -# If a non-NULL column follows the zeroblob, then the content of -# the zeroblob must be instantiated. -# -do_test zeroblob-1.3 { - set ::sqlite4_max_blobsize 0 - execsql { - INSERT INTO t1 VALUES(3,4,zeroblob(10000),5); - } - set ::sqlite4_max_blobsize -} {10010} -do_test zeroblob-1.4 { - execsql { - SELECT length(c), length(d) FROM t1 - } -} {1 1000000 10000 1} - -# Multiple zeroblobs can appear at the end of record. No instantiation -# of the blob content occurs on the stack. -# -do_test zeroblob-1.5 { - set ::sqlite4_max_blobsize 0 - execsql { - INSERT INTO t1 VALUES(4,5,zeroblob(10000),zeroblob(10000)); - } - set ::sqlite4_max_blobsize -} {11} -do_test zeroblob-1.6 { - execsql { - SELECT length(c), length(d) FROM t1 - } -} {1 1000000 10000 1 10000 10000} - -# NULLs can follow the zeroblob() or be intermixed with zeroblobs and -# no instantiation of the zeroblobs occurs on the stack. -# -do_test zeroblob-1.7 { - set ::sqlite4_max_blobsize 0 - execsql { - INSERT INTO t1 VALUES(5,zeroblob(10000),NULL,zeroblob(10000)); - } - set ::sqlite4_max_blobsize -} {10} -do_test zeroblob-1.8 { - execsql { - SELECT length(b), length(d) FROM t1 WHERE a=5 - } -} {10000 10000} - -# Comparisons against zeroblobs work. -# -do_test zeroblob-2.1 { - execsql { - SELECT a FROM t1 WHERE b=zeroblob(10000) - } -} {5} - -# Comparisons against zeroblobs work even when indexed. -# -do_test zeroblob-2.2 { - execsql { - CREATE INDEX i1_1 ON t1(b); - SELECT a FROM t1 WHERE b=zeroblob(10000); - } -} {5} - -# DISTINCT works for zeroblobs -# -ifcapable bloblit&&subquery&&compound { - do_test zeroblob-3.1 { - execsql { - SELECT count(DISTINCT a) FROM ( - SELECT x'00000000000000000000' AS a - UNION ALL - SELECT zeroblob(10) AS a - ) - } - } {1} -} - -# Concatentation works with zeroblob -# -ifcapable bloblit { - do_test zeroblob-4.1 { - execsql { - SELECT hex(zeroblob(2) || x'61') - } - } {000061} -} - -# Check various CAST(...) operations on zeroblob. -# -do_test zeroblob-5.1 { - execsql { - SELECT CAST (zeroblob(100) AS REAL); - } -} {0.0} -do_test zeroblob-5.2 { - execsql { - SELECT CAST (zeroblob(100) AS INTEGER); - } -} {0} -do_test zeroblob-5.3 { - execsql { - SELECT CAST (zeroblob(100) AS TEXT); - } -} {{}} -do_test zeroblob-5.4 { - execsql { - SELECT CAST(zeroblob(100) AS BLOB); - } -} [execsql {SELECT zeroblob(100)}] - - -# Check for malicious use of zeroblob. Make sure nothing crashes. -# -do_test zeroblob-6.1.1 { - execsql {select zeroblob(-1)} -} {{}} -do_test zeroblob-6.1.2 { - execsql {select zeroblob(-10)} -} {{}} -do_test zeroblob-6.1.3 { - execsql {select zeroblob(-100)} -} {{}} -do_test zeroblob-6.2 { - execsql {select length(zeroblob(-1))} -} {0} -do_test zeroblob-6.3 { - execsql {select zeroblob(-1)|1} -} {1} -do_test zeroblob-6.4 { - catchsql {select length(zeroblob(2147483648))} -} {1 {string or blob too big}} -do_test zeroblob-6.5 { - catchsql {select zeroblob(2147483648)} -} {1 {string or blob too big}} -do_test zeroblob-6.6 { - execsql {select hex(zeroblob(-1))} -} {{}} -do_test zeroblob-6.7 { - execsql {select typeof(zeroblob(-1))} -} {blob} - -# Test bind_zeroblob() -# -sqlite4_memory_highwater 1 -unset -nocomplain memused -set memused [sqlite4_memory_used] -do_test zeroblob-7.1 { - set ::STMT [sqlite4_prepare $::DB "SELECT length(?)" -1 DUMMY] - set ::sqlite4_max_blobsize 0 - sqlite4_bind_zeroblob $::STMT 1 450000 - sqlite4_step $::STMT -} {SQLITE4_ROW} -do_test zeroblob-7.2 { - sqlite4_column_int $::STMT 0 -} {450000} -do_test zeroblob-7.3 { - sqlite4_finalize $::STMT -} {SQLITE4_OK} -do_test zeroblob-7.4 { - set ::sqlite4_max_blobsize -} {0} -do_test zeroblob-7.5 { - expr {[sqlite4_memory_highwater]<$::memused+10000} -} {1} - -# Test that MakeRecord can handle a value with some real content -# and a zero-blob tail. -# -do_test zeroblob-8.1 { - llength [execsql { - SELECT 'hello' AS a, zeroblob(10) as b from t1 ORDER BY a, b; - }] -} {8} - - -# Ticket #3965 -# zeroblobs on either size of an IN operator -# -do_test zeroblob-9.1 { - db eval {SELECT x'0000' IN (x'000000')} -} {0} -do_test zeroblob-9.2 { - db eval {SELECT x'0000' IN (x'0000')} -} {1} -do_test zeroblob-9.3 { - db eval {SELECT zeroblob(2) IN (x'000000')} -} {0} -do_test zeroblob-9.4 { - db eval {SELECT zeroblob(2) IN (x'0000')} -} {1} -do_test zeroblob-9.5 { - db eval {SELECT x'0000' IN (zeroblob(3))} -} {0} -do_test zeroblob-9.6 { - db eval {SELECT x'0000' IN (zeroblob(2))} -} {1} -do_test zeroblob-9.7 { - db eval {SELECT zeroblob(2) IN (zeroblob(3))} -} {0} -do_test zeroblob-9.8 { - db eval {SELECT zeroblob(2) IN (zeroblob(2))} -} {1} - - -finish_test Index: tool/lsmview.tcl ================================================================== --- tool/lsmview.tcl +++ tool/lsmview.tcl @@ -399,11 +399,11 @@ $myText delete 0.0 end # Delete the existing tree entries. $myTree delete [$myTree children {}] - set nBlksz [exec_lsmtest_show -c $myCfg $myDb blocksize] + set nBlksz [expr [exec_lsmtest_show -c $myCfg $myDb blocksize] * 1024] set nPgsz [exec_lsmtest_show -c $myCfg $myDb pagesize] if {[regexp {c=1} $myCfg] || [regexp {co=1} $myCfg] || [regexp {com=1} $myCfg] || [regexp {comp=1} $myCfg] || [regexp {compr=1} $myCfg] || [regexp {compres=1} $myCfg] Index: tool/mksqlite4c.tcl ================================================================== --- tool/mksqlite4c.tcl +++ tool/mksqlite4c.tcl @@ -213,16 +213,18 @@ # foreach file { sqliteInt.h global.c + env.c ctime.c status.c date.c os.c fault.c + mem.c mem0.c mem1.c mem2.c mem3.c mem5.c Index: www/index.wiki ================================================================== --- www/index.wiki +++ www/index.wiki @@ -1,8 +1,9 @@ SQLite4 * [./design.wiki | The Design Of SQLite4] + * [./prog-intro.wiki | Introduction to SQLite4 for Programmers] * [./env.wiki | The Run-time Environment Object] * [./varint.wiki | The Varint format] * [./data_encoding.wiki | The Data Encoding] * [./key_encoding.wiki | The Key Encoding] * [./decimal.wiki | Internal representation of numeric values] Index: www/key_encoding.wiki ================================================================== --- www/key_encoding.wiki +++ www/key_encoding.wiki @@ -59,18 +59,28 @@ two strings where one is a prefix of the other that the shorter string will sort first.

Binary Encoding

-Each SQL value that is BINARY begins with a single byte of 0x25 and +The encoding of binaries fields is different depending on whether or not +the value to be encoded is the last value (the right-most value) in the key. + +Each SQL value that is BINARY that is not the last value of the key +begins with a single byte of 0x25 and ends with a single byte of 0x00. There are zero or more intervening bytes that encode the binary value. None of the intervening bytes may be zero. Each of the intervening bytes contains 7 bits of blob content with a 1 in the high-order bit (the 0x80 bit). The final byte before the 0x00 contains any left-over bits of the blob content. -The initial byte of a binary value, 0x25, is larger than the initial +When the very last value of a key is BINARY, then it is encoded as a single +byte of 0x26 and is followed by a byte-for-byte copy of the BINARY value. +This alternative encoding is more efficient, but it only works if there are +no subsequent values in the key, since there is no termination mark on the +BLOB being encoded. + +The initial byte of a binary value, 0x25 or 0x26, is larger than the initial byte of a text value, 0x24, ensuring that every binary value will sort after every text value.

Numeric Encoding

@@ -186,6 +196,7 @@ positive medium 0x17+E, M positive large 0x22, E, M positive infinity 0x23 text 0x24, T binary 0x25, B + final binary 0x26, X
Index: www/lsm.wiki ================================================================== --- www/lsm.wiki +++ www/lsm.wiki @@ -1,10 +1,39 @@ LSM Design Overview -

1. Summary

+ + + + +
+      1. Summary
+      2. Locks
+      3. Database Connect and Disconnect Operations
+            3.1. Read-write clients
+            3.2. Read-only clients
+      4. Data Structures
+            4.1. Database file
+                  4.1.1. Sorted Runs
+                  4.1.2. Levels
+                  4.1.3. Snapshots
+            4.2. In-Memory Tree
+                  4.2.1. Memory Allocation
+                  4.2.2. Header Fields
+            4.3. Other Shared-Memory Fields
+            4.4. Log file
+      5. Database Operations
+            5.1. Reading
+            5.2. Writing
+            5.3. Working
+                  5.3.1. Free-block list management
+            5.4. Checkpoint Operations
+ +
+ +

1. Summary

The LSM embedded database software stores data in three distinct data structures:
    @@ -44,57 +73,217 @@ in-memory tree. Once the in-memory tree has grown large enough, its contents are written into the database file as a new sorted run. To reduce the number of sorted runs in the database file, chronologically adjacent sorted runs may be merged together into a single run, either automatically or on demand. -

    2. Data Structures

    - -

    Locks

    -

    +

    2. Locks

    Read/write (shared/exclusive) file locks are used to control concurrent -access. LSM uses the following file-locks: - -
      -

    • The DMS1 and DMS2 locking regions. These are used to - implement the "dead-man-switch" mechanism copied from SQLite's WAL - mode for safely connecting to and disconnecting from a database. - See "Database Recovery and Shutdown" below. - -

    • Several (say 3) READER locking regions. Database clients - hold a SHARED lock one of the READER locking regions while reading the - database. As in SQLite WAL mode, each reader lock is paired with a - value indicating the version of the database that the client process - is using. The value consists of two fields: - -

        -
      • A 64-bit snapshot id. This identifies the version of the database file - that the reader is accessing. -
      • A 32-bit shared-memory chunk id. This identifies the version of the - in-memory tree the reader is reading. -
      - -

    • The WRITER locking region. A database client holds an - EXCLUSIVE lock on this locking region while writing data to the - database. Outside of recovery, only clients holding this lock may - modify the contents of the in-memory b-tree. - -

    • The WORKER lock. A database client holds an EXCLUSIVE lock - on this locking region while writing data into the body of the - database file. - -

    • The CHECKPOINTER lock. A database client holds an EXCLUSIVE - lock on this locking region while writing to the database file header. -

    +access. LSM uses the following "locking regions". Each locking region may +be locked and unlocked separately. + +

    + +
    DMS1 +

    + This locking region is used to serialize all connection and + disconnection operations performed by read-write database + connections. An EXCLUSIVE lock is taken for the duration of all such + operations. + +

    Additionally, read-only connections take a SHARED lock on this + locking region while attempting to connect to a database. This ensures + that a read-only connection does not attempt to connect to the + database while a read-write clients connection or disconnection + operation is ongoing. + +

    DMS2 +

    + Read-write connections hold a SHARED lock on this locking region for + as long as they are connected to the database. + +

    DMS3 +

    + Read-only connections hold a SHARED lock on this locking region for + as long as they are connected to the database. + +

    RWCLIENT(n) +

    + There are a total of 16 RWCLIENT locking regions. After a read-write + client connects to the database it attempts to find a free RWCLIENT + locking slot to take an EXCLUSIVE lock on. If it cannot find one, + this is not an error. If it can, then the lock is held for as long + as the read-write client is connected to the database for. + +

    The sole purpose of these locks is that they allow a read-only + client to detect whether or not there exists at least one read-write + client connected to the database. Of course if large numbers of + read-write clients connect and disconnect from the system in an + inconvenient order the system may enter a state where there exists + one or more connected read-write clients but none of them hold a + RWCLIENT lock. This is not important - if a read-only client fails + to detect that the system has read-write clients it may be less + efficient, but will not malfunction. + +

    WRITER +

    + A database client holds an EXCLUSIVE lock on this locking region + while writing data to the database. Outside of recovery, only clients + holding this lock may modify the contents of the in-memory b-tree. + Holding this lock is synonymous with having an open write transaction + on the database. + +

    WORKER +

    + A database client holds an EXCLUSIVE lock on this locking region + while performing database work (writing data into the body of the + database file). + +

    CHECKPOINTER +

    + A database client holds an EXCLUSIVE lock on this locking region + while performing a checkpoint (syncing the database file and + writing to the database header). + +

    ROTRANS +

    + A read-only database client holds a SHARED lock on this locking region + while reading from a non-live database system. + +

    READER(n) +

    + There are a total of 6 READER locking regions. Unless it is a + read-only client reading from a non-live database, a client holds a + SHARED lock on one of these while it has an open read transaction. + Each READER lock is associated with a pair of id values identifying + the regions of the in-memory tree and database file that may be read + by clients holding such SHARED locks. +

    + +

    3. Database Connect and Disconnect Operations

    + +

    3.1. Read-write clients

    + +

    When an LSM database connection is opened (i.e. lsm_open() is called): + +

    +  lock(DMS1, EXCLUSIVE)                # Block until successful
+    trylock(DMS2+DMS3, EXCLUSIVE)
+    if( trylock() successful ){
+      zero shared memory and run recovery
+    }
+    if( no error during recovery ){
+      lock(DMS2, SHARED)               # "cannot" fail
+      lock(RWCLIENT(x), EXCLUSIVE)     # see comment below
+    }
+  unlock(DMS1)
+
    + +

    +Running recovery involves reading the database file header and log file +to initialize the contents of shared-memory. Recovery is only run when the +first connection connects to the database file. There are no circumstances +(including the unexpected failure of a writer process) that may cause +recovery to take place after the first client has successfully connected. + +

    +After the SHARED lock on DMS2 is taken, an effort is made to find a free +RWCLIENT locking region and take an EXCLUSIVE lock on it. If no such region +can be found, this step is omitted. It is not an error if this happens. + +

    +Assuming recovery is successful (or not required), the database client is +left holding a SHARED lock on DMS2 and, possibly, an EXCLUSIVE lock on one +of the RWCLIENT locks. These locks are held for as long as the database +connection remains open. + +

    +When disconnecting from a database (i.e. an lsm_close() call following a +successful lsm_open()): + +

    +  lock(DMS1, EXCLUSIVE)           # Block until successful
+    trylock(DMS2, EXCLUSIVE)
+    if( trylock() successful ){
+      flush in-memory tree
+      checkpoint database
+
+      trylock(DMS3, EXCLUSIVE)
+      if( trylock() successful ){
+        delete shared memory
+      }
+
+      trylock(ROTRANS, EXCLUSIVE)
+      if( trylock() successful ){
+        unlink log file
+      }
+    }
+    unlock(RWCLIENT(x))           # If holding RWCLIENT lock
+    unlock(DMS2)
+  unlock(DMS1)
+
    + +

    3.2. Read-only clients

    + +

    It is assumed that read-only clients: + +

      +
    • may take SHARED locks only, +
    • may not write to shared-memory, the database file or the log file, and +
    • may not use the trylock(x, EXCLUSIVE) primitive to detect SHARED locks + held by other clients. +
    + +

    A read-only client does not attempt to connect to the database from within +the lsm_open() call. Instead, this is defered until the first time the client +attempts to read from the database. + +

    +  lock(DMS1, SHARED)              # Block until successful
+    trylock(RWCLIENT(all), SHARED)
+    if( trylock() successful ){
+      # System is not live. The database must be read directly from disk.
+      lock(ROTRANS, SHARED)
+    }else{
+      # Client is now connected. The read transaction may now be opened
+      # in the same way as it would be for a read-write client.
+      lock(DMS3, SHARED)
+    }
+  unlock(DMS1)
+
    + +

    +Assuming no error occurs, the procedure above leads to one of two possible +outcomes: + +

      +

    • DMS3 is locked and the read-only client is now connected to the + database. From this point on, the read-only client uses the same + procedure to open a read transaction as any other client. The lock on + DMS3 is held until the client disconnects from the database. + +

    • ROTRANS is locked and the read-only client is still disconnected. + Holding the lock on ROTRANS guarantees that no read-write client + will overwrite any existing data in the database file or log file. + This allows the read-only client to run a disconnected read + transaction - it recovers any data in the log file into private memory, + then reads data as required from the database file for the duration + of the users read transaction. +

    + +

    A disconnected read transaction is closed by dropping the ROTRANS lock. + + +

    4. Data Structures

    In the following sections, "the WRITER lock", refers to an exclusive lock on the WRITER locking region. For example "holding the WRITER lock" is equivalent to "holding an exclusive lock on the WRITER locking region". Similar interpretations apply to "the WORKER lock" and "the CHECKPOINTER lock". -

    Database file

    +

    4.1. Database file

    This section summarizes the contents of the database file informally. A detailed description is found in the header comments for source code files lsm_file.c (blocks, pages etc.), @@ -124,11 +313,11 @@ As with an SQLite database file, each page in the database may be addressed by its 32-bit page number. This means the maximum database size is roughly (pgsz * 2^32) bytes. The first and last pages in each block are 4 bytes smaller than the others. This is to make room for a single page-number. -

    Sorted Runs

    +

    4.1.1. Sorted Runs

    A single sorted run is spread across one or more database pages (each page is a part of at most one sorted run). Given the page number of a page in a sorted run the following statements are true: @@ -166,11 +355,11 @@ it is possible to traverse the entire run in either direction or query for arbitrary values.

    TODO: Embedded pointers. -

    Levels

    +

    4.1.2. Levels

    Each sorted run is assigned to a "level". Normally, a level consists of a single sorted run. However, a level may also consist of a set of sorted runs being incrementally merged into a single run. @@ -227,11 +416,11 @@ -

    Snapshots

    +

    4.1.3. Snapshots

    Each meta page may contain a database snapshot. A snapshot contains all the information required to interpret the remainder of the database file (the sorted runs and free space). Specifically, it contains: @@ -252,11 +441,11 @@

    A more detailed description is available in the header comments in source code file lsm_ckpt.c -

    In-Memory Tree

    +

    4.2. In-Memory Tree

    The in-memory tree is an append-only b-tree of order 4 (each node may have up to 4 children), which is more or less equivalent to a red-black tree. An append-only tree is convenient, as it naturally supports the @@ -266,11 +455,11 @@ The implementation includes some optimizations to reduce the number of interior nodes that are updated when a leaf node is written that are not described here. See header comments in source code file lsm_tree.c for details. -

    Memory Allocation

    +

    4.2.1. Memory Allocation

    More than one in-memory tree may exist in shared-memory at any time. For example in the following scenario: @@ -332,11 +521,11 @@ reconstruct the linked list. Any sequence ids assigned by the failed writer are reverted (perhaps not to their original values, but to values that put them at the start of the linked list - before those chunks that may still be in use by existing readers). -

    Header Fields

    +

    4.2.2. Header Fields

    As well as the in-memory tree data, the following fixed-size fields stored in well-known locations in shared-memory are part of the in-memory tree. Like the in-memory tree data, outside of recovery these fields are only ever written to by clients holding the WRITER lock. @@ -353,77 +542,27 @@ occur mid-transaction. It is only ever read (or written) by clients that hold the WRITER lock.

-

Other Shared-Memory Fields

+

4.3. Other Shared-Memory Fields

  • Snapshot 1.
  • Snapshot 2.
  • The meta-page pointer. This value is either 1 or 2. It indicates which of the two meta-pages contains the most recent database snapshot.
  • READER lock values.
-

Log file

+

4.4. Log file

lsm_log.c. -

3. Database Recovery and Shutdown

- -

-Exclusive locks on locking region DMS1 are used to serialize all connect and -disconnect operations. - -

When an LSM database connection is opened (i.e. lsm_open() is called): - -

-  lock(DMS1, EXCLUSIVE)           # Block until successful
-    lock(DMS2, EXCLUSIVE)         # Abandon if not immediately successful
-    if( DMS2 successfully locked ){
-      zero *-shm file (or equivalent)
-      run recovery
-    }
-    if( error during recovery ){
-      unlock(DMS2)
-    }else{
-      lock(DMS2, SHARED)          # "cannot" fail
-    }
-  unlock(DMS1)
-
- -

-Running recovery involves reading the database file header and log file -to initialize the contents of shared-memory. Recovery is only run when the -first connection connects to the database file. There are no circumstances -(including the unexpected failure of a writer process) that may cause -recovery to take place after the first client has successfully connected. - -

-Assuming recovery is successful (or not required), the database client is -left holding a SHARED lock on DMS2. This lock is held for as long as the -database connection remains open. - -

-When disconnecting from a database (i.e. an lsm_close() call following a -successful lsm_open()): - -

-  lock(DMS1, EXCLUSIVE)           # Block until successful
-    lock(DMS2, EXCLUSIVE)         # Abandon if not immediately successful
-    if( DMS2 successfully locked ){
-      ...TODO...
-      delete *-shm file (or equivalent)
-    }
-    unlock(DMS2)
-  unlock(DMS1)
-
- -

4. Database Operations

- -

Reading

+

5. Database Operations

+ +

5.1. Reading

Opening a read transaction:

    @@ -430,12 +569,12 @@

  1. Load the current tree-header from shared-memory.

  2. Load the current snapshot from shared-memory.

    Steps 1 and 2 are similar. In both cases, there are two copies of the data structure being read in shared memory. No lock is held to prevent -another client updating them while the read is taking place using the following -pattern: +another client updating them while the read is taking place. Updaters use the +following pattern:

    1. Update copy 2.
    2. Invoke xShmBarrier().
    3. Update copy 1. @@ -514,11 +653,11 @@ lock is held.

      To close a read transaction all that is required is to drop the SHARED lock held on the READER slot. -

      Writing

      +

      5.2. Writing

      To open a write transaction:

        @@ -562,72 +701,11 @@ that it is consistent with the current tree-header.
      1. Clear the writer flag.
      -

      Flushing the in-memory tree to disk

      - -

      -For the purposes of writing, the database file and the in-memory tree are -largely independent. Processes holding the WRITER lock write to the in-memory -tree, and processes holding the WORKER lock write to the database file. - -

        -
      1. If a write transaction is open, write the new tree-header to - shared-memory (as described above). Otherwise, if no write-transaction - is open, open one - repairing the tree headers if required. - -
      2. Execute a WORKER transaction (see "Working" below) to add a new level - to the LSM. The new level contains the current contents of the - in-memory tree. - -
      3. Update the private copy of the tree-header to reflect a new, empty tree. - -
      4. Commit the write transaction, writing the new, empty tree to - shared-memory. -
      - -

      Shared-memory management

      - -

      -A writer client may have to allocate new shared-memory chunks. This can be -done either by extending the shared-memory region or by recycling the first -chunk in the linked-list. To check if the first chunk in the linked-list may -be reused, the writer must check that: - -

        -
      • The chunk is not part of the current in-memory tree (the one being - appended to by the writer). A writer can check this by examining its - private copy of the tree-header. - -
      • The chunk is not part of an in-memory tree being used by an existing - reader. A writer checks this by scanning (and possibly updating) the - values associated with the READER locks - similar to the way SQLite - does in WAL mode. -
      - -

      Log file management

      - -

      -A writer client also writes to the log file. All information required to write -to the log file (the offset to write to and the initial checksum values) is -embedded in the tree-header. Except, in order to reuse log file space (wrap -around to the start of the log file), a writer needs to know that the space -being recycled will not be required by any recovery process in the future. -In other words, that the information contained in the transactions being -overwritten has been written into the database file and is part of the -snapshot written into the database file by a checkpointer (see "Checkpoint -Operations" below). - -

      -To determine whether or not the log file can be wrapped, the writer requires -access to information stored in the newest snapshot written into the database -header. Their exists a shared-memory variable indicating which of the two -meta-pages contain this snapshot, but the writer process still has to read the -snapshot data and verify its checksum from disk. - -

      Working

      +

      5.3. Working

      Working is similar to writing. The difference is that a "writer" modifies the in-memory tree. A "worker" modifies the contents of the database file. @@ -647,11 +725,11 @@

    4. Update snapshot-1 in shared-memory.

    5. Release the WORKER lock.

    -

    Free-block list management

    +

    5.3.1. Free-block list management

    Worker clients occasionally need to allocate new database blocks or move existing blocks to the free-block list. Along with the block number of each free block, the free-block list contains the snapshot-id of the first @@ -677,11 +755,11 @@ shared-memory variable. -

    Checkpoint Operations

    +

    5.4. Checkpoint Operations

    1. Take CHECKPOINTER lock.
    2. Load snapshot-1 from shared-memory. If the checksum does not match @@ -703,52 +781,10 @@ step 5.
    3. Drop the CHECKPOINTER lock.
    -

    5. Scheduling Policies

    - -

    -When a client writes to a database, the in-memory tree and log file are -updated by the client itself before the lsm_write() call returns. Eventually, -once sufficient writes have accumulated in memory, the client marks the -current tree as "old", and subsequent writes are accumulated in a new tree. - -

    -In order to prevent the in-memory tree and log file from growing indefinitely, -at some point in the future the following must occur: - -

      -
    • The contents of the old tree must be written into the database file - (a WORKER lock operation). Once this is done the memory used to store the - old tree is available for reuse. - -
    • A checkpoint operation must take place to sync the data into the - database file and update the database header (a CHECKPOINT lock - operation). Once this has been done the log file space that was used - to store the data may be reclaimed. -
    - -

    -In addition to the above, it is necessary to perform a certain amount of -work on the database to merge existing levels together. This is not just -to speed up queries - there is a hard limit of roughly 40 levels to stop -database snapshots from growing overly large. - -

    Explicit Calls to lsm_work() and lsm_checkpoint() - -

    Compulsory work - -

      -

    • If a writer tries to mark a tree as "old", but there is already an - old tree in-memory, the writer attempts to grab the WORKER lock and - write both the old and new tree to a new database level. - -

      If the WORKER lock cannot be obtained immediately, block until it - can be -

    - -

    Auto work + Index: www/lsmapi.wiki ================================================================== --- www/lsmapi.wiki +++ www/lsmapi.wiki @@ -19,11 +19,11 @@

    1. Database Runtime Environment
    2. LSM Error Codes
    3. Creating and Destroying Database Connection Handles
    4. Connecting to a Database -
    5. Obtaining pointers to databases environments +
    6. Obtaining pointers to database environments
    7. Configuring a database connection.
    8. Compression and/or Encryption Hooks
    9. Allocating and Freeing Memory
    10. Querying a Connection For Operational Data
    11. Opening and Closing Write Transactions @@ -65,10 +65,11 @@ lsm_rollback lsm_work

      All LSM API Types

      lsm_compress +lsm_compress lsm_env

      All LSM API Constants

      LSM_BUSY LSM_CANTOPEN @@ -76,26 +77,27 @@ LSM_CONFIG_AUTOFLUSH LSM_CONFIG_AUTOMERGE LSM_CONFIG_AUTOWORK LSM_CONFIG_BLOCK_SIZE LSM_CONFIG_GET_COMPRESSION -LSM_CONFIG_LOG_SIZE LSM_CONFIG_MAX_FREELIST LSM_CONFIG_MMAP LSM_CONFIG_MULTIPLE_PROCESSES LSM_CONFIG_PAGE_SIZE LSM_CONFIG_SAFETY LSM_CONFIG_SET_COMPRESSION +LSM_CONFIG_SET_COMPRESSION_FACTORY LSM_CONFIG_USE_LOG LSM_CORRUPT LSM_ERROR LSM_FULL LSM_INFO_ARRAY_PAGES LSM_INFO_ARRAY_STRUCTURE LSM_INFO_CHECKPOINT_SIZE LSM_INFO_DB_STRUCTURE LSM_INFO_FREELIST +LSM_INFO_FREELIST_SIZE LSM_INFO_LOG_STRUCTURE LSM_INFO_NREAD LSM_INFO_NWRITE LSM_INFO_PAGE_ASCII_DUMP LSM_INFO_PAGE_HEX_DUMP @@ -180,49 +182,63 @@

      Open and close a database connection handle.

      Connecting to a Database

      int lsm_open(lsm_db *pDb, const char *zFilename); -

      Obtaining pointers to databases environments

      +

      Obtaining pointers to database environments

      lsm_env *lsm_get_env(lsm_db *pDb); lsm_env *lsm_default_env(void);

      Return a pointer to the environment used by the database connection passed as the first argument. Assuming the argument is valid, this function always returns a valid environment pointer - it cannot fail. The lsm_default_env() function returns a pointer to the default LSM environment for the current platform. -

      Configuring a database connection.

      +

      Configuring a database connection.

      int lsm_config(lsm_db *, int, ...); -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_LOG_SIZE 6 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_AUTOFLUSH 1 +#define LSM_CONFIG_PAGE_SIZE 2 +#define LSM_CONFIG_SAFETY 3 +#define LSM_CONFIG_BLOCK_SIZE 4 +#define LSM_CONFIG_AUTOWORK 5 +#define LSM_CONFIG_MMAP 7 +#define LSM_CONFIG_USE_LOG 8 +#define LSM_CONFIG_AUTOMERGE 9 +#define LSM_CONFIG_MAX_FREELIST 10 +#define LSM_CONFIG_MULTIPLE_PROCESSES 11 +#define LSM_CONFIG_AUTOCHECKPOINT 12 +#define LSM_CONFIG_SET_COMPRESSION 13 +#define LSM_CONFIG_GET_COMPRESSION 14 +#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 #define LSM_SAFETY_OFF 0 #define LSM_SAFETY_NORMAL 1 #define LSM_SAFETY_FULL 2

      The lsm_config() function is used to configure a database connection. The following values may be passed as the second argument to lsm_config(). -

      LSM_CONFIG_AUTOFLUSH
      A read/write integer parameter. This value determines the maximum amount -of space (in bytes) used to accumulate writes in main-memory before -they are flushed to a level 0 segment. +

      LSM_CONFIG_AUTOFLUSH
      A read/write integer parameter. +

      This value determines the amount of data allowed to accumulate in a +live in-memory tree before it is marked as old. After committing a +transaction, a connection checks if the size of the live in-memory tree, +including data structure overhead, is greater than the value of this +option in KB. If it is, and there is not already an old in-memory tree, +the live in-memory tree is marked as old. +

      The maximum allowable value is 1048576 (1GB). There is no minimum +value. If this parameter is set to zero, then an attempt is made to +mark the live in-memory tree as old after each transaction is committed. +

      The default value is 1024 (1MB).

      LSM_CONFIG_PAGE_SIZE
      A read/write integer parameter. This parameter may only be set before lsm_open() has been called. -

      LSM_CONFIG_BLOCK_SIZE
      A read/write integer parameter. This parameter may only be set before -lsm_open() has been called. -

      LSM_CONFIG_LOG_SIZE
      A read/write integer parameter. +

      LSM_CONFIG_BLOCK_SIZE
      A read/write integer parameter. +

      This parameter may only be set before lsm_open() has been called. It +must be set to a power of two between 64 and 65536, inclusive (block +sizes between 64KB and 64MB). +

      If the connection creates a new database, the block size of the new +database is set to the value of this option in KB. After lsm_open() +has been called, querying this parameter returns the actual block +size of the opened database. +

      The default value is 1024 (1MB blocks).

      LSM_CONFIG_SAFETY
      A read/write integer parameter. Valid values are 0, 1 (the default) and 2. This parameter determines how robust the database is in the face of a system crash (e.g. a power failure or operating system crash). As follows:

      0 (off): No robustness. A system crash may corrupt the database. @@ -232,10 +248,21 @@

      2 (full): Full robustness. A system crash may not corrupt the database file. Following recovery the database file contains all successfully committed transactions.

      LSM_CONFIG_AUTOWORK
      A read/write integer parameter.

      LSM_CONFIG_AUTOCHECKPOINT
      A read/write integer parameter. +

      If this option is set to non-zero value N, then a checkpoint is +automatically attempted after each N KB of data have been written to +the database file. +

      The amount of uncheckpointed data already written to the database file +is a global parameter. After performing database work (writing to the +database file), the process checks if the total amount of uncheckpointed +data exceeds the value of this paramter. If so, a checkpoint is performed. +This means that this option may cause the connection to perform a +checkpoint even if the current connection has itself written very little +data into the database file. +

      The default value is 2048 (checkpoint every 2MB).

      LSM_CONFIG_MMAP
      A read/write integer parameter. True to use mmap() to access the database file. False otherwise.

      LSM_CONFIG_USE_LOG
      A read/write boolean parameter. True (the default) to use the log file normally. False otherwise.

      LSM_CONFIG_AUTOMERGE
      A read/write integer parameter. The minimum number of segments to @@ -257,17 +284,25 @@ structures contents.

      This option may only be used before lsm_open() is called. Invoking it after lsm_open() has been called results in an LSM_MISUSE error.

      LSM_CONFIG_GET_COMPRESSION
      Query the compression methods used to compress and decompress database content. -

      Compression and/or Encryption Hooks

      +

      LSM_CONFIG_SET_COMPRESSION_FACTORY
      Configure a factory method to be invoked in case of an LSM_MISMATCH +error. +

      Compression and/or Encryption Hooks

      struct lsm_compress { void *pCtx; unsigned int iId; int (*xBound)(void *, int nSrc); int (*xCompress)(void *, char *, int *, const char *, int); int (*xUncompress)(void *, char *, int *, const char *, int); + void (*xFree)(void *pCtx); +}; +struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *, lsm_db *, u32); + void (*xFree)(void *pCtx); };

      Allocating and Freeing Memory

      void *lsm_malloc(lsm_env*, size_t); void *lsm_realloc(lsm_env*, void *, size_t); @@ -274,11 +309,11 @@ void lsm_free(lsm_env*, void *);

      Invoke the memory allocation functions that belong to environment pEnv. Or the system defaults if no memory allocation functions have been registered. -

      Querying a Connection For Operational Data

      +

      Querying a Connection For Operational Data

      int lsm_info(lsm_db *, int, ...); #define LSM_INFO_NWRITE 1 #define LSM_INFO_NREAD 2 #define LSM_INFO_DB_STRUCTURE 3 #define LSM_INFO_LOG_STRUCTURE 4 @@ -287,10 +322,11 @@ #define LSM_INFO_PAGE_HEX_DUMP 7 #define LSM_INFO_FREELIST 8 #define LSM_INFO_ARRAY_PAGES 9 #define LSM_INFO_CHECKPOINT_SIZE 10 #define LSM_INFO_TREE_SIZE 11 +#define LSM_INFO_FREELIST_SIZE 12

      Query a database connection for operational statistics or data. The following values may be passed as the second argument to lsm_info().

      LSM_INFO_NWRITE
      The third parameter should be of type (int *). The location pointed to by the third parameter is set to the number of 4KB pages written to @@ -352,11 +388,11 @@ string should be eventually freed by the caller using lsm_free().

      The Tcl structure returned is a list containing one element for each free block in the database. The element itself consists of two integers - the block number and the id of the snapshot that freed it.

      LSM_INFO_CHECKPOINT_SIZE
      The third argument should be of type (int *). The location pointed to -by this argument is populated with the number of bytes written to the +by this argument is populated with the number of KB written to the database file since the most recent checkpoint.

      LSM_INFO_TREE_SIZE
      If this value is passed as the second argument to an lsm_info() call, it should be followed by two arguments of type (int *) (for a total of four arguments).

      At any time, there are either one or two tree structures held in shared @@ -365,11 +401,11 @@ information on them). One tree structure - the current tree - is used to accumulate new data written to the database. The other tree structure - the old tree - is a read-only tree holding older data and may be flushed to disk at any time.

      Assuming no error occurs, the location pointed to by the first of the two -(int *) arguments is set to the size of the old in-memory tree in bytes. +(int *) arguments is set to the size of the old in-memory tree in KB. The second is set to the size of the current, or live in-memory tree.

      Opening and Closing Write Transactions

      int lsm_begin(lsm_db *pDb, int iLevel); int lsm_commit(lsm_db *pDb, int iLevel); int lsm_rollback(lsm_db *pDb, int iLevel); @@ -407,21 +443,21 @@ Delete all database entries with keys that are greater than (pKey1/nKey1) and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and (pKey2/nKey2) themselves, if they exist in the database, are not deleted.

      Return LSM_OK if successful, or an LSM error code otherwise.

      Explicit Database Work and Checkpointing

      -int lsm_work(lsm_db *pDb, int nMerge, int nPage, int *pnWrite); +int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); int lsm_flush(lsm_db *pDb); -int lsm_checkpoint(lsm_db *pDb, int *pnByte); +int lsm_checkpoint(lsm_db *pDb, int *pnKB);

      This function is called by a thread to work on the database structure. Attempt to checkpoint the current database snapshot. Return an LSM error code if an error occurs or LSM_OK otherwise.

      If the current snapshot has already been checkpointed, calling this -function is a no-op. In this case if pnByte is not NULL, *pnByte is +function is a no-op. In this case if pnKB is not NULL, *pnKB is set to 0. Or, if the current snapshot is successfully checkpointed -by this function and pbCkpt is not NULL, *pnByte is set to the number +by this function and pbKB is not NULL, *pnKB is set to the number of bytes written to the database file since the previous checkpoint (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query).

      Opening and Closing Database Cursors

      int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); int lsm_csr_close(lsm_cursor *pCsr); Index: www/lsmusr.wiki ================================================================== --- www/lsmusr.wiki +++ www/lsmusr.wiki @@ -159,11 +159,11 @@

      LSM is not currently built or distributed independently. Instead, it is part of the SQLite4 library. To use LSM in an application, the application links against libsqlite4 and includes the header file "lsm.h" in any files that access the LSM API. -

      Pointer to build instructions for sqlite4 +

      Pointer to build instructions for sqlite4

      3. Basic Usage

      3.1. Opening and Closing Database Connections

      @@ -372,17 +372,17 @@ lsm_csr_valid() returns 0, and the loop is finished. API function lsm_csr_key() is used to retrieve the key associated with each database entry visited. - for(rc = lsm_csr_first(csr); lsm_csr_valid(csr); rc = lsm_csr_next(csr)){ + for(rc=lsm_csr_first(csr); rc==LSM_OK && lsm_csr_valid(csr); rc=lsm_csr_next(csr)){ const void *pKey; int nKey; const void *pVal; int nVal; rc = lsm_csr_key(csr, &pKey, &nKey); if( rc==LSM_OK ) rc = lsm_csr_value(csr, &pVal, &nVal); - if( rc!=LSM_OK ) break; + if( rc==LSM_OK ) break; /* At this point pKey points to the current key (size nKey bytes) and ** pVal points to the corresponding value (size nVal bytes). */ } @@ -725,51 +725,84 @@ int (*xUncompress)(void *pCtx, void *pOut, int *pnOut, const void *pIn, int nIn); void (*xFree)(void *pCtx); };       -

      Explain how the hooks work here (same as zipvfs) +

      Explain how the hooks work here (same as zipvfs) + -

      Example code? Using zlib? Or something simple like an RLE -implementation? +

      Example code? Using zlib? Or something simple like an +RLE implementation?

      The database file header of any LSM database contains a 32-bit unsigned "compression id" field. If the database is not a compressed database, this field is set to 1. Otherwise, it is set to an application supplied value identifying the compression and/or encryption scheme in use. Application compression scheme ids must be greater than or equal to 10000. Values smaller than 10000 are reserved for internal use. -

      The lsm_compression_id() API may be used to read the compression id from -a database connection. Because the compression id is stored in the database +

      The lsm_info() API may be used to read the compression id from a database +connection as follows: + + + unsigned int iCompressionId; + rc = lsm_info(db, LSM_INFO_COMPRESSION_ID, &iCompressionId); + if( rc==LSM_OK ){ + /* Variable iCompressionId now contains the db compression id */ + } + + +Because the compression id is stored in the database header, it may be read before any required compression or encryption hooks are configured. #define LSM_COMPRESSION_EMPTY 0 #define LSM_COMPRESSION_NONE 1 - int lsm_compression_id(lsm_db *db, u32 *piId);

      When a database is opened for the first time, before it is first written, -the compression id field is set to LSM_COMPRESSION_EMPTY (0). The first time -a transaction is committed, the database compression id is set to a copy of -the lsm_compress.iId field of the compression hooks for the database handle -committing the transaction, or to LSM_COMPRESSION_NONE (1) if no compression -hooks are configured. +the compression id field is set to LSM_COMPRESSION_EMPTY (0). After data is +written into the database file, the database compression id is set to a copy +of the lsm_compress.iId field of the compression hooks for the database handle +doing the writing, or to LSM_COMPRESSION_NONE (1) if no compression hooks +are configured.

      Once the compression id is set to something other than -LSM_COMPRESSION_EMPTY, when a database handle opens a read or write -transaction on the database, the compression id is compared against the -lsm_compress.iId field of the configured compression hooks, or against LSM_COMPRESSION_NONE if no compression hooks are configured. If the compression id -does not match, then an LSM_MISMATCH error is returned and the operation -fails (no transaction or database cursor is opened). - -

      Maybe there should be a way to register a mismatch-handler callback. -Otherwise, applications have to handle LSM_MISMATCH everywhere... - - +LSM_COMPRESSION_EMPTY, when a database handle attempts to read or write the +database file, the compression id is compared against the lsm_compress.iId +field of the configured compression hooks, or against LSM_COMPRESSION_NONE if +no compression hooks are configured. If the compression id does not match, then +an LSM_MISMATCH error is returned and the operation fails (no transaction or +database cursor is opened). + +

      It is also possible to register a compression factory callback with a +database handle. If one is registered, the compression factory callback is +invoked instead of returning LSM_MISMATCH if the configured compression hooks +do not match the compression id of a database. If the callback registers +compatible compression hooks with the database handle (using the normal +lsm_config() interface), then the database read or write operation resumes +after it returns. Otherwise, if the compression factory callback does not +register new, compatible, compression hooks with the database handle, +LSM_MISMATCH is returned to the user. + +

      A compression factory callback is registered with a database handle +by calling lsm_config() with the second argument set to +LSM_CONFIG_SET_COMPRESSION_FACTORY, and the third argument set to point to +an instance of structure lsm_compress_factory. The lsm_config() copies the +contents of the structure - it does not retain a pointer to it. + + + typedef struct lsm_compress_factory lsm_compress_factory; + struct lsm_compress_factory { + void *pCtx; + int (*xFactory)(void *pCtx, lsm_db *db, unsigned int iCompressionId); + void (*xFree)(void *pCtx); + }; + + +

      Explain how the xFactory hook works here.

      6. Performance Tuning

      This section describes the various measures that can be taken in order to fine-tune LSM in order to improve performance in specific circumstances. @@ -951,11 +984,15 @@ amount of data written to the database file. Decreasing it increases the amount of data written to the file, but also decreases the average number of segments present in the file, which can improve the performance of database read operations. -

      If auto-work is not enabled... +

      Additionally, whether or not auto-work is enabled, this option is used + to determine the maximum number of segments of a given age that are + allowed to accumulate in the database file. This is described in the + compulsary work and checkpoints + section below.

      The default value is 4. This option must be set to a value between 2 and 8, inclusive.

      LSM_CONFIG_AUTOWORK @@ -962,12 +999,12 @@

      This option may be set to either 1 (true) or 0 (false). If it is set to true, then work and checkpoint operations are automatically scheduled within calls to lsm_insert(), lsm_delete(), lsm_delete_range() and lsm_commit(). Otherwise, if it is set to false, these operations must - be explicitly invoked by the application. See some - link here for details. + be explicitly scheduled by the + application.

      The default value is 1.

      LSM_CONFIG_MMAP

      @@ -978,12 +1015,12 @@ it is false, data is accessed using ordinary OS file read and write primitives. Memory mapping the database file can significantly improve the performance of read operations, as database pages do not have to be copied from operating system buffers into user space buffers before they can be examined. -

      This option can only be set before lsm_open() is called on the database - connection. +

      This option may not be set if there is a read or write transaction + open on the database.

      The default value is 1 (true) on a 64-bit platform, and 0 otherwise.

      LSM_CONFIG_MULTIPLE_PROCESSES

      This option may also be set to either 1 (true) or 0 (false). The default @@ -1144,11 +1181,11 @@ clients should set the LSM_CONFIG_AUTOWORK parameter to zero. int rc; lsm_db *db; - int nCkpt = 4*1024*1024; + int nCkpt = 4*1024; /* 4096KB == 4MB */ /* Open a database connection to database "test.db". ** ** Configure the connection to automatically checkpoint the database after ** writing each 4MB of data to it (instead of the default 2MB). As well @@ -1162,11 +1199,11 @@ while( 1 ){ int nWrite; /* Attempt up to 512KB of work. Set nWrite to the number of bytes ** actually written to disk. */ - rc = lsm_work(db, 2, 512*1024, &nWrite); + rc = lsm_work(db, 2, 512, &nWrite); if( rc!=LSM_OK && rc!=LSM_BUSY ){ /* Anything other than LSM_OK or LSM_BUSY is a problem. LSM_BUSY ** indicates that some other client has taken the WORKER lock. Any ** other error indicates something has gone quite wrong. */ lsm_close(db); @@ -1283,11 +1320,11 @@ the following are true:

      • All database content is stored in a single segment. This makes the - database effectively equivalent to an optimally packed b-tree stucture + data structure equivalent to an optimally packed b-tree stucture for search operations - minimizing the number of disk sectors that need to be visted when searching the database.

      • The database file contains no (or as little as possible) free space. In other words, it is no larger than required to contain the single @@ -1294,25 +1331,21 @@ segment.

      In order to optimize the database, lsm_work() should be called with the nMerge argument set to 1 and the third parameter set to a negative value -(interpreted as - keep working until there is no more work to do). For +(interpreted as "keep working until there is no more work to do"). For example: rc = lsm_work(db, 1, -1, 0); -

      todo: the -1 as the 3rd argument above is currently -not supported -

      When optimizing the database as above, either the LSM_CONFIG_AUTOCHECKPOINT parameter should be set to a non-zero value or lsm_checkpoint() should be called periodically. Otherwise, no checkpoints will be performed, preventing the library from reusing any space occupied by old segments even after their content has been merged into the new segment. The result - a database file that is optimized, except that it is up to twice as large as it otherwise would be. - ADDED www/prog-intro.wiki Index: www/prog-intro.wiki ================================================================== --- /dev/null +++ www/prog-intro.wiki @@ -0,0 +1,289 @@ +Introduction For Programmers + +

      1.0 Overview

      + +This document provides a broad overview of the Application Programming +Interface (API) for SQLite4. If you are new to SQLite and are wondering +how to write a program that uses SQLite or how to incorporate SQLite into +your existing program, this document is a good starting point. + +

      2.0 Implementation Language

      + +SQLite is written in ANSI-C. +The C programming language is used because it is the universal assembly +language - it can run on just about any hardware and on just about any +operating system. No matter what programming +language is used for the application code, it can usually interface easily +with a library written in C. + +SQLite is written in an Object-Oriented (OO) style. +This may comes as a surprise to readers who have been trained to think +that C++ is necessary for OO programming. While it is true that C++ +has lots of syntactic sugar designed to make OO-style programming easier, +there is nothing missing from generic ANSI-C that disqualifies it from +using OO ideas and practices. + +In this document, there will be much talk of "objects", which are implemented +as ordinary C structures, of course. There is also talk of "methods" which +are really just procedures which take a pointer to the object as their +first parameter. SQLite also uses object inheritance, though inheritance +is only used in fringe areas of the API that do not fall within the scope +of this document. + +

      3.0 Principal Objects

      + +Here are four object classes that every programmer who uses SQLite4 needs +to be aware of: + + 1. sqlite4_mm - A memory allocator + 2. sqlite4_env - A run-time environment + 3. sqlite4 - A database connection + 4. sqlite4_stmt - A prepared statement + +The most important of these are the latter two, the database connection +and prepared statement objects. +SQLite4 provides default instances of a memory allocator and run-time +environment which are adequate for most system programming tasks. + + +

      3.1 The Memory Allocator Object

      + +A memory allocator object is used to allocate and deallocate bulk memory. +SQLite4 supplies a default memory allocator object (obtained by calling +the sqlite4_mm_default() function) which is just a wrapper around the +system malloc()/realloc()/free() routines. Most applications will get +along fine using just this default. But for applications with more advanced +needs, other memory allocator objects are available, including: + + * Memory allocators that use fixed pool of memory (provided at start time) + rather than the heap. + + * Meta-memory allocators that use one memory allocator first but then + fail-over to a second one if the first one runs out of memory or is + otherwise unable to satisfy an allocation. + + * Memory allocators operating out of a fixed pool of memory and that + provide robust guarantees against memory fragmentation. + + * Memory allocators that provide lots of information about how much + memory is being used. + + * Memory allocators that can simulate out-of-memory (OOM) faults for + testing purposes. + + * Memory allocators that aggressively check for misuse of the memory + allocation system, by (for example) overwriting memory as it is freed, + and initializing newly allocated memory to a random bit pattern. + + * Completely customized and application-specific memory allocators. + +The various memory allocators mentioned above are implemented by subclassing +the top-level memory allocator object. Unless you are implementing your +own customized memory allocator, you do not really need to know about this +subclassing. At the application programming level, you will only encounter +pointers to the sqlite4_mm object base class. + +There are various methods on the sqlite4_mm object, including: + + * sqlite4_mm_malloc() - allocate bulk memory + * sqlite4_mm_free() - release memory previously allocated + * sqlite4_mm_realloc() - change the size of a prior allocation + * sqlite4_mm_msize() - report the size of an allocation + * sqlite4_mm_stat() - report usage statistics on a memory allocator + +A memory allocator object is also passed as parameter to certain +interfaces in order to be used as a destructor. For example, when +binding a string to a parameter in a prepared statement using the +sqlite4_bind_text() method on the prepared statement object, there +is a parameter which is a pointer to a memory allocator object. If +that parameter is not NULL (and is not one of a handful of other +values with special meanings) then SQLite4 will invoke the sqlite3_mm_free() +method of the memory allocator on the string when it has finished using +the string. + +

      3.2 The Run-Time Environment Object

      + +An instance of the run-time environment object defines how SQLite4 +interfaces to the rest of the system. Each run-time environment object +includes a pointer to a memory allocator object which is the default +memory allocator for that environment. The run-time environment also +includes information such as: + + * How to create and use mutexes + * Which storage engines are available for use on the backend + * A library of SQL functions available for use by all database connections + * Methods for obtaining the current time and for obtaining randomness + * State information that is shared across multiple database connections + * Other "global" configuration settings and start-time settings + +SQLite4 supplies a default run-time environment object +(obtained by calling the sqlite4_env_default() function) +that is suitable for use in most applications. +In fact, for many routines that require a pointer to a +run-time environment object, you can pass in a NULL pointer and the +NULL pointer will be automatically converted into a pointer to the default +run-time environment. + +Usually the default run-time environment object is adequate, but some +applications, especially those running on custom platforms or on small +resource-limited devices, may want to create and use a new run-time +environment object with non-standard settings. A reasonable design is +to create a single instance of an appropriate run-time environment object, +then store a pointer to that object in a global variable where it is +readily accessible to all parts of the application. + +It is possible to use two or more run-time environment objects within +the same application. But use caution here. Due to a long-standing +bug in the design of Posix Advisor Locks (NB: the bug is in posix, not +in SQLite) if you open two or more database connections to the same +database using different run-time environment objects, then file locks +will be broken in unexpected ways, which can lead to database corruption. +Opening connections with different run-time environments to different +databases is harmless and safe. Opening multiple connections to the +same database using the same run-time environment is also harmless and safe. +But opening multiple connections to the same database from different +run-time environments can cause problems under posix. Therefore, if +you are programming for a posix system (iOS, Android, QNX, Linux, MacOS, +etc.) you are advised to stick to using a single run-time environment +for your application. Windows does not have the posix advisory locking +bug and so you are free to mix and match run-time environment objects, +database connections, and databases with wild abandon there, if you are +willing to sacrifice portability. + +

      3.3 The Database Connection Object

      + +The most important object is the database connection object: sqlite4. +Each database connection object represents an open connection to one or +more database files. (The database connection starts out associated with +just a single database file, but more databases can be added using the +ATTACH statement.) Many applications will have just a single database +connection object, since they are only talking to a single database. +But there are no arbitrary restrictions on the number of database objects +that can be open at once. (The number of open database connections might +be constrained by system resource limitations such as the maximum number of +open file descriptors.) Multiple database connections can be opened on +the same database file, if desired. + +Every database connection object is associated with a single run-time +environment object. The environment of a database connection determines +how the database connection interacts with the operating system. The +run-time environment for the database connection is specified when the +database connection is opened and cannot be changed. In fact, the +procedure for creating (for "opening") a database connection object is +really a method on the run-time environment object. + +The life-cycle of a database connection object usually goes something +like this: + + # Create the database connection object (also referred to as "opening" the + database connection) using the sqlite4_open() method of the + sqlite4_env object. + # Run SQL statements against the database connection using either the + direct interface (sqlite4_exec()) or by creating prepared statement + objects, one for each SQL statement, and evaluating the prepared + statement objects. + # Destroy the database connection object (also referred to as + "closing" the database connection) using the sqlite4_close() method + of the sqlite4 object. + +Much SQLite programming centers around the use of database connection objects +and programmers who use the SQLite library will quickly become familar with +this object. It is the traditional (at least among the SQLite developers +themselves) to use the name "db" for variable that are pointers to a +database connection object. + +

      3.4 The Prepared Statement Object

      + +A prepared statement object, sqlite4_stmt, represents a single statement +of SQL that is to be run against a particular database connection. Every +prepared statement object is associated with a particular database connection +object. All prepared statements for a database connection must be destroyed +(or "finalized") before the database connection can be destroyed (or "closed"). + +The use of prepared statements is not strictly necessary in SQLite4. One can +do any type of SQL statement desired using the sqlite4_exec() method of the +database connection object. However, prepared statements are very handy and +every programmer who wants to make serious use of the SQLite4 library needs +to be familiar with them. + +A prepared statement object is created using the sqlite4_prepare() method +on the database connection object. The sqlite4_prepare() method takes +a string as input which contains the text of the SQL statement, and outputs +a pointer to a prepared statement object. Think of the SQL statement text +as source code and the prepared statement object as object code and the +sqlite4_prepare() method as the compiler. + +A prepared statement is evaluated or executed using the sqlite4_step() +method on the prepared statement object. Each call to sqlite4_step() +advances the prepared +statement until it either (1) determines the next row of output or (2) +completely finishes evaluating the SQL statement or (3) encounters an +error. The integer return value from sqlite4_step() lets the application +know which of these three outcomes occurred. For an SQL statement like +an INSERT, UPDATE, or DELETE that does not return a result, the +sqlite4_step() statement is usually called just a single time and reports +completion of the SQL statement after the first step. For a SELECT +statement that returns multiple rows of output, on the other hand, +the sqlite4_step() statement will be called multiple times, once for +each row of output, then one last time to complete the evaluation. + +At any time, though typically after the prepared statement completes, +the prepared statement object can be destroyed (or "finalized") by +calling the sqlite4_finalize() method. The perpared statement can also +be reset back to the beginning of its execution program at any time by +using the sqlite4_reset() method. A common paradigm is to prepare +a prepared statement once, then run the same statement many times +using calls to sqlite4_step() for each execution with calls to +sqlite4_reset() to rewind the program back to the beginning whenever +it completes, followed by a single call to sqlite4_finalize() to destroy +the prepared statement object at the very end. + +The SQL text used to create a prepared statement might include +"parameter" - placeholders for unknown values. For example: + +
      +INSERT INTO tab_xyz(x1,y2,z3) VALUES(:x, :y, :z); +
      + +In the INSERT statement above, the three values to be inserted as a new +row into tab_xyz are specified as parameters ":x", ":y", and ":z". +Parameters can also be named as just a question mark ("?") or as a +question mark followed by a small integer (ex: "?17") or as identifiers +that come after "$" or "@" (ex: "$var1" or "@xyz"). + +When a prepared statement contains parameters, values (strings, blobs, +integers, floating point numbers) can be "bound" to those parameters +using methods on the prepared statement object. For example, the +sqlite4_bind_text() method will bind a UTF8 string to a parameter. +The sqlite4_bind_int64() method will bind a signed 64-bit integer to +a parameter. The sqlite4_bind_blob() method will bind a BLOB to the +parameter. And so forth. +A parameter can be bound multiple times. Each binding overrides the +previous. + +The use of parameters has important advantages: + + 1. Prepared statements can be compiled once and then reused many times, + by resetting the prepared statement and rebinding new values into + the parameters. This reduces the number of calls to sqlite4_prepare() + and thus improves application performance. + + 2. Values bound to parameters do not need to be escaped or encoded in + any way. One binds raw content. This is both more efficient than + having to encode the value as SQL text, and it completely eliminates + the possibility of an SQL injection attack. + +It is stated above that the use of prepared statements is not required +since all SQL statements can be run using the sqlite4_exec() method on +the database connection object. That statement is true, but does not +give the whole story. In reality, the use of a prepared statement is +the only way to evaluate SQL statements in SQLite4. The sqlite4_exec() +method is just a convenience wrapper that does all of the low-level work +of preparing, stepping, and finalizing the prepared statements automatically, +and out of sight. So even though an application can avoid having to use +prepared statements directly by using just sqlite4_exec(), prepared statement +objects are still being used behind the scenes. + +

      4.0 Usage Example

      + +TBD..