Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -4815,18 +4815,28 @@ /* ** Advance the cursor to the next entry in the database. If ** successful then set *pRes=0. If the cursor ** was already pointing to the last entry in the database before ** this routine was called, then set *pRes=1. +** +** The calling function will set *pRes to 0 or 1. The initial *pRes value +** will be 1 if the cursor being stepped corresponds to an SQL index and +** if this routine could have been skipped if that SQL index had been +** a unique index. Otherwise the caller will have set *pRes to zero. +** Zero is the common case. The btree implementation is free to use the +** initial *pRes value as a hint to improve performance, but the current +** SQLite btree implementation does not. (Note that the comdb2 btree +** implementation does use this hint, however.) */ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ int rc; int idx; MemPage *pPage; assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); if( pCur->eState!=CURSOR_VALID ){ rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ *pRes = 0; @@ -4901,17 +4911,27 @@ /* ** Step the cursor to the back to the previous entry in the database. If ** successful then set *pRes=0. If the cursor ** was already pointing to the first entry in the database before ** this routine was called, then set *pRes=1. +** +** The calling function will set *pRes to 0 or 1. The initial *pRes value +** will be 1 if the cursor being stepped corresponds to an SQL index and +** if this routine could have been skipped if that SQL index had been +** a unique index. Otherwise the caller will have set *pRes to zero. +** Zero is the common case. The btree implementation is free to use the +** initial *pRes value as a hint to improve performance, but the current +** SQLite btree implementation does not. (Note that the comdb2 btree +** implementation does use this hint, however.) */ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ int rc; MemPage *pPage; assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); pCur->atLast = 0; if( pCur->eState!=CURSOR_VALID ){ if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ rc = btreeRestoreCursorPosition(pCur); @@ -7148,11 +7168,11 @@ ** from the internal node. The 'previous' entry is used for this instead ** of the 'next' entry, as the previous entry is always a part of the ** sub-tree headed by the child page of the cell being deleted. This makes ** balancing the tree following the delete operation easier. */ if( !pPage->leaf ){ - int notUsed; + int notUsed = 0; rc = sqlite3BtreePrevious(pCur, ¬Used); if( rc ) return rc; } /* Save the positions of any other cursors open on this table before Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -3094,11 +3094,11 @@ ** Reset the PRNG back to its uninitialized state. The next call ** to sqlite3_randomness() will reseed the PRNG using a single call ** to the xRandomness method of the default VFS. */ case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3PrngResetState(); + sqlite3_randomness(0,0); break; } /* ** sqlite3_test_control(BITVEC_TEST, size, program) Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -258,10 +258,16 @@ */ char aPadding[32]; #endif }; +/* This variable holds the process id (pid) from when the xRandomness() +** method was called. If xOpen() is called from a different process id, +** indicating that a fork() has occurred, the PRNG will be reset. +*/ +static int randomnessPid = 0; + /* ** Allowed values for the unixFile.ctrlFlags bitmask: */ #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ @@ -5638,10 +5644,20 @@ assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); + + /* Detect a pid change and reset the PRNG. There is a race condition + ** here such that two or more threads all trying to open databases at + ** the same instant might all reset the PRNG. But multiple resets + ** are harmless. + */ + if( randomnessPid!=getpid() ){ + randomnessPid = getpid(); + sqlite3_randomness(0,0); + } memset(p, 0, sizeof(unixFile)); if( eType==SQLITE_OPEN_MAIN_DB ){ UnixUnusedFd *pUnused; @@ -6026,22 +6042,22 @@ ** When testing, initializing zBuf[] to zero is all we do. That means ** that we always use the same random number sequence. This makes the ** tests repeatable. */ memset(zBuf, 0, nBuf); + randomnessPid = getpid(); #if !defined(SQLITE_TEST) { - int pid, fd, got; + int fd, got; fd = robust_open("/dev/urandom", O_RDONLY, 0); if( fd<0 ){ time_t t; time(&t); memcpy(zBuf, &t, sizeof(t)); - pid = getpid(); - memcpy(&zBuf[sizeof(t)], &pid, sizeof(pid)); - assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf ); - nBuf = sizeof(t) + sizeof(pid); + memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid)); + assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(randomnessPid); }else{ do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); robust_close(0, fd, __LINE__); } } Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3117,11 +3117,11 @@ winModeBit(pFile, WINFILE_PSOW, (int*)pArg); OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_VFSNAME: { - *(char**)pArg = sqlite3_mprintf("win32"); + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_WIN32_AV_RETRY: { int *a = (int*)pArg; @@ -3222,11 +3222,11 @@ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } static void winShmLeaveMutex(void){ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } -#ifdef SQLITE_DEBUG +#ifndef NDEBUG static int winShmMutexHeld(void) { return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #endif Index: src/random.c ================================================================== --- src/random.c +++ src/random.c @@ -49,10 +49,16 @@ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); sqlite3_mutex_enter(mutex); #endif + + if( N<=0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not ** trying to do secure encryption or anything like that... @@ -77,19 +83,20 @@ wsdPrng.s[i] = t; } wsdPrng.isInit = 1; } - while( N-- ){ + assert( N>0 ); + do{ wsdPrng.i++; t = wsdPrng.s[wsdPrng.i]; wsdPrng.j += t; wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; wsdPrng.s[wsdPrng.j] = t; t += wsdPrng.s[wsdPrng.i]; *(zBuf++) = wsdPrng.s[t]; - } + }while( --N ); sqlite3_mutex_leave(mutex); } #ifndef SQLITE_OMIT_BUILTIN_TEST /* @@ -114,9 +121,6 @@ &GLOBAL(struct sqlite3PrngType, sqlite3Prng), &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), sizeof(sqlite3Prng) ); } -void sqlite3PrngResetState(void){ - GLOBAL(struct sqlite3PrngType, sqlite3Prng).isInit = 0; -} #endif /* SQLITE_OMIT_BUILTIN_TEST */ Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -2368,15 +2368,17 @@ ** already uses the largest possible [ROWID]. The PRNG is also used for ** the build-in random() and randomblob() SQL functions. This interface allows ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. +** ^If N is less than one, then P can be a NULL pointer. ** -** ^The first time this routine is invoked (either internally or by -** the application) the PRNG is seeded using randomness obtained -** from the xRandomness method of the default [sqlite3_vfs] object. -** ^On all subsequent invocations, the pseudo-randomness is generated +** ^If this routine has not been previously called or if the previous +** call had N less than one, then the PRNG is seeded using randomness +** obtained from the xRandomness method of the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more then +** the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ void sqlite3_randomness(int N, void *P); Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -2955,11 +2955,10 @@ void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); Vdbe *sqlite3GetVdbe(Parse*); void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); -void sqlite3PrngResetState(void); void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); void sqlite3CommitTransaction(Parse*); Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -3578,19 +3578,21 @@ #ifdef SQLITE_TEST sqlite3_search_count++; #endif if( oc>=OP_SeekGe ){ assert( oc==OP_SeekGe || oc==OP_SeekGt ); if( res<0 || (res==0 && oc==OP_SeekGt) ){ + res = 0; rc = sqlite3BtreeNext(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; pC->rowidIsValid = 0; }else{ res = 0; } }else{ assert( oc==OP_SeekLt || oc==OP_SeekLe ); if( res>0 || (res==0 && oc==OP_SeekLt) ){ + res = 0; rc = sqlite3BtreePrevious(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; pC->rowidIsValid = 0; }else{ /* res might be negative because the table is empty. Check to @@ -4440,50 +4442,60 @@ pc = pOp->p2 - 1; } break; } -/* Opcode: Next P1 P2 * * P5 +/* Opcode: Next P1 P2 P3 * P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its ** table or index. If there are no more key/value pairs then fall through ** to the following instruction. But if the cursor advance was successful, ** jump immediately to P2. ** ** The P1 cursor must be for a real table, not a pseudo-table. P1 must have ** been opened prior to this opcode or the program will segfault. +** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. ** ** P4 is always of type P4_ADVANCE. The function pointer points to ** sqlite3BtreeNext(). ** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. ** ** See also: Prev, NextIfOpen */ -/* Opcode: NextIfOpen P1 P2 * * P5 +/* Opcode: NextIfOpen P1 P2 P3 * P5 ** ** This opcode works just like OP_Next except that if cursor P1 is not ** open it behaves a no-op. */ -/* Opcode: Prev P1 P2 * * P5 +/* Opcode: Prev P1 P2 P3 * P5 ** ** Back up cursor P1 so that it points to the previous key/data pair in its ** table or index. If there is no previous key/value pairs then fall through ** to the following instruction. But if the cursor backup was successful, ** jump immediately to P2. ** ** The P1 cursor must be for a real table, not a pseudo-table. If P1 is ** not open then the behavior is undefined. +** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. ** ** P4 is always of type P4_ADVANCE. The function pointer points to ** sqlite3BtreePrevious(). ** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ -/* Opcode: PrevIfOpen P1 P2 * * P5 +/* Opcode: PrevIfOpen P1 P2 P3 * P5 ** ** This opcode works just like OP_Prev except that if cursor P1 is not ** open it behaves a no-op. */ case OP_SorterNext: { /* jump */ @@ -4501,13 +4513,16 @@ case OP_Prev: /* jump */ case OP_Next: /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pOp->p5aCounter) ); pC = p->apCsr[pOp->p1]; + res = pOp->p3; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); assert( pC->pCursor ); + assert( res==0 || (res==1 && pC->isTable==0) ); + testcase( res==1 ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); assert( pOp->opcode!=OP_PrevIfOpen || pOp->p4.xAdvance==sqlite3BtreePrevious); rc = pOp->p4.xAdvance(pC->pCursor, &res); Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -3180,10 +3180,12 @@ pLevel->op = OP_Prev; }else{ pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; + assert( (WHERE_UNQ_WANTED>>16)==1 ); + pLevel->p3 = (pLoop->wsFlags>>16)&1; if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; }else{ assert( pLevel->p5==0 ); } @@ -3926,10 +3928,11 @@ */ if( pTerm==0 && saved_nEq==saved_nSkip && saved_nEq+1nKeyCol && pProbe->aiRowEst[saved_nEq+1]>=18 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; pNew->u.btree.nEq++; pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; @@ -3975,16 +3978,17 @@ assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 || nInMul==0 ); pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 - || (pProbe->onError!=OE_None && nInMul==0 - && pNew->u.btree.nEq==pProbe->nKeyCol-1) - ){ + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1)){ assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); - pNew->wsFlags |= WHERE_ONEROW; + if( iCol>=0 && pProbe->onError==OE_None ){ + pNew->wsFlags |= WHERE_UNQ_WANTED; + }else{ + pNew->wsFlags |= WHERE_ONEROW; + } } pNew->u.btree.nEq++; pNew->nOut = nRowEst + nInMul; }else if( pTerm->eOperator & (WO_ISNULL) ){ pNew->wsFlags |= WHERE_COLUMN_NULL; @@ -5770,11 +5774,11 @@ int addr; pLevel = &pWInfo->a[i]; pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ - sqlite3VdbeAddOp2(v, pLevel->op, pLevel->p1, pLevel->p2); + sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); } if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; Index: src/whereInt.h ================================================================== --- src/whereInt.h +++ src/whereInt.h @@ -68,11 +68,11 @@ int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ u8 iFrom; /* Which entry in the FROM clause */ - u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ + u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ int p1, p2; /* Operands of the opcode used to ends the loop */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ struct InLoop { @@ -455,5 +455,6 @@ #define WHERE_IN_ABLE 0x00000800 /* Able to support an IN operator */ #define WHERE_ONEROW 0x00001000 /* Selects no more than one row */ #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ +#define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ Index: test/skipscan1.test ================================================================== --- test/skipscan1.test +++ test/skipscan1.test @@ -184,7 +184,29 @@ } {/* INDEX sqlite_autoindex_t3_1 (ANY(a) AND b=?)*/} do_execsql_test skipscan1-3.2sort { EXPLAIN QUERY PLAN SELECT a,b,c,d,'|' FROM t3 WHERE b=345 ORDER BY a; } {~/*ORDER BY*/} + +# Ticket 520070ec7fbaac: Array overrun in the skip-scan optimization +# 2013-12-22 +# +do_execsql_test skipscan1-4.1 { + CREATE TABLE t4(a,b,c,d,e,f,g,h,i); + CREATE INDEX t4all ON t4(a,b,c,d,e,f,g,h); + INSERT INTO t4 VALUES(1,2,3,4,5,6,7,8,9); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1 + VALUES('t4','t4all','655360 163840 40960 10240 2560 640 160 40 10'); + ANALYZE sqlite_master; + SELECT i FROM t4 WHERE a=1; + SELECT i FROM t4 WHERE b=2; + SELECT i FROM t4 WHERE c=3; + SELECT i FROM t4 WHERE d=4; + SELECT i FROM t4 WHERE e=5; + SELECT i FROM t4 WHERE f=6; + SELECT i FROM t4 WHERE g=7; + SELECT i FROM t4 WHERE h=8; +} {9 9 9 9 9 9 9 9} finish_test Index: test/win32longpath.test ================================================================== --- test/win32longpath.test +++ test/win32longpath.test @@ -16,16 +16,24 @@ if {$tcl_platform(platform)!="windows"} return set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix win32longpath + +do_test 1.0 { + file_control_vfsname db +} win32 db close set path [file nativename [get_pwd]] sqlite3 db [file join $path test.db] -vfs win32-longpath do_test 1.1 { + file_control_vfsname db +} win32-longpath + +do_test 1.2 { db eval { BEGIN EXCLUSIVE; CREATE TABLE t1(x); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2); @@ -45,17 +53,17 @@ set longPath(3) $longPath(2)\\[string repeat Y 255] make_win32_dir $longPath(3) set fileName $longPath(3)\\test.db -do_test 1.2 { +do_test 1.3 { list [catch {sqlite3 db2 [string range $fileName 4 end]} msg] $msg } {1 {unable to open database file}} sqlite3 db3 $fileName -vfs win32-longpath -do_test 1.3 { +do_test 1.4 { db3 eval { BEGIN EXCLUSIVE; CREATE TABLE t1(x); INSERT INTO t1 VALUES(5); INSERT INTO t1 VALUES(6); @@ -69,17 +77,17 @@ db3 close # puts " Database exists \{[exists_win32_path $fileName]\}" sqlite3 db3 $fileName -vfs win32-longpath -do_test 1.4 { +do_test 1.5 { db3 eval { PRAGMA journal_mode = WAL; } } {wal} -do_test 1.5 { +do_test 1.6 { db3 eval { BEGIN EXCLUSIVE; INSERT INTO t1 VALUES(9); INSERT INTO t1 VALUES(10); INSERT INTO t1 VALUES(11);