Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -2500,30 +2500,37 @@ ** parameter bDescDoclist should be false. If they are sorted in ascending ** order, it should be passed a non-zero value. ** ** The right-hand input doclist is overwritten by this function. */ -static void fts3DoclistPhraseMerge( +static int fts3DoclistPhraseMerge( int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ - char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; + char *aRight = *paRight; char *pEnd1 = &aLeft[nLeft]; char *pEnd2 = &aRight[*pnRight]; char *p1 = aLeft; char *p2 = aRight; char *p; int bFirstOut = 0; - char *aOut = aRight; + char *aOut; assert( nDist>0 ); - + if( bDescDoclist ){ + aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 && p2 ){ sqlite3_int64 iDiff = DOCID_CMP(i1, i2); @@ -2548,10 +2555,16 @@ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; } /* ** Argument pList points to a position list nList bytes in size. This ** function checks to see if the position list contains any entries for @@ -2672,12 +2685,26 @@ char *aDoclist, /* Pointer to doclist */ int nDoclist /* Size of aDoclist in bytes */ ){ if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist); + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); }else{ return SQLITE_NOMEM; @@ -3929,18 +3956,21 @@ ** It is merged into the main doclist stored in p->doclist.aAll/nAll. ** ** This function assumes that pList points to a buffer allocated using ** sqlite3_malloc(). This function takes responsibility for eventually ** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. */ -static void fts3EvalPhraseMergeToken( +static int fts3EvalPhraseMergeToken( Fts3Table *pTab, /* FTS Table pointer */ Fts3Phrase *p, /* Phrase to merge pList/nList into */ int iToken, /* Token pList/nList corresponds to */ char *pList, /* Pointer to doclist */ int nList /* Number of bytes in pList */ ){ + int rc = SQLITE_OK; assert( iToken!=p->iDoclistToken ); if( pList==0 ){ sqlite3_free(p->doclist.aAll); p->doclist.aAll = 0; @@ -3975,17 +4005,20 @@ pLeft = pList; nLeft = nList; nDiff = p->iDoclistToken - iToken; } - fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); sqlite3_free(pLeft); p->doclist.aAll = pRight; p->doclist.nAll = nRight; } if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; } /* ** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist ** does not take deferred tokens into account. @@ -4007,11 +4040,11 @@ if( pToken->pSegcsr ){ int nThis = 0; char *pThis = 0; rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ - fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } } assert( pToken->pSegcsr==0 ); } @@ -4809,13 +4842,17 @@ Fts3PhraseToken *pToken = pTC->pToken; int nList = 0; char *pList = 0; rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } if( rc==SQLITE_OK ){ int nCount; - fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); nCount = fts3DoclistCountDocids( pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll ); if( ii==0 || nCountctrlFlags |= UNIXFILE_BLOCK; + /* pFile->ctrlFlags |= UNIXFILE_BLOCK; // Deferred feature */ return SQLITE_OK; } case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -1184,10 +1184,24 @@ sNC.pParse = pParse; if( sqlite3ResolveExprNames(&sNC, p->pLimit) || sqlite3ResolveExprNames(&sNC, p->pOffset) ){ return WRC_Abort; } + + /* If the SF_Converted flags is set, then this Select object was + ** was created by the convertCompoundSelectToSubquery() function. + ** In this case the ORDER BY clause (p->pOrderBy) should be resolved + ** as if it were part of the sub-query, not the parent. This block + ** moves the pOrderBy down to the sub-query. It will be moved back + ** after the names have been resolved. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + assert( p->pSrc->nSrc==1 && isCompound==0 && p->pOrderBy ); + assert( pSub->pPrior && pSub->pOrderBy==0 ); + pSub->pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + } /* Recursively resolve names in all subqueries */ for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; @@ -1265,10 +1279,21 @@ /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries */ sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg; + + /* If this is a converted compound query, move the ORDER BY clause from + ** the sub-query back to the parent query. At this point each term + ** within the ORDER BY clause has been transformed to an integer value. + ** These integers will be replaced by copies of the corresponding result + ** set expressions by the call to resolveOrderGroupBy() below. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + p->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + } /* Process the ORDER BY clause for singleton SELECT statements. ** The ORDER BY clause for compounds SELECT statements is handled ** below, after all of the result-sets for all of the elements of ** the compound have been resolved. Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -3882,10 +3882,12 @@ pNew->pHaving = 0; pNew->pOrderBy = 0; p->pPrior = 0; p->pNext = 0; p->selFlags &= ~SF_Compound; + assert( (p->selFlags & SF_Converted)==0 ); + p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); pNew->pPrior->pNext = pNew; pNew->pLimit = 0; pNew->pOffset = 0; return WRC_Continue; Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -2395,10 +2395,11 @@ #define SF_AllValues 0x0100 /* All terms of compound are VALUES */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ #define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ +#define SF_Converted 0x2000 /* By convertCompoundSelectToSubquery() */ /* ** The results of a SELECT can be distributed in several ways, as defined ** by one of the following macros. The "SRT" prefix means "SELECT Result Index: test/e_walauto.test ================================================================== --- test/e_walauto.test +++ test/e_walauto.test @@ -13,10 +13,18 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/wal_common.tcl set testprefix e_walauto +# Do not run this test on OpenBSD, as it depends on read() and mmap both +# accessing the same coherent view of the "test.db-shm" file. This doesn't +# work on OpenBSD. +# +if {$tcl_platform(os) == "OpenBSD"} { + finish_test + return +} proc read_nbackfill {} { seek $::shmfd 96 binary scan [read $::shmfd 4] n nBackfill set nBackfill Index: test/fts3fault2.test ================================================================== --- test/fts3fault2.test +++ test/fts3fault2.test @@ -152,7 +152,26 @@ execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' } } -test { faultsim_test_result {0 {1 2}} } } + +reset_db +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t6 USING fts4(x,order=DESC); + INSERT INTO t6(docid, x) VALUES(-1,'a b'); + INSERT INTO t6(docid, x) VALUES(1, 'b'); + } + faultsim_save_and_close +} {} + +do_faultsim_test 6.1 -faults oom* -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_master} +} -body { + execsql { SELECT docid FROM t6 WHERE t6 MATCH '"a* b"' } +} -test { + faultsim_test_result {0 -1} +} finish_test Index: test/fts3prefix.test ================================================================== --- test/fts3prefix.test +++ test/fts3prefix.test @@ -271,7 +271,25 @@ INSERT INTO t2 VALUES('He dressed himself in cycling clothes'); } {} do_execsql_test 6.5.2 { SELECT md5sum(quote(root)) FROM t1_segdir; } [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}] + + +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t6 USING fts4(x,order=DESC); + INSERT INTO t6(docid, x) VALUES(-1,'a b'); + INSERT INTO t6(docid, x) VALUES(1, 'b'); +} +do_execsql_test 7.1 { + SELECT docid FROM t6 WHERE t6 MATCH '"a* b"'; +} {-1} +do_execsql_test 7.2 { + SELECT docid FROM t6 WHERE t6 MATCH 'a*'; +} {-1} +do_execsql_test 7.3 { + SELECT docid FROM t6 WHERE t6 MATCH 'a* b'; +} {-1} + + finish_test Index: test/selectA.test ================================================================== --- test/selectA.test +++ test/selectA.test @@ -1372,7 +1372,67 @@ UNION ALL SELECT a, b FROM t6 ORDER BY 1,2 } {/2 . 3 . 4 . 5 . 6 . 7 ./} + +proc strip_rnd {explain} { + regexp -all {sqlite_sq_[0123456789ABCDEF]*} $explain sqlite_sq +} + +proc do_same_test {tn q1 args} { + set r2 [strip_rnd [db eval "EXPLAIN $q1"]] + set i 1 + foreach q $args { + set tst [subst -nocommands {strip_rnd [db eval "EXPLAIN $q"]}] + uplevel do_test $tn.$i [list $tst] [list $r2] + incr i + } +} + +do_execsql_test 5.0 { + CREATE TABLE t8(a, b); + CREATE TABLE t9(c, d); +} {} + +do_same_test 5.1 { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY a; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY t8.a; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY 1; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY c; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY t9.c; +} + +do_same_test 5.2 { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY a COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY t8.a COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY 1 COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY c COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY t9.c COLLATE NOCASE +} + +do_same_test 5.3 { + SELECT a, b FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY b, c COLLATE NOCASE +} { + SELECT a, b FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY 2, 1 COLLATE NOCASE +} { + SELECT a, b FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY d, a COLLATE NOCASE +} { + SELECT a, b FROM t8 EXCEPT SELECT * FROM t9 ORDER BY t9.d, c COLLATE NOCASE +} { + SELECT * FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY d, t8.a COLLATE NOCASE +} + +do_catchsql_test 5.4 { + SELECT * FROM t8 UNION SELECT * FROM t9 ORDER BY a+b COLLATE NOCASE +} {1 {1st ORDER BY term does not match any column in the result set}} + finish_test Index: test/walblock.test ================================================================== --- test/walblock.test +++ test/walblock.test @@ -13,10 +13,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl +finish_test; return; # Feature currently not implemented. ifcapable !wal {finish_test ; return } if {$::tcl_platform(platform)!="unix"} { finish_test ; return } set testprefix walblock catch { db close } @@ -109,9 +110,5 @@ set ::out } {1 2 3 4 5 6 7 8 9 10} finish_test - - - - ADDED tool/showlocks.c Index: tool/showlocks.c ================================================================== --- tool/showlocks.c +++ tool/showlocks.c @@ -0,0 +1,64 @@ +/* +** This file implements a simple command-line utility that shows all of the +** Posix Advisory Locks on a file. +** +** Usage: +** +** showlocks FILENAME +** +** To compile: gcc -o showlocks showlocks.c +*/ +#include +#include +#include +#include +#include + +/* This utility only looks for locks in the first 2 billion bytes */ +#define MX_LCK 2147483647 + +/* +** Print all locks on the inode of "fd" that occur in between +** lwr and upr, inclusive. +*/ +static int showLocksInRange(int fd, off_t lwr, off_t upr){ + int cnt = 0; + struct flock x; + + x.l_type = F_WRLCK; + x.l_whence = SEEK_SET; + x.l_start = lwr; + x.l_len = upr-lwr; + fcntl(fd, F_GETLK, &x); + if( x.l_type==F_UNLCK ) return 0; + printf("start: %-12d len: %-5d pid: %-5d type: %s\n", + (int)x.l_start, (int)x.l_len, + x.l_pid, x.l_type==F_WRLCK ? "WRLCK" : "RDLCK"); + cnt++; + if( x.l_start>lwr ){ + cnt += showLocksInRange(fd, lwr, x.l_start-1); + } + if( x.l_start+x.l_len