Index: src/analyze.c ================================================================== --- src/analyze.c +++ src/analyze.c @@ -447,11 +447,13 @@ int regStat1 = iMem++; /* The stat column of sqlite_stat1 */ #ifdef SQLITE4_ENABLE_STAT3 int regNumEq = regStat1; /* Number of instances. Same as regStat1 */ int regNumLt = iMem++; /* Number of keys less than regSample */ int regNumDLt = iMem++; /* Number of distinct keys less than regSample */ +#endif int regSample = iMem++; /* The next sample value */ +#ifdef SQLITE4_ENABLE_STAT3 int regAccum = iMem++; /* Register to hold Stat3Accum object */ int regLoop = iMem++; /* Loop counter */ int regCount = iMem++; /* Number of rows in the table or index */ int regTemp1 = iMem++; /* Intermediate register */ int regTemp2 = iMem++; /* Intermediate register */ Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -340,11 +340,11 @@ ** the scan is complete because deleting an item can change the scan ** order. */ sqlite4VdbeAddOp2(v, OP_Null, 0, regSet); VdbeComment((v, "initialize RowSet")); pWInfo = sqlite4WhereBegin( - pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK + pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK, 0 ); if( pWInfo==0 ) goto delete_from_cleanup; sqlite4VdbeAddOp2(v, OP_RowKey, iCur, regKey); sqlite4VdbeAddOp3(v, OP_RowSetAdd, regSet, 0, regKey); sqlite4WhereEnd(pWInfo); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -1464,10 +1464,11 @@ ** ** in order to avoid running the ** test more often than is necessary. */ #ifndef SQLITE4_OMIT_SUBQUERY +#if 0 int sqlite4FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ Index *pIdx; int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ Vdbe *v = sqlite4GetVdbe(pParse); /* Virtual machine being coded */ @@ -1509,10 +1510,183 @@ return eType; } #endif +/* +** This function is used by the implementation of the IN (...) operator. +** The pX parameter is the expression on the RHS of the IN operator, which +** might be either a list of expressions or a subquery. +** +** The job of this routine is to find or create a b-tree object that can +** be used either to test for membership in the RHS set or to iterate through +** all members of the RHS set, skipping duplicates. +** +** A cursor is opened on the b-tree object that the RHS of the IN operator +** and pX->iTable is set to the index of that cursor. +** +** The returned value of this function indicates the b-tree type, as follows: +** +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. +** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. +** +** An existing b-tree might be used if the RHS expression pX is a simple +** subquery such as: +** +** SELECT FROM +** +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephermeral table instead of an +** existing table. +** +** If the prNotFound parameter is 0, then the b-tree will be used to iterate +** through the set members, skipping any duplicates. In this case an +** epheremal table must be used unless the selected is guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or it +** has a UNIQUE constraint or UNIQUE index. +** +** If the prNotFound parameter is not 0, then the b-tree will be used +** for fast set membership tests. In this case an epheremal table must +** be used unless is an INTEGER PRIMARY KEY or an index can +** be found with as its left-most column. +** +** When the b-tree is being used for membership tests, the calling function +** needs to know whether or not the structure contains an SQL NULL +** value in order to correctly evaluate expressions like "X IN (Y, Z)". +** If there is any chance that the (...) might contain a NULL value at +** runtime, then a register is allocated and the register number written +** to *prNotFound. If there is no chance that the (...) contains a +** NULL value, then *prNotFound is left unchanged. +** +** If a register is allocated and its location stored in *prNotFound, then +** its initial value is NULL. If the (...) does not remain constant +** for the duration of the query (i.e. the SELECT within the (...) +** is a correlated subquery) then the value of the allocated register is +** reset to NULL each time the subquery is rerun. This allows the +** caller to use vdbe code equivalent to the following: +** +** if( register==NULL ){ +** has_null = +** register = 1 +** } +** +** in order to avoid running the +** test more often than is necessary. +*/ +int sqlite4FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ + Select *p; /* SELECT to the right of IN operator */ + int eType = 0; /* Type of RHS table. IN_INDEX_* */ + int iTab = pParse->nTab++; /* Cursor of the RHS table */ + int mustBeUnique = (prNotFound==0); /* True if RHS must be unique */ + Vdbe *v = sqlite4GetVdbe(pParse); /* Virtual machine being coded */ + + assert( pX->op==TK_IN ); + + /* Check to see if an existing table or index can be used to + ** satisfy the query. This is preferable to generating a new + ** ephemeral table. + */ + p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); + if( ALWAYS(pParse->nErr==0) && isCandidateForInOpt(p) && prNotFound ){ + sqlite4 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table
. */ + Expr *pExpr; /* Expression */ + int iCol; /* Index of column */ + int iDb; /* Database idx for pTab */ + + assert( p ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + iCol = pExpr->iColumn; + + /* Code an OP_VerifyCookie for
. */ + iDb = sqlite4SchemaToIndex(db, pTab->pSchema); + sqlite4CodeVerifySchema(pParse, iDb); + + /* This function is only called from two places. In both cases the vdbe + ** has already been allocated. So assume sqlite4GetVdbe() is always + ** successful here. + */ + assert(v); +#if 0 + if( iCol<0 ){ + int iAddr; + + iAddr = sqlite4CodeOnce(pParse); + + sqlite4OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + eType = IN_INDEX_ROWID; + + sqlite4VdbeJumpHere(v, iAddr); + }else +#endif + { + Index *pIdx; /* Iterator variable */ + + /* The collation sequence used by the comparison. If an index is to + ** be used in place of a temp-table, it must be ordered according + ** to this collation sequence. */ + CollSeq *pReq = sqlite4BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); + + /* Check that the affinity that will be used to perform the + ** comparison is the same as the affinity of the column. If + ** it is not, it is not possible to use any index. + */ + int affinity_ok = sqlite4IndexAffinityOk(pX, pTab->aCol[iCol].affinity); + + for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ + if( (pIdx->aiColumn[0]==iCol) + && sqlite4FindCollSeq(db, pIdx->azColl[0], 0)==pReq + && (!mustBeUnique || (pIdx->nColumn==1 && pIdx->onError!=OE_None)) + ){ + int iAddr; + + iAddr = sqlite4CodeOnce(pParse); + sqlite4OpenIndex(pParse, iTab, iDb, pIdx, OP_OpenRead); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + sqlite4VdbeJumpHere(v, iAddr); + if( prNotFound && !pTab->aCol[iCol].notNull ){ + *prNotFound = ++pParse->nMem; + sqlite4VdbeAddOp2(v, OP_Null, 0, *prNotFound); + } + } + } + } + } + + if( eType==0 ){ + /* Could not found an existing table or index to use as the RHS b-tree. + ** We will have to generate an ephemeral table to do the job. + */ + u32 savedNQueryLoop = pParse->nQueryLoop; + int rMayHaveNull = 0; + eType = IN_INDEX_EPH; + if( prNotFound ){ + *prNotFound = rMayHaveNull = ++pParse->nMem; + sqlite4VdbeAddOp2(v, OP_Null, 0, *prNotFound); + }else{ + testcase( pParse->nQueryLoop>0 ); + pParse->nQueryLoop = 0; + } + sqlite4CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); + pParse->nQueryLoop = savedNQueryLoop; + }else{ + pX->iTable = iTab; + } + return eType; +} +#endif + /* ** Generate code for scalar subqueries used as a subquery expression, EXISTS, ** or IN operators. Examples: ** ** (SELECT a FROM b) -- subquery Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -511,11 +511,11 @@ sqlite4ResolveExprNames(&sNameContext, pWhere); /* Create VDBE to loop through the entries in pSrc that match the WHERE ** clause. For each row found, increment the relevant constraint counter ** by nIncr. */ - pWInfo = sqlite4WhereBegin(pParse, pSrc, pWhere, 0, 0, 0); + pWInfo = sqlite4WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); if( nIncr>0 && pFKey->isDeferred==0 ){ sqlite4ParseToplevel(pParse)->mayAbort = 1; } sqlite4VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -3956,13 +3956,16 @@ /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ ExprList *pDist = (isDistinct ? p->pEList : 0); /* Begin the database scan. */ - pWInfo = sqlite4WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0); + pWInfo = sqlite4WhereBegin(pParse, pTabList, pWhere, pOrderBy, pDist, 0, 0); if( pWInfo==0 ) goto select_end; - if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; + if( sqlite4WhereOutputRowCount(pWInfo)nSelectRow ){ + p->nSelectRow = sqlite4WhereOutputRowCount(pWInfo); + } + if( pOrderBy && sqlite4WhereIsOrdered(pWInfo) ) pOrderBy = 0; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ @@ -3969,22 +3972,22 @@ if( addrSortIndex>=0 && pOrderBy==0 ){ sqlite4VdbeChangeToNoop(v, addrSortIndex); p->addrOpenEphm[2] = -1; } - if( pWInfo->eDistinct ){ + if( sqlite4WhereIsDistinct(pWInfo) ){ VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ assert( addrDistinctIndex>=0 ); pOp = sqlite4VdbeGetOp(v, addrDistinctIndex); assert( isDistinct ); - assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED - || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE + assert( sqlite4WhereIsDistinct(pWInfo)==WHERE_DISTINCT_ORDERED + || sqlite4WhereIsDistinct(pWInfo)==WHERE_DISTINCT_UNIQUE ); distinct = -1; - if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ + if( sqlite4WhereIsDistinct(pWInfo)==WHERE_DISTINCT_ORDERED ){ int iJump; int iExpr; int iFlag = ++pParse->nMem; int iBase = pParse->nMem+1; int iBase2 = iBase + pEList->nExpr; @@ -4003,11 +4006,11 @@ CollSeq *pColl = sqlite4ExprCollSeq(pParse, pEList->a[iExpr].pExpr); sqlite4VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); sqlite4VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); sqlite4VdbeChangeP5(v, SQLITE4_NULLEQ); } - sqlite4VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); + sqlite4VdbeAddOp2(v, OP_Goto, 0, sqlite4WhereContinueLabel(pWInfo)); sqlite4VdbeAddOp2(v, OP_Integer, 0, iFlag); assert( sqlite4VdbeCurrentAddr(v)==iJump ); sqlite4VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); }else{ @@ -4015,11 +4018,12 @@ } } /* Use the standard inner loop. */ selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, - pWInfo->iContinue, pWInfo->iBreak); + sqlite4WhereContinueLabel(pWInfo), sqlite4WhereBreakLabel(pWInfo) + ); /* End the database scan loop. */ sqlite4WhereEnd(pWInfo); }else{ @@ -4125,18 +4129,17 @@ ** This might involve two separate loops with an OP_Sort in between, or ** it might be a single loop that uses an index to extract information ** in the right order to begin with. */ sqlite4VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite4WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0); + pWInfo = sqlite4WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, 0, 0); if( pWInfo==0 ) goto select_end; - if( pGroupBy==0 ){ + if( sqlite4WhereIsOrdered(pWInfo) ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be - ** cancelled later because we still need to use the pKeyInfo + ** cancelled later because we still need to use the pKeyInfo. */ - pGroupBy = p->pGroupBy; groupBySort = 0; /* Evaluate the current GROUP BY terms and store in b0, b1, b2... ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth) ** Then compare the current GROUP BY terms against the GROUP BY terms @@ -4337,18 +4340,18 @@ /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite4WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag); + pWInfo = sqlite4WhereBegin(pParse, pTabList, pWhere, pMinMax, 0,flag,0); if( pWInfo==0 ){ sqlite4ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); - if( !pMinMax && flag ){ - sqlite4VdbeAddOp2(v, OP_Goto, 0, pWInfo->iBreak); + if( sqlite4WhereIsOrdered(pWInfo) && flag ){ + sqlite4VdbeAddOp2(v, OP_Goto, 0, sqlite4WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); } sqlite4WhereEnd(pWInfo); finalizeAggFunctions(pParse, &sAggInfo); Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -347,10 +347,16 @@ */ #ifndef offsetof #define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) #endif +/* +** Macros to compute minimum and maximum of two numbers. +*/ +#define MIN(A,B) ((A)<(B)?(A):(B)) +#define MAX(A,B) ((A)>(B)?(A):(B)) + /* ** Check to see if this machine uses EBCDIC. (Yes, believe it or ** not, there are still machines out there that use EBCDIC.) */ #if 'A' == '\301' @@ -529,10 +535,15 @@ ** A convenience macro that returns the number of elements in ** an array. */ #define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) +/* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + /* ** The following macros are used to suppress compiler warnings and to ** make it clear to human readers when a function parameter is deliberately ** left unused within the body of a function. This usually happens when ** a function is called via a function pointer. For example the @@ -921,10 +932,17 @@ #define SQLITE4_FactorOutConst 0x40 /* Disable factoring out constants */ #define SQLITE4_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ #define SQLITE4_DistinctOpt 0x80 /* DISTINCT using indexes */ #define SQLITE4_OptMask 0xff /* Mask of all disablable opts */ +/* +** Some new things pulled in from SQLite3 use these macros. todo: Replace +** them with working versions. +*/ +#define OptimizationDisabled(db, mask) 0 +#define OptimizationEnabled(db, mask) 1 + /* ** Possible values for the sqlite.magic field. ** The numbers are obtained at random and have no special meaning, other ** than being distinct from one another. */ @@ -1384,10 +1402,12 @@ int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ #endif Fts5Index *pFts; /* Fts5 data (or NULL if this is not an fts index) */ + + unsigned bUnordered:1; /* Use this index for == or IN queries only */ }; /* Index.eIndexType must be set to one of the following. */ #define SQLITE4_INDEX_USER 0 /* Index created by CREATE INDEX statement */ #define SQLITE4_INDEX_UNIQUE 1 /* Index created by UNIQUE constraint */ @@ -1842,10 +1862,11 @@ struct WhereTerm *pTerm; /* WHERE clause term for OR-search */ sqlite4_index_info *pVtabIdx; /* Virtual table index to use */ } u; }; +#if 0 /* ** For each nested loop in a WHERE clause implementation, the WhereInfo ** structure contains a single instance of this structure. This structure ** is intended to be private the the where.c module and should not be ** access or modified by other modules. @@ -1885,11 +1906,13 @@ ** a convenient place since there is one WhereLevel for each FROM clause ** element. */ sqlite4_index_info *pIdxInfo; /* Index info for n-th source table */ }; +#endif +#if 0 /* ** Flags appropriate for the wctrlFlags parameter of sqlite4WhereBegin() ** and the WhereInfo.wctrlFlags member. */ #define WHERE_ORDERBY_NORMAL 0x0000 /* No-op */ @@ -1899,18 +1922,40 @@ #define WHERE_DUPLICATES_OK 0x0008 /* Ok to return a row more than once */ #define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ #define WHERE_NO_AUTOINDEX 0x0020 /* Do not use an auto-index search */ #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ #define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ +#endif + + +/* +** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() +** and the WhereInfo.wctrlFlags member. +*/ +#define WHERE_ORDERBY_NORMAL 0x0000 /* No-op */ +#define WHERE_ORDERBY_MIN 0x0001 /* ORDER BY processing for min() func */ +#define WHERE_ORDERBY_MAX 0x0002 /* ORDER BY processing for max() func */ +#define WHERE_ONEPASS_DESIRED 0x0004 /* Want to do one-pass UPDATE/DELETE */ +#define WHERE_DUPLICATES_OK 0x0008 /* Ok to return a row more than once */ +#define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ +#define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ +#define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ +#define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ +#define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ +#define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ +#define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ + + /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second ** half does the tail of the WHERE loop. An instance of ** this structure is returned by the first half and passed ** into the second half to give some continuity. */ +#if 0 struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ u16 wctrlFlags; /* Flags originally passed to sqlite4WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ @@ -1923,13 +1968,16 @@ struct WhereClause *pWC; /* Decomposition of the WHERE clause */ double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ double nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +#endif -#define WHERE_DISTINCT_UNIQUE 1 -#define WHERE_DISTINCT_ORDERED 2 +#define WHERE_DISTINCT_NOOP 0 +#define WHERE_DISTINCT_UNIQUE 1 +#define WHERE_DISTINCT_ORDERED 2 +#define WHERE_DISTINCT_UNORDERED 3 /* ** A NameContext defines a context in which to resolve table and column ** names. The context consists of a list of tables (the pSrcList) field and ** a list of named expression (pEList). The named expression list may @@ -2724,12 +2772,19 @@ && !defined(SQLITE4_OMIT_SUBQUERY) Expr *sqlite4LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); #endif void sqlite4DeleteFrom(Parse*, SrcList*, Expr*); void sqlite4Update(Parse*, SrcList*, ExprList*, Expr*, int); -WhereInfo *sqlite4WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); +WhereInfo *sqlite4WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); + void sqlite4WhereEnd(WhereInfo*); +u64 sqlite4WhereOutputRowCount(WhereInfo*); +int sqlite4WhereIsDistinct(WhereInfo*); +int sqlite4WhereIsOrdered(WhereInfo*); +int sqlite4WhereContinueLabel(WhereInfo*); +int sqlite4WhereBreakLabel(WhereInfo*); +int sqlite4WhereOkOnePass(WhereInfo*); int sqlite4ExprCodeGetColumn(Parse*, Table*, int, int, int); void sqlite4ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite4ExprCodeMove(Parse*, int, int, int); void sqlite4ExprCodeCopy(Parse*, int, int, int); void sqlite4ExprCacheStore(Parse*, int, int, int); @@ -3127,11 +3182,12 @@ #define sqlite4EndBenignMalloc(X) #endif #define IN_INDEX_ROWID 1 #define IN_INDEX_EPH 2 -#define IN_INDEX_INDEX 3 +#define IN_INDEX_INDEX_ASC 3 +#define IN_INDEX_INDEX_DESC 4 int sqlite4FindInIndex(Parse *, Expr *, int*); Index *sqlite4FindExistingInIndex(Parse *, Expr *, int); #if SQLITE4_MAX_EXPR_DEPTH>0 Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -341,13 +341,13 @@ ** clause indicates that the loop will visit at most one row, then the ** RowSet object is bypassed and the primary key of the single row (if ** any) left in register regOldKey. This is called the "one-pass" ** approach. Set okOnePass to true if it can be used in this case. */ sqlite4VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldKey); - pWInfo = sqlite4WhereBegin(pParse, pSrc, pWhere, 0, 0, WHERE_ONEPASS_DESIRED); + pWInfo = sqlite4WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); if( pWInfo==0 ) goto update_cleanup; - okOnePass = pWInfo->okOnePass; + okOnePass = sqlite4WhereOkOnePass(pWInfo); sqlite4VdbeAddOp2(v, OP_RowKey, iCur+iPk, regOldKey); if( !okOnePass ){ sqlite4VdbeAddOp3(v, OP_RowSetAdd, regRowSet, 0, regOldKey); } sqlite4WhereEnd(pWInfo); Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -2148,11 +2148,11 @@ }else{ sqlite4VdbeMemSetNull(pDest); } UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); - assert( rc<100 ); + assert( rc<100 ); break; } /* Opcode: Affinity P1 P2 * P4 * ** Index: src/vdbecodec.c ================================================================== --- src/vdbecodec.c +++ src/vdbecodec.c @@ -476,12 +476,12 @@ if( enlargeEncoderAllocation(p, pMem->n*4 + 2) ) return SQLITE4_NOMEM; p->aOut[p->nOut++] = 0x24; /* Text */ if( pColl==0 || pColl->xMkKey==0 ){ const char *z = (const char *)sqlite4ValueText(pMem, SQLITE4_UTF8); if( z ){ - char *zCsr = z; - char *zEnd = &z[pMem->n]; + const char *zCsr = z; + const char *zEnd = &z[pMem->n]; while( *zCsr && zCsraOut+p->nOut, z, (zCsr-z)); p->nOut += (zCsr-z); } }else{ Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -19,30 +19,173 @@ #include "sqliteInt.h" /* For VdbeCodecEncodeKey() - revisit this */ #include "vdbeInt.h" - /* ** Trace output macros */ #if defined(SQLITE4_TEST) || defined(SQLITE4_DEBUG) -int sqlite4WhereTrace = 0; +/***/ int sqlite4WhereTrace = 0; #endif -#if defined(SQLITE4_TEST) && defined(SQLITE4_DEBUG) -# define WHERETRACE(X) if(sqlite4WhereTrace) sqlite4DebugPrintf X +#if defined(SQLITE4_DEBUG) \ + && (defined(SQLITE4_TEST) || defined(SQLITE4_ENABLE_WHERETRACE)) +# define WHERETRACE(K,X) if(sqlite4WhereTrace&(K)) sqlite4DebugPrintf X +# define WHERETRACE_ENABLED 1 #else -# define WHERETRACE(X) +# define WHERETRACE(K,X) #endif /* Forward reference */ typedef struct WhereClause WhereClause; typedef struct WhereMaskSet WhereMaskSet; typedef struct WhereOrInfo WhereOrInfo; typedef struct WhereAndInfo WhereAndInfo; -typedef struct WhereCost WhereCost; +typedef struct WhereLevel WhereLevel; +typedef struct WhereLoop WhereLoop; +typedef struct WherePath WherePath; +typedef struct WhereTerm WhereTerm; +typedef struct WhereLoopBuilder WhereLoopBuilder; +typedef struct WhereScan WhereScan; + +/* +** Cost X is tracked as 10*log2(X) stored in a 16-bit integer. The +** maximum cost for ordinary tables is 64*(2**63) which becomes 6900. +** (Virtual tables can return a larger cost, but let's assume they do not.) +** So all costs can be stored in a 16-bit unsigned integer without risk +** of overflow. +** +** Costs are estimates, so don't go to the computational trouble to compute +** 10*log2(X) exactly. Instead, a close estimate is used. Any value of +** X<=1 is stored as 0. X=2 is 10. X=3 is 16. X=1000 is 99. etc. +** +** The tool/wherecosttest.c source file implements a command-line program +** that will convert between WhereCost to integers and do addition and +** multiplication on WhereCost values. That command-line program is a +** useful utility to have around when working with this module. +*/ +typedef unsigned short int WhereCost; + +/* +** This object contains information needed to implement a single nested +** loop in WHERE clause. +** +** Contrast this object with WhereLoop. This object describes the +** implementation of the loop. WhereLoop describes the algorithm. +** This object contains a pointer to the WhereLoop algorithm as one of +** its elements. +** +** The WhereInfo object contains a single instance of this object for +** each term in the FROM clause (which is to say, for each of the +** nested loops as implemented). The order of WhereLevel objects determines +** the loop nested order, with WhereInfo.a[0] being the outer loop and +** WhereInfo.a[WhereInfo.nLevel-1] being the inner loop. +*/ +struct WhereLevel { + int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ + int iTabCur; /* The VDBE cursor used to access the table */ + int iIdxCur; /* The VDBE cursor used to access pIdx */ + int addrBrk; /* Jump here to break out of the loop */ + int addrNxt; /* Jump here to start the next IN combination */ + int addrCont; /* Jump here to continue with the next loop cycle */ + int addrFirst; /* First instruction of interior of the loop */ + u8 iFrom; /* Which entry in the FROM clause */ + u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ + int p1, p2; /* Operands of the opcode used to ends the loop */ + union { /* Information that depends on pWLoop->wsFlags */ + struct { + int nIn; /* Number of entries in aInLoop[] */ + struct InLoop { + int iCur; /* The VDBE cursor used by this IN operator */ + int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ + } *aInLoop; /* Information about each nested IN operator */ + } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ + Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ + } u; + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ +}; + +/* +** Each instance of this object represents an algorithm for evaluating one +** term of a join. Every term of the FROM clause will have at least +** one corresponding WhereLoop object (unless INDEXED BY constraints +** prevent a query solution - which is an error) and many terms of the +** FROM clause will have multiple WhereLoop objects, each describing a +** potential way of implementing that FROM-clause term, together with +** dependencies and cost estimates for using the chosen algorithm. +** +** Query planning consists of building up a collection of these WhereLoop +** objects, then computing a particular sequence of WhereLoop objects, with +** one WhereLoop object per FROM clause term, that satisfy all dependencies +** and that minimize the overall cost. +*/ +struct WhereLoop { + Bitmask prereq; /* Bitmask of other loops that must run first */ + Bitmask maskSelf; /* Bitmask identifying table iTab */ +#ifdef SQLITE4_DEBUG + char cId; /* Symbolic ID of this loop for debugging use */ +#endif + u8 iTab; /* Position in FROM clause of table for this loop */ + u8 iSortIdx; /* Sorting index number. 0==None */ + WhereCost rSetup; /* One-time setup cost (ex: create transient index) */ + WhereCost rRun; /* Cost of running each loop */ + WhereCost nOut; /* Estimated number of output rows */ + union { + struct { /* Information for internal btree tables */ + int nEq; /* Number of equality constraints */ + Index *pIndex; /* Index used, or NULL */ + } btree; + struct { /* Information for virtual tables */ + int idxNum; /* Index number */ + u8 needFree; /* True if sqlite4_free(idxStr) is needed */ + u8 isOrdered; /* True if satisfies ORDER BY */ + u16 omitMask; /* Terms that may be omitted */ + char *idxStr; /* Index identifier string */ + } vtab; + } u; + u32 wsFlags; /* WHERE_* flags describing the plan */ + u16 nLTerm; /* Number of entries in aLTerm[] */ + /**** whereLoopXfer() copies fields above ***********************/ +# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) + u16 nLSlot; /* Number of slots allocated for aLTerm[] */ + WhereTerm **aLTerm; /* WhereTerms used */ + WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ + WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ +}; + +/* Forward declaration of methods */ +static int whereLoopResize(sqlite4*, WhereLoop*, int); + +/* +** Each instance of this object holds a sequence of WhereLoop objects +** that implement some or all of a query plan. +** +** Think of each WhereLoop object as a node in a graph with arcs +** showing dependences and costs for travelling between nodes. (That is +** not a completely accurate description because WhereLoop costs are a +** vector, not a scalar, and because dependences are many-to-one, not +** one-to-one as are graph nodes. But it is a useful visualization aid.) +** Then a WherePath object is a path through the graph that visits some +** or all of the WhereLoop objects once. +** +** The "solver" works by creating the N best WherePath objects of length +** 1. Then using those as a basis to compute the N best WherePath objects +** of length 2. And so forth until the length of WherePaths equals the +** number of nodes in the FROM clause. The best (lowest cost) WherePath +** at the end is the choosen query plan. +*/ +struct WherePath { + Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ + Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ + WhereCost nRow; /* Estimated number of rows generated by this path */ + WhereCost rCost; /* Total cost of this path */ + u8 isOrdered; /* True if this path satisfies ORDER BY */ + u8 isOrderedValid; /* True if the isOrdered field is valid */ + WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ +}; /* ** The query generator uses an array of instances of this structure to ** help it analyze the subexpressions of the WHERE clause. Each WHERE ** clause subexpression is separated from the others by AND operators, @@ -91,19 +234,18 @@ ** ** The number of terms in a join is limited by the number of bits ** in prereqRight and prereqAll. The default is 64 bits, hence SQLite ** is only able to process joins with 64 or fewer tables. */ -typedef struct WhereTerm WhereTerm; struct WhereTerm { Expr *pExpr; /* Pointer to the subexpression that is this term */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X " */ union { int leftColumn; /* Column number of X in "X " */ - WhereOrInfo *pOrInfo; /* Extra information if eOperator==WO_OR */ - WhereAndInfo *pAndInfo; /* Extra information if eOperator==WO_AND */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; u16 eOperator; /* A WO_xx value describing */ u8 wtFlags; /* TERM_xxx bit flags. See below */ u8 nChild; /* Number of children that must disable us */ WhereClause *pWC; /* The clause this term is part of */ @@ -125,10 +267,26 @@ # define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ #else # define TERM_VNULL 0x00 /* Disabled if not using stat3 */ #endif +/* +** An instance of the WhereScan object is used as an iterator for locating +** terms in the WHERE clause that are useful to the query planner. +*/ +struct WhereScan { + WhereClause *pOrigWC; /* Original, innermost WhereClause */ + WhereClause *pWC; /* WhereClause currently being scanned */ + char *zCollName; /* Required collating sequence, if not NULL */ + char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char nEquiv; /* Number of entries in aEquiv[] */ + unsigned char iEquiv; /* Next unused slot in aEquiv[] */ + u32 opMask; /* Acceptable operators */ + int k; /* Resume scanning at this->pWC->a[this->k] */ + int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ +}; + /* ** An instance of the following structure holds all information about a ** WHERE clause. Mostly this is a container for one or more WhereTerms. ** ** Explanation of pOuter: For a WHERE clause of the form @@ -138,16 +296,13 @@ ** There are separate WhereClause objects for the whole clause and for ** the subclauses "(b AND c)" and "(d AND e)". The pOuter field of the ** subclauses points to the WhereClause object for the whole clause. */ struct WhereClause { - Parse *pParse; /* The parser context */ - WhereMaskSet *pMaskSet; /* Mapping of table cursor numbers to bitmasks */ - Bitmask vmask; /* Bitmask identifying virtual table cursors */ + WhereInfo *pWInfo; /* WHERE clause processing context */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ - u16 wctrlFlags; /* Might include WHERE_AND_ONLY */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ #if defined(SQLITE4_SMALL_STACK) WhereTerm aStatic[1]; /* Initial static space for a[] */ @@ -176,11 +331,11 @@ /* ** An instance of the following structure keeps track of a mapping ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. ** ** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** SrcListItem.iCursor and Expr.iTable fields. For any given WHERE ** clause, the cursor numbers might not begin with 0 and they might ** contain gaps in the numbering sequence. But we want to make maximum ** use of the bits in our bitmasks. This structure provides a mapping ** from the sparse cursor numbers into consecutive integers beginning ** with 0. @@ -203,23 +358,59 @@ int n; /* Number of assigned cursor values */ int ix[BMS]; /* Cursor assigned to each bit */ }; /* -** A WhereCost object records a lookup strategy and the estimated -** cost of pursuing that strategy. +** This object is a convenience wrapper holding all information needed +** to construct WhereLoop objects for a particular query. */ -struct WhereCost { - WherePlan plan; /* The lookup strategy */ - double rCost; /* Overall cost of pursuing this search strategy */ - Bitmask used; /* Bitmask of cursors used by this plan */ +struct WhereLoopBuilder { + WhereInfo *pWInfo; /* Information about this WHERE */ + WhereClause *pWC; /* WHERE clause terms */ + ExprList *pOrderBy; /* ORDER BY clause */ + WhereLoop *pNew; /* Template WhereLoop */ + WhereLoop *pBest; /* If non-NULL, store single best loop here */ +}; + +/* +** The WHERE clause processing routine has two halves. The +** first part does the start of the WHERE loop and the second +** half does the tail of the WHERE loop. An instance of +** this structure is returned by the first half and passed +** into the second half to give some continuity. +** +** An instance of this object holds the complete state of the query +** planner. +*/ +struct WhereInfo { + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pResultSet; /* Result set. DISTINCT operates on these */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + WhereCost nRowOut; /* Estimated number of output rows */ + u16 wctrlFlags; /* Flags originally passed to sqlite4WhereBegin() */ + u8 bOBSat; /* ORDER BY satisfied by indices */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + u8 nLevel; /* Number of nested loop */ + int iTop; /* The very beginning of the WHERE loop */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; /* -** Bitmasks for the operators that indices are able to exploit. An +** Bitmasks for the operators on WhereTerm objects. These are all +** operators that are of interest to the query planner. An ** OR-ed combination of these values can be used when searching for -** terms in the where clause. +** particular WhereTerms within a WhereClause. */ #define WO_IN 0x001 #define WO_EQ 0x002 #define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) #define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) @@ -227,64 +418,113 @@ #define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) #define WO_MATCH 0x040 #define WO_ISNULL 0x080 #define WO_OR 0x100 /* Two or more OR-connected terms */ #define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ #define WO_NOOP 0x800 /* This term does not restrict search space */ #define WO_ALL 0xfff /* Mask of all possible WO_* values */ #define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ /* -** Value for wsFlags returned by bestIndex() and stored in -** WhereLevel.wsFlags. These flags determine which search -** strategies are appropriate. -** -** The least significant 12 bits is reserved as a mask for WO_ values above. -** The WhereLevel.wsFlags field is usually set to WO_IN|WO_EQ|WO_ISNULL. -** But if the table is the right table of a left join, WhereLevel.wsFlags -** is set to WO_IN|WO_EQ. The WhereLevel.wsFlags field can then be used as -** the "op" parameter to findTerm when we are resolving equality constraints. -** ISNULL constraints will then not be used on the right table of a left -** join. Tickets #2177 and #2189. +** These are definitions of bits in the WhereLoop.wsFlags field. +** The particular combination of bits in each WhereLoop help to +** determine the algorithm that WhereLoop represents. */ -#define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ -#define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ -#define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ -#define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ -#define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ -#define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ -#define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ -#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xEXPR */ +#define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ +#define WHERE_CONSTRAINT 0x0000000f /* Any of the WHERE_COLUMN_xxx values */ +#define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and x=5 ) n -= 2; + else if( n>=1 ) n -= 1; + if( x>=3 ) return (n+8)<<(x-3); + return (n+8)>>(3-x); +} + +/* +** Return the estimated number of output rows from a WHERE clause +*/ +u64 sqlite4WhereOutputRowCount(WhereInfo *pWInfo){ + return whereCostToInt(pWInfo->nRowOut); +} + +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +int sqlite4WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} + +/* +** Return TRUE if the WHERE clause returns rows in ORDER BY order. +** Return FALSE if the output needs to be sorted. +*/ +int sqlite4WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->bOBSat!=0; +} + +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +int sqlite4WhereContinueLabel(WhereInfo *pWInfo){ + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +int sqlite4WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return TRUE if an UPDATE or DELETE statement can operate directly on +** the rowids returned by a WHERE clause. Return FALSE if doing an +** UPDATE or DELETE might change subsequent WHERE clause results. +*/ +int sqlite4WhereOkOnePass(WhereInfo *pWInfo){ + return pWInfo->okOnePass; +} /* ** Initialize a preallocated WhereClause structure. */ static void whereClauseInit( WhereClause *pWC, /* The WhereClause to be initialized */ - Parse *pParse, /* The parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmasks */ - u16 wctrlFlags /* Might include WHERE_AND_ONLY */ + WhereInfo *pWInfo /* The WHERE processing context */ ){ - pWC->pParse = pParse; - pWC->pMaskSet = pMaskSet; + pWC->pWInfo = pWInfo; pWC->pOuter = 0; pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; - pWC->vmask = 0; - pWC->wctrlFlags = wctrlFlags; } /* Forward reference */ static void whereClauseClear(WhereClause*); @@ -309,11 +549,11 @@ ** itself is not freed. This routine is the inverse of whereClauseInit(). */ static void whereClauseClear(WhereClause *pWC){ int i; WhereTerm *a; - sqlite4 *db = pWC->pParse->db; + sqlite4 *db = pWC->pWInfo->pParse->db; for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ if( a->wtFlags & TERM_DYNAMIC ){ sqlite4ExprDelete(db, a->pExpr); } if( a->wtFlags & TERM_ORINFO ){ @@ -324,10 +564,29 @@ } if( pWC->a!=pWC->aStatic ){ sqlite4DbFree(db, pWC->a); } } + + +/* +** Skip over any TK_COLLATE and/or TK_AS operators at the root of +** an expression. +*/ +Expr *sqlite4ExprSkipCollate(Expr *pExpr){ + while( pExpr && (pExpr->op==TK_COLLATE || pExpr->op==TK_AS) ){ + pExpr = pExpr->pLeft; + } + return pExpr; +} + +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) + + /* ** Add a single new WhereTerm entry to the WhereClause object pWC. ** The new WhereTerm object is constructed from Expr p and with wtFlags. ** The index in pWC->a[] of the new WhereTerm is returned on success. @@ -350,11 +609,11 @@ WhereTerm *pTerm; int idx; testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; - sqlite4 *db = pWC->pParse->db; + sqlite4 *db = pWC->pWInfo->pParse->db; pWC->a = sqlite4DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ sqlite4ExprDelete(db, p); } @@ -366,11 +625,11 @@ sqlite4DbFree(db, pOld); } pWC->nSlot = sqlite4DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); } pTerm = &pWC->a[idx = pWC->nTerm++]; - pTerm->pExpr = p; + pTerm->pExpr = sqlite4ExprSkipCollate(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; return idx; } @@ -390,12 +649,12 @@ ** ** In the previous sentence and in the diagram, "slot[]" refers to ** the WhereClause.a[] array. The slot[] array grows as needed to contain ** all terms of the WHERE clause. */ -static void whereSplit(WhereClause *pWC, Expr *pExpr, int op){ - pWC->op = (u8)op; +static void whereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + pWC->op = op; if( pExpr==0 ) return; if( pExpr->op!=op ){ whereClauseInsert(pWC, pExpr, 0); }else{ whereSplit(pWC, pExpr->pLeft, op); @@ -402,13 +661,13 @@ whereSplit(pWC, pExpr->pRight, op); } } /* -** Initialize an expression mask set (a WhereMaskSet object) +** Initialize a WhereMaskSet object */ -#define initMaskSet(P) memset(P, 0, sizeof(*P)) +#define initMaskSet(P) (P)->n=0 /* ** Return the bitmask for the given cursor number. Return 0 if ** iCursor is not in the set. */ @@ -415,11 +674,11 @@ static Bitmask getMask(WhereMaskSet *pMaskSet, int iCursor){ int i; assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); for(i=0; in; i++){ if( pMaskSet->ix[i]==iCursor ){ - return ((Bitmask)1)<n < ArraySize(pMaskSet->ix) ); pMaskSet->ix[pMaskSet->n++] = iCursor; } /* -** This routine walks (recursively) an expression tree and generates +** These routine walk (recursively) an expression tree and generates ** a bitmask indicating which tables are used in that expression ** tree. -** -** In order for this routine to work, the calling function must have -** previously invoked sqlite4ResolveExprNames() on the expression. See -** the header comment on that routine for additional information. -** The sqlite4ResolveExprNames() routines looks for column names and -** sets their opcodes to TK_COLUMN and their Expr.iTable fields to -** the VDBE cursor number of the table. This routine just has to -** translate the cursor numbers into bitmask values and OR all -** the bitmasks together. */ static Bitmask exprListTableUsage(WhereMaskSet*, ExprList*); static Bitmask exprSelectTableUsage(WhereMaskSet*, Select*); static Bitmask exprTableUsage(WhereMaskSet *pMaskSet, Expr *p){ Bitmask mask = 0; @@ -500,11 +750,11 @@ } /* ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", and "IN". +** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" ** ** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be ** of one of the following forms: column = expression column > expression ** column >= expression column < expression column <= expression ** expression = column expression > column expression >= column @@ -526,27 +776,35 @@ /* ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". ** -** If a collation sequence is associated with either the left or right -** side of the comparison, it remains associated with the same side after -** the commutation. So "Y collate NOCASE op X" becomes -** "X collate NOCASE op Y". This is because any collation sequence on +** If left/right precedence rules come into play when determining the +** collating sequence, then COLLATE operators are adjusted to ensure +** that the collating sequence does not change. For example: +** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on ** the left hand side of a comparison overrides any collation sequence ** attached to the right. For the same reason the EP_ExpCollate flag ** is not commuted. */ static void exprCommute(Parse *pParse, Expr *pExpr){ u16 expRight = (pExpr->pRight->flags & EP_ExpCollate); u16 expLeft = (pExpr->pLeft->flags & EP_ExpCollate); assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - pExpr->pRight->pColl = sqlite4ExprCollSeq(pParse, pExpr->pRight); - pExpr->pLeft->pColl = sqlite4ExprCollSeq(pParse, pExpr->pLeft); - SWAP(CollSeq*,pExpr->pRight->pColl,pExpr->pLeft->pColl); - pExpr->pRight->flags = (pExpr->pRight->flags & ~EP_ExpCollate) | expLeft; - pExpr->pLeft->flags = (pExpr->pLeft->flags & ~EP_ExpCollate) | expRight; + if( expRight==expLeft ){ + /* Either X and Y both have COLLATE operator or neither do */ + if( expRight ){ + /* Both X and Y have COLLATE operators. Make sure X is always + ** used by clearing the EP_ExpCollate flag from Y. */ + pExpr->pRight->flags &= ~EP_ExpCollate; + }else if( sqlite4ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ + /* Neither X nor Y have COLLATE operators, but X has a non-default + ** collating sequence. So add the EP_ExpCollate marker on X to cause + ** it to be searched first. */ + pExpr->pLeft->flags |= EP_ExpCollate; + } + } SWAP(Expr*,pExpr->pRight,pExpr->pLeft); if( pExpr->op>=TK_GT ){ assert( TK_LT==TK_GT+2 ); assert( TK_GE==TK_LE+2 ); assert( TK_GT>TK_EQ ); @@ -577,83 +835,249 @@ assert( op!=TK_LE || c==WO_LE ); assert( op!=TK_GT || c==WO_GT ); assert( op!=TK_GE || c==WO_GE ); return c; } + +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + int iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ + + while( pScan->iEquiv<=pScan->nEquiv ){ + iCur = pScan->aEquiv[pScan->iEquiv-2]; + iColumn = pScan->aEquiv[pScan->iEquiv-1]; + while( (pWC = pScan->pWC)!=0 ){ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaEquiv) + ){ + int j; + pX = sqlite4ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); + for(j=0; jnEquiv; j+=2){ + if( pScan->aEquiv[j]==pX->iTable + && pScan->aEquiv[j+1]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aEquiv[j] = pX->iTable; + pScan->aEquiv[j+1] = pX->iColumn; + pScan->nEquiv += 2; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite4IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite4BinaryCompareCollSeq(pParse, + pX->pLeft, pX->pRight); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite4_stricmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & WO_EQ)!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aEquiv[0] + && pX->iColumn==pScan->aEquiv[1] + ){ + continue; + } + pScan->k = k+1; + return pTerm; + } + } + } + pScan->pWC = pScan->pWC->pOuter; + k = 0; + } + pScan->pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv += 2; + } + return 0; +} + +/* +** Return the table column number of the iIdxCol'th field in the index +** keys used by index pIdx, including any appended PRIMARY KEY fields. +** If there is no iIdxCol'th field in index pIdx, return -2. +** +** Example: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)); +** CREATE INDEX i1 ON t1(c); +** +** Index i1 in the example above consists of three fields - the indexed +** field "c" followed by the two primary key fields. The automatic PRIMARY +** KEY index consists of two fields only. +*/ +static int idxColumnNumber(Index *pIdx, Index *pPk, int iIdxCol){ + int iRet = -2; + if( iIdxColnColumn ){ + iRet = pIdx->aiColumn[iIdxCol]; + }else if( pPk && iIdxCol<(pIdx->nColumn + pPk->nColumn) ){ + iRet = pPk->aiColumn[iIdxCol - pIdx->nColumn]; + } + return iRet; +} + +/* +** Return a pointer to a buffer containing the name of the collation +** sequence used with the iIdxCol'th field in index pIdx, including any +** appended PRIMARY KEY fields. +*/ +static char *idxColumnCollation(Index *pIdx, Index *pPk, int iIdxCol){ + char *zColl; + assert( iIdxCol<(pIdx->nColumn + pPk->nColumn) ); + if( iIdxColnColumn ){ + zColl = pIdx->azColl[iIdxCol]; + }else if( pPk && iIdxCol<(pIdx->nColumn + pPk->nColumn) ){ + zColl = pPk->azColl[iIdxCol - pIdx->nColumn]; + } + return zColl; +} + +/* +** Return the sort order (SQLITE4_SO_ASC or DESC) used by the iIdxCol'th +** field in index pIdx, including any appended PRIMARY KEY fields. +*/ +static int idxColumnSortOrder(Index *pIdx, Index *pPk, int iIdxCol){ + int iRet = SQLITE4_SO_ASC; + if( iIdxColnColumn ){ + iRet = pIdx->aSortOrder[iIdxCol]; + } + return iRet; +} + +/* +** Return the total number of fields in the index pIdx, including any +** trailing primary key fields. +*/ +static int idxColumnCount(Index *pIdx, Index *pPk){ + return (pIdx->nColumn + (pIdx==pPk ? 0 : pPk->nColumn)); +} + +/* +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. The must be one of the operators described by opMask. +** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. +*/ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ +){ + int j; + + /* memset(pScan, 0, sizeof(*pScan)); */ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + if( pIdx && iColumn>=0 ){ + Index *pPk = sqlite4FindPrimaryKey(pIdx->pTable, 0); + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + for(j=0; idxColumnNumber(pIdx, pPk, j)!=iColumn; j++){ + if( NEVER(j>=idxColumnCount(pIdx, pPk)) ) return 0; + } + pScan->zCollName = idxColumnCollation(pIdx, pPk, j); + }else{ + pScan->idxaff = 0; + pScan->zCollName = 0; + } + pScan->opMask = opMask; + pScan->k = 0; + pScan->aEquiv[0] = iCur; + pScan->aEquiv[1] = iColumn; + pScan->nEquiv = 2; + pScan->iEquiv = 2; + return whereScanNext(pScan); +} /* ** Search for a term in the WHERE clause that is of the form "X " ** where X is a reference to the iColumn of table iCur and is one of ** the WO_xx operator codes specified by the op parameter. ** Return a pointer to the term. Return 0 if not found. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aEquiv[] array holds X and all its equivalents, with each SQL variable +** taking up two slots in aEquiv[]. The first slot is for the cursor number +** and the second is for the column number. There are 22 slots in aEquiv[] +** so that means we can look for X plus up to 10 other equivalent values. +** Hence a search for X will return if X=A1 and A1=A2 and A2=A3 +** and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. */ static WhereTerm *findTerm( WhereClause *pWC, /* The WHERE clause to be searched */ int iCur, /* Cursor number of LHS */ int iColumn, /* Column number of LHS */ Bitmask notReady, /* RHS must not overlap with this mask */ u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ){ - sqlite4 *db = pWC->pParse->db; /* Database handle */ - WhereTerm *pTerm; - int k; - - assert( iCur>=0 ); - op &= WO_ALL; - for(; pWC; pWC=pWC->pOuter){ - for(pTerm=pWC->a, k=pWC->nTerm; k; k--, pTerm++){ - if( pTerm->leftCursor==iCur - && (pTerm->prereqRight & notReady)==0 - && pTerm->u.leftColumn==iColumn - && (pTerm->eOperator & op)!=0 - ){ - if( iColumn>=0 && pIdx && pTerm->eOperator!=WO_ISNULL ){ - Table *pTab = pIdx->pTable; - const char *zColl; /* Collation sequence used by index */ - CollSeq *pColl; /* Collation sequence used by expression */ - Expr *pX = pTerm->pExpr; - int j; - Parse *pParse = pWC->pParse; - - if( !sqlite4IndexAffinityOk(pX, pTab->aCol[iColumn].affinity) ){ - continue; - } - - /* Figure out the collation sequence used by expression pX. Store - ** this in pColl. Also the collation sequence used by the index. - ** Store this one in zColl. */ - assert(pX->pLeft); - pColl = sqlite4BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - for(j=0; pIdx->aiColumn[j]!=iColumn && jnColumn; j++); - if( j>=pIdx->nColumn ){ - zColl = pTab->aCol[iColumn].zColl; - }else{ - zColl = pIdx->azColl[j]; - } - - /* If the collation sequence used by the index is not the same as - ** that used by the expression, then this term is not a match. */ - if( pColl!=sqlite4FindCollSeq(db, zColl, 0) ) continue; - } - return pTerm; - } - } - } - return 0; + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; + + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&WO_EQ)!=0 ){ + return p; + } + if( pResult==0 ) pResult = p; + } + p = whereScanNext(&scan); + } + return pResult; } /* Forward reference */ static void exprAnalyze(SrcList*, WhereClause*, int); /* ** Call exprAnalyze on all terms in a WHERE clause. -** -** Note that exprAnalyze() might add new virtual terms onto the end of -** the WHERE clause. We do not want to analyze these virtual terms, so -** start analyzing at the end and work forward so that the added virtual -** terms are never processed. */ static void exprAnalyzeAll( SrcList *pTabList, /* the FROM clause */ WhereClause *pWC /* the WHERE clause to be analyzed */ ){ @@ -695,11 +1119,14 @@ #ifdef SQLITE4_EBCDIC if( *pnoCase ) return 0; #endif pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN || sqlite4ExprAffinity(pLeft)!=SQLITE4_AFF_TEXT ){ + if( pLeft->op!=TK_COLUMN + || sqlite4ExprAffinity(pLeft)!=SQLITE4_AFF_TEXT + || IsVirtual(pLeft->pTab) + ){ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ return 0; } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ @@ -712,11 +1139,11 @@ if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; int iCol = pRight->iColumn; pVal = sqlite4VdbeGetValue(pReprepare, iCol, SQLITE4_AFF_NONE); if( pVal && sqlite4_value_type(pVal)==SQLITE4_TEXT ){ - z = sqlite4_value_text(pVal, 0); + z = (char *)sqlite4_value_text(pVal, 0); } sqlite4VdbeSetVarmask(pParse->pVdbe, iCol); assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ z = pRight->u.zToken; @@ -823,11 +1250,11 @@ ** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') ** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) ** ** CASE 1: ** -** If all subterms are of the form T.C=expr for some single column of C +** If all subterms are of the form T.C=expr for some single column of C and ** a single table T (as shown in example B above) then create a new virtual ** term that is an equivalent IN expression. In other words, if the term ** being analyzed is: ** ** x = expr1 OR expr2 = x OR x = expr3 @@ -878,15 +1305,15 @@ static void exprAnalyzeOrTerm( SrcList *pSrc, /* the FROM clause */ WhereClause *pWC, /* the complete WHERE clause */ int idxTerm /* Index of the OR-term to be analyzed */ ){ - Parse *pParse = pWC->pParse; /* Parser context */ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ sqlite4 *db = pParse->db; /* Database connection */ WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ Expr *pExpr = pTerm->pExpr; /* The expression of the term */ - WhereMaskSet *pMaskSet = pWC->pMaskSet; /* Table use masks */ int i; /* Loop counters */ WhereClause *pOrWc; /* Breakup of pTerm into subterms */ WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ Bitmask chngToIN; /* Tables that might satisfy case 1 */ @@ -901,25 +1328,24 @@ assert( pExpr->op==TK_OR ); pTerm->u.pOrInfo = pOrInfo = sqlite4DbMallocZero(db, sizeof(*pOrInfo)); if( pOrInfo==0 ) return; pTerm->wtFlags |= TERM_ORINFO; pOrWc = &pOrInfo->wc; - whereClauseInit(pOrWc, pWC->pParse, pMaskSet, pWC->wctrlFlags); + whereClauseInit(pOrWc, pWInfo); whereSplit(pOrWc, pExpr, TK_OR); exprAnalyzeAll(pSrc, pOrWc); if( db->mallocFailed ) return; assert( pOrWc->nTerm>=2 ); /* ** Compute the set of tables that might satisfy cases 1 or 2. */ indexable = ~(Bitmask)0; - chngToIN = ~(pWC->vmask); + chngToIN = ~(Bitmask)0; for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ WhereAndInfo *pAndInfo; - assert( pOrTerm->eOperator==0 ); assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; pAndInfo = sqlite4DbMallocRaw(db, sizeof(*pAndInfo)); if( pAndInfo ){ WhereClause *pAndWC; @@ -928,20 +1354,20 @@ Bitmask b = 0; pOrTerm->u.pAndInfo = pAndInfo; pOrTerm->wtFlags |= TERM_ANDINFO; pOrTerm->eOperator = WO_AND; pAndWC = &pAndInfo->wc; - whereClauseInit(pAndWC, pWC->pParse, pMaskSet, pWC->wctrlFlags); + whereClauseInit(pAndWC, pWC->pWInfo); whereSplit(pAndWC, pOrTerm->pExpr, TK_AND); exprAnalyzeAll(pSrc, pAndWC); pAndWC->pOuter = pWC; testcase( db->mallocFailed ); if( !db->mallocFailed ){ for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ assert( pAndTerm->pExpr ); if( allowedOp(pAndTerm->pExpr->op) ){ - b |= getMask(pMaskSet, pAndTerm->leftCursor); + b |= getMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); } } } indexable &= b; } @@ -948,17 +1374,17 @@ }else if( pOrTerm->wtFlags & TERM_COPIED ){ /* Skip this term for now. We revisit it when we process the ** corresponding TERM_VIRTUAL term */ }else{ Bitmask b; - b = getMask(pMaskSet, pOrTerm->leftCursor); + b = getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); if( pOrTerm->wtFlags & TERM_VIRTUAL ){ WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; - b |= getMask(pMaskSet, pOther->leftCursor); + b |= getMask(&pWInfo->sMaskSet, pOther->leftCursor); } indexable &= b; - if( pOrTerm->eOperator!=WO_EQ ){ + if( (pOrTerm->eOperator & WO_EQ)==0 ){ chngToIN = 0; }else{ chngToIN &= b; } } @@ -1005,19 +1431,19 @@ ** and column is found but leave okToChngToIN false if not found. */ for(j=0; j<2 && !okToChngToIN; j++){ pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); pOrTerm->wtFlags &= ~TERM_OR_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and ** current term is from the first iteration. So skip this term. */ assert( j==1 ); continue; } - if( (chngToIN & getMask(pMaskSet, pOrTerm->leftCursor))==0 ){ + if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the ** chngToIN set but t1 is not. This term will be either preceeded ** or follwed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); @@ -1031,21 +1457,21 @@ } if( i<0 ){ /* No candidate table+column was found. This can only occur ** on the second iteration */ assert( j==1 ); - assert( (chngToIN&(chngToIN-1))==0 ); - assert( chngToIN==getMask(pMaskSet, iCursor) ); + assert( IsPowerOfTwo(chngToIN) ); + assert( chngToIN==getMask(&pWInfo->sMaskSet, iCursor) ); break; } testcase( j==1 ); /* We have found a candidate table and column. Check to see if that ** table and column is common to every term in the OR clause */ okToChngToIN = 1; for(; i>=0 && okToChngToIN; i--, pOrTerm++){ - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); if( pOrTerm->leftCursor!=iCursor ){ pOrTerm->wtFlags &= ~TERM_OR_OK; }else if( pOrTerm->u.leftColumn!=iColumn ){ okToChngToIN = 0; }else{ @@ -1077,15 +1503,15 @@ Expr *pLeft = 0; /* The LHS of the IN operator */ Expr *pNew; /* The complete IN operator */ for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; - assert( pOrTerm->eOperator==WO_EQ ); + assert( pOrTerm->eOperator & WO_EQ ); assert( pOrTerm->leftCursor==iCursor ); assert( pOrTerm->u.leftColumn==iColumn ); pDup = sqlite4ExprDup(db, pOrTerm->pExpr->pRight, 0); - pList = sqlite4ExprListAppend(pWC->pParse, pList, pDup); + pList = sqlite4ExprListAppend(pWInfo->pParse, pList, pDup); pLeft = pOrTerm->pExpr->pLeft; } assert( pLeft!=0 ); pDup = sqlite4ExprDup(db, pLeft, 0); pNew = sqlite4PExpr(pParse, TK_IN, pDup, 0, 0); @@ -1107,11 +1533,10 @@ } } } #endif /* !SQLITE4_OMIT_OR_OPTIMIZATION && !SQLITE4_OMIT_SUBQUERY */ - /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the ** subexpression and populate all the other fields of the WhereTerm ** structure. @@ -1131,10 +1556,11 @@ static void exprAnalyze( SrcList *pSrc, /* the FROM clause */ WhereClause *pWC, /* the WHERE clause */ int idxTerm /* Index of the term to be analyzed */ ){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ WhereTerm *pTerm; /* The term to be analyzed */ WhereMaskSet *pMaskSet; /* Set of table index masks */ Expr *pExpr; /* The expression to be analyzed */ Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ Bitmask prereqAll; /* Prerequesites of pExpr */ @@ -1141,19 +1567,20 @@ Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ int noCase = 0; /* LIKE/GLOB distinguishes case */ int op; /* Top-level operator. pExpr->op */ - Parse *pParse = pWC->pParse; /* Parsing context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite4 *db = pParse->db; /* Database connection */ if( db->mallocFailed ){ return; } pTerm = &pWC->a[idxTerm]; - pMaskSet = pWC->pMaskSet; + pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); prereqLeft = exprTableUsage(pMaskSet, pExpr->pLeft); op = pExpr->op; if( op==TK_IN ){ assert( pExpr->pRight==0 ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -1175,21 +1602,23 @@ } pTerm->prereqAll = prereqAll; pTerm->leftCursor = -1; pTerm->iParent = -1; pTerm->eOperator = 0; - if( allowedOp(op) && (pTerm->prereqRight & prereqLeft)==0 ){ - Expr *pLeft = pExpr->pLeft; - Expr *pRight = pExpr->pRight; + if( allowedOp(op) ){ + Expr *pLeft = sqlite4ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite4ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; if( pLeft->op==TK_COLUMN ){ pTerm->leftCursor = pLeft->iTable; pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op); + pTerm->eOperator = operatorMask(op) & opMask; } if( pRight && pRight->op==TK_COLUMN ){ WhereTerm *pNew; Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite4ExprDup(db, pExpr, 0); if( db->mallocFailed ){ sqlite4ExprDelete(db, pDup); @@ -1200,22 +1629,29 @@ pNew = &pWC->a[idxNew]; pNew->iParent = idxTerm; pTerm = &pWC->a[idxTerm]; pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; + if( pExpr->op==TK_EQ + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE4_Transitive) + ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } }else{ pDup = pExpr; pNew = pTerm; } exprCommute(pParse, pDup); - pLeft = pDup->pLeft; + pLeft = sqlite4ExprSkipCollate(pDup->pLeft); pNew->leftCursor = pLeft->iTable; pNew->u.leftColumn = pLeft->iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; - pNew->eOperator = operatorMask(pDup->op); + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } } #ifndef SQLITE4_OMIT_BETWEEN_OPTIMIZATION /* If a term is the BETWEEN operator, create two new virtual terms @@ -1284,11 +1720,11 @@ Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ Expr *pNewExpr1; Expr *pNewExpr2; int idxNew1; int idxNew2; - CollSeq *pColl; /* Collating sequence to use */ + Token sCollSeqName; /* Name of collating sequence */ pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite4ExprDup(db, pStr1, 0); if( !db->mallocFailed ){ u8 c, *pC; /* Last character before the first wildcard */ @@ -1306,20 +1742,21 @@ c = sqlite4UpperToLower[c]; } *pC = c + 1; } - pColl = sqlite4FindCollSeq(db, noCase ? "NOCASE" : "BINARY",0); - pNewExpr1 = sqlite4PExpr(pParse, TK_GE, - sqlite4ExprSetColl(sqlite4ExprDup(db,pLeft,0), pColl), - pStr1, 0); + sCollSeqName.z = noCase ? "NOCASE" : "BINARY"; + sCollSeqName.n = 6; + pNewExpr1 = sqlite4ExprDup(db, pLeft, 0); + sqlite4ExprSetCollByToken(pParse, pNewExpr1, &sCollSeqName); + pNewExpr1 = sqlite4PExpr(pParse, TK_GE, pNewExpr1, pStr1, 0); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite4PExpr(pParse, TK_LT, - sqlite4ExprSetColl(sqlite4ExprDup(db,pLeft,0), pColl), - pStr2, 0); + pNewExpr2 = sqlite4ExprDup(db, pLeft, 0); + sqlite4ExprSetCollByToken(pParse, pNewExpr2, &sCollSeqName); + pNewExpr2 = sqlite4PExpr(pParse, TK_LT, pNewExpr2, pStr2, 0); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ @@ -1379,10 +1816,11 @@ ** the start of the loop will prevent any results from being returned. */ if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 + && OptimizationEnabled(db, SQLITE4_Stat3) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; int idxNew; WhereTerm *pNewTerm; @@ -1413,34 +1851,12 @@ */ pTerm->prereqRight |= extraRight; } /* -** Return TRUE if any of the expressions in pList->a[iFirst...] contain -** a reference to any table other than the iBase table. -*/ -static int referencesOtherTables( - ExprList *pList, /* Search expressions in ths list */ - WhereMaskSet *pMaskSet, /* Mapping from tables to bitmaps */ - int iFirst, /* Be searching with the iFirst-th expression */ - int iBase /* Ignore references to this table */ -){ - Bitmask allowed = ~getMask(pMaskSet, iBase); - while( iFirstnExpr ){ - if( (exprTableUsage(pMaskSet, pList->a[iFirst++].pExpr)&allowed)!=0 ){ - return 1; - } - } - return 0; -} - -/* -** This function searches the expression list passed as the second argument -** for an expression of type TK_COLUMN that refers to the same column and -** uses the same collation sequence as the iCol'th column of index pIdx. -** Argument iBase is the cursor number used for the table that pIdx refers -** to. +** This function searches pList for a entry that matches the iCol-th column +** of index pIdx. ** ** If such an expression is found, its index in pList->a[] is returned. If ** no expression is found, -1 is returned. */ static int findIndexCol( @@ -1452,96 +1868,37 @@ ){ int i; const char *zColl = pIdx->azColl[iCol]; for(i=0; inExpr; i++){ - Expr *p = pList->a[i].pExpr; + Expr *p = sqlite4ExprSkipCollate(pList->a[i].pExpr); if( p->op==TK_COLUMN && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase ){ - CollSeq *pColl = sqlite4ExprCollSeq(pParse, p); - assert( pColl || p->iColumn==-1 ); - if( 0==pColl || 0==sqlite4_stricmp(pColl->zName, zColl) ){ + CollSeq *pColl = sqlite4ExprCollSeq(pParse, pList->a[i].pExpr); + if( ALWAYS(pColl) && 0==sqlite4_stricmp(pColl->zName, zColl) ){ return i; } } } return -1; } - -/* -** This routine determines if pIdx can be used to assist in processing a -** DISTINCT qualifier. In other words, it tests whether or not using this -** index for the outer loop guarantees that rows with equal values for -** all expressions in the pDistinct list are delivered grouped together. -** -** For example, the query -** -** SELECT DISTINCT a, b, c FROM tbl WHERE a = ? -** -** can benefit from any index on columns "b" and "c". -*/ -static int isDistinctIndex( - Parse *pParse, /* Parsing context */ - WhereClause *pWC, /* The WHERE clause */ - Index *pIdx, /* The index being considered */ - int base, /* Cursor number for the table pIdx is on */ - ExprList *pDistinct, /* The DISTINCT expressions */ - int nEqCol /* Number of index columns with == */ -){ - Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ - int i; /* Iterator variable */ - - if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; - testcase( pDistinct->nExpr==BMS-1 ); - - /* Loop through all the expressions in the distinct list. If any of them - ** are not simple column references, return early. Otherwise, test if the - ** WHERE clause contains a "col=X" clause. If it does, the expression - ** can be ignored. If it does not, and the column does not belong to the - ** same table as index pIdx, return early. Finally, if there is no - ** matching "col=X" expression and the column is on the same table as pIdx, - ** set the corresponding bit in variable mask. - */ - for(i=0; inExpr; i++){ - WhereTerm *pTerm; - Expr *p = pDistinct->a[i].pExpr; - if( p->op!=TK_COLUMN ) return 0; - pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0); - if( pTerm ){ - Expr *pX = pTerm->pExpr; - CollSeq *p1 = sqlite4BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); - CollSeq *p2 = sqlite4ExprCollSeq(pParse, p); - if( p1==p2 ) continue; - } - if( p->iTable!=base ) return 0; - mask |= (((Bitmask)1) << i); - } - - for(i=nEqCol; mask && inColumn; i++){ - int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i); - if( iExpr<0 ) break; - mask &= ~(((Bitmask)1) << iExpr); - } - - return (mask==0); -} - /* ** Return true if the DISTINCT expression-list passed as the third argument -** is redundant. A DISTINCT list is redundant if the database contains a -** UNIQUE index that guarantees that the result of the query will be distinct -** anyway. +** is redundant. +** +** A DISTINCT list is redundant if the database contains some subset of +** columns that are unique and non-null. */ static int isDistinctRedundant( - Parse *pParse, - SrcList *pTabList, - WhereClause *pWC, - ExprList *pDistinct + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ ){ Table *pTab; Index *pIdx; int i; int iBase; @@ -1556,11 +1913,11 @@ /* If any of the expressions is an IPK column on table iBase, then return ** true. Note: The (p->iTable==iBase) part of this test may be false if the ** current SELECT is a correlated sub-query. */ for(i=0; inExpr; i++){ - Expr *p = pDistinct->a[i].pExpr; + Expr *p = sqlite4ExprSkipCollate(pDistinct->a[i].pExpr); if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; } /* Loop through all indices on the table, checking each to see if it makes ** the DISTINCT qualifier redundant. It does so if: @@ -1569,19 +1926,23 @@ ** ** 2. All of the columns in the index are either part of the pDistinct ** list, or else the WHERE clause contains a term of the form "col=X", ** where X is a constant value. The collation sequences of the ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( pIdx->onError==OE_None ) continue; for(i=0; inColumn; i++){ int iCol = pIdx->aiColumn[i]; - if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) - && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i) - ){ - break; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); + if( iIdxCol<0 || pTab->aCol[pIdx->aiColumn[i]].notNull==0 ){ + break; + } } } if( i==pIdx->nColumn ){ /* This index implies that the DISTINCT qualifier is redundant. */ return 1; @@ -1589,228 +1950,89 @@ } return 0; } - -/* -** Return the table column number of the iIdxCol'th field in the index -** keys used by index pIdx, including any appended PRIMARY KEY fields. -** If there is no iIdxCol'th field in index pIdx, return -2. -** -** Example: -** -** CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)); -** CREATE INDEX i1 ON t1(c); -** -** Index i1 in the example above consists of three fields - the indexed -** field "c" followed by the two primary key fields. The automatic PRIMARY -** KEY index consists of two fields only. -*/ -static int idxColumnNumber(Index *pIdx, Index *pPk, int iIdxCol){ - int iRet = -2; - if( iIdxColnColumn ){ - iRet = pIdx->aiColumn[iIdxCol]; - }else if( pPk && iIdxCol<(pIdx->nColumn + pPk->nColumn) ){ - iRet = pPk->aiColumn[iIdxCol - pIdx->nColumn]; - } - return iRet; -} - -/* -** Return the name of the iCol'th column of table pTab. Or, if iCol is less -** than zero, return a pointer to the constant string "rowid". -*/ -static const char *tblColumnName(Table *pTab, int iCol){ - if( iCol<0 ) return "rowid"; - return pTab->aCol[iCol].zName; -} - -/* -** This routine decides if pIdx can be used to satisfy the ORDER BY -** clause. If it can, it returns 1. If pIdx cannot satisfy the -** ORDER BY clause, this routine returns 0. -** -** pOrderBy is an ORDER BY clause from a SELECT statement. pTab is the -** left-most table in the FROM clause of that same SELECT statement and -** the table has a cursor number of "base". pIdx is an index on pTab. -** -** nEqCol is the number of columns of pIdx that are used as equality -** constraints. Any of these columns may be missing from the ORDER BY -** clause and the match can still be a success. -** -** All terms of the ORDER BY that match against the index must be either -** ASC or DESC. (Terms of the ORDER BY clause past the end of a UNIQUE -** index do not need to satisfy this constraint.) The *pbRev value is -** set to 1 if the ORDER BY clause is all DESC and it is set to 0 if -** the ORDER BY clause is all ASC. -*/ -static int isSortingIndex( - Parse *pParse, /* Parsing context */ - WhereMaskSet *pMaskSet, /* Mapping from table cursor numbers to bitmaps */ - Index *pIdx, /* The index we are testing */ - int base, /* Cursor number for the table to be sorted */ - ExprList *pOrderBy, /* The ORDER BY clause */ - int nEqCol, /* Number of index columns with == constraints */ - int wsFlags, /* Index usages flags */ - int *pbRev /* Set to 1 if ORDER BY is DESC */ -){ - sqlite4 *db = pParse->db; /* Database handle */ - int sortOrder = 0; /* XOR of index and ORDER BY sort direction */ - int nTerm; /* Number of ORDER BY terms */ - int iTerm; /* Used to iterate through nTerm terms */ - int iNext = nEqCol; /* Index of next unmatched column in index */ - int nIdxCol; /* Number of columns in index, incl. PK */ - Index *pPk; - Table *pTab; - - if( !pOrderBy ) return 0; - if( wsFlags & WHERE_COLUMN_IN ) return 0; - if( pIdx->fIndex & IDX_Unordered ) return 0; - - pTab = pIdx->pTable; - pPk = sqlite4FindPrimaryKey(pTab, 0); - nTerm = pOrderBy->nExpr; - nIdxCol = pIdx->nColumn + (pIdx==pPk ? 0 : pPk->nColumn); - - assert( nTerm>0 ); - assert( pIdx && pIdx->zName ); - - for(iTerm=0; iTerma[iTerm]; - pExpr = pTerm->pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=base ) break; - iColumn = pExpr->iColumn; - - /* Check that column iColumn is a part of the index. If it is not, then - ** this index may not be used as a sorting index. This block also checks - ** that column iColumn is either the iNext'th column of the index, or - ** else one of the nEqCol columns that the index guarantees will be - ** constant. */ - for(iIdxCol=0; iIdxCol=nEqCol && iIdxCol!=iNext) ) break; - - /* Check that the collation sequence used by the expression is the same - ** as the collation sequence used by the index. If not, this is not a - ** sorting index. */ - pColl = sqlite4ExprCollSeq(pParse, pExpr); - if( !pColl ) pColl = db->pDfltColl; - if( iIdxColnColumn ){ - zColl = pIdx->azColl[iIdxCol]; - }else if( iColumn>=0 ) { - zColl = pTab->aCol[iColumn].zColl; - }else{ - zColl = 0; - } - if( pColl!=sqlite4FindCollSeq(db, zColl, 0) ) break; - - if( iIdxCol==iNext ){ - u8 reqSortOrder; - u8 idxSortOrder = SQLITE4_SO_ASC; - if( iIdxColnColumn ) idxSortOrder = pIdx->aSortOrder[iIdxCol]; - assert( idxSortOrder==SQLITE4_SO_ASC || idxSortOrder==SQLITE4_SO_DESC ); - - reqSortOrder = (idxSortOrder ^ pTerm->sortOrder); - if( iNext==nEqCol ){ - sortOrder = reqSortOrder; - }else if( sortOrder!=reqSortOrder ){ - break; - } - iNext++; - } - -#if 0 - if( iColumn<0 && !referencesOtherTables(pOrderBy, pMaskSet, j, base) ){ - /* If the indexed column is the primary key and everything matches - ** so far and none of the ORDER BY terms to the right reference other - ** tables in the join, then we are assured that the index can be used - ** to sort because the primary key is unique and so none of the other - ** columns will make any difference - */ - j = nTerm; - } -#endif - } - - *pbRev = sortOrder!=0; - - if( iTerm>=nTerm ){ - /* All terms of the ORDER BY clause are covered by this index. The - ** index can therefore be used for sorting. */ - return 1; - } - - if( pIdx->onError!=OE_None - && iNext>=pIdx->nColumn - && (wsFlags & WHERE_COLUMN_NULL)==0 - && !referencesOtherTables(pOrderBy, pMaskSet, iTerm, base) - ){ - - if( iNext==nIdxCol ){ - /* All columns indexed by this UNIQUE index, and all PK columns are - ** are matched by a prefix of the ORDER BY clause. And since the PK - ** columns are guaranteed to be unique and NOT NULL, there is no way - ** for the trailing ORDER BY terms to affect the sort order. Therefore, - ** we have a sorting index. */ - return 1; - }else{ - int i; - for(i=nEqCol; inColumn; i++){ - int iCol = pIdx->aiColumn[i]; - if( iCol>=0 && pTab->aCol[iCol].notNull==0 ) break; - } - - /* All columns indexed by this UNIQUE index are matched by a prefix - ** of the ORDER BY clause. And there is reason to believe that none - ** of the expressions in the ORDER BY prefix will evalulate to NULL. - ** The index may be used for sorting in this case too since it is - ** guaranteed that none of the trailing, unmatched ORDER BY terms - ** affect the sort order. */ - return (i>=pIdx->nColumn); - } - } - - return 0; -} - -/* -** Prepare a crude estimate of the logarithm of the input value. -** The results need not be exact. This is only used for estimating -** the total cost of performing operations with O(logN) or O(NlogN) -** complexity. Because N is just a guess, it is no great tragedy if -** logN is a little off. -*/ -static double estLog(double N){ - double logN = 1; - double x = 10; - while( N>x ){ - logN += 1; - x *= 10; - } - return logN; +/* +** The (an approximate) sum of two WhereCosts. This computation is +** not a simple "+" operator because WhereCost is stored as a logarithmic +** value. +** +*/ +static WhereCost whereCostAdd(WhereCost a, WhereCost b){ + static const unsigned char x[] = { + 10, 10, /* 0,1 */ + 9, 9, /* 2,3 */ + 8, 8, /* 4,5 */ + 7, 7, 7, /* 6,7,8 */ + 6, 6, 6, /* 9,10,11 */ + 5, 5, 5, /* 12-14 */ + 4, 4, 4, 4, /* 15-18 */ + 3, 3, 3, 3, 3, 3, /* 19-24 */ + 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ + }; + if( a>=b ){ + if( a>b+49 ) return a; + if( a>b+31 ) return a+1; + return a+x[a-b]; + }else{ + if( b>a+49 ) return b; + if( b>a+31 ) return b+1; + return b+x[b-a]; + } +} + +/* +** Convert an integer into a WhereCost. In other words, compute a +** good approximatation for 10*log2(x). +*/ +static WhereCost whereCost(tRowcnt x){ + static WhereCost a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + WhereCost y = 40; + if( x<8 ){ + if( x<2 ) return 0; + while( x<8 ){ y -= 10; x <<= 1; } + }else{ + while( x>255 ){ y += 40; x >>= 4; } + while( x>15 ){ y += 10; x >>= 1; } + } + return a[x&7] + y - 10; +} + +#ifndef SQLITE4_OMIT_VIRTUALTABLE +/* +** Convert a double (as received from xBestIndex of a virtual table) +** into a WhereCost. In other words, compute an approximation for +** 10*log2(x). +*/ +static WhereCost whereCostFromDouble(double x){ + u64 a; + WhereCost e; + assert( sizeof(x)==8 && sizeof(a)==8 ); + if( x<=1 ) return 0; + if( x<=2000000000 ) return whereCost((tRowcnt)x); + memcpy(&a, &x, 8); + e = (a>>52) - 1022; + return e*10; +} +#endif /* SQLITE4_OMIT_VIRTUALTABLE */ + +/* +** Estimate the logarithm of the input value to base 2. +*/ +static WhereCost estLog(WhereCost N){ + WhereCost x = whereCost(N); + return x>33 ? x - 33 : 0; } /* ** Two routines for printing the content of an sqlite4_index_info ** structure. Used for testing and debugging only. If neither ** SQLITE4_TEST or SQLITE4_DEBUG are defined, then these routines ** are no-ops. */ -#if !defined(SQLITE4_OMIT_VIRTUALTABLE) && defined(SQLITE4_DEBUG) +#if !defined(SQLITE4_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) static void TRACE_IDX_INPUTS(sqlite4_index_info *p){ int i; if( !sqlite4WhereTrace ) return; for(i=0; inConstraint; i++){ sqlite4DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", @@ -1844,210 +2066,29 @@ #else #define TRACE_IDX_INPUTS(A) #define TRACE_IDX_OUTPUTS(A) #endif -/* -** Required because bestIndex() is called by bestOrClauseIndex() -*/ -static void bestIndex( - Parse*, WhereClause*, SrcListItem*, - Bitmask, Bitmask, ExprList*, WhereCost*); - -/* -** This routine attempts to find an scanning strategy that can be used -** to optimize an 'OR' expression that is part of a WHERE clause. -** -** The table associated with FROM clause term pSrc may be either a -** regular B-Tree table or a virtual table. -*/ -static void bestOrClauseIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - SrcListItem *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ -#ifndef SQLITE4_OMIT_OR_OPTIMIZATION - const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ - const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ - WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - - /* The OR-clause optimization is disallowed if the INDEXED BY or - ** NOT INDEXED clauses are used or if the WHERE_AND_ONLY bit is set. */ - if( pSrc->notIndexed || pSrc->pIndex!=0 ){ - return; - } - if( pWC->wctrlFlags & WHERE_AND_ONLY ){ - return; - } - - /* Search the WHERE clause terms for a usable WO_OR term. */ - for(pTerm=pWC->a; pTermeOperator==WO_OR - && ((pTerm->prereqAll & ~maskSrc) & notReady)==0 - && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 - ){ - WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; - WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; - WhereTerm *pOrTerm; - int flags = WHERE_MULTI_OR; - double rTotal = 0; - double nRow = 0; - Bitmask used = 0; - - for(pOrTerm=pOrWC->a; pOrTerma), (pTerm - pWC->a) - )); - if( pOrTerm->eOperator==WO_AND ){ - WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); - }else if( pOrTerm->leftCursor==iCur ){ - WhereClause tempWC; - tempWC.pParse = pWC->pParse; - tempWC.pMaskSet = pWC->pMaskSet; - tempWC.pOuter = pWC; - tempWC.op = TK_AND; - tempWC.a = pOrTerm; - tempWC.wctrlFlags = 0; - tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); - }else{ - continue; - } - rTotal += sTermCost.rCost; - nRow += sTermCost.plan.nRow; - used |= sTermCost.used; - if( rTotal>=pCost->rCost ) break; - } - - /* If there is an ORDER BY clause, increase the scan cost to account - ** for the cost of the sort. */ - if( pOrderBy!=0 ){ - WHERETRACE(("... sorting increases OR cost %.9g to %.9g\n", - rTotal, rTotal+nRow*estLog(nRow))); - rTotal += nRow*estLog(nRow); - } - - /* If the cost of scanning using this OR term for optimization is - ** less than the current cost stored in pCost, replace the contents - ** of pCost. */ - WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); - if( rTotalrCost ){ - pCost->rCost = rTotal; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = flags; - pCost->plan.u.pTerm = pTerm; - } - } - } -#endif /* SQLITE4_OMIT_OR_OPTIMIZATION */ -} - -#ifndef SQLITE4_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it ** could be used with an index to access pSrc, assuming an appropriate ** index existed. */ static int termCanDriveIndex( WhereTerm *pTerm, /* WHERE clause term to check */ - SrcListItem *pSrc, /* Table we are trying to access */ + struct SrcListItem *pSrc, /* Table we are trying to access */ Bitmask notReady /* Tables in outer loops of the join */ ){ char aff; if( pTerm->leftCursor!=pSrc->iCursor ) return 0; - if( pTerm->eOperator!=WO_EQ ) return 0; + if( (pTerm->eOperator & WO_EQ)==0 ) return 0; if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.leftColumn<0 ) return 0; aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; if( !sqlite4IndexAffinityOk(pTerm->pExpr, aff) ) return 0; return 1; } -#endif - -#ifndef SQLITE4_OMIT_AUTOMATIC_INDEX -/* -** If the query plan for pSrc specified in pCost is a full table scan -** and indexing is allows (if there is no NOT INDEXED clause) and it -** possible to construct a transient index that would perform better -** than a full table scan even when the cost of constructing the index -** is taken into account, then alter the query plan to use the -** transient index. -*/ -static void bestAutomaticIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - SrcListItem *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ - WhereCost *pCost /* Lowest cost query plan */ -){ - double nTableRow; /* Rows in the input table */ - double logN; /* log(nTableRow) */ - double costTempIdx; /* per-query cost of the transient index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ - WhereTerm *pWCEnd; /* End of pWC->a[] */ - Table *pTable; /* Table tht might be indexed */ - - if( pParse->nQueryLoop<=(double)1 ){ - /* There is no point in building an automatic index for a single scan */ - return; - } - if( (pParse->db->flags & SQLITE4_AutoIndex)==0 ){ - /* Automatic indices are disabled at run-time */ - return; - } - if( (pWC->wctrlFlags & WHERE_NO_AUTOINDEX)!=0 ){ - return; - } - if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){ - /* We already have some kind of index in use for this query. */ - return; - } - if( pSrc->notIndexed ){ - /* The NOT INDEXED clause appears in the SQL. */ - return; - } - if( pSrc->isCorrelated ){ - /* The source is a correlated sub-query. No point in indexing it. */ - return; - } - - assert( pParse->nQueryLoop >= (double)1 ); - pTable = pSrc->pTab; - nTableRow = pTable->nRowEst; - logN = estLog(nTableRow); - costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); - if( costTempIdx>=pCost->rCost ){ - /* The cost of creating the transient table would be greater than - ** doing the full table scan */ - return; - } - - /* Search for any equality comparison term */ - pWCEnd = &pWC->a[pWC->nTerm]; - for(pTerm=pWC->a; pTermrCost, costTempIdx)); - pCost->rCost = costTempIdx; - pCost->plan.nRow = logN + 1; - pCost->plan.wsFlags = WHERE_TEMP_INDEX; - pCost->plan.u.pIdx = 0; - pCost->used = pTerm->prereqRight; - break; - } - } -} -#else -# define bestAutomaticIndex(A,B,C,D,E) /* no-op */ -#endif /* SQLITE4_OMIT_AUTOMATIC_INDEX */ #ifndef SQLITE4_OMIT_AUTOMATIC_INDEX /* ** Generate code to construct the Index object for an automatic index @@ -2055,80 +2096,113 @@ ** makes use of the automatic index. */ static void constructAutomaticIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ - SrcListItem *pSrc, /* The FROM clause term to get the next index */ + struct SrcListItem *pSrc, /* The FROM clause term to get the next index */ Bitmask notReady, /* Mask of cursors that are not available */ WhereLevel *pLevel /* Write new index here */ ){ - int nCol = 0; /* Number of columns in index keys */ + int nColumn; /* Number of columns in the constructed index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ WhereTerm *pWCEnd; /* End of pWC->a[] */ int nByte; /* Byte of memory needed for pIdx */ Index *pIdx; /* Object describing the transient index */ Vdbe *v; /* Prepared statement under construction */ - int addrOnce; /* Address of the initialization bypass jump */ + int addrInit; /* Address of the initialization bypass jump */ Table *pTable; /* The table being indexed */ KeyInfo *pKeyinfo; /* Key information for the index */ - int addrRewind; /* Top of the index fill loop */ + int addrTop; /* Top of the index fill loop */ int regRecord; /* Register holding an index record */ - int regKey; /* Register holding an index key */ int n; /* Column counter */ + int i; /* Loop counter */ + int mxBitCol; /* Maximum column in pSrc->colUsed */ CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ Bitmask idxCols; /* Bitmap of columns used for indexing */ - int iPkCur = pLevel->iTabCur; /* Primary key cursor to read data from */ + Bitmask extraCols; /* Bitmap of additional columns */ + u8 sentWarning = 0; /* True if a warnning has been issued */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ v = pParse->pVdbe; assert( v!=0 ); - addrOnce = sqlite4CodeOnce(pParse); + addrInit = sqlite4CodeOnce(pParse); - /* Count the number of columns that will be encoded into the index keys. - ** set nCol to this value. Use the idxCols mask to ensure that the same - ** column is not added to the index more than once. */ + /* Count the number of columns that will be added to the index + ** and used to match WHERE clause constraints */ + nColumn = 0; pTable = pSrc->pTab; pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); testcase( iCol==BMS ); testcase( iCol==BMS-1 ); + if( !sentWarning ){ + sqlite4_log(SQLITE4_WARNING_AUTOINDEX, + "automatic index on %s(%s)", pTable->zName, + pTable->aCol[iCol].zName); + sentWarning = 1; + } if( (idxCols & cMask)==0 ){ - nCol++; + if( whereLoopResize(pParse->db, pLoop, nColumn+1) ) return; + pLoop->aLTerm[nColumn++] = pTerm; idxCols |= cMask; } } } - assert( nCol>0 ); - pLevel->plan.nEq = nCol; - pLevel->plan.wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WO_EQ; + assert( nColumn>0 ); + pLoop->u.btree.nEq = pLoop->nLTerm = nColumn; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_AUTO_INDEX; + + /* Count the number of additional columns needed to create a + ** covering index. A "covering index" is an index that contains all + ** columns that are needed by the query. With a covering index, the + ** original table never needs to be accessed. Automatic indices must + ** be a covering index because the index will not be updated if the + ** original table changes and the index and table cannot both be used + ** if they go out of sync. + */ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + testcase( pTable->nCol==BMS-1 ); + testcase( pTable->nCol==BMS-2 ); + for(i=0; icolUsed & MASKBIT(BMS-1) ){ + nColumn += pTable->nCol - BMS + 1; + } + pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ - nByte = sizeof(Index); /* Index */ - nByte += nCol*sizeof(int); /* Index.aiColumn */ - nByte += nCol*sizeof(char*); /* Index.azColl */ - nByte += nCol; /* Index.aSortOrder */ + nByte = sizeof(Index); + nByte += nColumn*sizeof(int); /* Index.aiColumn */ + nByte += nColumn*sizeof(char*); /* Index.azColl */ + nByte += nColumn; /* Index.aSortOrder */ pIdx = sqlite4DbMallocZero(pParse->db, nByte); if( pIdx==0 ) return; - pLevel->plan.u.pIdx = pIdx; - pIdx->eIndexType = SQLITE4_INDEX_TEMP; + pLoop->u.btree.pIndex = pIdx; pIdx->azColl = (char**)&pIdx[1]; - pIdx->aiColumn = (int*)&pIdx->azColl[nCol]; - pIdx->aSortOrder = (u8*)&pIdx->aiColumn[nCol]; + pIdx->aiColumn = (int*)&pIdx->azColl[nColumn]; + pIdx->aSortOrder = (u8*)&pIdx->aiColumn[nColumn]; pIdx->zName = "auto-index"; - pIdx->nColumn = nCol; + pIdx->nColumn = nColumn; pIdx->pTable = pTable; n = 0; idxCols = 0; for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? ((Bitmask)1)<<(BMS-1) : ((Bitmask)1)<=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); if( (idxCols & cMask)==0 ){ Expr *pX = pTerm->pExpr; idxCols |= cMask; pIdx->aiColumn[n] = pTerm->u.leftColumn; pColl = sqlite4BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); @@ -2135,33 +2209,51 @@ pIdx->azColl[n] = ALWAYS(pColl) ? pColl->zName : "BINARY"; n++; } } } - assert( (u32)n==pLevel->plan.nEq ); + assert( (u32)n==pLoop->u.btree.nEq ); - /* Open the automatic index cursor */ + /* Add additional columns needed to make the automatic index into + ** a covering index */ + for(i=0; iaiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; + } + } + if( pSrc->colUsed & MASKBIT(BMS-1) ){ + for(i=BMS-1; inCol; i++){ + pIdx->aiColumn[n] = i; + pIdx->azColl[n] = "BINARY"; + n++; + } + } + assert( n==nColumn ); + + /* Create the automatic index */ pKeyinfo = sqlite4IndexKeyinfo(pParse, pIdx); assert( pLevel->iIdxCur>=0 ); - sqlite4VdbeAddOp3(v, OP_OpenAutoindex, pLevel->iIdxCur, 0, 0); - sqlite4VdbeChangeP4(v, -1, (char*)pKeyinfo, P4_KEYINFO_HANDOFF); + pLevel->iIdxCur = pParse->nTab++; + sqlite4VdbeAddOp4(v, OP_OpenAutoindex, pLevel->iIdxCur, nColumn+1, 0, + (char*)pKeyinfo, P4_KEYINFO_HANDOFF); VdbeComment((v, "for %s", pTable->zName)); - /* Populate the automatic index */ - regRecord = sqlite4GetTempRange(pParse, 2); - regKey = regRecord+1; - addrRewind = sqlite4VdbeAddOp1(v, OP_Rewind, iPkCur); - sqlite4EncodeIndexKey(pParse, 0, iPkCur, pIdx, pLevel->iIdxCur, 1, regKey); - sqlite4VdbeAddOp2(v, OP_RowData, iPkCur, regRecord); - sqlite4VdbeAddOp3(v, OP_IdxInsert, pLevel->iIdxCur, regRecord, regKey); - sqlite4VdbeAddOp2(v, OP_Next, iPkCur, addrRewind+1); + /* Fill the automatic index with content */ + addrTop = sqlite4VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); + regRecord = sqlite4GetTempReg(pParse); + sqlite4GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 1); + sqlite4VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); + sqlite4VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite4VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); sqlite4VdbeChangeP5(v, SQLITE4_STMTSTATUS_AUTOINDEX); - sqlite4VdbeJumpHere(v, addrRewind); - sqlite4ReleaseTempRange(pParse, regRecord, 2); + sqlite4VdbeJumpHere(v, addrTop); + sqlite4ReleaseTempReg(pParse, regRecord); /* Jump here when skipping the initialization */ - sqlite4VdbeJumpHere(v, addrOnce); + sqlite4VdbeJumpHere(v, addrInit); } #endif /* SQLITE4_OMIT_AUTOMATIC_INDEX */ #ifndef SQLITE4_OMIT_VIRTUALTABLE /* @@ -2168,13 +2260,13 @@ ** Allocate and populate an sqlite4_index_info structure. It is the ** responsibility of the caller to eventually release the structure ** by passing the pointer returned by this function to sqlite4_free(). */ static sqlite4_index_info *allocateIndexInfo( - Parse *pParse, + Parse *pParse, WhereClause *pWC, - SrcListItem *pSrc, + struct SrcListItem *pSrc, ExprList *pOrderBy ){ int i, j; int nTerm; struct sqlite4_index_constraint *pIdxCons; @@ -2182,20 +2274,18 @@ struct sqlite4_index_constraint_usage *pUsage; WhereTerm *pTerm; int nOrderBy; sqlite4_index_info *pIdxInfo; - WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName)); - /* Count the number of possible WHERE clause constraints referring ** to this virtual table */ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; } /* If the ORDER BY clause contains only columns in the current @@ -2202,16 +2292,17 @@ ** virtual table then allocate space for the aOrderBy part of ** the sqlite4_index_info structure. */ nOrderBy = 0; if( pOrderBy ){ - for(i=0; inExpr; i++){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; } - if( i==pOrderBy->nExpr ){ - nOrderBy = pOrderBy->nExpr; + if( i==n){ + nOrderBy = n; } } /* Allocate the sqlite4_index_info structure */ @@ -2218,11 +2309,10 @@ pIdxInfo = sqlite4DbMallocZero(pParse->db, sizeof(*pIdxInfo) + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm + sizeof(*pIdxOrderBy)*nOrderBy ); if( pIdxInfo==0 ){ sqlite4ErrorMsg(pParse, "out of memory"); - /* (double)0 In case of SQLITE4_OMIT_FLOATING_POINT... */ return 0; } /* Initialize the structure. The sqlite4_index_info structure contains ** many fields that are declared "const" to prevent xBestIndex from @@ -2238,29 +2328,32 @@ *(struct sqlite4_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; *(struct sqlite4_index_constraint_usage**)&pIdxInfo->aConstraintUsage = pUsage; for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; - assert( (pTerm->eOperator&(pTerm->eOperator-1))==0 ); - testcase( pTerm->eOperator==WO_IN ); - testcase( pTerm->eOperator==WO_ISNULL ); - if( pTerm->eOperator & (WO_IN|WO_ISNULL) ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + if( pTerm->eOperator & (WO_ISNULL) ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; - pIdxCons[j].op = (u8)pTerm->eOperator; + op = (u8)pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + pIdxCons[j].op = op; /* The direct assignment in the previous line is possible only because ** the WO_ and SQLITE4_INDEX_CONSTRAINT_ codes are identical. The ** following asserts verify this fact. */ assert( WO_EQ==SQLITE4_INDEX_CONSTRAINT_EQ ); assert( WO_LT==SQLITE4_INDEX_CONSTRAINT_LT ); assert( WO_LE==SQLITE4_INDEX_CONSTRAINT_LE ); assert( WO_GT==SQLITE4_INDEX_CONSTRAINT_GT ); assert( WO_GE==SQLITE4_INDEX_CONSTRAINT_GE ); assert( WO_MATCH==SQLITE4_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); j++; } for(i=0; ia[i].pExpr; pIdxOrderBy[i].iColumn = pExpr->iColumn; @@ -2271,12 +2364,12 @@ } /* ** The table object reference passed as the second argument to this function ** must represent a virtual table. This function invokes the xBestIndex() -** method of the virtual table with the sqlite4_index_info pointer passed -** as the argument. +** method of the virtual table with the sqlite4_index_info object that +** comes in as the 3rd argument to this function. ** ** If an error occurs, pParse is populated with an error message and a ** non-zero value is returned. Otherwise, 0 is returned and the output ** part of the sqlite4_index_info structure is left populated. ** @@ -2287,11 +2380,10 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite4_index_info *p){ sqlite4_vtab *pVtab = sqlite4GetVTable(pParse->db, pTab)->pVtab; int i; int rc; - WHERETRACE(("xBestIndex for %s\n", pTab->zName)); TRACE_IDX_INPUTS(p); rc = pVtab->pModule->xBestIndex(pVtab, p); TRACE_IDX_OUTPUTS(p); if( rc!=SQLITE4_OK ){ @@ -2313,166 +2405,12 @@ } } return pParse->nErr; } - - -/* -** Compute the best index for a virtual table. -** -** The best index is computed by the xBestIndex method of the virtual -** table module. This routine is really just a wrapper that sets up -** the sqlite4_index_info structure that is used to communicate with -** xBestIndex. -** -** In a join, this routine might be called multiple times for the -** same virtual table. The sqlite4_index_info structure is created -** and initialized on the first invocation and reused on all subsequent -** invocations. The sqlite4_index_info structure is also used when -** code is generated to access the virtual table. The whereInfoDelete() -** routine takes care of freeing the sqlite4_index_info structure after -** everybody has finished with it. -*/ -static void bestVirtualIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - SrcListItem *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for index */ - Bitmask notValid, /* Cursors not valid for any purpose */ - ExprList *pOrderBy, /* The order by clause */ - WhereCost *pCost, /* Lowest cost query plan */ - sqlite4_index_info **ppIdxInfo /* Index information passed to xBestIndex */ -){ - Table *pTab = pSrc->pTab; - sqlite4_index_info *pIdxInfo; - struct sqlite4_index_constraint *pIdxCons; - struct sqlite4_index_constraint_usage *pUsage; - WhereTerm *pTerm; - int i, j; - int nOrderBy; - double rCost; - - /* Make sure wsFlags is initialized to some sane value. Otherwise, if the - ** malloc in allocateIndexInfo() fails and this function returns leaving - ** wsFlags in an uninitialized state, the caller may behave unpredictably. - */ - memset(pCost, 0, sizeof(*pCost)); - pCost->plan.wsFlags = WHERE_VIRTUALTABLE; - - /* If the sqlite4_index_info structure has not been previously - ** allocated and initialized, then allocate and initialize it now. - */ - pIdxInfo = *ppIdxInfo; - if( pIdxInfo==0 ){ - *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy); - } - if( pIdxInfo==0 ){ - return; - } - - /* At this point, the sqlite4_index_info structure that pIdxInfo points - ** to will have been initialized, either during the current invocation or - ** during some prior invocation. Now we just have to customize the - ** details of pIdxInfo for the current invocation and pass it to - ** xBestIndex. - */ - - /* The module name must be defined. Also, by this point there must - ** be a pointer to an sqlite4_vtab structure. Otherwise - ** sqlite4ViewGetColumnNames() would have picked up the error. - */ - assert( pTab->azModuleArg && pTab->azModuleArg[0] ); - assert( sqlite4GetVTable(pParse->db, pTab) ); - - /* Set the aConstraint[].usable fields and initialize all - ** output variables to zero. - ** - ** aConstraint[].usable is true for constraints where the right-hand - ** side contains only references to tables to the left of the current - ** table. In other words, if the constraint is of the form: - ** - ** column = expr - ** - ** and we are evaluating a join, then the constraint on column is - ** only valid if all tables referenced in expr occur to the left - ** of the table containing column. - ** - ** The aConstraints[] array contains entries for all constraints - ** on the current table. That way we only have to compute it once - ** even though we might try to pick the best index multiple times. - ** For each attempt at picking an index, the order of tables in the - ** join might be different so we have to recompute the usable flag - ** each time. - */ - pIdxCons = *(struct sqlite4_index_constraint**)&pIdxInfo->aConstraint; - pUsage = pIdxInfo->aConstraintUsage; - for(i=0; inConstraint; i++, pIdxCons++){ - j = pIdxCons->iTermOffset; - pTerm = &pWC->a[j]; - pIdxCons->usable = (pTerm->prereqRight¬Ready) ? 0 : 1; - } - memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); - if( pIdxInfo->needToFreeIdxStr ){ - sqlite4_free(pIdxInfo->idxStr); - } - pIdxInfo->idxStr = 0; - pIdxInfo->idxNum = 0; - pIdxInfo->needToFreeIdxStr = 0; - pIdxInfo->orderByConsumed = 0; - /* ((double)2) In case of SQLITE4_OMIT_FLOATING_POINT... */ - pIdxInfo->estimatedCost = SQLITE4_BIG_DBL / ((double)2); - nOrderBy = pIdxInfo->nOrderBy; - if( !pOrderBy ){ - pIdxInfo->nOrderBy = 0; - } - - if( vtabBestIndex(pParse, pTab, pIdxInfo) ){ - return; - } - - pIdxCons = *(struct sqlite4_index_constraint**)&pIdxInfo->aConstraint; - for(i=0; inConstraint; i++){ - if( pUsage[i].argvIndex>0 ){ - pCost->used |= pWC->a[pIdxCons[i].iTermOffset].prereqRight; - } - } - - /* If there is an ORDER BY clause, and the selected virtual table index - ** does not satisfy it, increase the cost of the scan accordingly. This - ** matches the processing for non-virtual tables in bestKVIndex(). - */ - rCost = pIdxInfo->estimatedCost; - if( pOrderBy && pIdxInfo->orderByConsumed==0 ){ - rCost += estLog(rCost)*rCost; - } - - /* The cost is not allowed to be larger than SQLITE4_BIG_DBL (the - ** inital value of lowestCost in this loop. If it is, then the - ** (costrCost = (SQLITE4_BIG_DBL/((double)2)); - }else{ - pCost->rCost = rCost; - } - pCost->plan.u.pVtabIdx = pIdxInfo; - if( pIdxInfo->orderByConsumed ){ - pCost->plan.wsFlags |= WHERE_ORDERBY; - } - pCost->plan.nEq = 0; - pIdxInfo->nOrderBy = nOrderBy; - - /* Try to find a more efficient access pattern by using multiple indexes - ** to optimize an OR expression within the WHERE clause. - */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); -} -#endif /* SQLITE4_OMIT_VIRTUALTABLE */ +#endif /* !defined(SQLITE4_OMIT_VIRTUALTABLE) */ + #ifdef SQLITE4_ENABLE_STAT3 /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: @@ -2483,40 +2421,121 @@ ** Return SQLITE4_OK on success. */ static int whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ - sqlite4_buffer *pBuf, /* Buffer containing encoded value to consider */ + sqlite4_value *pVal, /* Value to consider */ int roundUp, /* Round up if true. Round down if false */ tRowcnt *aStat /* OUT: stats written here */ ){ tRowcnt n; IndexSample *aSample; - int i; + int i, eType; int isEq = 0; + i64 v; + double r, rS; assert( roundUp==0 || roundUp==1 ); assert( pIdx->nSample>0 ); - assert( pBuf->n>0 ); - + if( pVal==0 ) return SQLITE4_ERROR; n = pIdx->aiRowEst[0]; aSample = pIdx->aSample; - - - /* Set variable i to the index of the first sample equal to or larger - ** than the value in pBuf. Set isEq to true if the value is equal, or - ** false otherwise. */ - for(i=0; inSample; i++){ - int res; - int n = pBuf->n; - if( n>aSample[i].nVal ) n = aSample[i].nVal; - - res = memcmp(pBuf->p, aSample[i].aVal, n); - if( res==0 ) res = pBuf->n - aSample[i].nVal; - if( res<=0 ){ - isEq = (res==0); - break; + eType = sqlite4_value_type(pVal); + + if( eType==SQLITE4_INTEGER ){ + v = sqlite4_value_int64(pVal); + r = (i64)v; + for(i=0; inSample; i++){ + if( aSample[i].eType==SQLITE4_NULL ) continue; + if( aSample[i].eType>=SQLITE4_TEXT ) break; + if( aSample[i].eType==SQLITE4_INTEGER ){ + if( aSample[i].u.i>=v ){ + isEq = aSample[i].u.i==v; + break; + } + }else{ + assert( aSample[i].eType==SQLITE4_FLOAT ); + if( aSample[i].u.r>=r ){ + isEq = aSample[i].u.r==r; + break; + } + } + } + }else if( eType==SQLITE4_FLOAT ){ + r = sqlite4_value_double(pVal); + for(i=0; inSample; i++){ + if( aSample[i].eType==SQLITE4_NULL ) continue; + if( aSample[i].eType>=SQLITE4_TEXT ) break; + if( aSample[i].eType==SQLITE4_FLOAT ){ + rS = aSample[i].u.r; + }else{ + rS = aSample[i].u.i; + } + if( rS>=r ){ + isEq = rS==r; + break; + } + } + }else if( eType==SQLITE4_NULL ){ + i = 0; + if( aSample[0].eType==SQLITE4_NULL ) isEq = 1; + }else{ + assert( eType==SQLITE4_TEXT || eType==SQLITE4_BLOB ); + for(i=0; inSample; i++){ + if( aSample[i].eType==SQLITE4_TEXT || aSample[i].eType==SQLITE4_BLOB ){ + break; + } + } + if( inSample ){ + sqlite4 *db = pParse->db; + CollSeq *pColl; + const u8 *z; + if( eType==SQLITE4_BLOB ){ + z = (const u8 *)sqlite4_value_blob(pVal); + pColl = db->pDfltColl; + assert( pColl->enc==SQLITE4_UTF8 ); + }else{ + pColl = sqlite4GetCollSeq(pParse, SQLITE4_UTF8, 0, *pIdx->azColl); + /* If the collating sequence was unavailable, we should have failed + ** long ago and never reached this point. But we'll check just to + ** be doubly sure. */ + if( NEVER(pColl==0) ) return SQLITE4_ERROR; + z = (const u8 *)sqlite4ValueText(pVal, pColl->enc); + if( !z ){ + return SQLITE4_NOMEM; + } + assert( z && pColl && pColl->xCmp ); + } + n = sqlite4ValueBytes(pVal, pColl->enc); + + for(; inSample; i++){ + int c; + int eSampletype = aSample[i].eType; + if( eSampletypeenc!=SQLITE4_UTF8 ){ + int nSample; + char *zSample = sqlite4Utf8to16( + db, pColl->enc, aSample[i].u.z, aSample[i].nByte, &nSample + ); + if( !zSample ){ + assert( db->mallocFailed ); + return SQLITE4_NOMEM; + } + c = pColl->xCmp(pColl->pUser, nSample, zSample, n, z); + sqlite4DbFree(db, zSample); + }else +#endif + { + c = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z); + } + if( c>=0 ){ + if( c==0 ) isEq = 1; + break; + } + } } } /* At this point, aSample[i] is the first sample that is greater than ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less @@ -2551,75 +2570,44 @@ return SQLITE4_OK; } #endif /* SQLITE4_ENABLE_STAT3 */ /* -** If expression pExpr represents a literal value, extract it and apply -** the affinity aff to it. Then encode the value using the database index -** key encoding and write the result into buffer pBuf. +** If expression pExpr represents a literal value, set *pp to point to +** an sqlite4_value structure containing the same value, with affinity +** aff applied to it, before returning. It is the responsibility of the +** caller to eventually release this structure by passing it to +** sqlite4ValueFree(). ** ** If the current parse is a recompile (sqlite4Reprepare()) and pExpr ** is an SQL variable that currently has a non-NULL value bound to it, -** do the same with the bound value. +** create an sqlite4_value structure containing this value, again with +** affinity aff applied to it, instead. ** -** If neither of the above apply, leave the buffer empty. +** If neither of the above apply, set *pp to NULL. ** ** If an error occurs, return an error code. Otherwise, SQLITE4_OK. */ #ifdef SQLITE4_ENABLE_STAT3 static int valueFromExpr( - Parse *pParse, /* Parse context */ - KeyInfo *pKeyinfo, /* Collation sequence and sort order */ - Expr *pExpr, /* Expression to extract value from */ - u8 aff, /* Affinity to apply to value */ - sqlite4_buffer *pBuf /* Buffer to populate */ + Parse *pParse, + Expr *pExpr, + u8 aff, + sqlite4_value **pp ){ - int rc = SQLITE4_OK; - sqlite4 *db = pParse->db; - sqlite4_value *pVal = 0; - - assert( pBuf->n==0 ); - if( pExpr->op==TK_VARIABLE || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) ){ int iVar = pExpr->iColumn; sqlite4VdbeSetVarmask(pParse->pVdbe, iVar); - pVal = sqlite4VdbeGetValue(pParse->pReprepare, iVar, aff); - }else{ - rc = sqlite4ValueFromExpr(db, pExpr, SQLITE4_UTF8, aff, &pVal); - } - - if( pVal && rc==SQLITE4_OK ){ - u8 *aOut; - int nOut; - rc = sqlite4VdbeEncodeKey(db, pVal, 1, 2, -1, pKeyinfo, &aOut, &nOut, 0); - if( rc==SQLITE4_OK ){ - rc = sqlite4_buffer_set(pBuf, aOut, nOut); - } - sqlite4DbFree(db, aOut); - } - - sqlite4ValueFree(pVal); - return SQLITE4_OK; + *pp = sqlite4VdbeGetValue(pParse->pReprepare, iVar, aff); + return SQLITE4_OK; + } + return sqlite4ValueFromExpr(pParse->db, pExpr, SQLITE4_UTF8, aff, pp); } #endif -static int whereSampleKeyinfo(Parse *pParse, Index *p, KeyInfo *pKeyInfo){ - CollSeq *pColl; - memset(pKeyInfo, 0, sizeof(KeyInfo)); - pKeyInfo->db = pParse->db; - pKeyInfo->enc = SQLITE4_UTF8; - pKeyInfo->nField = p->nColumn; - pKeyInfo->nPK = 1; - pKeyInfo->nData = 0; - pKeyInfo->aSortOrder = p->aSortOrder; - pKeyInfo->aColl[0] = pColl = sqlite4LocateCollSeq(pParse, p->azColl[0]); - pKeyInfo->aColl[0] = pColl; - return pColl ? SQLITE4_OK : SQLITE4_ERROR; -} - /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper ** bound, a lower bound, or both. The WHERE clause terms that set the upper ** and lower bounds are represented by pLower and pUpper respectively. For @@ -2661,72 +2649,73 @@ Parse *pParse, /* Parsing & code generating context */ Index *p, /* The index containing the range-compared column; "x" */ int nEq, /* index into p->aCol[] of the range-compared column */ WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - double *pRangeDiv /* OUT: Reduce search space by this divisor */ + WhereCost *pRangeDiv /* OUT: Reduce search space by this divisor */ ){ int rc = SQLITE4_OK; #ifdef SQLITE4_ENABLE_STAT3 - if( nEq==0 && p->nSample ){ - sqlite4 *db = pParse->db; - KeyInfo keyinfo; - sqlite4_buffer buf; /* Buffer used for index sample */ + if( nEq==0 && p->nSample && OptimizationEnabled(pParse->db, SQLITE4_Stat3) ){ + sqlite4_value *pRangeVal; tRowcnt iLower = 0; tRowcnt iUpper = p->aiRowEst[0]; tRowcnt a[2]; u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity; - sqlite4_buffer_init(&buf, db->pEnv->pMM); - rc = whereSampleKeyinfo(pParse, p, &keyinfo); - - if( rc==SQLITE4_OK && pLower ){ + if( pLower ){ Expr *pExpr = pLower->pExpr->pRight; - rc = valueFromExpr(pParse, &keyinfo, pExpr, aff, &buf); - assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE ); - if( rc==SQLITE4_OK && buf.n - && whereKeyStats(pParse, p, &buf, 0, a)==SQLITE4_OK + rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + if( rc==SQLITE4_OK + && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE4_OK ){ iLower = a[0]; - if( pLower->eOperator==WO_GT ) iLower += a[1]; + if( (pLower->eOperator & WO_GT)!=0 ) iLower += a[1]; } - sqlite4_buffer_set(&buf, 0, 0); + sqlite4ValueFree(pRangeVal); } if( rc==SQLITE4_OK && pUpper ){ Expr *pExpr = pUpper->pExpr->pRight; - rc = valueFromExpr(pParse, &keyinfo, pExpr, aff, &buf); - assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE ); - if( rc==SQLITE4_OK && buf.n - && whereKeyStats(pParse, p, &buf, 1, a)==SQLITE4_OK + rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + if( rc==SQLITE4_OK + && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE4_OK ){ iUpper = a[0]; - if( pUpper->eOperator==WO_LE ) iUpper += a[1]; + if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } + sqlite4ValueFree(pRangeVal); } - sqlite4_buffer_clear(&buf); if( rc==SQLITE4_OK ){ - if( iUpper<=iLower ){ - *pRangeDiv = (double)p->aiRowEst[0]; - }else{ - *pRangeDiv = (double)p->aiRowEst[0]/(double)(iUpper - iLower); + WhereCost iBase = whereCost(p->aiRowEst[0]); + if( iUpper>iLower ){ + iBase -= whereCost(iUpper - iLower); } - WHERETRACE(("range scan regions: %u..%u div=%g\n", - (u32)iLower, (u32)iUpper, *pRangeDiv)); + *pRangeDiv = iBase; + WHERETRACE(0x100, ("range scan regions: %u..%u div=%d\n", + (u32)iLower, (u32)iUpper, *pRangeDiv)); return SQLITE4_OK; } } #else UNUSED_PARAMETER(pParse); UNUSED_PARAMETER(p); UNUSED_PARAMETER(nEq); #endif assert( pLower || pUpper ); - *pRangeDiv = (double)1; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *pRangeDiv *= (double)4; - if( pUpper ) *pRangeDiv *= (double)4; + *pRangeDiv = 0; + /* TUNING: Each inequality constraint reduces the search space 4-fold. + ** A BETWEEN operator, therefore, reduces the search space 16-fold */ + if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ + *pRangeDiv += 20; assert( 20==whereCost(4) ); + } + if( pUpper ){ + *pRangeDiv += 20; assert( 20==whereCost(4) ); + } return rc; } #ifdef SQLITE4_ENABLE_STAT3 /* @@ -2748,43 +2737,34 @@ */ static int whereEqualScanEst( Parse *pParse, /* Parsing & code generating context */ Index *p, /* The index whose left-most column is pTerm */ Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ - double *pnRow /* Write the revised row estimate here */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ - sqlite4_buffer buf; + sqlite4_value *pRhs = 0; /* VALUE on right-hand side of pTerm */ u8 aff; /* Column affinity */ int rc; /* Subfunction return code */ tRowcnt a[2]; /* Statistics */ assert( p->aSample!=0 ); assert( p->nSample>0 ); - - sqlite4_buffer_init(&buf, pParse->db->pEnv->pMM); aff = p->pTable->aCol[p->aiColumn[0]].affinity; if( pExpr ){ - KeyInfo keyinfo; - rc = whereSampleKeyinfo(pParse, p, &keyinfo); - if( rc==SQLITE4_OK ){ - rc = valueFromExpr(pParse, &keyinfo, pExpr, aff, &buf); - if( buf.n==0 ) return SQLITE4_NOTFOUND; - } - }else{ - /* Populate the buffer with a NULL. */ - u8 aNull[2] = {0x05, 0xfa}; /* ASC, DESC */ - rc = sqlite4_buffer_set(&buf, &aNull[p->aSortOrder[0]], 1); - } - if( rc ) goto whereEqualScanEst_cancel; - - rc = whereKeyStats(pParse, p, &buf, 0, a); - if( rc==SQLITE4_OK ){ - WHERETRACE(("equality scan regions: %d\n", (int)a[1])); + rc = valueFromExpr(pParse, pExpr, aff, &pRhs); + if( rc ) goto whereEqualScanEst_cancel; + }else{ + pRhs = sqlite4ValueNew(pParse->db); + } + if( pRhs==0 ) return SQLITE4_NOTFOUND; + rc = whereKeyStats(pParse, p, pRhs, 0, a); + if( rc==SQLITE4_OK ){ + WHERETRACE(0x100,("equality scan regions: %d\n", (int)a[1])); *pnRow = a[1]; } whereEqualScanEst_cancel: - sqlite4_buffer_clear(&buf); + sqlite4ValueFree(pRhs); return rc; } #endif /* defined(SQLITE4_ENABLE_STAT3) */ #ifdef SQLITE4_ENABLE_STAT3 @@ -2806,16 +2786,16 @@ */ static int whereInScanEst( Parse *pParse, /* Parsing & code generating context */ Index *p, /* The index whose left-most column is pTerm */ ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ - double *pnRow /* Write the revised row estimate here */ + tRowcnt *pnRow /* Write the revised row estimate here */ ){ - int rc = SQLITE4_OK; /* Subfunction return code */ - double nEst; /* Number of rows for a single term */ - double nRowEst = (double)0; /* New estimate of the number of rows */ - int i; /* Loop counter */ + int rc = SQLITE4_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ assert( p->aSample!=0 ); for(i=0; rc==SQLITE4_OK && inExpr; i++){ nEst = p->aiRowEst[0]; rc = whereEqualScanEst(pParse, p, pList->a[i].pExpr, &nEst); @@ -2822,581 +2802,16 @@ nRowEst += nEst; } if( rc==SQLITE4_OK ){ if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; *pnRow = nRowEst; - WHERETRACE(("IN row estimate: est=%g\n", nRowEst)); + WHERETRACE(0x100,("IN row estimate: est=%g\n", nRowEst)); } return rc; } #endif /* defined(SQLITE4_ENABLE_STAT3) */ -/* -** Try to find a MATCH expression that constrains the pTabItem table in the -** WHERE clause. If one exists, set *piTerm to the index in the pWC->a[] array -** and return non-zero. If no such expression exists, return 0. -*/ -static int findMatchExpr( - Parse *pParse, - WhereClause *pWC, - SrcListItem *pTabItem, - int *piTerm -){ - int i; - int iCsr = pTabItem->iCursor; - - for(i=0; inTerm; i++){ - Expr *pMatch = pWC->a[i].pExpr; - if( pMatch->iTable==iCsr && pMatch->op==TK_MATCH ) break; - } - if( i==pWC->nTerm ) return 0; - - *piTerm = i; - return 1; -} - -static int bestMatchIdx( - Parse *pParse, - WhereClause *pWC, - SrcListItem *pTabItem, - Bitmask notReady, - WhereCost *pCost -){ - int iTerm; - - if( 0==findMatchExpr(pParse, pWC, pTabItem, &iTerm) ) return 0; - - /* Check that the MATCH expression is not composed using values from any - ** tables that are not ready. If it does, return 0. */ - if( notReady & pWC->a[iTerm].prereqAll ) return 0; - - pCost->used = pWC->a[iTerm].prereqAll; - pCost->rCost = 1.0; - pCost->plan.wsFlags = WHERE_INDEXED; - pCost->plan.nEq = 0; - pCost->plan.nRow = 10; - pCost->plan.u.pIdx = pWC->a[iTerm].pExpr->pIdx; - return 1; -} - -/* -** Find the best query plan for accessing a particular table. Write the -** best query plan and its cost into the WhereCost object supplied as the -** last parameter. -** -** The lowest cost plan wins. The cost is an estimate of the amount of -** CPU and disk I/O needed to process the requested result. -** Factors that influence cost include: -** -** * The estimated number of rows that will be retrieved. (The -** fewer the better.) -** -** * Whether or not sorting must occur. -** -** * Whether or not there must be separate lookups in the -** index and in the main table. -** -** If there was an INDEXED BY clause (pSrc->pIndex) attached to the table in -** the SQL statement, then this function only considers plans using the -** named index. If no such plan is found, then the returned cost is -** SQLITE4_BIG_DBL. If a plan is found that uses the named index, -** then the cost is calculated in the usual way. -** -** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table -** in the SELECT statement, then no indexes are considered. However, the -** selected plan may still take advantage of the built-in rowid primary key -** index. -*/ -static void bestKVIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - SrcListItem *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - ExprList *pDistinct, /* The select-list if query is DISTINCT */ - WhereCost *pCost /* Lowest cost query plan */ -){ - int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ - Index *pProbe; /* An index we are evaluating */ - Index *pFirst; /* First index to evaluate */ - Index *pPk; /* Primary Key index */ - int eqTermMask; /* Current mask of valid equality operators */ - int idxEqTermMask; /* Index mask of valid equality operators */ - - /* Initialize the cost to a worst-case value */ - memset(pCost, 0, sizeof(*pCost)); - pCost->rCost = SQLITE4_BIG_DBL; - pPk = sqlite4FindPrimaryKey(pSrc->pTab, 0); - - /* If the pSrc table is the right table of a LEFT JOIN then we may not - ** use an index to satisfy IS NULL constraints on that table. This is - ** because columns might end up being NULL if the table does not match - - ** a circumstance which the index cannot help us discover. Ticket #2177. - */ - if( pSrc->jointype & JT_LEFT ){ - idxEqTermMask = WO_EQ|WO_IN; - }else{ - idxEqTermMask = WO_EQ|WO_IN|WO_ISNULL; - } - - /* Normally, this function considers all indexes attached to the table - ** being queried. Except, if an INDEXED BY clause is specified then only - ** the named index is considered. And if a NOT INDEXED clause was present - ** only the PRIMARY KEY index may be considered. - */ - if( pSrc->notIndexed ){ - pFirst = pPk; - }else if( pSrc->pIndex ){ - pFirst = pSrc->pIndex; - }else{ - pFirst = pSrc->pTab->pIndex; - } - eqTermMask = idxEqTermMask; - - /* Loop over all indices looking for the best one to use */ - for(pProbe=pFirst; pProbe; pProbe=pProbe->pNext){ - const tRowcnt * const aiRowEst = pProbe->aiRowEst; - double cost; /* Cost of using pProbe */ - double nRow; /* Estimated number of rows in result set */ - double log10N = (double)1; /* base-10 logarithm of nRow (inexact) */ - int rev; /* True to scan in reverse order */ - int wsFlags = 0; - Bitmask used = 0; - - /* The following variables are populated based on the properties of - ** index being evaluated. They are then used to determine the expected - ** cost and number of rows returned. - ** - ** nEq: - ** Number of equality terms that can be implemented using the index. - ** In other words, the number of initial fields in the index that - ** are used in == or IN or NOT NULL constraints of the WHERE clause. - ** - ** nInMul: - ** The "in-multiplier". This is an estimate of how many seek operations - ** SQLite must perform on the index in question. For example, if the - ** WHERE clause is: - ** - ** WHERE a IN (1, 2, 3) AND b IN (4, 5, 6) - ** - ** SQLite must perform 9 lookups on an index on (a, b), so nInMul is - ** set to 9. Given the same schema and either of the following WHERE - ** clauses: - ** - ** WHERE a = 1 - ** WHERE a >= 2 - ** - ** nInMul is set to 1. - ** - ** If there exists a WHERE term of the form "x IN (SELECT ...)", then - ** the sub-select is assumed to return 25 rows for the purposes of - ** determining nInMul. - ** - ** bInEst: - ** Set to true if there was at least one "x IN (SELECT ...)" term used - ** in determining the value of nInMul. Note that the RHS of the - ** IN operator must be a SELECT, not a value list, for this variable - ** to be true. - ** - ** rangeDiv: - ** An estimate of a divisor by which to reduce the search space due - ** to inequality constraints. In the absence of sqlite_stat3 ANALYZE - ** data, a single inequality reduces the search space to 1/4rd its - ** original size (rangeDiv==4). Two inequalities reduce the search - ** space to 1/16th of its original size (rangeDiv==16). - ** - ** bSort: - ** Boolean. True if there is an ORDER BY clause that will require an - ** external sort (i.e. scanning the index being evaluated will not - ** correctly order records). - ** - ** bLookup: - ** Boolean. True if a table lookup is required for each index entry - ** visited. In other words, true if this is not a covering index. - ** This is always false for the rowid primary key index of a table. - ** For other indexes, it is true unless all the columns of the table - ** used by the SELECT statement are present in the index (such an - ** index is sometimes described as a covering index). - ** For example, given the index on (a, b), the second of the following - ** two queries requires table b-tree lookups in order to find the value - ** of column c, but the first does not because columns a and b are - ** both available in the index. - ** - ** SELECT a, b FROM tbl WHERE a = 1; - ** SELECT a, b, c FROM tbl WHERE a = 1; - */ - int nEq; /* Number of == or IN terms matching index */ - int bInEst = 0; /* True if "x IN (SELECT...)" seen */ - int nInMul = 1; /* Number of distinct equalities to lookup */ - double rangeDiv = (double)1; /* Estimated reduction in search space */ - int nBound = 0; /* Number of range constraints seen */ - int bSort = !!pOrderBy; /* True if external sort required */ - int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ - int bLookup = 0; /* True if not the PK index */ - WhereTerm *pTerm; /* A single term of the WHERE clause */ -#ifdef SQLITE4_ENABLE_STAT3 - WhereTerm *pFirstTerm = 0; /* First term matching the index */ -#endif - int nCol = pProbe->nColumn; /* Total columns in index record */ - - if( pProbe->eIndexType==SQLITE4_INDEX_FTS5 ) continue; - - /* Unless pProbe is the primary key index, then the encoded PK column - ** values are at the end of each record. Set variable nCol to the total - ** number of columns encoded into each index record, including the PK - ** columns. */ - if( pProbe!=pPk ) nCol += pPk->nColumn; - - /* Determine the values of nEq and nInMul */ - for(nEq=0; nEqpWC!=pWC ); - if( pTerm->eOperator & WO_IN ){ - Expr *pExpr = pTerm->pExpr; - wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* "x IN (SELECT ...)": Assume the SELECT returns 25 rows */ - nInMul *= 25; - bInEst = 1; - }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ - /* "x IN (value, value, ...)" */ - nInMul *= pExpr->x.pList->nExpr; - } - }else if( pTerm->eOperator & WO_ISNULL ){ - wsFlags |= WHERE_COLUMN_NULL; - } -#ifdef SQLITE4_ENABLE_STAT3 - if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm; -#endif - used |= pTerm->prereqRight; - } - - /* If the index being considered is UNIQUE, and there is an equality - ** constraint for all columns in the index, then this search will find - ** at most a single row. In this case set the WHERE_UNIQUE flag to - ** indicate this to the caller. - ** - ** Otherwise, if the search may find more than one row, test to see if - ** there is a range constraint on indexed column (nEq+1) that can be - ** optimized using the index. - */ - if( nEq>=pProbe->nColumn && pProbe->onError!=OE_None ){ - testcase( wsFlags & WHERE_COLUMN_IN ); - testcase( wsFlags & WHERE_COLUMN_NULL ); - if( (wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_NULL))==0 ){ - wsFlags |= WHERE_UNIQUE; - } - }else if( (pProbe->fIndex & IDX_Unordered)==0 ){ - int j = idxColumnNumber(pProbe, pPk, nEq); - if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pProbe) ){ - WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pProbe); - WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pProbe); - whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv); - if( pTop ){ - nBound = 1; - wsFlags |= WHERE_TOP_LIMIT; - used |= pTop->prereqRight; - testcase( pTop->pWC!=pWC ); - } - if( pBtm ){ - nBound++; - wsFlags |= WHERE_BTM_LIMIT; - used |= pBtm->prereqRight; - testcase( pBtm->pWC!=pWC ); - } - wsFlags |= WHERE_COLUMN_RANGE; - } - } - - /* If there is an ORDER BY clause and the index being considered will - ** naturally scan rows in the required order, set the appropriate flags - ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index - ** will scan rows in a different order, set the bSort variable. */ - if( isSortingIndex( - pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) - ){ - bSort = 0; - wsFlags |= WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); - } - - /* If there is a DISTINCT qualifier and this index will scan rows in - ** order of the DISTINCT expressions, clear bDist and set the appropriate - ** flags in wsFlags. */ - if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){ - bDist = 0; - wsFlags |= WHERE_COLUMN_RANGE|WHERE_DISTINCT; - } - - /* If currently calculating the cost of using an index (not the PK - ** index), determine if all required column data may be obtained without - ** using the main table (i.e. if the index is a covering - ** index for this query). If it is, set the WHERE_IDX_ONLY flag in - ** wsFlags. Otherwise, set the bLookup variable to true. - ** - ** TODO: Not clear if this optimization can be applied in SQLite 4. Fix - ** this block once that is figured out. - */ -#if 0 - if( wsFlags ){ - Bitmask m = pSrc->colUsed; - int j; - for(j=0; jnColumn; j++){ - int x = pProbe->aiColumn[j]; - if( xeIndexType!=SQLITE4_INDEX_PRIMARYKEY); - - /* - ** Estimate the number of rows of output. For an "x IN (SELECT...)" - ** constraint, do not let the estimate exceed half the rows in the table. - */ - nRow = (double)(aiRowEst[nEq] * nInMul); - if( bInEst && nRow*2>aiRowEst[0] ){ - nRow = aiRowEst[0]/2; - nInMul = (int)(nRow / aiRowEst[nEq]); - } - -#ifdef SQLITE4_ENABLE_STAT3 - /* If the constraint is of the form x=VALUE or x IN (E1,E2,...) - ** and we do not think that values of x are unique and if histogram - ** data is available for column x, then it might be possible - ** to get a better estimate on the number of rows based on - ** VALUE and how common that value is according to the histogram. - */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ - assert( (pFirstTerm->eOperator & (WO_EQ|WO_ISNULL|WO_IN))!=0 ); - if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ - testcase( pFirstTerm->eOperator==WO_EQ ); - testcase( pFirstTerm->eOperator==WO_ISNULL ); - whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow); - }else if( bInEst==0 ){ - assert( pFirstTerm->eOperator==WO_IN ); - whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow); - } - } -#endif /* SQLITE4_ENABLE_STAT3 */ - - /* Adjust the number of output rows and downward to reflect rows - ** that are excluded by range constraints. - */ - nRow = nRow/rangeDiv; - if( nRow<1 ) nRow = 1; - - /* Experiments run on real SQLite databases show that the time needed - ** to do a binary search to locate a row in a table or index is roughly - ** log10(N) times the time to move from one row to the next row within - ** a table or index. The actual times can vary, with the size of - ** records being an important factor. Both moves and searches are - ** slower with larger records, presumably because fewer records fit - ** on one page and hence more pages have to be fetched. - ** - ** The ANALYZE command and the sqlite_stat1 and sqlite_stat3 tables do - ** not give us data on the relative sizes of table and index records. - ** So this computation assumes table records are about twice as big - ** as index records - */ - if( (wsFlags & WHERE_NOT_FULLSCAN)==0 ){ - /* The cost of a full table scan is a number of move operations equal - ** to the number of rows in the table. - ** - ** We add an additional 4x penalty to full table scans. This causes - ** the cost function to err on the side of choosing an index over - ** choosing a full scan. This 4x full-scan penalty is an arguable - ** decision and one which we expect to revisit in the future. But - ** it seems to be working well enough at the moment. - */ - cost = aiRowEst[0]*4; - }else{ - log10N = estLog(aiRowEst[0]); - cost = nRow; - if( bLookup ){ - /* For an index lookup followed by a table lookup: - ** nInMul index searches to find the start of each index range - ** + nRow steps through the index - ** + nRow table searches to lookup the table entry using the PK - */ - cost += (nInMul + nRow)*log10N; - }else{ - /* For a covering index: - ** nInMul index searches to find the initial entry - ** + nRow steps through the index - */ - cost += nInMul*log10N; - } - } - - /* Add in the estimated cost of sorting the result. Actual experimental - ** measurements of sorting performance in SQLite show that sorting time - ** adds C*N*log10(N) to the cost, where N is the number of rows to be - ** sorted and C is a factor between 1.95 and 4.3. We will split the - ** difference and select C of 3.0. - */ - if( bSort ){ - cost += nRow*estLog(nRow)*3; - } - if( bDist ){ - cost += nRow*estLog(nRow)*3; - } - - /**** Cost of using this index has now been computed ****/ - - /* If there are additional constraints on this table that cannot - ** be used with the current index, but which might lower the number - ** of output rows, adjust the nRow value accordingly. This only - ** matters if the current index is the least costly, so do not bother - ** with this step if we already know this index will not be chosen. - ** Also, never reduce the output row count below 2 using this step. - ** - ** It is critical that the notValid mask be used here instead of - ** the notReady mask. When computing an "optimal" index, the notReady - ** mask will only have one bit set - the bit for the current table. - ** The notValid mask, on the other hand, always has all bits set for - ** tables that are not in outer loops. If notReady is used here instead - ** of notValid, then a optimal index that depends on inner joins loops - ** might be selected even when there exists an optimal index that has - ** no such dependency. - */ - if( nRow>2 && cost<=pCost->rCost ){ - int k; /* Loop counter */ - int nSkipEq = nEq; /* Number of == constraints to skip */ - int nSkipRange = nBound; /* Number of < constraints to skip */ - Bitmask thisTab; /* Bitmap for pSrc */ - - thisTab = getMask(pWC->pMaskSet, iCur); - for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ - if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notValid)!=thisTab ) continue; - if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ - if( nSkipEq ){ - /* Ignore the first nEq equality matches since the index - ** has already accounted for these */ - nSkipEq--; - }else{ - /* Assume each additional equality match reduces the result - ** set size by a factor of 10 */ - nRow /= 10; - } - }else if( pTerm->eOperator & (WO_LT|WO_LE|WO_GT|WO_GE) ){ - if( nSkipRange ){ - /* Ignore the first nSkipRange range constraints since the index - ** has already accounted for these */ - nSkipRange--; - }else{ - /* Assume each additional range constraint reduces the result - ** set size by a factor of 3. Indexed range constraints reduce - ** the search space by a larger factor: 4. We make indexed range - ** more selective intentionally because of the subjective - ** observation that indexed range constraints really are more - ** selective in practice, on average. */ - nRow /= 3; - } - }else if( pTerm->eOperator!=WO_NOOP ){ - /* Any other expression lowers the output row count by half */ - nRow /= 2; - } - } - if( nRow<2 ) nRow = 2; - } - - - WHERETRACE(( - "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n" - " notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n", - pSrc->pTab->zName, pProbe->zName, - nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags, - notReady, log10N, nRow, cost, used - )); - - /* If this index is the best we have seen so far, then record this - ** index and its cost in the pCost structure. - */ - if( (pProbe==pFirst || wsFlags || pProbe==pPk) - && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) - ){ - pCost->rCost = cost; - pCost->used = used; - pCost->plan.nRow = nRow; - pCost->plan.wsFlags = wsFlags; - pCost->plan.nEq = nEq; - pCost->plan.u.pIdx = pProbe; - } - - /* If there was an INDEXED BY or NOT INDEXED clause, only one index is - ** considered. */ - if( pSrc->pIndex || pSrc->notIndexed ) break; - } - - /* If there is no ORDER BY clause and the SQLITE4_ReverseOrder flag - ** is set, then reverse the order that the index will be scanned - ** in. This is used for application testing, to help find cases - ** where application behaviour depends on the (undefined) order that - ** SQLite outputs rows in in the absence of an ORDER BY clause. */ - if( !pOrderBy && pParse->db->flags & SQLITE4_ReverseOrder ){ - pCost->plan.wsFlags |= WHERE_REVERSE; - } - - assert( pOrderBy || (pCost->plan.wsFlags&WHERE_ORDERBY)==0 ); - assert( pSrc->pIndex==0 - || pCost->plan.u.pIdx==0 - || pCost->plan.u.pIdx==pSrc->pIndex - ); - - WHERETRACE(("best index is: %s\n", - ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : - pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") - )); - - bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); - bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); - pCost->plan.wsFlags |= eqTermMask; -} - -/* -** Find the query plan for accessing table pSrc->pTab. Write the -** best query plan and its cost into the WhereCost object supplied -** as the last parameter. This function may calculate the cost of -** both real and virtual table scans. -*/ -static void bestIndex( - Parse *pParse, /* The parsing context */ - WhereClause *pWC, /* The WHERE clause */ - SrcListItem *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors not available for indexing */ - Bitmask notValid, /* Cursors not available for any purpose */ - ExprList *pOrderBy, /* The ORDER BY clause */ - WhereCost *pCost /* Lowest cost query plan */ -){ -#ifndef SQLITE4_OMIT_VIRTUALTABLE - if( IsVirtual(pSrc->pTab) ){ - sqlite4_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); - if( p->needToFreeIdxStr ){ - sqlite4_free(p->idxStr); - } - sqlite4DbFree(pParse->db, p); - }else -#endif - { - bestKVIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost); - } -} - /* ** Disable a term in the WHERE clause. Except, do not disable the term ** if it controls a LEFT OUTER JOIN and it did not originate in the ON ** or USING clause of that join. ** @@ -3488,11 +2903,13 @@ ** this routine sets up a loop that will iterate over all values of X. */ static int codeEqualityTerm( Parse *pParse, /* The parsing context */ WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ - WhereLevel *pLevel, /* When level of the FROM clause we are working on */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; Vdbe *v = pParse->pVdbe; int iReg; /* Register holding results */ @@ -3503,76 +2920,53 @@ }else if( pX->op==TK_ISNULL ){ iReg = iTarget; sqlite4VdbeAddOp2(v, OP_Null, 0, iReg); #ifndef SQLITE4_OMIT_SUBQUERY }else{ - /* Code a loop that iterates through the set of distinct, non-null - ** values in the set on the right-hand-side of the IN(...) operator. - ** There are two ways to do this: - ** - ** * If the SELECT statement is of the form "SELECT x FROM tbl", - ** and column x is subject to a UNIQUE constraint, and the - ** default affinity and collation sequence of column "x" match - ** those required by the comparison, iterate through the PK - ** index. - ** - ** * Otherwise, materialize the set into an ephemeral index using - ** "x" as both the key and value. Then loop through the contents - ** of the ephemeral index. - */ - sqlite4 *db = pParse->db; + int eType; int iTab; - int iCol; /* Column to read from cursor iTab */ struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } assert( pX->op==TK_IN ); iReg = iTarget; - - if( sqlite4FindExistingInIndex(pParse, pX, 1) ){ - /* This branch is taken if the rhs of the IN is a select of the - ** form "SELECT x FROM tble" and column x is subject to a UNIQUE - ** constraint that uses the same collation sequence and affinity as - ** this IN (...) test. In this case just loop through all values of - ** "x", skipping any NULLs. */ - Table *pTab = pX->x.pSelect->pSrc->a[0].pTab; - int iDb = sqlite4SchemaToIndex(db, pTab->pSchema); - iTab = pX->iTable = pParse->nTab++; - sqlite4OpenPrimaryKey(pParse, iTab, iDb, pTab, OP_OpenRead); - iCol = pX->pLeft->iColumn; - }else{ - /* Set Parse.nQueryLoop to 1 before calling sqlite4CodeSubselect(). - ** This informs the optimizer that there is no point in constructing - ** any automatic indexes for the outer loop of the sub-select, as it - ** will only be run once. See also bestAutomaticIndex(). */ - int nQueryLoopSave = pParse->nQueryLoop; - pParse->nQueryLoop = (double)1; - sqlite4CodeSubselect(pParse, pX, 0, 0); - pParse->nQueryLoop = nQueryLoopSave; - iTab = pX->iTable; - iCol = 0; - } - sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0); - assert( pLevel->plan.wsFlags & WHERE_IN_ABLE ); - - if( pLevel->u.in.nIn==0 ) pLevel->addrNxt = sqlite4VdbeMakeLabel(v); + eType = sqlite4FindInIndex(pParse, pX, 0); + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + iTab = pX->iTable; + sqlite4VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite4VdbeMakeLabel(v); + } pLevel->u.in.nIn++; - pLevel->u.in.aInLoop = sqlite4DbReallocOrFree(db, pLevel->u.in.aInLoop, - (sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn) - ); + pLevel->u.in.aInLoop = + sqlite4DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); pIn = pLevel->u.in.aInLoop; - if( pIn ){ pIn += pLevel->u.in.nIn - 1; pIn->iCur = iTab; - if( iCol>=0 ){ - pIn->addrInTop = sqlite4VdbeAddOp3(v, OP_Column, iTab, iCol, iReg); - }else{ + if( eType==IN_INDEX_ROWID ){ pIn->addrInTop = sqlite4VdbeAddOp2(v, OP_Rowid, iTab, iReg); + }else{ + pIn->addrInTop = sqlite4VdbeAddOp3(v, OP_Column, iTab, 0, iReg); } + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; sqlite4VdbeAddOp1(v, OP_IsNull, iReg); }else{ - assert( db->mallocFailed ); pLevel->u.in.nIn = 0; } #endif } disableTerm(pLevel, pTerm); @@ -3619,53 +3013,54 @@ ** string in this example would be set to SQLITE4_AFF_NONE. */ static int codeAllEqualityTerms( Parse *pParse, /* Parsing context */ WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ - WhereClause *pWC, /* The WHERE clause */ - Bitmask notReady, /* Which parts of FROM have not yet been coded */ + int bRev, /* Reverse the order of IN operators */ int nExtraReg, /* Number of extra registers to allocate */ char **pzAff /* OUT: Set to point to affinity string */ ){ - int nEq = pLevel->plan.nEq; /* The number of == or IN constraints to code */ + int nEq; /* The number of == or IN constraints to code */ Vdbe *v = pParse->pVdbe; /* The vm under construction */ Index *pIdx; /* The index being used for this loop */ - int iCur = pLevel->iTabCur; /* The cursor of the table */ WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ int j; /* Loop counter */ int regBase; /* Base register */ int nReg; /* Number of registers to allocate */ char *zAff; /* Affinity string to return */ /* This module is only called on query plans that use an index. */ - assert( pLevel->plan.wsFlags & WHERE_INDEXED ); - pIdx = pLevel->plan.u.pIdx; + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); /* Figure out how many memory cells we will need then allocate them. */ regBase = pParse->nMem + 1; - nReg = pLevel->plan.nEq + nExtraReg; + nReg = pLoop->u.btree.nEq + nExtraReg; pParse->nMem += nReg; zAff = sqlite4DbStrDup(pParse->db, sqlite4IndexAffinityStr(v, pIdx)); if( !zAff ){ pParse->db->mallocFailed = 1; } /* Evaluate the equality constraints */ - assert( pIdx->nColumn>=nEq ); + assert( idxColumnCount(pIdx, sqlite4FindPrimaryKey(pIdx->pTable, 0))>=nEq ); for(j=0; jaiColumn[j]; - pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx); - if( NEVER(pTerm==0) ) break; + pTerm = pLoop->aLTerm[j]; + assert( pTerm!=0 ); /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ - r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ sqlite4ReleaseTempReg(pParse, regBase); regBase = r1; }else{ @@ -3729,38 +3124,36 @@ ** ** The returned pointer points to memory obtained from sqlite4DbMalloc(). ** It is the responsibility of the caller to free the buffer when it is ** no longer required. */ -static char *explainIndexRange(sqlite4 *db, WhereLevel *pLevel, Table *pTab){ - WherePlan *pPlan = &pLevel->plan; - Index *pPk; - Index *pIdx = pPlan->u.pIdx; - int nEq = pPlan->nEq; - int i; +static char *explainIndexRange(sqlite4 *db, WhereLoop *pLoop, Table *pTab){ + Index *pIndex = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + int i, j; + Column *aCol = pTab->aCol; + int *aiColumn = pIndex->aiColumn; StrAccum txt; - pPk = sqlite4FindPrimaryKey(pTab, 0); - if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ + if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ return 0; } sqlite4StrAccumInit(&txt, 0, 0, SQLITE4_MAX_LENGTH); txt.db = db; - txt.pEnv = db->pEnv; - sqlite4StrAccumAppend(&txt, " (", 2); for(i=0; i"); - } - if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ - const char *zCol = tblColumnName(pTab, idxColumnNumber(pIdx, pPk, nEq)); - explainAppendTerm(&txt, i, zCol, "<"); + explainAppendTerm(&txt, i, aCol[aiColumn[i]].zName, "="); + } + + j = i; + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(&txt, i++, z, ">"); + } + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + char *z = (j==pIndex->nColumn ) ? "rowid" : aCol[aiColumn[j]].zName; + explainAppendTerm(&txt, i, z, "<"); } sqlite4StrAccumAppend(&txt, ")", 1); return sqlite4StrAccumFinish(&txt); } @@ -3777,24 +3170,26 @@ int iLevel, /* Value for "level" column of output */ int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite4WhereBegin() */ ){ if( pParse->explain==2 ){ - u32 flags = pLevel->plan.wsFlags; - SrcListItem *pItem = &pTabList->a[pLevel->iFrom]; + struct SrcListItem *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite4 *db = pParse->db; /* Database handle */ char *zMsg; /* Text to add to EQP output */ - sqlite4_int64 nRow; /* Expected number of rows visited by scan */ int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; - isSearch = (pLevel->plan.nEq>0) - || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); zMsg = sqlite4MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); if( pItem->pSelect ){ zMsg = sqlite4MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); }else{ @@ -3802,80 +3197,87 @@ } if( pItem->zAlias ){ zMsg = sqlite4MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); } - if( (flags & WHERE_INDEXED)!=0 ){ - char *zWhere = explainIndexRange(db, pLevel, pItem->pTab); - Index *pIdx = pLevel->plan.u.pIdx; - const char *zName = ""; - const char *zType = "INDEX"; - - if( pIdx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){ - zType = "PRIMARY KEY"; - }else if( 0==(flags & WHERE_TEMP_INDEX) ){ - zName = pIdx->zName; - } - zMsg = sqlite4MAppendf(db, zMsg, "%s USING %s%s%s%s%s", zMsg, - ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""), - zType, (zName[0] ? " " : ""), zName, zWhere - ); + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 + && ALWAYS(pLoop->u.btree.pIndex!=0) + ){ + char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); + zMsg = sqlite4MAppendf(db, zMsg, + ((flags & WHERE_AUTO_INDEX) ? + "%s USING AUTOMATIC %sINDEX%.0s%s" : + "%s USING %sINDEX %s%s"), + zMsg, ((flags & WHERE_IDX_ONLY) ? "COVERING " : ""), + pLoop->u.btree.pIndex->zName, zWhere); sqlite4DbFree(db, zWhere); + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + zMsg = sqlite4MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); + + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zMsg = sqlite4MAppendf(db, zMsg, "%s (rowid=?)", zMsg); + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zMsg = sqlite4MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); + }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){ + zMsg = sqlite4MAppendf(db, zMsg, "%s (rowidplan.u.pVtabIdx; zMsg = sqlite4MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, - pVtabIdx->idxNum, pVtabIdx->idxStr); + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif - if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ - testcase( wctrlFlags & WHERE_ORDERBY_MIN ); - nRow = 1; - }else{ - nRow = (sqlite4_int64)pLevel->plan.nRow; - } - zMsg = sqlite4MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); + zMsg = sqlite4MAppendf(db, zMsg, "%s", zMsg); sqlite4VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); } } #else # define explainOneScan(u,v,w,x,y,z) #endif /* SQLITE4_OMIT_EXPLAIN */ + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. */ static Bitmask codeOneLoopStart( WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ - u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ - Bitmask notReady, /* Which tables are currently available */ - Expr *pWhere /* Complete WHERE clause */ + Bitmask notReady /* Which tables are currently available */ ){ int j, k; /* Loop counters */ int iCur; /* The VDBE cursor for the table */ int addrNxt; /* Where to jump to continue with the next IN case */ + int omitTable; /* True if we use the index only */ int bRev; /* True if we need to scan in reverse order */ WhereLevel *pLevel; /* The where level to be coded */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ WhereClause *pWC; /* Decomposition of the entire WHERE clause */ WhereTerm *pTerm; /* A WHERE clause term */ Parse *pParse; /* Parsing context */ Vdbe *v; /* The prepared stmt under constructions */ - SrcListItem *pTabItem; /* FROM clause term being coded */ + struct SrcListItem *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ int addrCont; /* Jump here to continue with next cycle */ - int iReleaseReg = 0; /* Temp register to free before returning */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ + Bitmask newNotReady; /* Return value */ pParse = pWInfo->pParse; v = pParse->pVdbe; - pWC = pWInfo->pWC; + pWC = &pWInfo->sWC; pLevel = &pWInfo->a[iLevel]; + pLoop = pLevel->pWLoop; pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; iCur = pTabItem->iCursor; - bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; + bRev = (pWInfo->revMask>>iLevel)&1; + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. ** Jump to cont to go immediately to the next iteration of the ** loop. @@ -3896,65 +3298,54 @@ pLevel->iLeftJoin = ++pParse->nMem; sqlite4VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); VdbeComment((v, "init LEFT JOIN no-match flag")); } - if( (pLevel->plan.wsFlags & WHERE_INDEXED) - && (pLevel->plan.u.pIdx->eIndexType==SQLITE4_INDEX_FTS5) - ){ - /* Case -1: An FTS query */ - int iTerm; - int rMatch; - int rFree; - findMatchExpr(pParse, pWC, pTabItem, &iTerm); - - rMatch = sqlite4ExprCodeTemp(pParse, pWC->a[iTerm].pExpr->pRight, &rFree); - pWC->a[iTerm].wtFlags |= TERM_CODED; - sqlite4Fts5CodeQuery(pParse, - pLevel->plan.u.pIdx, pLevel->iIdxCur, addrBrk, rMatch - ); - sqlite4ReleaseTempReg(pParse, rFree); - - pLevel->p2 = sqlite4VdbeCurrentAddr(v); - sqlite4VdbeAddOp3(v, OP_SeekPk, iCur, 0, pLevel->iIdxCur); - pLevel->op = OP_FtsNext; - pLevel->p1 = pLevel->iIdxCur; - }else +#if 0 + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite4VdbeAddOp2(v, OP_Integer, pTabItem->addrFillSub-1, regYield); + pLevel->p2 = sqlite4VdbeAddOp1(v, OP_Yield, regYield); + VdbeComment((v, "next row of co-routine %s", pTabItem->pTab->zName)); + sqlite4VdbeAddOp2(v, OP_If, regYield+1, addrBrk); + pLevel->op = OP_Goto; + }else +#endif + #ifndef SQLITE4_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - /* Case 0: The table is a virtual-table. Use the VFilter and VNext + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext ** to access the data. */ int iReg; /* P3 Value for OP_VFilter */ - sqlite4_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; - int nConstraint = pVtabIdx->nConstraint; - struct sqlite4_index_constraint_usage *aUsage = - pVtabIdx->aConstraintUsage; - const struct sqlite4_index_constraint *aConstraint = - pVtabIdx->aConstraint; + int addrNotFound; + int nConstraint = pLoop->nLTerm; sqlite4ExprCachePush(pParse); iReg = sqlite4GetTempRange(pParse, nConstraint+2); - for(j=1; j<=nConstraint; j++){ - for(k=0; ka[iTerm].pExpr->pRight, iReg+j+1); - break; - } - } - if( k==nConstraint ) break; - } - sqlite4VdbeAddOp2(v, OP_Integer, pVtabIdx->idxNum, iReg); - sqlite4VdbeAddOp2(v, OP_Integer, j-1, iReg+1); - sqlite4VdbeAddOp4(v, OP_VFilter, iCur, addrBrk, iReg, pVtabIdx->idxStr, - pVtabIdx->needToFreeIdxStr ? P4_MPRINTF : P4_STATIC); - pVtabIdx->needToFreeIdxStr = 0; + addrNotFound = pLevel->addrBrk; for(j=0; ja[iTerm]); + int iTarget = iReg+j+2; + pTerm = pLoop->aLTerm[j]; + if( pTerm==0 ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + sqlite4ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + } + } + sqlite4VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite4VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite4VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + pLoop->u.vtab.needFree = 0; + for(j=0; ju.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pLoop->aLTerm[j]); } } pLevel->op = OP_VNext; pLevel->p1 = iCur; pLevel->p2 = sqlite4VdbeCurrentAddr(v); @@ -3961,12 +3352,119 @@ sqlite4ReleaseTempRange(pParse, iReg, nConstraint+2); sqlite4ExprCachePop(pParse, 1); }else #endif /* SQLITE4_OMIT_VIRTUALTABLE */ - if( pLevel->plan.wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ) ){ - /* Case 3: A scan using an index. + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + assert( 0 ); + + /* Case 2: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + assert( pLoop->u.btree.nEq==1 ); + iReleaseReg = sqlite4GetTempReg(pParse); + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite4VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + sqlite4VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + sqlite4ExprCacheAffinityChange(pParse, iRowidReg, 1); + sqlite4ExprCacheStore(pParse, iCur, -1, iRowidReg); + VdbeComment((v, "pk")); + pLevel->op = OP_Noop; + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; + + assert( 0 ); + + assert( omitTable==0 ); + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ + + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGt, + /* TK_LE */ OP_SeekLe, + /* TK_LT */ OP_SeekLt, + /* TK_GE */ OP_SeekGe + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + pX = pStart->pExpr; + assert( pX!=0 ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ + r1 = sqlite4ExprCodeTemp(pParse, pX->pRight, &rTemp); + sqlite4VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); + VdbeComment((v, "pk")); + sqlite4ExprCacheAffinityChange(pParse, r1, 1); + sqlite4ReleaseTempReg(pParse, rTemp); + disableTerm(pLevel, pStart); + }else{ + sqlite4VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + } + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ + memEndValue = ++pParse->nMem; + sqlite4ExprCode(pParse, pX->pRight, memEndValue); + if( pX->op==TK_LT || pX->op==TK_GT ){ + testOp = bRev ? OP_Le : OP_Ge; + }else{ + testOp = bRev ? OP_Lt : OP_Gt; + } + disableTerm(pLevel, pEnd); + } + start = sqlite4VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + assert( pLevel->p5==0 ); + if( testOp!=OP_Noop ){ + iRowidReg = iReleaseReg = sqlite4GetTempReg(pParse); + sqlite4VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite4ExprCacheStore(pParse, iCur, -1, iRowidReg); + sqlite4VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + sqlite4VdbeChangeP5(v, SQLITE4_AFF_NUMERIC | SQLITE4_JUMPIFNULL); + } + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. ** ** The WHERE clause may contain zero or more equality ** terms ("==" or "IN" operators) that refer to the N ** left-most columns of the index. It may also contain ** inequality constraints (>, <, >= or <=) on the indexed @@ -4011,12 +3509,12 @@ OP_IdxLE, /* 2: (end_constraints && !endEq && bRev) */ OP_IdxGT, /* 3: (end_constraints && endEq && !bRev) */ OP_IdxLT /* 4: (end_constraints && endEq && bRev) */ }; - int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ - int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ + int nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ int r1; /* Temp register */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ int startEq; /* True if range start uses ==, >= or <= */ @@ -4031,11 +3529,11 @@ char *zEndAff; /* Affinity for end of range constraint */ int regEndKey; /* Register for end-key */ int iIneq; /* The table column subject to inequality */ Index *pPk; /* Primary key index on same table as pIdx */ - pIdx = pLevel->plan.u.pIdx; + pIdx = pLoop->u.btree.pIndex; pPk = sqlite4FindPrimaryKey(pIdx->pTable, 0); iIneq = idxColumnNumber(pIdx, pPk, nEq); iIdxCur = pLevel->iIdxCur; assert( iCur==pLevel->iTabCur ); @@ -4045,57 +3543,56 @@ ** a single iteration. This means that the first row returned ** should not have a NULL value stored in 'x'. If column 'x' is ** the first one after the nEq equality constraints in the index, ** this requires some special handling. */ - if( (wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pLevel->plan.wsFlags&WHERE_ORDERBY) + if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 + && (pWInfo->bOBSat!=0) && (pIdx->nColumn>nEq) ){ /* assert( pOrderBy->nExpr==1 ); */ /* assert( pOrderBy->a[0].pExpr->iColumn==pIdx->aiColumn[nEq] ); */ isMinQuery = 1; nExtraReg = 1; } /* Find any inequality constraint terms for the start and end - ** of the range. */ - if( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ){ - pRangeEnd = findTerm(pWC, iCur, iIneq, notReady, (WO_LT|WO_LE), pIdx); + ** of the range. + */ + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; nExtraReg = 1; } - if( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ){ - pRangeStart = findTerm(pWC, iCur, iIneq, notReady, (WO_GT|WO_GE), pIdx); + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; nExtraReg = 1; } /* Generate code to evaluate all constraint terms using == or IN ** and store the values of those terms in an array of registers - ** starting at regBase. Ensure that nExtraReg registers are allocated - ** immediately following the array. + ** starting at regBase. */ - regBase = codeAllEqualityTerms( - pParse, pLevel, pWC, notReady, nExtraReg, &zStartAff - ); + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); assert( (regBase+nEq+nExtraReg-1)<=pParse->nMem ); - zEndAff = sqlite4DbStrDup(pParse->db, zStartAff); addrNxt = pLevel->addrNxt; /* If we are doing a reverse order scan on an ascending index, or ** a forward order scan on a descending index, interchange the - ** start and end terms (pRangeStart and pRangeEnd). */ + ** start and end terms (pRangeStart and pRangeEnd). + */ if( (nEqnColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE4_SO_ASC)) || (bRev && pIdx->nColumn==nEq) ){ SWAP(WhereTerm *, pRangeEnd, pRangeStart); } - testcase( pRangeStart && pRangeStart->eOperator & WO_LE ); - testcase( pRangeStart && pRangeStart->eOperator & WO_GE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_LE ); - testcase( pRangeEnd && pRangeEnd->eOperator & WO_GE ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); start_constraints = pRangeStart || nEq>0; /* Seek the index cursor to the start of the range. */ @@ -4133,11 +3630,11 @@ testcase( op==OP_SeekGt ); testcase( op==OP_SeekGe ); testcase( op==OP_SeekLe ); testcase( op==OP_SeekLt ); sqlite4VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - if( (pIdx->nColumn + (pIdx==pPk ? 0 : pPk->nColumn))>nEq ){ + if( nEqpExpr->pRight; sqlite4ExprCacheRemove(pParse, regBase+nEq, 1); sqlite4ExprCode(pParse, pRight, regBase+nEq); @@ -4162,12 +3660,12 @@ sqlite4ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); } if( zEndAff ){ if( sqlite4CompareAffinity(pRight, zEndAff[nEq])==SQLITE4_AFF_NONE){ /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE4_AFF_NONE. */ + ** applied to the operands, set the affinity to apply to pRight to + ** SQLITE4_AFF_NONE. */ zEndAff[nEq] = SQLITE4_AFF_NONE; } if( sqlite4ExprNeedsNoAffinityChange(pRight, zEndAff[nEq]) ){ zEndAff[nEq] = SQLITE4_AFF_NONE; } @@ -4180,10 +3678,11 @@ /* Now compute an end-key using OP_MakeIdxKey */ regEndKey = ++pParse->nMem; sqlite4VdbeAddOp4Int( v, OP_MakeIdxKey, iIdxCur, regBase, regEndKey, nConstraint ); + } sqlite4DbFree(pParse->db, zStartAff); sqlite4DbFree(pParse->db, zEndAff); @@ -4202,38 +3701,43 @@ ){ sqlite4VdbeAddOp3(v, OP_SeekPk, iCur, 0, iIdxCur); } /* If there are inequality constraints, check that the value - ** of the table column that the inequality constrains is not NULL. - ** If it is, jump to the next iteration of the loop. */ + ** of the table column that the inequality contrains is not NULL. + ** If it is, jump to the next iteration of the loop. + */ r1 = sqlite4GetTempReg(pParse); - testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); - testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); - if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ + testcase( pLoop->wsFlags & WHERE_BTM_LIMIT ); + testcase( pLoop->wsFlags & WHERE_TOP_LIMIT ); + if( (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ sqlite4ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iIneq, r1); sqlite4VdbeAddOp2(v, OP_IsNull, r1, addrCont); } sqlite4ReleaseTempReg(pParse, r1); /* Record the instruction used to terminate the loop. Disable ** WHERE clause terms made redundant by the index range scan. */ - if( pLevel->plan.wsFlags & WHERE_UNIQUE ){ + if( pLoop->wsFlags & WHERE_ONEROW ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; }else{ pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; - + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE4_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } }else #ifndef SQLITE4_OMIT_OR_OPTIMIZATION - if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ - /* Case 4: Two or more separately indexed terms connected by OR + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR ** ** Example: ** ** CREATE TABLE t1(a,b,c,d); ** CREATE INDEX i1 ON t1(a); @@ -4270,10 +3774,12 @@ ** B: ** */ WhereClause *pOrWc; /* The OR-clause broken out into subterms */ SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ int regKeyset = 0; /* Register for RowSet object */ int regKey = 0; /* Register holding key */ int iLoopBody = sqlite4VdbeMakeLabel(v); /* Start of loop body */ @@ -4280,13 +3786,13 @@ int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - pTerm = pLevel->plan.u.pTerm; + pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); - assert( pTerm->eOperator==WO_OR ); + assert( pTerm->eOperator & WO_OR ); assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); pOrWc = &pTerm->u.pOrInfo->wc; pLevel->op = OP_Return; pLevel->p1 = regReturn; @@ -4294,16 +3800,16 @@ ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. ** This becomes the SrcList in the recursive call to sqlite4WhereBegin(). */ if( pWInfo->nLevel>1 ){ int nNotReady; /* The number of notReady tables */ - SrcListItem *origSrc; /* Original list of tables */ + struct SrcListItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; pOrTab = sqlite4StackAllocRaw(pParse->db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; - pOrTab->nAlloc = (i16)(nNotReady + 1); + pOrTab->nAlloc = (u8)(nNotReady + 1); pOrTab->nSrc = pOrTab->nAlloc; memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); origSrc = pWInfo->pTabList->a; for(k=1; k<=nNotReady; k++){ memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); @@ -4311,21 +3817,21 @@ }else{ pOrTab = pWInfo->pTabList; } /* Initialize the keyset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. + ** equivalent to an empty keyset. ** ** Also initialize regReturn to contain the address of the instruction ** immediately following the OP_Return at the bottom of the loop. This ** is required in a few obscure LEFT JOIN cases where control jumps ** over the top of the loop into the body of it. In this case the ** correct response for the end-of-loop code (the OP_Return) is to ** fall through to the next instruction, just as an OP_Next does if ** called on an uninitialized cursor. */ - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ regKeyset = ++pParse->nMem; regKey = ++pParse->nMem; sqlite4VdbeAddOp2(v, OP_Null, 0, regKeyset); } iRetInit = sqlite4VdbeAddOp2(v, OP_Integer, 0, regReturn); @@ -4332,56 +3838,105 @@ /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y ** Then for every term xN, evaluate as the subexpression: xN AND z ** That way, terms in y that are factored into the disjunction will ** be picked up by the recursive calls to sqlite4WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 */ if( pWC->nTerm>1 ){ - pAndExpr = sqlite4ExprAlloc(pParse->db, TK_AND, 0, 0); - pAndExpr->pRight = pWhere; + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; + if( pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_ORINFO) ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + pExpr = sqlite4ExprDup(pParse->db, pExpr, 0); + pAndExpr = sqlite4ExprAnd(pParse->db, pAndExpr, pExpr); + } + if( pAndExpr ){ + pAndExpr = sqlite4PExpr(pParse, TK_AND, 0, pAndExpr, 0); + } } for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; - if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; - if( pAndExpr ){ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ pSubWInfo = sqlite4WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_NO_AUTOINDEX | WHERE_ONETABLE_ONLY); + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur); + assert( pSubWInfo || pParse->nErr || pParse->db->mallocFailed ); if( pSubWInfo ){ + WhereLoop *pSubLoop; explainOneScan( pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 ); - if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int addrJump; + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); sqlite4VdbeAddOp2(v, OP_RowKey, iCur, regKey); - addrJump = sqlite4VdbeCurrentAddr(v) + 2; - sqlite4VdbeAddOp4Int(v, OP_RowSetTest, - regKeyset, addrJump, regKey, ((ii==pOrWc->nTerm-1)?-1:ii) - ); + sqlite4VdbeAddOp4Int(v, OP_RowSetTest, regKeyset, + sqlite4VdbeCurrentAddr(v)+2, regKey, iSet); } sqlite4VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); /* The pSubWInfo->untestedTerms flag means that this OR term ** contained one or more AND term from a notReady table. The ** terms from the notReady table could not be tested and will ** need to be tested later. */ if( pSubWInfo->untestedTerms ) untestedTerms = 1; + + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite4WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite4WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ +#if 0 + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + }else{ + pCov = 0; + } +#endif /* Finish the loop through table entries that match term pOrTerm. */ sqlite4WhereEnd(pSubWInfo); } } } - sqlite4DbFree(pParse->db, pAndExpr); + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite4ExprDelete(pParse->db, pAndExpr); + } sqlite4VdbeChangeP1(v, iRetInit, sqlite4VdbeCurrentAddr(v)); sqlite4VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); sqlite4VdbeResolveLabel(v, iLoopBody); if( pWInfo->nLevel>1 ) sqlite4StackFree(pParse->db, pOrTab); @@ -4388,11 +3943,14 @@ if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE4_OMIT_OR_OPTIMIZATION */ { - /* Case 5: There is no usable index. We must do a complete + /* TODO: This case is currently being used. Why can't it use the + ** index case instead? */ + + /* Case 6: There is no usable index. We must do a complete ** scan of the entire table. */ static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); @@ -4399,11 +3957,11 @@ pLevel->op = aStep[bRev]; pLevel->p1 = iCur; pLevel->p2 = 1 + sqlite4VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); pLevel->p5 = SQLITE4_STMTSTATUS_FULLSCAN_STEP; } - notReady &= ~getMask(pWC->pMaskSet, iCur); + newNotReady = notReady & ~getMask(&pWInfo->sMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. ** ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through @@ -4413,11 +3971,11 @@ for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ Expr *pE; testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); pWInfo->untestedTerms = 1; continue; } @@ -4427,10 +3985,40 @@ continue; } sqlite4ExprIfFalse(pParse, pE, addrCont, SQLITE4_JUMPIFNULL); pTerm->wtFlags |= TERM_CODED; } + + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + WhereTerm *pAlt; + Expr sEq; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pLevel->iLeftJoin ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & newNotReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IN ); + VdbeNoopComment((v, "begin transitive constraint")); + sEq = *pAlt->pExpr; + sEq.pLeft = pE->pLeft; + sqlite4ExprIfFalse(pParse, &sEq, addrCont, SQLITE4_JUMPIFNULL); + } /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ if( pLevel->iLeftJoin ){ @@ -4440,11 +4028,11 @@ sqlite4ExprCacheClear(pParse); for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; } assert( pTerm->pExpr ); sqlite4ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE4_JUMPIFNULL); @@ -4451,55 +4039,1583 @@ pTerm->wtFlags |= TERM_CODED; } } sqlite4ReleaseTempReg(pParse, iReleaseReg); - return notReady; -} - -#if defined(SQLITE4_TEST) -/* -** The following variable holds a text description of query plan generated -** by the most recent call to sqlite4WhereBegin(). Each call to WhereBegin -** overwrites the previous. This information is used for testing and -** analysis only. -*/ -char sqlite4_query_plan[BMS*2*40]; /* Text of the join */ -static int nQPlan = 0; /* Next free slow in _query_plan[] */ - -#endif /* SQLITE4_TEST */ - + return newNotReady; +} + +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +*/ +static void whereLoopPrint(WhereLoop *p, SrcList *pTabList){ + int nb = 1+(pTabList->nSrc+7)/8; + struct SrcListItem *pItem = pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + sqlite4DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq); + sqlite4DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + if( p->u.btree.pIndex ){ + const char *zName = p->u.btree.pIndex->zName; + if( zName==0 ) zName = "ipk"; + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ + int i = sqlite4Strlen30(zName) - 1; + while( zName[i]!='_' ) i--; + zName += i; + } + sqlite4DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); + }else{ + sqlite4DebugPrintf("%20s",""); + } + }else{ + char *z; + if( p->u.vtab.idxStr ){ + z = sqlite4_mprintf(0, "(%d,\"%s\",%x)", + p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); + }else{ + z = sqlite4_mprintf(0, "(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); + } + sqlite4DebugPrintf(" %-19s", z); + sqlite4_free(0, z); + } + sqlite4DebugPrintf(" f %04x N %d", p->wsFlags, p->nLTerm); + sqlite4DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); +} +#endif + +/* +** Convert bulk memory into a valid WhereLoop that can be passed +** to whereLoopClear harmlessly. +*/ +static void whereLoopInit(WhereLoop *p){ + p->aLTerm = p->aLTermSpace; + p->nLTerm = 0; + p->nLSlot = ArraySize(p->aLTermSpace); + p->wsFlags = 0; +} + +/* +** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. +*/ +static void whereLoopClearUnion(sqlite4 *db, WhereLoop *p){ + if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ +#if 0 + sqlite4_free(p->u.vtab.idxStr); +#endif + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite4DbFree(db, p->u.btree.pIndex->zColAff); + sqlite4DbFree(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; + } + } +} + +/* +** Deallocate internal memory used by a WhereLoop object +*/ +static void whereLoopClear(sqlite4 *db, WhereLoop *p){ + if( p->aLTerm!=p->aLTermSpace ) sqlite4DbFree(db, p->aLTerm); + whereLoopClearUnion(db, p); + whereLoopInit(p); +} + +/* +** Increase the memory allocation for pLoop->aLTerm[] to be at least n. +*/ +static int whereLoopResize(sqlite4 *db, WhereLoop *p, int n){ + WhereTerm **paNew; + if( p->nLSlot>=n ) return SQLITE4_OK; + n = (n+7)&~7; + paNew = sqlite4DbMallocRaw(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE4_NOMEM; + memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); + if( p->aLTerm!=p->aLTermSpace ) sqlite4DbFree(db, p->aLTerm); + p->aLTerm = paNew; + p->nLSlot = n; + return SQLITE4_OK; +} + +/* +** Transfer content from the second pLoop into the first. +*/ +static int whereLoopXfer(sqlite4 *db, WhereLoop *pTo, WhereLoop *pFrom){ + if( whereLoopResize(db, pTo, pFrom->nLTerm) ) return SQLITE4_NOMEM; + whereLoopClearUnion(db, pTo); + memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); + if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ + pFrom->u.vtab.needFree = 0; + }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + pFrom->u.btree.pIndex = 0; + } + return SQLITE4_OK; +} + +/* +** Delete a WhereLoop object +*/ +static void whereLoopDelete(sqlite4 *db, WhereLoop *p){ + whereLoopClear(db, p); + sqlite4DbFree(db, p); +} /* ** Free a WhereInfo structure */ static void whereInfoFree(sqlite4 *db, WhereInfo *pWInfo){ if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - sqlite4_index_info *pInfo = pWInfo->a[i].pIdxInfo; - if( pInfo ){ - /* assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed ); */ - if( pInfo->needToFreeIdxStr ){ - sqlite4_free(db->pEnv, pInfo->idxStr); - } - sqlite4DbFree(db, pInfo); - } - if( pWInfo->a[i].plan.wsFlags & WHERE_TEMP_INDEX ){ - Index *pIdx = pWInfo->a[i].plan.u.pIdx; - if( pIdx ){ - assert( pIdx->eIndexType==SQLITE4_INDEX_TEMP ); - sqlite4DbFree(db, pIdx->zColAff); - sqlite4DbFree(db, pIdx); - } - } - } - whereClauseClear(pWInfo->pWC); + whereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } sqlite4DbFree(db, pWInfo); } } +/* +** Insert or replace a WhereLoop entry using the template supplied. +** +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based on the template. +** +** If pBuilder->pBest is not NULL then we only care about the very +** best template and that template should be stored in pBuilder->pBest. +** If pBuilder->pBest is NULL then a list of the best templates are stored +** in pBuilder->pWInfo->pLoops. +** +** When accumulating multiple loops (when pBuilder->pBest is NULL) we +** still might overwrite similar loops with the new template if the +** template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop +** (5) The template uses more terms of the same index but has no additional +** dependencies +*/ +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p, *pNext = 0; + WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite4 *db = pWInfo->pParse->db; + + assert( pTemplate->u.btree.pIndex || !(pTemplate->wsFlags & WHERE_INDEXED) ); + + /* If pBuilder->pBest is defined, then only keep track of the single + ** best WhereLoop. pBuilder->pBest->maskSelf==0 indicates that no + ** prior WhereLoops have been evaluated and that the current pTemplate + ** is therefore the first and hence the best and should be retained. + */ + if( (p = pBuilder->pBest)!=0 ){ + if( p->maskSelf!=0 ){ + WhereCost rCost = whereCostAdd(p->rRun,p->rSetup); + WhereCost rTemplate = whereCostAdd(pTemplate->rRun,pTemplate->rSetup); + if( rCost < rTemplate ){ + testcase( rCost==rTemplate-1 ); + goto whereLoopInsert_noop; + } + if( rCost==rTemplate && (p->prereq & pTemplate->prereq)==p->prereq ){ + goto whereLoopInsert_noop; + } + } +#if WHERETRACE_ENABLED + if( sqlite4WhereTrace & 0x8 ){ + sqlite4DebugPrintf(p->maskSelf==0 ? "ins-init: " : "ins-best: "); + whereLoopPrint(pTemplate, pWInfo->pTabList); + } +#endif + whereLoopXfer(db, p, pTemplate); + return SQLITE4_OK; + } + + /* Search for an existing WhereLoop to overwrite, or which takes + ** priority over pTemplate. + */ + for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; + } + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); + + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); + + if( (p->prereq & pTemplate->prereq)==p->prereq + && p->rSetup<=pTemplate->rSetup + && p->rRun<=pTemplate->rRun + ){ + /* This branch taken when p is equal or better than pTemplate in + ** all of (1) dependences (2) setup-cost, and (3) run-cost. */ + assert( p->rSetup==pTemplate->rSetup ); + if( p->nLTermnLTerm + && (p->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && p->u.btree.pIndex==pTemplate->u.btree.pIndex + && p->prereq==pTemplate->prereq + ){ + /* Overwrite an existing WhereLoop with an similar one that uses + ** more terms of the index */ + pNext = p->pNextLoop; + break; + }else{ + /* pTemplate is not helpful. + ** Return without changing or adding anything */ + goto whereLoopInsert_noop; + } + } + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq + && p->rRun>=pTemplate->rRun + && ALWAYS(p->rSetup>=pTemplate->rSetup) /* See SETUP-INVARIANT above */ + ){ + /* Overwrite an existing WhereLoop with a better one: one that is + ** better at one of (1) dependences, (2) setup-cost, or (3) run-cost + ** and is no worse in any of those categories. */ + pNext = p->pNextLoop; + break; + } + } + + /* If we reach this point it means that either p[] should be overwritten + ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new + ** WhereLoop and insert it. + */ +#if WHERETRACE_ENABLED + if( sqlite4WhereTrace & 0x8 ){ + if( p!=0 ){ + sqlite4DebugPrintf("ins-del: "); + whereLoopPrint(p, pWInfo->pTabList); + } + sqlite4DebugPrintf("ins-new: "); + whereLoopPrint(pTemplate, pWInfo->pTabList); + } +#endif + if( p==0 ){ + p = sqlite4DbMallocRaw(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE4_NOMEM; + whereLoopInit(p); + } + whereLoopXfer(db, p, pTemplate); + p->pNextLoop = pNext; + *ppPrev = p; + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + Index *pIndex = p->u.btree.pIndex; + if( pIndex && pIndex->tnum==0 ){ + p->u.btree.pIndex = 0; + } + } + return SQLITE4_OK; + + /* Jump here if the insert is a no-op */ +whereLoopInsert_noop: +#if WHERETRACE_ENABLED + if( sqlite4WhereTrace & 0x8 ){ + sqlite4DebugPrintf(pBuilder->pBest ? "ins-skip: " : "ins-noop: "); + whereLoopPrint(pTemplate, pWInfo->pTabList); + } +#endif + return SQLITE4_OK; +} + + +/* +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. +** Try to match one more. +*/ +static int whereLoopAddBtreeIndex( + WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ + struct SrcListItem *pSrc, /* FROM clause term being analyzed */ + Index *pProbe, /* An index on pSrc */ + WhereCost nInMul /* log(Number of iterations due to IN) */ +){ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite4 *db = pParse->db; /* Database connection malloc context */ + WhereLoop *pNew; /* Template WhereLoop under construction */ + WhereTerm *pTerm; /* A WhereTerm under consideration */ + int opMask; /* Valid operators for constraints */ + WhereScan scan; /* Iterator for WHERE terms */ + Bitmask saved_prereq; /* Original value of pNew->prereq */ + u16 saved_nLTerm; /* Original value of pNew->nLTerm */ + int saved_nEq; /* Original value of pNew->u.btree.nEq */ + u32 saved_wsFlags; /* Original value of pNew->wsFlags */ + WhereCost saved_nOut; /* Original value of pNew->nOut */ + int iCol; /* Index of the column in the table */ + int rc = SQLITE4_OK; /* Return code */ + WhereCost nRowEst; /* Estimated index selectivity */ + WhereCost rLogSize; /* Logarithm of table size */ + WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ + + assert( pProbe->eIndexType==SQLITE4_INDEX_USER + || pProbe->eIndexType==SQLITE4_INDEX_UNIQUE + || pProbe->eIndexType==SQLITE4_INDEX_PRIMARYKEY + ); + + pNew = pBuilder->pNew; + if( db->mallocFailed ) return SQLITE4_NOMEM; + + assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); + if( pNew->wsFlags & WHERE_BTM_LIMIT ){ + opMask = WO_LT|WO_LE; + }else if( pProbe->tnum<=0 || (pSrc->jointype & JT_LEFT)!=0 ){ + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE; + }else{ + opMask = WO_EQ|WO_IN|WO_ISNULL|WO_GT|WO_GE|WO_LT|WO_LE; + } + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + + if( pNew->u.btree.nEq < pProbe->nColumn ){ + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + nRowEst = whereCost(pProbe->aiRowEst[pNew->u.btree.nEq+1]); + if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; + }else if( pProbe->eIndexType!=SQLITE4_INDEX_PRIMARYKEY ){ + Index *pPk; + pPk = sqlite4FindPrimaryKey(pProbe->pTable, 0); + iCol = idxColumnNumber(pProbe, pPk, pNew->u.btree.nEq); + nRowEst = 0; + }else{ + return SQLITE4_OK; + } + assert( iCol>=-1 ); + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, + opMask, pProbe); + saved_nEq = pNew->u.btree.nEq; + saved_nLTerm = pNew->nLTerm; + saved_wsFlags = pNew->wsFlags; + saved_prereq = pNew->prereq; + saved_nOut = pNew->nOut; + pNew->rSetup = 0; + rLogSize = estLog(whereCost(pProbe->aiRowEst[0])); + for(; rc==SQLITE4_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + int nIn = 0; + if( pTerm->prereqRight & pNew->maskSelf ) continue; +#ifdef SQLITE4_ENABLE_STAT3 + if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ + continue; /* skip IS NOT NULL constraints on a NOT NULL column */ + } +#endif + pNew->wsFlags = saved_wsFlags; + pNew->u.btree.nEq = saved_nEq; + pNew->nLTerm = saved_nLTerm; + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTerm; + pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; + pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ + if( pTerm->eOperator & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + pNew->wsFlags |= WHERE_COLUMN_IN; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + nIn = 46; assert( 46==whereCost(25) ); + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = whereCost(pExpr->x.pList->nExpr); + } + pNew->rRun += nIn; + pNew->u.btree.nEq++; + pNew->nOut = nRowEst + nInMul + nIn; + }else if( pTerm->eOperator & (WO_EQ) ){ + assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 + || nInMul==0 ); + pNew->wsFlags |= WHERE_COLUMN_EQ; + if( iCol<0 + || (pProbe->onError!=OE_None && nInMul==0 + && pNew->u.btree.nEq==pProbe->nColumn-1) + ){ + assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); + pNew->wsFlags |= WHERE_ONEROW; + } + pNew->u.btree.nEq++; + pNew->nOut = nRowEst + nInMul; + }else if( pTerm->eOperator & (WO_ISNULL) ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; + pNew->u.btree.nEq++; + /* TUNING: IS NULL selects 2 rows */ + nIn = 10; assert( 10==whereCost(2) ); + pNew->nOut = nRowEst + nInMul + nIn; + }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ + testcase( pTerm->eOperator & WO_GT ); + testcase( pTerm->eOperator & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pBtm = pTerm; + pTop = 0; + }else{ + assert( pTerm->eOperator & (WO_LT|WO_LE) ); + testcase( pTerm->eOperator & WO_LT ); + testcase( pTerm->eOperator & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + /* Adjust nOut and rRun for STAT3 range values */ + WhereCost rDiv; + whereRangeScanEst(pParse, pProbe, pNew->u.btree.nEq, + pBtm, pTop, &rDiv); + pNew->nOut = saved_nOut>rDiv+10 ? saved_nOut - rDiv : 10; + } +#ifdef SQLITE4_ENABLE_STAT3 + if( pNew->u.btree.nEq==1 && pProbe->nSample + && OptimizationEnabled(db, SQLITE4_Stat3) ){ + tRowcnt nOut = 0; + if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( pTerm->eOperator & WO_EQ ); + testcase( pTerm->eOperator & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pProbe, pTerm->pExpr->pRight, &nOut); + }else if( (pTerm->eOperator & WO_IN) + && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ + rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut); + } + if( rc==SQLITE4_OK ) pNew->nOut = whereCost(nOut); + } +#endif + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + /* Each row involves a step of the index, then a binary search of + ** the main table */ + pNew->rRun = whereCostAdd(pNew->rRun, rLogSize>27 ? rLogSize-17 : 10); + } + /* Step cost for each output row */ + pNew->rRun = whereCostAdd(pNew->rRun, pNew->nOut); + /* TBD: Adjust nOut for additional constraints */ + rc = whereLoopInsert(pBuilder, pNew); + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 + && pNew->u.btree.nEq<(pProbe->nColumn + (pProbe->zName!=0)) + ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); + } + } + pNew->prereq = saved_prereq; + pNew->u.btree.nEq = saved_nEq; + pNew->wsFlags = saved_wsFlags; + pNew->nOut = saved_nOut; + pNew->nLTerm = saved_nLTerm; + return rc; +} + +/* +** Return True if it is possible that pIndex might be useful in +** implementing the ORDER BY clause in pBuilder. +** +** Return False if pBuilder does not contain an ORDER BY clause or +** if there is no way for pIndex to be useful in implementing that +** ORDER BY clause. +*/ +static int indexMightHelpWithOrderBy( + WhereLoopBuilder *pBuilder, + Index *pIndex, + int iCursor +){ + ExprList *pOB; + int ii, jj; + + if( pIndex->bUnordered ) return 0; + if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = sqlite4ExprSkipCollate(pOB->a[ii].pExpr); + if( pExpr->op!=TK_COLUMN ) return 0; + if( pExpr->iTable==iCursor ){ + for(jj=0; jjnColumn; jj++){ + if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; + } + } + } + return 0; +} + +/* +** Return a bitmask where 1s indicate that the corresponding column of +** the table is used by an index. Only the first 63 columns are considered. +*/ +static Bitmask columnsInIndex(Index *pIdx){ + Bitmask m = 0; + int j; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xpNew->iTab. That table is guaranteed to be +** a b-tree table, not a virtual table. +*/ +static int whereLoopAddBtree( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mExtra /* Extra prerequesites for using this table */ +){ + WhereInfo *pWInfo; /* WHERE analysis context */ + Index *pProbe; /* An index we are evaluating */ + Index *pPk; /* Primary key index for table pSrc */ + tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + int aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + SrcList *pTabList; /* The FROM clause */ + struct SrcListItem *pSrc; /* The FROM clause btree term to add */ + WhereLoop *pNew; /* Template WhereLoop object */ + int rc = SQLITE4_OK; /* Return code */ + int iSortIdx = 1; /* Index number */ + int b; /* A boolean value */ + WhereCost rSize; /* number of rows in the table */ + WhereCost rLogSize; /* Logarithm of the number of rows in the table */ + + pNew = pBuilder->pNew; + pWInfo = pBuilder->pWInfo; + pTabList = pWInfo->pTabList; + pSrc = pTabList->a + pNew->iTab; + assert( !IsVirtual(pSrc->pTab) ); + pPk = sqlite4FindPrimaryKey(pSrc->pTab, 0); + + if( pSrc->pIndex ){ + /* An INDEXED BY clause specifies a particular index to use */ + pProbe = pSrc->pIndex; + }else if( pSrc->notIndexed ){ + /* A NOT INDEXED clause means use the PK index */ + pProbe = pPk; + }else{ + /* Otherwise, consider all indexes */ + pProbe = pSrc->pTab->pIndex; + } + + rSize = whereCost(pSrc->pTab->nRowEst); + rLogSize = estLog(rSize); + +#ifndef SQLITE4_OMIT_AUTOMATIC_INDEX + /* Automatic indexes */ + if( !pBuilder->pBest + && (pWInfo->pParse->db->flags & SQLITE4_AutoIndex)!=0 + && pSrc->pIndex==0 +#if 0 + && !pSrc->viaCoroutine +#endif + && !pSrc->notIndexed + && !pSrc->isCorrelated + ){ + /* Generate auto-index WhereLoops */ + WhereClause *pWC = pBuilder->pWC; + WhereTerm *pTerm; + WhereTerm *pWCEnd = pWC->a + pWC->nTerm; + for(pTerm=pWC->a; rc==SQLITE4_OK && pTermprereqRight & pNew->maskSelf ) continue; + if( termCanDriveIndex(pTerm, pSrc, 0) ){ + pNew->u.btree.nEq = 1; + pNew->u.btree.pIndex = 0; + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + /* TUNING: One-time cost for computing the automatic index is + ** approximately 7*N*log2(N) where N is the number of rows in + ** the table being indexed. */ + pNew->rSetup = rLogSize + rSize + 28; assert( 28==whereCost(7) ); + /* TUNING: Each index lookup yields 20 rows in the table. This + ** is more than the usual guess of 10 rows, since we have no way + ** of knowning how selective the index will ultimately be. It would + ** not be unreasonable to make this value much larger. */ + pNew->nOut = 43; assert( 43==whereCost(20) ); + pNew->rRun = whereCostAdd(rLogSize,pNew->nOut); + pNew->wsFlags = WHERE_AUTO_INDEX; + pNew->prereq = mExtra | pTerm->prereqRight; + rc = whereLoopInsert(pBuilder, pNew); + } + } + } +#endif /* ifndef SQLITE4_OMIT_AUTOMATIC_INDEX */ + + /* If this table has no primary key, then it is either a materialized + ** view or ephemeral table. Either way, add a WhereLoop for a full-scan + ** of it. */ + if( pPk==0 ){ + assert( pSrc->pTab->pSelect || (pSrc->pTab->tabFlags & TF_Ephemeral) ); + pNew->u.btree.nEq = 0; + pNew->nLTerm = 0; + pNew->iSortIdx = 0; + pNew->rSetup = 0; + pNew->prereq = mExtra; + pNew->nOut = rSize; + pNew->u.btree.pIndex = 0; + pNew->wsFlags = 0; + pNew->rRun = whereCostAdd(rSize,rLogSize) + 16; + rc = whereLoopInsert(pBuilder, pNew); + } + + /* Loop through the set of indices being considered. */ + for(; rc==SQLITE4_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ + + if( pProbe->eIndexType==SQLITE4_INDEX_FTS5 ) continue; + assert( pProbe->tnum>0 ); + + pNew->u.btree.nEq = 0; + pNew->nLTerm = 0; + pNew->rSetup = 0; + pNew->prereq = mExtra; + pNew->nOut = rSize; + pNew->u.btree.pIndex = pProbe; + pNew->wsFlags = WHERE_INDEXED; + + b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); + /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ + assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); + pNew->iSortIdx = b ? iSortIdx : 0; + + if( pProbe==pPk || b ){ + /* Add a WhereLoop for full-scan via primary key index. */ + + /* TUNING: Cost of full table scan is 3*(N + log2(N)). + ** + The extra 3 factor is to encourage the use of indexed lookups + ** over full scans. A smaller constant 2 is used for covering + ** index scans so that a covering index scan will be favored over + ** a table scan. */ + /* TODO: Fix tuning for src4 as described in comment immediately above. */ + pNew->rRun = whereCostAdd(rSize,rLogSize) + 16; + rc = whereLoopInsert(pBuilder, pNew); + if( rc ) break; + } + + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + + /* If there was an INDEXED BY or NOT INDEXED clause, then only one + ** index is considered. */ + if( pSrc->pIndex || pSrc->notIndexed ) break; + } + return rc; +} + +#ifndef SQLITE4_OMIT_VIRTUALTABLE +/* +** Add all WhereLoop objects for a table of the join identified by +** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. +*/ +static int whereLoopAddVirtual( + WhereLoopBuilder *pBuilder /* WHERE clause information */ +){ + WhereInfo *pWInfo; /* WHERE analysis context */ + Parse *pParse; /* The parsing context */ + WhereClause *pWC; /* The WHERE clause */ + struct SrcListItem *pSrc; /* The FROM clause term to search */ + Table *pTab; + sqlite4 *db; + sqlite4_index_info *pIdxInfo; + struct sqlite4_index_constraint *pIdxCons; + struct sqlite4_index_constraint_usage *pUsage; + WhereTerm *pTerm; + int i, j; + int iTerm, mxTerm; + int nConstraint; + int seenIn = 0; /* True if an IN operator is seen */ + int seenVar = 0; /* True if a non-constant constraint is seen */ + int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ + WhereLoop *pNew; + int rc = SQLITE4_OK; + + pWInfo = pBuilder->pWInfo; + pParse = pWInfo->pParse; + db = pParse->db; + pWC = pBuilder->pWC; + pNew = pBuilder->pNew; + pSrc = &pWInfo->pTabList->a[pNew->iTab]; + pTab = pSrc->pTab; + assert( IsVirtual(pTab) ); + pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pBuilder->pOrderBy); + if( pIdxInfo==0 ) return SQLITE4_NOMEM; + pNew->prereq = 0; + pNew->rSetup = 0; + pNew->wsFlags = WHERE_VIRTUALTABLE; + pNew->nLTerm = 0; + pNew->u.vtab.needFree = 0; + pUsage = pIdxInfo->aConstraintUsage; + nConstraint = pIdxInfo->nConstraint; + if( whereLoopResize(db, pNew, nConstraint) ){ + sqlite4DbFree(db, pIdxInfo); + return SQLITE4_NOMEM; + } + + for(iPhase=0; iPhase<=3; iPhase++){ + if( !seenIn && (iPhase&1)!=0 ){ + iPhase++; + if( iPhase>3 ) break; + } + if( !seenVar && iPhase>1 ) break; + pIdxCons = *(struct sqlite4_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pIdxCons++){ + j = pIdxCons->iTermOffset; + pTerm = &pWC->a[j]; + switch( iPhase ){ + case 0: /* Constants without IN operator */ + pIdxCons->usable = 0; + if( (pTerm->eOperator & WO_IN)!=0 ){ + seenIn = 1; + } + if( pTerm->prereqRight!=0 ){ + seenVar = 1; + }else if( (pTerm->eOperator & WO_IN)==0 ){ + pIdxCons->usable = 1; + } + break; + case 1: /* Constants with IN operators */ + assert( seenIn ); + pIdxCons->usable = (pTerm->prereqRight==0); + break; + case 2: /* Variables without IN */ + assert( seenVar ); + pIdxCons->usable = (pTerm->eOperator & WO_IN)==0; + break; + default: /* Variables with IN */ + assert( seenVar && seenIn ); + pIdxCons->usable = 1; + break; + } + } + memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); + if( pIdxInfo->needToFreeIdxStr ) sqlite4_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->needToFreeIdxStr = 0; + pIdxInfo->orderByConsumed = 0; + pIdxInfo->estimatedCost = SQLITE4_BIG_DBL / (double)2; + rc = vtabBestIndex(pParse, pTab, pIdxInfo); + if( rc ) goto whereLoopAddVtab_exit; + pIdxCons = *(struct sqlite4_index_constraint**)&pIdxInfo->aConstraint; + pNew->prereq = 0; + mxTerm = -1; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; iaLTerm[i] = 0; + pNew->u.vtab.omitMask = 0; + for(i=0; i=0 ){ + j = pIdxCons->iTermOffset; + if( iTerm>=nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aLTerm[iTerm]!=0 + ){ + rc = SQLITE4_ERROR; + sqlite4ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName); + goto whereLoopAddVtab_exit; + } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + assert( iTermnLSlot ); + pNew->aLTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); + if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit==0 ){ + /* Do not attempt to use an IN constraint if the virtual table + ** says that the equivalent EQ constraint cannot be safely omitted. + ** If we do attempt to use such a constraint, some rows might be + ** repeated in the output. */ + break; + } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + } + } + } + if( i>=nConstraint ){ + pNew->nLTerm = mxTerm+1; + assert( pNew->nLTerm<=pNew->nLSlot ); + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) + && pIdxInfo->orderByConsumed); + pNew->rSetup = 0; + pNew->rRun = whereCostFromDouble(pIdxInfo->estimatedCost); + /* TUNING: Every virtual table query returns 25 rows */ + pNew->nOut = 46; assert( 46==whereCost(25) ); + whereLoopInsert(pBuilder, pNew); + if( pNew->u.vtab.needFree ){ + sqlite4_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; + } + } + } + +whereLoopAddVtab_exit: + if( pIdxInfo->needToFreeIdxStr ) sqlite4_free(pIdxInfo->idxStr); + sqlite4DbFree(db, pIdxInfo); + return rc; +} +#endif /* SQLITE4_OMIT_VIRTUALTABLE */ + +/* +** Add WhereLoop entries to handle OR terms. This works for either +** btrees or virtual tables. +*/ +static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ + WhereInfo *pWInfo = pBuilder->pWInfo; + WhereClause *pWC; + WhereLoop *pNew; + WhereTerm *pTerm, *pWCEnd; + int rc = SQLITE4_OK; + int iCur; + WhereClause tempWC; + WhereLoopBuilder sSubBuild; + WhereLoop sBest; + struct SrcListItem *pItem; + + pWC = pBuilder->pWC; + if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE4_OK; + pWCEnd = pWC->a + pWC->nTerm; + pNew = pBuilder->pNew; + + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 + && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + ){ + WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; + WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; + WhereTerm *pOrTerm; + WhereCost rTotal = 0; + WhereCost nRow = 0; + Bitmask prereq = mExtra; + + whereLoopInit(&sBest); + pItem = pWInfo->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; + sSubBuild = *pBuilder; + sSubBuild.pOrderBy = 0; + sSubBuild.pBest = &sBest; + + for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ + sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; + }else if( pOrTerm->leftCursor==iCur ){ + tempWC.pWInfo = pWC->pWInfo; + tempWC.pOuter = pWC; + tempWC.op = TK_AND; + tempWC.nTerm = 1; + tempWC.a = pOrTerm; + sSubBuild.pWC = &tempWC; + }else{ + continue; + } + sBest.maskSelf = 0; + sBest.rSetup = 0; + sBest.rRun = 0; +#ifndef SQLITE4_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(&sSubBuild); + }else +#endif + { + rc = whereLoopAddBtree(&sSubBuild, mExtra); + } + /* sBest.maskSelf is always zero if an error occurs */ + assert( rc==SQLITE4_OK || sBest.maskSelf==0 ); + if( sBest.maskSelf==0 ) break; + assert( sBest.rSetup==0 ); + rTotal = whereCostAdd(rTotal, sBest.rRun); + nRow = whereCostAdd(nRow, sBest.nOut); + prereq |= sBest.prereq; + } + assert( pNew->nLSlot>=1 ); + if( sBest.maskSelf ){ + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = 0; + /* TUNING: Multiple by 3.5 for the secondary table lookup */ + pNew->rRun = rTotal + 18; assert( 18==whereCost(7)-whereCost(2) ); + pNew->nOut = nRow; + pNew->prereq = prereq; + memset(&pNew->u, 0, sizeof(pNew->u)); + rc = whereLoopInsert(pBuilder, pNew); + } + whereLoopClear(pWInfo->pParse->db, &sBest); + } + } + return rc; +} + +/* +** Add all WhereLoop objects for all tables +*/ +static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo = pBuilder->pWInfo; + Bitmask mExtra = 0; + Bitmask mPrior = 0; + int iTab; + SrcList *pTabList = pWInfo->pTabList; + struct SrcListItem *pItem; + sqlite4 *db = pWInfo->pParse->db; + int nTabList = pWInfo->nLevel; + int rc = SQLITE4_OK; + u8 priorJoinType = 0; + WhereLoop *pNew; + + /* Loop over the tables in the join, from left to right */ + pNew = pBuilder->pNew; + whereLoopInit(pNew); + for(iTab=0, pItem=pTabList->a; iTabiTab = iTab; + pNew->maskSelf = getMask(&pWInfo->sMaskSet, pItem->iCursor); + if( ((pItem->jointype|priorJoinType) & (JT_LEFT|JT_CROSS))!=0 ){ + mExtra = mPrior; + } + priorJoinType = pItem->jointype; +#ifndef SQLITE4_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(pBuilder); + }else +#endif + { + rc = whereLoopAddBtree(pBuilder, mExtra); + } + if( rc==SQLITE4_OK ){ + rc = whereLoopAddOr(pBuilder, mExtra); + } + mPrior |= pNew->maskSelf; + if( rc || db->mallocFailed ) break; + } + whereLoopClear(db, pNew); + return rc; +} + +/* +** Examine a WherePath (with the addition of the extra WhereLoop of the 5th +** parameters) to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate sort operation. Return: +** +** 0: ORDER BY is not satisfied. Sorting required +** 1: ORDER BY is satisfied. Omit sorting +** -1: Unknown at this time +** +** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as +** strict. With GROUP BY and DISTINCT the only requirement is that +** equivalent rows appear immediately adjacent to one another. GROUP BY +** and DISTINT do not require rows to appear in any particular order as long +** as equivelent rows are grouped together. Thus for GROUP BY and DISTINCT +** the pOrderBy terms can be matched in any order. With ORDER BY, the +** pOrderBy terms must be matched in strict left-to-right order. +*/ +static int wherePathSatisfiesOrderBy( + WhereInfo *pWInfo, /* The WHERE clause */ + ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ + WherePath *pPath, /* The WherePath to check */ + u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 nLoop, /* Number of entries in pPath->aLoop[] */ + WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ + Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ +){ + u8 revSet; /* True if rev is known */ + u8 rev; /* Composite sort order */ + u8 revIdx; /* Index sort order */ + u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ + u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 nColumn; /* Number of columns in pIndex */ + u16 nOrderBy; /* Number terms in the ORDER BY clause */ + int iLoop; /* Index of WhereLoop in pPath being processed */ + int i, j; /* Loop counters */ + int iCur; /* Cursor number for current WhereLoop */ + int iColumn; /* A column number within table iCur */ + WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + Expr *pOBExpr; /* An expression from the ORDER BY clause */ + CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ + Index *pIndex; /* The index associated with pLoop */ + sqlite4 *db = pWInfo->pParse->db; /* Database connection */ + Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ + Bitmask obDone; /* Mask of all ORDER BY terms */ + Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ + Bitmask ready; /* Mask of inner loops */ + + /* + ** We say the WhereLoop is "one-row" if it generates no more than one + ** row of output. A WhereLoop is one-row if all of the following are true: + ** (a) All index columns match with WHERE_COLUMN_EQ. + ** (b) The index is unique + ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. + ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. + ** + ** We say the WhereLoop is "order-distinct" if the set of columns from + ** that WhereLoop that are in the ORDER BY clause are different for every + ** row of the WhereLoop. Every one-row WhereLoop is automatically + ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause + ** is not order-distinct. To be order-distinct is not quite the same as being + ** UNIQUE since a UNIQUE column or index can have multiple rows that + ** are NULL and NULL values are equivalent for the purpose of order-distinct. + ** To be order-distinct, the columns must be UNIQUE and NOT NULL. + ** + ** The rowid for a table is always UNIQUE and NOT NULL so whenever the + ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is + ** automatically order-distinct. + */ + + assert( pOrderBy!=0 ); + + /* Sortability of virtual tables is determined by the xBestIndex method + ** of the virtual table itself */ + if( pLast->wsFlags & WHERE_VIRTUALTABLE ){ + testcase( nLoop>0 ); /* True when outer loops are one-row and match + ** no ORDER BY terms */ + return pLast->u.vtab.isOrdered; + } + if( nLoop && OptimizationDisabled(db, SQLITE4_OrderByIdxJoin) ) return 0; + + nOrderBy = pOrderBy->nExpr; + testcase( nOrderBy==BMS-1 ); + if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ + isOrderDistinct = 1; + obDone = MASKBIT(nOrderBy)-1; + orderDistinctMask = 0; + ready = 0; + for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; + pLoop = iLoopaLoop[iLoop] : pLast; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; + + /* Mark off any ORDER BY term X that is a column in the table of + ** the current loop for which there is term in the WHERE + ** clause of the form X IS NULL or X=? that reference only outer + ** loops. + */ + for(i=0; ia[i].pExpr); + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + pTerm = findTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, WO_EQ|WO_ISNULL, 0); + if( pTerm==0 ) continue; + if( (pTerm->eOperator&WO_EQ)!=0 && pOBExpr->iColumn>=0 ){ + const char *z1, *z2; + pColl = sqlite4ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + z1 = pColl->zName; + pColl = sqlite4ExprCollSeq(pWInfo->pParse, pTerm->pExpr); + if( !pColl ) pColl = db->pDfltColl; + z2 = pColl->zName; + if( sqlite4_stricmp(z1, z2)!=0 ) continue; + } + obSat |= MASKBIT(i); + } + + if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ + Index *pPk = 0; + if( pLoop->wsFlags & WHERE_IPK ){ + pIndex = 0; + nColumn = 0; + }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ + return 0; + }else{ + isOrderDistinct = pIndex->onError!=OE_None; + pPk = sqlite4FindPrimaryKey(pIndex->pTable, 0); + nColumn = idxColumnCount(pIndex, pPk); + } + + /* Loop through all columns of the index and deal with the ones + ** that are not constrained by == or IN. + */ + rev = revSet = 0; + for(j=0; ju.btree.nEq + && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 + ){ + if( i & WO_ISNULL ){ + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } + continue; + } + + /* Get the column number in the table (iColumn) and sort order + ** (revIdx) for the j-th column of the index. + */ + if( j=0 + && j>=pLoop->u.btree.nEq + && pIndex->pTable->aCol[iColumn].notNull==0 + ){ + isOrderDistinct = 0; + } + + /* Find the ORDER BY term that corresponds to the j-th column + ** of the index and and mark that ORDER BY term off + */ + bOnce = 1; + isMatch = 0; + for(i=0; bOnce && ia[i].pExpr); + testcase( wctrlFlags & WHERE_GROUPBY ); + testcase( wctrlFlags & WHERE_DISTINCTBY ); + if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=0 ){ + const char *zIdxColl; + pColl = sqlite4ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( !pColl ) pColl = db->pDfltColl; + zIdxColl = idxColumnCollation(pIndex, pPk, j); + if( sqlite4_stricmp(pColl->zName, zIdxColl)!=0 ) continue; + } + isMatch = 1; + break; + } + if( isMatch ){ + obSat |= MASKBIT(i); + if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) return 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } + }else{ + /* No match found */ + if( j==0 || jmaskSelf; + for(i=0; ia[i].pExpr; + if( (exprTableUsage(&pWInfo->sMaskSet, p)&~orderDistinctMask)==0 ){ + obSat |= MASKBIT(i); + } + } + } + } /* End the loop over all WhereLoops from outer-most down to inner-most */ + if( obSat==obDone ) return 1; + if( !isOrderDistinct ) return 0; + return -1; +} + +#ifdef WHERETRACE_ENABLED +/* For debugging use only: */ +static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ + static char zName[65]; + int i; + for(i=0; iaLoop[i]->cId; } + if( pLast ) zName[i++] = pLast->cId; + zName[i] = 0; + return zName; +} +#endif + + +/* +** Given the list of WhereLoop objects at pWInfo->pLoops, this routine +** attempts to find the lowest cost path that visits each WhereLoop +** once. This path is then loaded into the pWInfo->a[].pWLoop fields. +** +** Assume that the total number of output rows that will need to be sorted +** will be nRowEst (in the 10*log2 representation). Or, ignore sorting +** costs if nRowEst==0. +** +** Return SQLITE4_OK on success or SQLITE4_NOMEM of a memory allocation +** error occurs. +*/ +static int wherePathSolver(WhereInfo *pWInfo, WhereCost nRowEst){ + int mxChoice; /* Maximum number of simultaneous paths tracked */ + int nLoop; /* Number of terms in the join */ + Parse *pParse; /* Parsing context */ + sqlite4 *db; /* The database connection */ + int iLoop; /* Loop counter over the terms of the join */ + int ii, jj; /* Loop counters */ + WhereCost rCost; /* Cost of a path */ + WhereCost mxCost = 0; /* Maximum cost of a set of paths */ + WhereCost rSortCost; /* Cost to do a sort */ + int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ + WherePath *aFrom; /* All nFrom paths at the previous level */ + WherePath *aTo; /* The nTo best paths at the current level */ + WherePath *pFrom; /* An element of aFrom[] that we are working on */ + WherePath *pTo; /* An element of aTo[] that we are working on */ + WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop **pX; /* Used to divy up the pSpace memory */ + char *pSpace; /* Temporary memory used by this routine */ + + pParse = pWInfo->pParse; + db = pParse->db; + nLoop = pWInfo->nLevel; + /* TUNING: For simple queries, only the best path is tracked. + ** For 2-way joins, the 5 best paths are followed. + ** For joins of 3 or more tables, track the 10 best paths */ + mxChoice = (nLoop==1) ? 1 : (nLoop==2 ? 5 : 10); + assert( nLoop<=pWInfo->pTabList->nSrc ); + WHERETRACE(0x002, ("---- begin solver\n")); + + /* Allocate and initialize space for aTo and aFrom */ + ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + pSpace = sqlite4DbMallocRaw(db, ii); + if( pSpace==0 ) return SQLITE4_NOMEM; + aTo = (WherePath*)pSpace; + aFrom = aTo+mxChoice; + memset(aFrom, 0, sizeof(aFrom[0])); + pX = (WhereLoop**)(aFrom+mxChoice); + for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ + pFrom->aLoop = pX; + } + + /* Seed the search with a single WherePath containing zero WhereLoops. + ** + ** TUNING: Do not let the number of iterations go above 25. If the cost + ** of computing an automatic index is not paid back within the first 25 + ** rows, then do not use the automatic index. */ + aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==whereCost(25) ); + nFrom = 1; + + /* Precompute the cost of sorting the final result set, if the caller + ** to sqlite4WhereBegin() was concerned about sorting */ + rSortCost = 0; + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ + aFrom[0].isOrderedValid = 1; + }else{ + /* TUNING: Estimated cost of sorting is N*log2(N) where N is the + ** number of output rows. */ + rSortCost = nRowEst + estLog(nRowEst); + WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); + } + + /* Compute successively longer WherePaths using the previous generation + ** of WherePaths as the basis for the next. Keep track of the mxChoice + ** best paths at each generation */ + for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + Bitmask maskNew; + Bitmask revMask = 0; + u8 isOrderedValid = pFrom->isOrderedValid; + u8 isOrdered = pFrom->isOrdered; + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + /* At this point, pWLoop is a candidate to be the next loop. + ** Compute its cost */ + rCost = whereCostAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rCost = whereCostAdd(rCost, pFrom->rCost); + maskNew = pFrom->maskLoop | pWLoop->maskSelf; + if( !isOrderedValid ){ + switch( wherePathSatisfiesOrderBy(pWInfo, + pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, + iLoop, pWLoop, &revMask) ){ + case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ + isOrdered = 1; + isOrderedValid = 1; + break; + case 0: /* No. pFrom+pWLoop will require a separate sort */ + isOrdered = 0; + isOrderedValid = 1; + rCost = whereCostAdd(rCost, rSortCost); + break; + default: /* Cannot tell yet. Try again on the next iteration */ + break; + } + }else{ + revMask = pFrom->revLoop; + } + /* Check to see if pWLoop should be added to the mxChoice best so far */ + for(jj=0, pTo=aTo; jjmaskLoop==maskNew && pTo->isOrderedValid==isOrderedValid ){ + testcase( jj==nTo-1 ); + break; + } + } + if( jj>=nTo ){ + if( nTo>=mxChoice && rCost>=mxCost ){ +#ifdef WHERETRACE_ENABLED + if( sqlite4WhereTrace&0x4 ){ + sqlite4DebugPrintf("Skip %s cost=%3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + } +#endif + continue; + } + /* Add a new Path to the aTo[] set */ + if( nTo0); } + } + pTo = &aTo[jj]; +#ifdef WHERETRACE_ENABLED + if( sqlite4WhereTrace&0x4 ){ + sqlite4DebugPrintf("New %s cost=%-3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + } +#endif + }else{ + if( pTo->rCost<=rCost ){ +#ifdef WHERETRACE_ENABLED + if( sqlite4WhereTrace&0x4 ){ + sqlite4DebugPrintf( + "Skip %s cost=%-3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + sqlite4DebugPrintf(" vs %s cost=%-3d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, + pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + } +#endif + testcase( pTo->rCost==rCost ); + continue; + } + testcase( pTo->rCost==rCost+1 ); + /* A new and better score for a previously created equivalent path */ +#ifdef WHERETRACE_ENABLED + if( sqlite4WhereTrace&0x4 ){ + sqlite4DebugPrintf( + "Update %s cost=%-3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, + isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + sqlite4DebugPrintf(" was %s cost=%-3d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, + pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + } +#endif + } + /* pWLoop is a winner. Add it to the set of best so far */ + pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->revLoop = revMask; + pTo->nRow = pFrom->nRow + pWLoop->nOut; + pTo->rCost = rCost; + pTo->isOrderedValid = isOrderedValid; + pTo->isOrdered = isOrdered; + memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); + pTo->aLoop[iLoop] = pWLoop; + if( nTo>=mxChoice ){ + mxCost = aTo[0].rCost; + for(jj=1, pTo=&aTo[1]; jjrCost>mxCost ) mxCost = pTo->rCost; + } + } + } + } + +#ifdef WHERETRACE_ENABLED + if( sqlite4WhereTrace>=2 ){ + sqlite4DebugPrintf("---- after round %d ----\n", iLoop); + for(ii=0, pTo=aTo; iirCost, pTo->nRow, + pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + if( pTo->isOrderedValid && pTo->isOrdered ){ + sqlite4DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite4DebugPrintf("\n"); + } + } + } +#endif + + /* Swap the roles of aFrom and aTo for the next generation */ + pFrom = aTo; + aTo = aFrom; + aFrom = pFrom; + nFrom = nTo; + } + + if( nFrom==0 ){ + sqlite4ErrorMsg(pParse, "no query solution"); + sqlite4DbFree(db, pSpace); + return SQLITE4_ERROR; + } + + /* Find the lowest cost path. pFrom will be left pointing to that path */ + pFrom = aFrom; + assert( nFrom==1 ); +#if 0 /* The following is needed if nFrom is ever more than 1 */ + for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; + } +#endif + assert( pWInfo->nLevel==nLoop ); + /* Load the lowest cost path into pWInfo */ + for(iLoop=0; iLoopa + iLoop; + pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; + pLevel->iFrom = pWLoop->iTab; + pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; + } + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 + && pWInfo->eDistinct==WHERE_DISTINCT_NOOP + && nRowEst + ){ + Bitmask notUsed; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, + WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); + if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + if( pFrom->isOrdered ){ + if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + }else{ + pWInfo->bOBSat = 1; + pWInfo->revMask = pFrom->revLoop; + } + } + pWInfo->nRowOut = pFrom->nRow; + + /* Free temporary memory and return success */ + sqlite4DbFree(db, pSpace); + return SQLITE4_OK; +} + +/* +** Most queries use only a single table (they are not joins) and have +** simple == constraints against indexed fields. This routine attempts +** to plan those simple cases using much less ceremony than the +** general-purpose query planner, and thereby yield faster sqlite4_prepare() +** times for the common case. +** +** Return non-zero on success, if this query can be handled by this +** no-frills query planner. Return zero if this query needs the +** general-purpose query planner. +*/ +static int whereShortCut(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo; + struct SrcListItem *pItem; + WhereClause *pWC; + WhereTerm *pTerm; + WhereLoop *pLoop; + int iCur; + int j; + Table *pTab; + Index *pIdx; + + return 0; + + pWInfo = pBuilder->pWInfo; + if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; + assert( pWInfo->pTabList->nSrc>=1 ); + pItem = pWInfo->pTabList->a; + pTab = pItem->pTab; + if( IsVirtual(pTab) ) return 0; + if( pItem->zIndex ) return 0; + iCur = pItem->iCursor; + pWC = &pWInfo->sWC; + pLoop = pBuilder->pNew; + pLoop->wsFlags = 0; + pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); + if( pTerm ){ + assert( 0 ); + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; + pLoop->aLTerm[0] = pTerm; + pLoop->nLTerm = 1; + pLoop->u.btree.nEq = 1; + /* TUNING: Cost of a rowid lookup is 10 */ + pLoop->rRun = 33; /* 33==whereCost(10) */ + }else{ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_None ) continue; + for(j=0; jnColumn; j++){ + pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); + if( pTerm==0 ) break; + whereLoopResize(pWInfo->pParse->db, pLoop, j); + pLoop->aLTerm[j] = pTerm; + } + if( j!=pIdx->nColumn ) continue; + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; + if( (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + pLoop->wsFlags |= WHERE_IDX_ONLY; + } + pLoop->nLTerm = j; + pLoop->u.btree.nEq = j; + pLoop->u.btree.pIndex = pIdx; + /* TUNING: Cost of a unique index lookup is 15 */ + pLoop->rRun = 39; /* 39==whereCost(15) */ + break; + } + } + if( pLoop->wsFlags ){ + pLoop->nOut = (WhereCost)1; + pWInfo->a[0].pWLoop = pLoop; + pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); + pWInfo->a[0].iTabCur = iCur; + pWInfo->nRowOut = 1; + if( pWInfo->pOrderBy ) pWInfo->bOBSat = 1; + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } +#ifdef SQLITE4_DEBUG + pLoop->cId = '0'; +#endif + return 1; + } + return 0; +} /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains ** information needed to terminate the loop. Later, the calling routine @@ -4529,11 +5645,11 @@ ** appear in the FROM clause if a different order is better able to make ** use of indices. Note also that when the IN operator appears in ** the WHERE clause, it might result in additional nested loops for ** scanning through all values on the right-hand side of the IN. ** -** There are cursors associated with each table. t1 uses cursor +** There are Btree cursors associated with each table. t1 uses cursor ** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. ** And so forth. This routine generates code to open those VDBE cursors ** and sqlite4WhereEnd() generates the code to close them. ** ** The code that sqlite4WhereBegin() generates leaves the cursors named @@ -4572,44 +5688,50 @@ ** fi ** end ** ** ORDER BY CLAUSE PROCESSING ** -** *ppOrderBy is a pointer to the ORDER BY clause of a SELECT statement, +** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause +** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement ** if there is one. If there is no ORDER BY clause or if this routine -** is called from an UPDATE or DELETE statement, then ppOrderBy is NULL. -** -** If an index can be used so that the natural output order of the table -** scan is correct for the ORDER BY clause, then that index is used and -** *ppOrderBy is set to NULL. This is an optimization that prevents an -** unnecessary sort of the result set if an index appropriate for the -** ORDER BY clause already exists. -** -** If the where clause loops cannot be arranged to provide the correct -** output order, then the *ppOrderBy is unchanged. +** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. */ WhereInfo *sqlite4WhereBegin( Parse *pParse, /* The parser context */ - SrcList *pTabList, /* A list of all tables to be scanned */ + SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ - ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ - ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ - u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ + ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pResultSet, /* Result set of the query */ + u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ + int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ ){ - int i; /* Loop counter */ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ int nTabList; /* Number of elements in pTabList */ WhereInfo *pWInfo; /* Will become the return value of this function */ Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ + WhereLoopBuilder sWLB; /* The WhereLoop builder */ WhereMaskSet *pMaskSet; /* The expression mask set */ - WhereClause *pWC; /* Decomposition of the WHERE clause */ - SrcListItem *pTabItem; /* A single entry from pTabList */ - WhereLevel *pLevel; /* A single level in the pWInfo list */ - int iFrom; /* First unused FROM clause element */ - int andFlags; /* AND-ed combination of all pWC->a[].wtFlags */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ + int ii; /* Loop counter */ sqlite4 *db; /* Database connection */ + int rc; /* Return code */ + + /* src4: In SQLite3, the caller would set this flag. */ + if( pResultSet ) wctrlFlags |= WHERE_WANT_DISTINCT; + + /* Variable initialization */ + db = pParse->db; + memset(&sWLB, 0, sizeof(sWLB)); + sWLB.pOrderBy = pOrderBy; + + /* Disable the DISTINCT optimization if SQLITE4_DistinctOpt is set via + ** sqlite4_test_ctrl(SQLITE4_TESTCTRL_OPTIMIZATIONS,...) */ + if( OptimizationDisabled(db, SQLITE4_DistinctOpt) ){ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } /* The number of tables in the FROM clause is limited by the number of ** bits in a Bitmask */ testcase( pTabList->nSrc==BMS ); @@ -4630,50 +5752,59 @@ ** struct, the contents of WhereInfo.a[], the WhereClause structure ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - db = pParse->db; nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite4DbMallocZero(db, - nByteWInfo + - sizeof(WhereClause) + - sizeof(WhereMaskSet) - ); + pWInfo = sqlite4DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite4DbFree(db, pWInfo); pWInfo = 0; goto whereBeginError; } pWInfo->nLevel = nTabList; pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; + pWInfo->pOrderBy = pOrderBy; + pWInfo->pResultSet = pResultSet; pWInfo->iBreak = sqlite4VdbeMakeLabel(v); - pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo]; pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; - pMaskSet = (WhereMaskSet*)&pWC[1]; - - /* Disable the DISTINCT optimization if SQLITE4_DistinctOpt is set via - ** sqlite4_test_ctrl(SQLITE4_TESTCTRL_OPTIMIZATIONS,...) */ - if( db->flags & SQLITE4_DistinctOpt ) pDistinct = 0; + pMaskSet = &pWInfo->sMaskSet; + sWLB.pWInfo = pWInfo; + sWLB.pWC = &pWInfo->sWC; + sWLB.pNew = (WhereLoop*)&pWInfo->a[nTabList]; + whereLoopInit(sWLB.pNew); +#ifdef SQLITE4_DEBUG + sWLB.pNew->cId = '*'; +#endif /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); - whereClauseInit(pWC, pParse, pMaskSet, wctrlFlags); + whereClauseInit(&pWInfo->sWC, pWInfo); sqlite4ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + whereSplit(&pWInfo->sWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ + sqlite4CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. */ if( pWhere && (nTabList==0 || sqlite4ExprIsConstantNotJoin(pWhere)) ){ sqlite4ExprIfFalse(pParse, pWhere, pWInfo->iBreak, SQLITE4_JUMPIFNULL); pWhere = 0; } + + /* Special case: No FROM clause + */ + if( nTabList==0 ){ + if( pOrderBy ) pWInfo->bOBSat = 1; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + } /* Assign a bit from the bitmask to every term in the FROM clause. ** ** When assigning bitmask values to FROM clause cursors, it must be ** the case that if X is the bitmask for the N-th FROM clause term then @@ -4682,407 +5813,259 @@ ** its Expr.iRightJoinTable value to find the bitmask of the right table ** of the join. Subtracting one from the right table bitmask gives a ** bitmask for all tables to the left of the join. Knowing the bitmask ** for all tables to the left of a left join is important. Ticket #3015. ** - ** Configure the WhereClause.vmask variable so that bits that correspond - ** to virtual table cursors are set. This is used to selectively disable - ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful - ** with virtual tables. - ** ** Note that bitmasks are created for all pTabList->nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally ** equal to pTabList->nSrc but might be shortened to 1 if the ** WHERE_ONETABLE_ONLY flag is set. */ - assert( pWC->vmask==0 && pMaskSet->n==0 ); - for(i=0; inSrc; i++){ - createMask(pMaskSet, pTabList->a[i].iCursor); -#ifndef SQLITE4_OMIT_VIRTUALTABLE - if( ALWAYS(pTabList->a[i].pTab) && IsVirtual(pTabList->a[i].pTab) ){ - pWC->vmask |= ((Bitmask)1 << i); - } -#endif + for(ii=0; iinSrc; ii++){ + createMask(pMaskSet, pTabList->a[ii].iCursor); } #ifndef NDEBUG { Bitmask toTheLeft = 0; - for(i=0; inSrc; i++){ - Bitmask m = getMask(pMaskSet, pTabList->a[i].iCursor); + for(ii=0; iinSrc; ii++){ + Bitmask m = getMask(pMaskSet, pTabList->a[ii].iCursor); assert( (m-1)==toTheLeft ); toTheLeft |= m; } } #endif - /* Analyze all of the subexpressions. */ - exprAnalyzeAll(pTabList, pWC); + /* Analyze all of the subexpressions. Note that exprAnalyze() might + ** add new virtual terms onto the end of the WHERE clause. We do not + ** want to analyze these virtual terms, so start analyzing at the end + ** and work forward so that the added virtual terms are never processed. + */ + exprAnalyzeAll(pTabList, &pWInfo->sWC); if( db->mallocFailed ){ goto whereBeginError; } - /* Check if the DISTINCT qualifier, if there is one, is redundant. - ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to - ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. - */ - if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ - pDistinct = 0; - pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; - } - - /* Chose the best index to use for each table in the FROM clause. - ** - ** This loop fills in the following fields: - ** - ** pWInfo->a[].pIdx The index to use for this level of the loop. - ** pWInfo->a[].wsFlags WHERE_xxx flags associated with pIdx - ** pWInfo->a[].nEq The number of == and IN constraints - ** pWInfo->a[].iFrom Which term of the FROM clause is being coded - ** pWInfo->a[].iTabCur The VDBE cursor for the database table - ** pWInfo->a[].iIdxCur The VDBE cursor for the index - ** pWInfo->a[].pTerm When wsFlags==WO_OR, the OR-clause term - ** - ** This loop also figures out the nesting order of tables in the FROM - ** clause. - */ - notReady = ~(Bitmask)0; - andFlags = ~0; - WHERETRACE(("*** Optimizer Start ***\n")); - for(i=iFrom=0, pLevel=pWInfo->a; i=0 && bestJ<0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ - for(j=iFrom, pTabItem=&pTabList->a[j]; jjointype & (JT_LEFT|JT_CROSS))!=0; - if( j!=iFrom && doNotReorder ) break; - m = getMask(pMaskSet, pTabItem->iCursor); - if( (m & notReady)==0 ){ - if( j==iFrom ) iFrom++; - continue; - } - mask = (isOptimal ? m : notReady); - pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); - pDist = (i==0 ? pDistinct : 0); - if( pTabItem->pIndex==0 ) nUnconstrained++; - - WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", - j, isOptimal)); - assert( pTabItem->pTab ); - - if( bestMatchIdx(pParse, pWC, pTabItem, notReady, &sCost) ){ - /* no-op */ - }else -#ifndef SQLITE4_OMIT_VIRTUALTABLE - if( IsVirtual(pTabItem->pTab) ){ - sqlite4_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost, pp); - }else -#endif - { - bestKVIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - pDist, &sCost); - } - - assert( isOptimal || (sCost.used¬Ready)==0 ); - - /* If an INDEXED BY clause is present, then the plan must use that - ** index if it uses any index at all */ - assert( pTabItem->pIndex==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || sCost.plan.u.pIdx==pTabItem->pIndex ); - - if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ - notIndexed |= m; - } - - /* Conditions under which this table becomes the best so far: - ** - ** (1) The table must not depend on other tables that have not - ** yet run. - ** - ** (2) A full-table-scan plan cannot supercede indexed plan unless - ** the full-table-scan is an "optimal" plan as defined above. - ** - ** (3) All tables have an INDEXED BY clause or this table lacks an - ** INDEXED BY clause or this table uses the specific - ** index specified by its INDEXED BY clause. This rule ensures - ** that a best-so-far is always selected even if an impossible - ** combination of INDEXED BY clauses are given. The error - ** will be detected and relayed back to the application later. - ** The NEVER() comes about because rule (2) above prevents - ** An indexable full-table-scan from reaching rule (3). - ** - ** (4) The plan cost must be lower than prior plans or else the - ** cost must be the same and the number of rows must be lower. - */ - if( (sCost.used¬Ready)==0 /* (1) */ - && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ - || (bestPlan.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 - || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) - && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ - || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) - && (bestJ<0 || sCost.rCost=0 ); - assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d" - " with cost=%g and nRow=%g\n", - bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); - /* The ALWAYS() that follows was added to hush up clang scan-build */ - if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 && ALWAYS(ppOrderBy) ){ - *ppOrderBy = 0; - } - if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ - assert( pWInfo->eDistinct==0 ); - pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; - } - andFlags &= bestPlan.plan.wsFlags; - pLevel->plan = bestPlan.plan; - testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); - testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX ); - if( bestPlan.plan.wsFlags & (WHERE_INDEXED|WHERE_TEMP_INDEX) ){ - pLevel->iIdxCur = pParse->nTab++; - }else{ - pLevel->iIdxCur = -1; - } - notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); - pLevel->iFrom = (u8)bestJ; - if( bestPlan.plan.nRow>=(double)1 ){ - pParse->nQueryLoop *= bestPlan.plan.nRow; - } - - /* Check that if the table scanned by this loop iteration had an - ** INDEXED BY clause attached to it, that the named index is being - ** used for the scan. If not, then query compilation has failed. - ** Return an error. - */ - pIdx = pTabList->a[bestJ].pIndex; - if( pIdx ){ - if( (bestPlan.plan.wsFlags & WHERE_INDEXED)==0 ){ - sqlite4ErrorMsg(pParse, "cannot use index: %s", pIdx->zName); - goto whereBeginError; - }else{ - /* If an INDEXED BY clause is used, the bestIndex() function is - ** guaranteed to find the index specified in the INDEXED BY clause - ** if it find an index at all. */ - assert( bestPlan.plan.u.pIdx==pIdx ); - } - } - } - WHERETRACE(("*** Optimizer Finished ***\n")); - if( pParse->nErr || db->mallocFailed ){ - goto whereBeginError; - } - - /* If the total query only selects a single row, then the ORDER BY - ** clause is irrelevant. - */ - if( (andFlags & WHERE_UNIQUE)!=0 && ppOrderBy ){ - *ppOrderBy = 0; - } + /* If the ORDER BY (or GROUP BY) clause contains references to general + ** expressions, then we won't be able to satisfy it using indices, so + ** go ahead and disable it now. + */ + if( pOrderBy && (wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ + for(ii=0; iinExpr; ii++){ + Expr *pExpr = sqlite4ExprSkipCollate(pOrderBy->a[ii].pExpr); + if( pExpr->op!=TK_COLUMN ){ + pWInfo->pOrderBy = pOrderBy = 0; + break; + }else if( pExpr->iColumn<0 ){ + break; + } + } + } + + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + /* The DISTINCT marking is pointless. Ignore it. */ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + }else if( pOrderBy==0 ){ + /* Try to ORDER BY the result set to make distinct processing easier */ + pWInfo->wctrlFlags |= WHERE_DISTINCTBY; + pWInfo->pOrderBy = pResultSet; + } + } + + /* Construct the WhereLoop objects */ + WHERETRACE(0xffff,("*** Optimizer Start ***\n")); + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + + /* Display all of the WhereLoop objects if wheretrace is enabled */ +#ifdef WHERETRACE_ENABLED + if( sqlite4WhereTrace ){ + WhereLoop *p; + int i; + static char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%sizeof(zLabel)]; + whereLoopPrint(p, pTabList); + } + } +#endif + + wherePathSolver(pWInfo, 0); + if( db->mallocFailed ) goto whereBeginError; + if( pWInfo->pOrderBy ){ + wherePathSolver(pWInfo, pWInfo->nRowOut+1); + if( db->mallocFailed ) goto whereBeginError; + } + } + if( pWInfo->pOrderBy==0 && (db->flags & SQLITE4_ReverseOrder)!=0 ){ + pWInfo->revMask = (Bitmask)(-1); + } + if( pParse->nErr || NEVER(db->mallocFailed) ){ + goto whereBeginError; + } +#ifdef WHERETRACE_ENABLED + if( sqlite4WhereTrace ){ + int ii; + sqlite4DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); + if( pWInfo->bOBSat ){ + sqlite4DebugPrintf(" ORDERBY=0x%llx", pWInfo->revMask); + } + switch( pWInfo->eDistinct ){ + case WHERE_DISTINCT_UNIQUE: { + sqlite4DebugPrintf(" DISTINCT=unique"); + break; + } + case WHERE_DISTINCT_ORDERED: { + sqlite4DebugPrintf(" DISTINCT=ordered"); + break; + } + case WHERE_DISTINCT_UNORDERED: { + sqlite4DebugPrintf(" DISTINCT=unordered"); + break; + } + } + sqlite4DebugPrintf("\n"); + for(ii=0; iinLevel; ii++){ + whereLoopPrint(pWInfo->a[ii].pWLoop, pTabList); + } + } +#endif + /* Attempt to omit tables from the join that do not effect the result */ + if( pWInfo->nLevel>=2 + && pResultSet!=0 + && OptimizationEnabled(db, SQLITE4_OmitNoopJoin) + ){ + Bitmask tabUsed = exprListTableUsage(pMaskSet, pResultSet); + if( pOrderBy ) tabUsed |= exprListTableUsage(pMaskSet, pOrderBy); + while( pWInfo->nLevel>=2 ){ + WhereTerm *pTerm, *pEnd; + pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; + if( (pWInfo->pTabList->a[pLoop->iTab].jointype & JT_LEFT)==0 ) break; + if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 + ){ + break; + } + if( (tabUsed & pLoop->maskSelf)!=0 ) break; + pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; + for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + ){ + break; + } + } + if( pTerm drop loop %c not used\n", pLoop->cId)); + pWInfo->nLevel--; + nTabList--; + } + } + WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. ** The one-pass algorithm only works if the WHERE clause constraints ** the statement to update a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 && (andFlags & WHERE_UNIQUE)!=0 ){ + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ pWInfo->okOnePass = 1; - pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY; + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; } /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ - sqlite4CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; - pWInfo->nRowOut = (double)1; - for(i=0, pLevel=pWInfo->a; ia; iia[pLevel->iFrom]; pTab = pTabItem->pTab; - pLevel->iTabCur = pTabItem->iCursor; - pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite4SchemaToIndex(db, pTab->pSchema); + pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ /* Do nothing */ }else #ifndef SQLITE4_OMIT_VIRTUALTABLE - if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ const char *pVTab = (const char *)sqlite4GetVTable(db, pTab); int iCur = pTabItem->iCursor; sqlite4VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else if( IsVirtual(pTab) ){ + /* noop */ }else #endif - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ int op = pWInfo->okOnePass ? OP_OpenWrite : OP_OpenRead; sqlite4OpenPrimaryKey(pParse, pTabItem->iCursor, iDb, pTab, op); - testcase( pTab->nCol==BMS-1 ); - testcase( pTab->nCol==BMS ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 ); + testcase( !pWInfo->okOnePass && pTab->nCol==BMS ); +#if 0 + if( !pWInfo->okOnePass && pTab->nColcolUsed; + int n = 0; + for(; b; b=b>>1, n++){} + sqlite4VdbeChangeP4(v, sqlite4VdbeCurrentAddr(v)-1, + SQLITE4_INT_TO_PTR(n), P4_INT32); + assert( n<=pTab->nCol ); + } +#endif + } +#if 0 + else{ + sqlite4TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } +#endif #ifndef SQLITE4_OMIT_AUTOMATIC_INDEX - if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - constructAutomaticIndex(pParse, pWC, pTabItem, notReady, pLevel); + if( (pLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + constructAutomaticIndex(pParse, &pWInfo->sWC, pTabItem, notReady, pLevel); }else #endif - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - Index *pIx = pLevel->plan.u.pIdx; + if( pLoop->wsFlags & WHERE_INDEXED ){ + Index *pIx = pLoop->u.btree.pIndex; if( pIx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){ pLevel->iIdxCur = pTabItem->iCursor; - }else if( pIx->eIndexType!=SQLITE4_INDEX_FTS5 ){ + } + else if( pIx->eIndexType!=SQLITE4_INDEX_FTS5 ){ KeyInfo *pKey = sqlite4IndexKeyinfo(pParse, pIx); - int iIdxCur = pLevel->iIdxCur; + /* FIXME: As an optimization use pTabItem->iCursor if WHERE_IDX_ONLY */ + int iIndexCur = pLevel->iIdxCur = iIdxCur ? iIdxCur : pParse->nTab++; assert( pIx->pSchema==pTab->pSchema ); - assert( iIdxCur>=0 ); - sqlite4VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIx->tnum, iDb, + assert( iIndexCur>=0 ); + sqlite4VdbeAddOp4(v, OP_OpenRead, iIndexCur, pIx->tnum, iDb, (char*)pKey, P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIx->zName)); } } sqlite4CodeVerifySchema(pParse, iDb); - notReady &= ~getMask(pWC->pMaskSet, pTabItem->iCursor); + notReady &= ~getMask(&pWInfo->sMaskSet, pTabItem->iCursor); } pWInfo->iTop = sqlite4VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; /* Generate the code to do the search. Each iteration of the for ** loop below generates code for a single nested loop of the VM ** program. */ notReady = ~(Bitmask)0; - for(i=0; ia[i]; - explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); - notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady, pWhere); + for(ii=0; iia[ii]; + explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; } -#ifdef SQLITE4_TEST /* For testing and debugging use only */ - /* Record in the query plan information about the current table - ** and the index used to access it (if any). If the table itself - ** is not used, its name is just '{}'. If no index is used - ** the index is listed as "{}". If the primary key is used the - ** index name is '*'. - */ - for(i=0; ia[i]; - pTabItem = &pTabList->a[pLevel->iFrom]; - z = pTabItem->zAlias; - if( z==0 ) z = pTabItem->pTab->zName; - n = sqlite4Strlen30(z); - if( n+nQPlan < sizeof(sqlite4_query_plan)-10 ){ - if( pLevel->plan.wsFlags & WHERE_IDX_ONLY ){ - memcpy(&sqlite4_query_plan[nQPlan], "{}", 2); - nQPlan += 2; - }else{ - memcpy(&sqlite4_query_plan[nQPlan], z, n); - nQPlan += n; - } - sqlite4_query_plan[nQPlan++] = ' '; - } - if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - n = sqlite4Strlen30(pLevel->plan.u.pIdx->zName); - if( n+nQPlan < sizeof(sqlite4_query_plan)-2 ){ - memcpy(&sqlite4_query_plan[nQPlan], pLevel->plan.u.pIdx->zName, n); - nQPlan += n; - sqlite4_query_plan[nQPlan++] = ' '; - } - }else{ - memcpy(&sqlite4_query_plan[nQPlan], "{} ", 3); - nQPlan += 3; - } - } - while( nQPlan>0 && sqlite4_query_plan[nQPlan-1]==' ' ){ - sqlite4_query_plan[--nQPlan] = 0; - } - sqlite4_query_plan[nQPlan] = 0; - nQPlan = 0; -#endif /* SQLITE4_TEST // Testing and debugging use only */ - - /* Record the continuation address in the WhereInfo structure. Then - ** clean up and return. - */ + /* Done. */ return pWInfo; /* Jump here if malloc fails */ whereBeginError: if( pWInfo ){ @@ -5099,44 +6082,46 @@ void sqlite4WhereEnd(WhereInfo *pWInfo){ Parse *pParse = pWInfo->pParse; Vdbe *v = pParse->pVdbe; int i; WhereLevel *pLevel; + WhereLoop *pLoop; SrcList *pTabList = pWInfo->pTabList; sqlite4 *db = pParse->db; /* Generate loop termination code. */ sqlite4ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ pLevel = &pWInfo->a[i]; + pLoop = pLevel->pWLoop; sqlite4VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ sqlite4VdbeAddOp2(v, pLevel->op, pLevel->p1, pLevel->p2); sqlite4VdbeChangeP5(v, pLevel->p5); } - if( pLevel->plan.wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; sqlite4VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite4VdbeJumpHere(v, pIn->addrInTop+1); - sqlite4VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop); + sqlite4VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); sqlite4VdbeJumpHere(v, pIn->addrInTop-1); } sqlite4DbFree(db, pLevel->u.in.aInLoop); } sqlite4VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->iLeftJoin ){ int addr; addr = sqlite4VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); - assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 - || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ); - if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){ + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ sqlite4VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); } - if( pLevel->iIdxCur>=0 ){ + if( pLoop->wsFlags & WHERE_INDEXED ){ sqlite4VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); } if( pLevel->op==OP_Return ){ sqlite4VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); }else{ @@ -5151,75 +6136,64 @@ */ sqlite4VdbeResolveLabel(v, pWInfo->iBreak); /* Close all of the cursors that were opened by sqlite4WhereBegin. */ - assert( pWInfo->nLevel==1 || pWInfo->nLevel==pTabList->nSrc ); + assert( pWInfo->nLevel<=pTabList->nSrc ); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ - SrcListItem *pTabItem = &pTabList->a[pLevel->iFrom]; + struct SrcListItem *pTabItem = &pTabList->a[pLevel->iFrom]; Table *pTab = pTabItem->pTab; assert( pTab!=0 ); + pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)==0 && pTab->pSelect==0 && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ - int ws = pLevel->plan.wsFlags; + int ws = pLoop->wsFlags; if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){ sqlite4VdbeAddOp1(v, OP_Close, pTabItem->iCursor); } - if( (ws & WHERE_INDEXED)!=0 && (ws & WHERE_TEMP_INDEX)==0 ){ + if( (ws & WHERE_INDEXED)!=0 && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 ){ if( pLevel->iIdxCur!=pTabItem->iCursor ){ sqlite4VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); } } } - /* If this scan uses an index, make code substitutions to read data - ** from the index in preference to the table. Sometimes, this means - ** the table need never be read from. This is a performance boost, - ** as the vdbe level waits until the table is read before actually - ** seeking the table cursor to the record corresponding to the current - ** position in the index. + /* If this scan uses an index, make VDBE code substitutions to read data + ** from the index instead of from the table where possible. In some cases + ** this optimization prevents the table from ever being read, which can + ** yield a significant performance boost. ** ** Calls to the code generator in between sqlite4WhereBegin and ** sqlite4WhereEnd will have created code that references the table ** directly. This loop scans all that code looking for opcodes ** that reference the table and converts them into opcodes that ** reference the index. */ - if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX) && !db->mallocFailed ){ - VdbeOp *pOp; - VdbeOp *pEnd; - - assert( pLevel->plan.u.pIdx ); - assert( pLevel->iTabCur!=pLevel->iIdxCur ); - pOp = sqlite4VdbeGetOp(v, pWInfo->iTop); - pEnd = &pOp[sqlite4VdbeCurrentAddr(v) - pWInfo->iTop]; - - while( pOpp1==pLevel->iTabCur && pOp->opcode==OP_Column ){ - pOp->p1 = pLevel->iIdxCur; - } - pOp++; - } - } - - if( (pLevel->plan.wsFlags & WHERE_INDEXED) - && (pLevel->plan.u.pIdx->eIndexType==SQLITE4_INDEX_FTS5) - ){ - VdbeOp *pOp; - VdbeOp *pEnd; - - assert( pLevel->iTabCur!=pLevel->iIdxCur ); - pOp = sqlite4VdbeGetOp(v, pWInfo->iTop); - pEnd = &pOp[sqlite4VdbeCurrentAddr(v) - pWInfo->iTop]; - - while( pOpp1==pLevel->iTabCur && pOp->opcode==OP_Mifunction ){ - pOp->p1 = pLevel->iIdxCur; - } - pOp++; + if( (pLoop->wsFlags & WHERE_AUTO_INDEX) && !db->mallocFailed ){ + int k, j, last; + VdbeOp *pOp; + Index *pIdx = pLoop->u.btree.pIndex; + + pOp = sqlite4VdbeGetOp(v, pWInfo->iTop); + last = sqlite4VdbeCurrentAddr(v); + for(k=pWInfo->iTop; kp1!=pLevel->iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + for(j=0; jnColumn; j++){ + if( pOp->p2==pIdx->aiColumn[j] ){ + pOp->p2 = j; + pOp->p1 = pLevel->iIdxCur; + break; + } + } + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || jnColumn ); + }else if( pOp->opcode==OP_Rowid ){ + pOp->p1 = pLevel->iIdxCur; + pOp->opcode = OP_IdxRowid; + } } } } /* Final cleanup Index: test/analyze4.test ================================================================== --- test/analyze4.test +++ test/analyze4.test @@ -36,11 +36,11 @@ ANALYZE; } # Should choose the t1a index since it is more specific than t1b. db eval {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=5 AND b IS NULL} -} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}} # Verify that the t1b index shows that it does not narrow down the # search any at all. # do_test analyze4-1.1 { Index: test/between.test ================================================================== --- test/between.test +++ test/between.test @@ -46,18 +46,28 @@ } } {} # This procedure executes the SQL. Then it appends to the result the # "sort" or "nosort" keyword depending on whether or not any sorting -# is done. Then it appends the ::sqlite_query_plan variable. +# is done. Then it appends the names of the table and index used. # proc queryplan {sql} { set ::sqlite_sort_count 0 set data [execsql $sql] if {$::sqlite_sort_count} {set x sort} {set x nosort} lappend data $x - return [concat $data $::sqlite_query_plan] + set eqp [execsql "EXPLAIN QUERY PLAN $sql"] + # puts eqp=$eqp + foreach {a b c x} $eqp { + if {[regexp { TABLE (\w+ AS )?(\w+) USING.* INDEX (\w+)\y} \ + $x all as tab idx]} { + lappend data $tab $idx + } elseif {[regexp { TABLE (\w+ AS )?(\w+)\y} $x all as tab]} { + lappend data $tab * + } + } + return $data } do_test between-1.1.1 { queryplan { SELECT * FROM t1 WHERE w BETWEEN 5 AND 6 ORDER BY +w @@ -65,36 +75,36 @@ } {5 2 36 38 6 2 49 51 sort t1 i1w} do_test between-1.1.2 { queryplan { SELECT * FROM t1 WHERE +w BETWEEN 5 AND 6 ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 t1} do_test between-1.2.1 { queryplan { SELECT * FROM t1 WHERE w BETWEEN 5 AND 65-y ORDER BY +w } } {5 2 36 38 6 2 49 51 sort t1 i1w} do_test between-1.2.2 { queryplan { SELECT * FROM t1 WHERE +w BETWEEN 5 AND 65-y ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 t1} do_test between-1.3.1 { queryplan { SELECT * FROM t1 WHERE w BETWEEN 41-y AND 6 ORDER BY +w } } {5 2 36 38 6 2 49 51 sort t1 i1w} do_test between-1.3.2 { queryplan { SELECT * FROM t1 WHERE +w BETWEEN 41-y AND 6 ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 t1} do_test between-1.4 { queryplan { SELECT * FROM t1 WHERE w BETWEEN 41-y AND 65-y ORDER BY +w } -} {5 2 36 38 6 2 49 51 sort t1 {}} +} {5 2 36 38 6 2 49 51 sort t1 t1} do_test between-1.5.1 { queryplan { SELECT * FROM t1 WHERE 26 BETWEEN y AND z ORDER BY +w } } {4 2 25 27 sort t1 i1zyx} @@ -105,9 +115,9 @@ } {4 2 25 27 sort t1 i1zyx} do_test between-1.5.3 { queryplan { SELECT * FROM t1 WHERE 26 BETWEEN y AND +z ORDER BY +w } -} {4 2 25 27 sort t1 {}} +} {4 2 25 27 sort t1 t1} finish_test Index: test/collate4.test ================================================================== --- test/collate4.test +++ test/collate4.test @@ -87,11 +87,11 @@ do_test collate4-1.1.5 { cksort {SELECT b FROM collate4t1 ORDER BY b COLLATE TEXT} } {{} A B a b nosort} do_test collate4-1.1.6 { cksort {SELECT b FROM collate4t1 ORDER BY b COLLATE NOCASE} -} {{} a A b B sort} +} {{} A a B b sort} do_test collate4-1.1.7 { execsql { CREATE TABLE collate4t2( a PRIMARY KEY COLLATE NOCASE, @@ -164,14 +164,14 @@ CREATE INDEX collate4i4 ON collate4t4(b COLLATE NOCASE); } } {} do_test collate4-1.1.22 { cksort {SELECT a FROM collate4t4 ORDER BY a} -} {{} a A b B sort} +} {{} A a B b sort} do_test collate4-1.1.23 { cksort {SELECT a FROM collate4t4 ORDER BY a COLLATE NOCASE} -} {{} a A b B sort} +} {{} A a B b sort} do_test collate4-1.1.24 { cksort {SELECT a FROM collate4t4 ORDER BY a COLLATE TEXT} } {{} A B a b nosort} do_test collate4-1.1.25 { cksort {SELECT b FROM collate4t4 ORDER BY b} @@ -215,11 +215,11 @@ do_test collate4-1.2.4 { cksort {SELECT a FROM collate4t1 ORDER BY a, b} } {{} A a B b nosort} do_test collate4-1.2.5 { cksort {SELECT a FROM collate4t1 ORDER BY a, b COLLATE nocase} -} {{} a A b B sort} +} {{} A a B b sort} do_test collate4-1.2.6 { cksort {SELECT a FROM collate4t1 ORDER BY a, b COLLATE text} } {{} A a B b nosort} do_test collate4-1.2.7 { @@ -264,14 +264,14 @@ CREATE INDEX collate4i2 ON collate4t3(a COLLATE TEXT, b COLLATE NOCASE); } } {} do_test collate4-1.2.15 { cksort {SELECT a FROM collate4t3 ORDER BY a} -} {{} a A b B sort} +} {{} A a B b sort} do_test collate4-1.2.16 { cksort {SELECT a FROM collate4t3 ORDER BY a COLLATE nocase} -} {{} a A b B sort} +} {{} A a B b sort} do_test collate4-1.2.17 { cksort {SELECT a FROM collate4t3 ORDER BY a COLLATE text} } {{} A B a b nosort} do_test collate4-1.2.18 { cksort {SELECT a FROM collate4t3 ORDER BY a COLLATE text, b} @@ -607,16 +607,16 @@ CREATE INDEX collate4i1 ON collate4t1(a COLLATE NUMERIC); } count { SELECT min(a) FROM collate4t1; } -} {10 5} +} {10 9} do_test collate4-4.6 { count { SELECT max(a) FROM collate4t1; } -} {20 5} +} {20 9} do_test collate4-4.7 { execsql { DROP TABLE collate4t1; } } {} Index: test/descidx3.test ================================================================== --- test/descidx3.test +++ test/descidx3.test @@ -109,15 +109,15 @@ ifcapable subquery { # If the subquery capability is not compiled in to the binary, then # the IN(...) operator is not available. Hence these tests cannot be # run. do_test descidx3-4.1 { - execsql { + lsort [execsql { UPDATE t1 SET a=2 WHERE i<6; SELECT i FROM t1 WHERE a IN (1,2) AND b>0 AND b<'zzz'; - } - } {8 6 2 4 3} + }] + } {2 3 4 6 8} do_test descidx3-4.2 { execsql { UPDATE t1 SET a=1; SELECT i FROM t1 WHERE a IN (1,2) AND b>0 AND b<'zzz'; } Index: test/e_createtable.test ================================================================== --- test/e_createtable.test +++ test/e_createtable.test @@ -1354,17 +1354,17 @@ CREATE TABLE t1(a, b PRIMARY KEY); CREATE TABLE t2(a, b, c, UNIQUE(b, c)); } do_createtable_tests 4.10 { 1 "EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b = 5" - {0 0 0 {SEARCH TABLE t1 USING PRIMARY KEY (b=?) (~1 rows)}} + {0 0 0 {SEARCH TABLE t1 USING INDEX t1 (b=?)}} 2 "EXPLAIN QUERY PLAN SELECT * FROM t2 ORDER BY b, c" - {0 0 0 {SCAN TABLE t2 USING INDEX sqlite_t2_unique1 (~1000000 rows)}} + {0 0 0 {SCAN TABLE t2 USING INDEX sqlite_t2_unique1}} 3 "EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE b=10 AND c>10" - {0 0 0 {SEARCH TABLE t2 USING INDEX sqlite_t2_unique1 (b=? AND c>?) (~2 rows)}} + {0 0 0 {SEARCH TABLE t2 USING INDEX sqlite_t2_unique1 (b=? AND c>?)}} } # EVIDENCE-OF: R-45493-35653 A CHECK constraint may be attached to a # column definition or specified as a table constraint. In practice it # makes no difference. Index: test/e_fkey.test ================================================================== --- test/e_fkey.test +++ test/e_fkey.test @@ -963,19 +963,19 @@ do_execsql_test e_fkey-25.2 { PRAGMA foreign_keys = OFF; EXPLAIN QUERY PLAN DELETE FROM artist WHERE 1; EXPLAIN QUERY PLAN SELECT rowid FROM track WHERE trackartist = ?; } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SCAN TABLE track (~100000 rows)} + 0 0 0 {SCAN TABLE artist USING INDEX artist} + 0 0 0 {SCAN TABLE track USING INDEX track} } do_execsql_test e_fkey-25.3 { PRAGMA foreign_keys = ON; EXPLAIN QUERY PLAN DELETE FROM artist WHERE 1; } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SCAN TABLE track (~100000 rows)} + 0 0 0 {SCAN TABLE artist USING INDEX artist} + 0 0 0 {SCAN TABLE track USING INDEX track} } do_test e_fkey-25.4 { execsql { INSERT INTO artist VALUES(5, 'artist 5'); INSERT INTO artist VALUES(6, 'artist 6'); @@ -1088,19 +1088,19 @@ eqp { INSERT INTO artist VALUES(?, ?) } } {} do_execsql_test e_fkey-27.3 { EXPLAIN QUERY PLAN UPDATE artist SET artistid = ?, artistname = ? } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SEARCH TABLE track USING INDEX trackindex (trackartist=?) (~10 rows)} - 0 0 0 {SEARCH TABLE track USING INDEX trackindex (trackartist=?) (~10 rows)} + 0 0 0 {SCAN TABLE artist USING INDEX artist} + 0 0 0 {SEARCH TABLE track USING INDEX trackindex (trackartist=?)} + 0 0 0 {SEARCH TABLE track USING INDEX trackindex (trackartist=?)} } do_execsql_test e_fkey-27.4 { EXPLAIN QUERY PLAN DELETE FROM artist } { - 0 0 0 {SCAN TABLE artist (~1000000 rows)} - 0 0 0 {SEARCH TABLE track USING INDEX trackindex (trackartist=?) (~10 rows)} + 0 0 0 {SCAN TABLE artist USING INDEX artist} + 0 0 0 {SEARCH TABLE track USING INDEX trackindex (trackartist=?)} } ########################################################################### ### SECTION 4.1: Composite Foreign Key Constraints Index: test/like.test ================================================================== --- test/like.test +++ test/like.test @@ -187,11 +187,11 @@ do_test like-3.1 { set sqlite_like_count 0 queryplan { SELECT x FROM t1 WHERE x LIKE 'abc%' ORDER BY 1; } -} {ABC {ABC abc xyz} abc abcd sort t1 *} +} {ABC {ABC abc xyz} abc abcd sort t1 t1} do_test like-3.2 { set sqlite_like_count } {12} # With an index on t1.x and case sensitivity on, optimize completely. @@ -300,11 +300,11 @@ DROP INDEX i1; } queryplan { SELECT x FROM t1 WHERE x LIKE 'abc%' ORDER BY 1; } -} {abc abcd sort t1 *} +} {abc abcd sort t1 t1} do_test like-3.16 { set sqlite_like_count } 12 # No GLOB optimization without an index. @@ -312,11 +312,11 @@ do_test like-3.17 { set sqlite_like_count 0 queryplan { SELECT x FROM t1 WHERE x GLOB 'abc*' ORDER BY 1; } -} {abc abcd sort t1 *} +} {abc abcd sort t1 t1} do_test like-3.18 { set sqlite_like_count } 12 # GLOB is optimized regardless of the case_sensitive_like setting. @@ -826,17 +826,17 @@ do_test like-11.1 { db eval {PRAGMA case_sensitive_like=OFF;} queryplan { SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; } -} {abc abcd ABC ABCD nosort t11 *} +} {abc abcd ABC ABCD nosort t11 t11} do_test like-11.2 { db eval {PRAGMA case_sensitive_like=ON;} queryplan { SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; } -} {abc abcd nosort t11 *} +} {abc abcd nosort t11 t11} do_test like-11.3 { db eval { PRAGMA case_sensitive_like=OFF; CREATE INDEX t11b ON t11(b); } @@ -847,11 +847,11 @@ do_test like-11.4 { db eval {PRAGMA case_sensitive_like=ON;} queryplan { SELECT b FROM t11 WHERE b LIKE 'abc%' ORDER BY a; } -} {abc abcd nosort t11 *} +} {abc abcd nosort t11 t11} do_test like-11.5 { db eval { PRAGMA case_sensitive_like=OFF; DROP INDEX t11b; CREATE INDEX t11bnc ON t11(b COLLATE nocase); Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -129,19 +129,18 @@ # quick # full # lappend ::testsuitelist xxx +# fts5expr1.test fts5query1.test fts5rnd1.test fts5create.test fts5snippet.test test_suite "src4" -prefix "" -description { } -files { simple.test simple2.test lsm1.test lsm2.test lsm3.test lsm4.test lsm5.test csr1.test ckpt1.test mc1.test - fts5expr1.test fts5query1.test fts5rnd1.test fts5create.test - fts5snippet.test alter.test alter3.test alter4.test analyze.test analyze3.test analyze4.test analyze5.test analyze6.test analyze7.test analyze8.test auth.test auth2.test auth3.test auth4.test Index: test/simple.test ================================================================== --- test/simple.test +++ test/simple.test @@ -695,11 +695,16 @@ CREATE TRIGGER tr1 INSTEAD OF DELETE ON v1 BEGIN INSERT INTO log VALUES(old.b, old.a); END; } do_execsql_test 38.3 { + SELECT * FROM v1; +} {3 4} +do_execsql_test 38.4 { DELETE FROM v1 WHERE a = 3; +} +do_execsql_test 38.5 { SELECT * FROM log; } {4 3} #------------------------------------------------------------------------- reset_db @@ -1359,15 +1364,15 @@ do_catchsql_test 70.2 { select * from maintable as m inner join joinme as j indexed by joinme_id_text_idx on ( m.id = j.id_int) -} {1 {cannot use index: joinme_id_text_idx}} +} {1 {no query solution}} do_catchsql_test 70.3 { select * from maintable, joinme INDEXED by joinme_id_text_idx -} {1 {cannot use index: joinme_id_text_idx}} +} {1 {no query solution}} #------------------------------------------------------------------------- # This is testing that the "phantom" runs feature works. # # UPDATE: Said feature was dropped early in development. But the test @@ -1613,9 +1618,55 @@ INSERT INTO t3 SELECT x, 'abc' || y || 'xyz' FROM t1; CREATE INDEX i3 ON t3(y); SELECT x FROM t3 WHERE y LIKE 'abcX%'; } {} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 86.0 { + SELECT * FROM sqlite_master; +} {} +do_execsql_test 86.1 { + CREATE TABLE t1(a PRIMARY KEY, b); +} +do_execsql_test 86.2 { + INSERT INTO t1 VALUES(1, 'one'); +} +do_execsql_test 86.3 { + SELECT * FROM t1; +} {1 one} +do_execsql_test 86.4 { + SELECT * FROM t1 WHERE a = 1; +} {1 one} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 87.1 { + CREATE TABLE t6(a INTEGER PRIMARY KEY, b TEXT); + CREATE INDEX t6i1 ON t6(b); +} {} +do_eqp_test 87.2 { + SELECT * FROM t6 ORDER BY b, a; +} {0 0 0 {SCAN TABLE t6 USING INDEX t6i1}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 88.1 { + CREATE TABLE t8(a INTEGER PRIMARY KEY, b TEXT); + CREATE UNIQUE INDEX t8i ON t8(b); +} +do_eqp_test 88.2 { + SELECT * FROM t8 x ORDER BY x.b, x.a, x.b||x.a +} {0 0 0 {SCAN TABLE t8 AS x USING INDEX t8i}} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 89.1 { + CREATE TABLE t1(a COLLATE NOCASE); + CREATE INDEX i1 ON t1(a); +} +do_eqp_test 89.2 { + SELECT * FROM t1 ORDER BY a; +} {0 0 0 {SCAN TABLE t1 USING INDEX i1}} finish_test Index: test/subquery.test ================================================================== --- test/subquery.test +++ test/subquery.test @@ -239,12 +239,15 @@ SELECT * FROM t4 WHERE x IN (SELECT a FROM t3); } } {10.0} do_test subquery-2.5.3.2 { # Verify that the t4i index was not used in the previous query - set ::sqlite_query_plan -} {t4 {}} + execsql { + EXPLAIN QUERY PLAN + SELECT * FROM t4 WHERE x IN (SELECT a FROM t3); + } +} {/SCAN TABLE t4 /} do_test subquery-2.5.4 { execsql { DROP TABLE t3; DROP TABLE t4; } @@ -328,17 +331,18 @@ do_test subquery-3.3.5 { execsql { SELECT a, (SELECT count(*) FROM t2 WHERE a=c) FROM t1; } } {1 1 2 1} + #------------------------------------------------------------------ # These tests - subquery-4.* - use the TCL statement cache to try # and expose bugs to do with re-using statements that have been # passed to sqlite4_reset(). # -# One problem was that VDBE memory cells were not being initialised +# One problem was that VDBE memory cells were not being initialized # to NULL on the second and subsequent executions. # do_test subquery-4.1.1 { execsql { SELECT (SELECT a FROM t1); Index: test/test_main.c ================================================================== --- test/test_main.c +++ test/test_main.c @@ -1497,11 +1497,11 @@ Tcl_Obj *pDel; }; typedef struct TestNeededX TestNeededX; static void testCollationNeeded(void *pCtx, sqlite4 *db, const char *zReq){ - TestNeededX *p = (TestCollationX *)pCtx; + TestNeededX *p = (TestNeededX *)pCtx; Tcl_Obj *pScript; int rc; pScript = Tcl_DuplicateObj(p->pNeeded); Tcl_IncrRefCount(pScript); @@ -4122,11 +4122,11 @@ Tcl_Obj *pRet; const void *zName16; const void *(*xFunc)(sqlite4_stmt*, int, int*); int dummy; - xFunc = (const void *(*)(sqlite4_stmt*, int))clientData; + xFunc = (const void *(*)(sqlite4_stmt*, int, int*))clientData; if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetString(objv[0]), " STMT column", 0); return TCL_ERROR; } @@ -4312,12 +4312,10 @@ #ifdef SQLITE4_DEBUG extern int sqlite4WhereTrace; extern int sqlite4OSTrace; #endif #ifdef SQLITE4_TEST - extern char sqlite4_query_plan[]; - static char *query_plan = sqlite4_query_plan; #ifdef SQLITE4_ENABLE_FTS3 extern int sqlite4_fts3_enable_parentheses; #endif #endif @@ -4351,14 +4349,10 @@ (char*)&sqlite4_xferopt_count, TCL_LINK_INT); #if SQLITE4_OS_WIN Tcl_LinkVar(interp, "sqlite_os_type", (char*)&sqlite4_os_type, TCL_LINK_INT); #endif -#ifdef SQLITE4_TEST - Tcl_LinkVar(interp, "sqlite_query_plan", - (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); -#endif #ifdef SQLITE4_DEBUG Tcl_LinkVar(interp, "sqlite_where_trace", (char*)&sqlite4WhereTrace, TCL_LINK_INT); #endif Tcl_LinkVar(interp, "sqlite_static_bind_value", Index: test/test_mem.c ================================================================== --- test/test_mem.c +++ test/test_mem.c @@ -14,12 +14,10 @@ #include #include "sqliteInt.h" #include "testInt.h" -#define MIN(x,y) ((x)<(y) ? (x) : (y)) - #if defined(__GLIBC__) extern int backtrace(void**,int); extern void backtrace_symbols_fd(void*const*,int,int); # define TM_BACKTRACE 12 #else Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -493,16 +493,29 @@ if {[catch {uplevel #0 "$cmd;\n"} result]} { puts "\nError: $result" fail_test $name } else { if {[regexp {^~?/.*/$} $expected]} { + # "expected" is of the form "/PATTERN/" then the result if correct if + # regular expression PATTERN matches the result. "~/PATTERN/" means + # the regular expression must not match. if {[string index $expected 0]=="~"} { set re [string map {# {[-0-9.]+}} [string range $expected 2 end-1]] set ok [expr {![regexp $re $result]}] } else { set re [string map {# {[-0-9.]+}} [string range $expected 1 end-1]] set ok [regexp $re $result] + } + } elseif {[regexp {^~?\*.*\*$} $expected]} { + # "expected" is of the form "*GLOB*" then the result if correct if + # glob pattern GLOB matches the result. "~/GLOB/" means + # the glob must not match. + if {[string index $expected 0]=="~"} { + set e [string range $expected 1 end] + set ok [expr {![string match $e $result]}] + } else { + set ok [string match $expected $result] } } else { set ok [expr {[string compare $result $expected]==0}] } if {!$ok} { Index: test/tkt3442.test ================================================================== --- test/tkt3442.test +++ test/tkt3442.test @@ -47,20 +47,20 @@ # and verify that the query plan is the same. # ifcapable explain { do_test tkt3442-1.2 { EQP { SELECT node FROM listhash WHERE id='5000' LIMIT 1; } - } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} + } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?)}} } # Some extra tests testing other permutations of 5000. # ifcapable explain { do_test tkt3442-1.4 { EQP { SELECT node FROM listhash WHERE id=5000 LIMIT 1; } - } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?) (~1 rows)}} + } {0 0 0 {SEARCH TABLE listhash USING INDEX ididx (id=?)}} } do_test tkt3442-1.5 { catchsql { SELECT node FROM listhash WHERE id=[5000] LIMIT 1; } Index: test/where.test ================================================================== --- test/where.test +++ test/where.test @@ -61,39 +61,38 @@ set res [execsql $sql] #puts "sql={$sql} seek=[kvwrap seek] step=[kvwrap step]" return [concat $res [expr [kvwrap step] + [kvwrap seek]]] } -# Verify that queries use an index. We are using the special variable -# "sqlite_search_count" which tallys the number of executions of MoveTo -# and Next operators in the VDBE. By verifing that the search count is -# small we can be assured that indices are being used properly. +# Verify that queries use an index. By verifing that the KVWrap layer +# xNext/xPrev/xSeek count is small we can be assured that indices are +# being used properly. # do_test where-1.1.1 { count {SELECT x, y, w FROM t1 WHERE w=10} } {3 121 10 3} -do_test where-1.1.2 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.1.2 { + SELECT x, y, w FROM t1 WHERE w=10 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.1.3 { db status step } {0} do_test where-1.1.4 { db eval {SELECT x, y, w FROM t1 WHERE +w=10} } {3 121 10} do_test where-1.1.5 { db status step } {99} -do_test where-1.1.6 { - set sqlite_query_plan -} {t1 {}} +do_eqp_test where-1.1.6 { + SELECT x, y, w FROM t1 WHERE +w=10 +} {*SCAN TABLE t1*} do_test where-1.1.7 { count {SELECT x, y, w AS abc FROM t1 WHERE abc=10} } {3 121 10 3} -do_test where-1.1.8 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.1.8 { + SELECT x, y, w AS abc FROM t1 WHERE abc=10 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.1.9 { db status step } {0} do_test where-1.2.1 { count {SELECT x, y, w FROM t1 WHERE w=11} @@ -108,50 +107,50 @@ count {SELECT x, y, w AS abc FROM t1 WHERE 11=abc} } {3 144 11 3} do_test where-1.4.1 { count {SELECT w, x, y FROM t1 WHERE 11=w AND x>2} } {11 3 144 3} -do_test where-1.4.2 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.4.2 { + SELECT w, x, y FROM t1 WHERE 11=w AND x>2 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.4.3 { count {SELECT w AS a, x AS b, y FROM t1 WHERE 11=a AND b>2} } {11 3 144 3} -do_test where-1.4.4 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.4.4 { + SELECT w AS a, x AS b, y FROM t1 WHERE 11=a AND b>2 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.5 { count {SELECT x, y FROM t1 WHERE y<200 AND w=11 AND x>2} } {3 144 3} -do_test where-1.5.2 { - set sqlite_query_plan -} {t1 i1w} +do_eqp_test where-1.5.2 { + SELECT x, y FROM t1 WHERE y<200 AND w=11 AND x>2 +} {*SEARCH TABLE t1 USING INDEX i1w (w=?)*} do_test where-1.6 { count {SELECT x, y FROM t1 WHERE y<200 AND x>2 AND w=11} } {3 144 3} do_test where-1.7 { count {SELECT x, y FROM t1 WHERE w=11 AND y<200 AND x>2} } {3 144 3} do_test where-1.8 { count {SELECT x, y FROM t1 WHERE w>10 AND y=144 AND x=3} } {3 144 3} -do_test where-1.8.2 { - set sqlite_query_plan -} {t1 i1xy} -do_test where-1.8.3 { - count {SELECT x, y FROM t1 WHERE y=144 AND x=3} - set sqlite_query_plan -} {t1 i1xy} +do_eqp_test where-1.8.2 { + SELECT x, y FROM t1 WHERE w>10 AND y=144 AND x=3 +} {*SEARCH TABLE t1 USING INDEX i1xy (x=? AND y=?)*} +do_eqp_test where-1.8.3 { + SELECT x, y FROM t1 WHERE y=144 AND x=3 +} {*SEARCH TABLE t1 USING INDEX i1xy (x=? AND y=?)*} do_test where-1.9 { count {SELECT x, y FROM t1 WHERE y=144 AND w>10 AND x=3} } {3 144 3} do_test where-1.10 { count {SELECT x, y FROM t1 WHERE x=3 AND w>=10 AND y=121} } {3 121 3} do_test where-1.11 { count {SELECT x, y FROM t1 WHERE x=3 AND y=100 AND w<10} } {3 100 3} + # New for SQLite version 2.1: Verify that that inequality constraints # are used correctly. # do_test where-1.12 { @@ -512,23 +511,21 @@ SELECT * FROM t3 WHERE a>0 ORDER BY a LIMIT 3 } } {1 100 4 2 99 9 3 98 16 nosort} do_test where-6.7 { - # UPDATE: src4 does a sort here. It picks a different index because it - # does not support the covering index optimization. cksort { SELECT * FROM t3 WHERE b>0 ORDER BY a LIMIT 3 } -} {1 100 4 2 99 9 3 98 16 sort} +} {1 100 4 2 99 9 3 98 16 nosort} ifcapable subquery { do_test where-6.8 { cksort { SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a LIMIT 3 } - } {1 100 4 2 99 9 3 98 16 sort} + } {1 100 4 2 99 9 3 98 16 nosort} } do_test where-6.9.1 { cksort { SELECT * FROM t3 WHERE a=1 AND c>0 ORDER BY a LIMIT 3 } @@ -1100,16 +1097,16 @@ INSERT INTO t8 VALUES(4,'four'); } cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.a, y.b } -} {1/4 1/1 4/4 4/1 sort} +} {1/4 1/1 4/4 4/1 nosort} do_test where-14.2 { cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.a, y.b DESC } -} {1/1 1/4 4/1 4/4 sort} +} {1/1 1/4 4/1 4/4 nosort} do_test where-14.3 { cksort { SELECT x.a || '/' || y.a FROM t8 x, t8 y ORDER BY x.a, x.b } } {1/1 1/4 4/1 4/4 nosort} Index: test/where7.test ================================================================== --- test/where7.test +++ test/where7.test @@ -23330,19 +23330,19 @@ CREATE INDEX t302_c8_c3 on t302(c8, c3); CREATE INDEX t302_c5 on t302(c5); EXPLAIN QUERY PLAN SELECT t302.c1 - FROM t302 JOIN t301 ON t302.c8 = t301.c8 + FROM t302 JOIN t301 ON t302.c8 = +t301.c8 WHERE t302.c2 = 19571 AND t302.c3 > 1287603136 AND (t301.c4 = 1407449685622784 OR t301.c8 = 1407424651264000) ORDER BY t302.c5 LIMIT 200; } { - 0 0 1 {SEARCH TABLE t301 USING INDEX t301_c4 (c4=?) (~5 rows)} - 0 0 1 {SEARCH TABLE t301 USING PRIMARY KEY (c8=?) (~1 rows)} - 0 1 0 {SEARCH TABLE t302 USING INDEX t302_c8_c3 (c8=? AND c3>?) (~2 rows)} + 0 0 1 {SEARCH TABLE t301 USING INDEX t301_c4 (c4=?)} + 0 0 1 {SEARCH TABLE t301 USING INDEX t301 (c8=?)} + 0 1 0 {SEARCH TABLE t302 USING INDEX t302_c8_c3 (c8=? AND c3>?)} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } finish_test Index: test/where8.test ================================================================== --- test/where8.test +++ test/where8.test @@ -254,11 +254,11 @@ SELECT a, d FROM t1, t2 WHERE (a = 2 OR b = 'three' OR c = 'IX') AND (d = a OR e = 'sixteen') ORDER BY t1.rowid } -} {2 2 2 4 3 3 3 4 9 9 9 4 0 0 seek=13 step=16} +} {2 2 2 4 3 3 3 4 9 9 9 4 9 0 seek=13 step=16} do_test where8-3.10 { execsql_status { SELECT d FROM t2 WHERE e IS NULL OR e = 'four' } } {1 3 5 10 2 0 0} @@ -270,11 +270,11 @@ } {1 1 2 2 3 3 4 2 4 4 0 0} do_test where8-3.12 { execsql_status { SELECT a, d FROM t1, t2 WHERE (a=d OR b=e) AND +a<5 ORDER BY a } -} {1 1 2 2 3 3 4 2 4 4 0 0} +} {1 1 2 2 3 3 4 2 4 4 9 0} do_test where8-3.13 { execsql_status { SELECT a, d FROM t1, t2 WHERE (a=d OR b=e) AND +a<5 } } {1 1 2 2 3 3 4 2 4 4 9 0}