Index: addopcodes.awk ================================================================== --- addopcodes.awk +++ addopcodes.awk @@ -28,6 +28,7 @@ printf "#define TK_%-29s %4d\n", "COLUMN", ++max printf "#define TK_%-29s %4d\n", "AGG_FUNCTION", ++max printf "#define TK_%-29s %4d\n", "AGG_COLUMN", ++max printf "#define TK_%-29s %4d\n", "UMINUS", ++max printf "#define TK_%-29s %4d\n", "UPLUS", ++max + printf "#define TK_%-29s %4d\n", "REGISTER", ++max } Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -1469,10 +1469,23 @@ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); } + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: ** @@ -1497,11 +1510,24 @@ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; pInfo->estimatedCost = 5000000; for(i=0; inConstraint; i++){ int bDocid; /* True if this constraint is on docid */ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ) continue; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ Index: ext/misc/spellfix.c ================================================================== --- ext/misc/spellfix.c +++ ext/misc/spellfix.c @@ -2049,29 +2049,32 @@ ** (B) langid == $langid ** (C) top = $top ** (D) scope = $scope ** (E) distance < $distance ** (F) distance <= $distance +** (G) rowid = $rowid ** ** The plan number is a bit mask formed with these bits: ** ** 0x01 (A) is found ** 0x02 (B) is found ** 0x04 (C) is found ** 0x08 (D) is found ** 0x10 (E) is found ** 0x20 (F) is found +** 0x40 (G) is found ** -** filter.argv[*] values contains $str, $langid, $top, and $scope, +** filter.argv[*] values contains $str, $langid, $top, $scope and $rowid ** if specified and in that order. */ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iPlan = 0; int iLangTerm = -1; int iTopTerm = -1; int iScopeTerm = -1; int iDistTerm = -1; + int iRowidTerm = -1; int i; const struct sqlite3_index_constraint *pConstraint; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ if( pConstraint->usable==0 ) continue; @@ -2120,10 +2123,19 @@ || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) ){ iPlan |= pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ? 16 : 32; iDistTerm = i; } + + /* Terms of the form: distance < $dist or distance <= $dist */ + if( (iPlan & 64)==0 + && pConstraint->iColumn<0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 64; + iRowidTerm = i; + } } if( iPlan&1 ){ int idx = 2; pIdxInfo->idxNum = iPlan; if( pIdxInfo->nOrderBy==1 @@ -2147,10 +2159,15 @@ if( iPlan&(16|32) ){ pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = idx++; pIdxInfo->aConstraintUsage[iDistTerm].omit = 1; } pIdxInfo->estimatedCost = 1e5; + }else if( (iPlan & 64) ){ + pIdxInfo->idxNum = 64; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 5; }else{ pIdxInfo->idxNum = 0; pIdxInfo->estimatedCost = 1e50; } return SQLITE_OK; @@ -2463,20 +2480,27 @@ spellfix1_cursor *pCur, int idxNum, int argc, sqlite3_value **argv ){ - int rc; + int rc = SQLITE_OK; char *zSql; spellfix1_vtab *pVTab = pCur->pVTab; spellfix1ResetCursor(pCur); + assert( idxNum==0 || idxNum==64 ); zSql = sqlite3_mprintf( - "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"", - pVTab->zDbName, pVTab->zTableName); + "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"%s", + pVTab->zDbName, pVTab->zTableName, + ((idxNum & 64) ? " WHERE rowid=?" : "") + ); if( zSql==0 ) return SQLITE_NOMEM; rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pFullScan, 0); sqlite3_free(zSql); + if( rc==SQLITE_OK && (idxNum & 64) ){ + assert( argc==1 ); + rc = sqlite3_bind_value(pCur->pFullScan, 1, argv[0]); + } pCur->nRow = pCur->iRow = 0; if( rc==SQLITE_OK ){ rc = sqlite3_step(pCur->pFullScan); if( rc==SQLITE_ROW ){ pCur->iRow = -1; rc = SQLITE_OK; } if( rc==SQLITE_DONE ){ rc = SQLITE_OK; } Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -3752,11 +3752,11 @@ static void assertCellInfo(BtCursor *pCur){ CellInfo info; int iPage = pCur->iPage; memset(&info, 0, sizeof(info)); btreeParseCell(pCur->apPage[iPage], pCur->aiIdx[iPage], &info); - assert( memcmp(&info, &pCur->info, sizeof(info))==0 ); + assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 ); } #else #define assertCellInfo(x) #endif #ifdef _MSC_VER @@ -4388,30 +4388,28 @@ if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; } pCur->iPage = 0; - - /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor - ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is - ** NULL, the caller expects a table b-tree. If this is not the case, - ** return an SQLITE_CORRUPT error. */ - assert( pCur->apPage[0]->intKey==1 || pCur->apPage[0]->intKey==0 ); - if( (pCur->pKeyInfo==0)!=pCur->apPage[0]->intKey ){ - return SQLITE_CORRUPT_BKPT; - } } - - /* Assert that the root page is of the correct type. This must be the - ** case as the call to this function that loaded the root-page (either - ** this call or a previous invocation) would have detected corruption - ** if the assumption were not true, and it is not possible for the flags - ** byte to have been modified while this cursor is holding a reference - ** to the page. */ pRoot = pCur->apPage[0]; assert( pRoot->pgno==pCur->pgnoRoot ); - assert( pRoot->isInit && (pCur->pKeyInfo==0)==pRoot->intKey ); + + /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor + ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is + ** NULL, the caller expects a table b-tree. If this is not the case, + ** return an SQLITE_CORRUPT error. + ** + ** Earlier versions of SQLite assumed that this test could not fail + ** if the root page was already loaded when this function was called (i.e. + ** if pCur->iPage>=0). But this is not so if the database is corrupted + ** in such a way that page pRoot is linked into a second b-tree table + ** (or the freelist). */ + assert( pRoot->intKey==1 || pRoot->intKey==0 ); + if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ + return SQLITE_CORRUPT_BKPT; + } pCur->aiIdx[0] = 0; pCur->info.nSize = 0; pCur->atLast = 0; pCur->validNKey = 0; @@ -5249,10 +5247,11 @@ releasePage(pTrunk); releasePage(pPrevTrunk); if( rc==SQLITE_OK ){ if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ releasePage(*ppPage); + *ppPage = 0; return SQLITE_CORRUPT_BKPT; } (*ppPage)->isInit = 0; }else{ *ppPage = 0; @@ -7348,18 +7347,20 @@ ){ MemPage *pPage; int rc; unsigned char *pCell; int i; + int hdr; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; + hdr = pPage->hdrOffset; for(i=0; inCell; i++){ pCell = findCell(pPage, i); if( !pPage->leaf ){ rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; @@ -7366,20 +7367,20 @@ } rc = clearCell(pPage, pCell); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange); + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ assert( pPage->intKey ); *pnChange += pPage->nCell; } if( freePageFlag ){ freePage(pPage, &rc); }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ - zeroPage(pPage, pPage->aData[0] | PTF_LEAF); + zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF); } cleardatabasepage_out: releasePage(pPage); return rc; Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -138,10 +138,11 @@ */ v = sqlite3GetVdbe(pParse); assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ + while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are ** set for each database that is used. Generate code to start a @@ -1449,14 +1450,14 @@ i = *pIdx; for(j=0; zIdent[j]; j++){ if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; } - needQuote = sqlite3Isdigit(zIdent[0]) || sqlite3KeywordCode(zIdent, j)!=TK_ID; - if( !needQuote ){ - needQuote = zIdent[j]; - } + needQuote = sqlite3Isdigit(zIdent[0]) + || sqlite3KeywordCode(zIdent, j)!=TK_ID + || zIdent[j]!=0 + || j==0; if( needQuote ) z[i++] = '"'; for(j=0; zIdent[j]; j++){ z[i++] = zIdent[j]; if( zIdent[j]=='"' ) z[i++] = '"'; @@ -2675,11 +2676,11 @@ ** records into the sorter. */ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 0, &iPartIdxLabel); + sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeResolveLabel(v, iPartIdxLabel); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); sqlite3VdbeJumpHere(v, addr1); if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); @@ -4195,5 +4196,75 @@ } } } return sqlite3KeyInfoRef(pIdx->pKeyInfo); } + +#ifndef SQLITE_OMIT_CTE +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. +*/ +With *sqlite3WithAdd( + Parse *pParse, /* Parsing context */ + With *pWith, /* Existing WITH clause, or NULL */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ + Select *pQuery /* Query used to initialize the table */ +){ + sqlite3 *db = pParse->db; + With *pNew; + char *zName; + + /* Check that the CTE name is unique within this WITH clause. If + ** not, store an error in the Parse structure. */ + zName = sqlite3NameFromToken(pParse->db, pName); + if( zName && pWith ){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName); + } + } + } + + if( pWith ){ + int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + pNew = sqlite3DbRealloc(db, pWith, nByte); + }else{ + pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + } + assert( zName!=0 || pNew==0 ); + assert( db->mallocFailed==0 || pNew==0 ); + + if( pNew==0 ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + sqlite3DbFree(db, zName); + pNew = pWith; + }else{ + pNew->a[pNew->nCte].pSelect = pQuery; + pNew->a[pNew->nCte].pCols = pArglist; + pNew->a[pNew->nCte].zName = zName; + pNew->a[pNew->nCte].zErr = 0; + pNew->nCte++; + } + + return pNew; +} + +/* +** Free the contents of the With object passed as the second argument. +*/ +void sqlite3WithDelete(sqlite3 *db, With *pWith){ + if( pWith ){ + int i; + for(i=0; inCte; i++){ + struct Cte *pCte = &pWith->a[i]; + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); + } + sqlite3DbFree(db, pWith); + } +} +#endif /* !defined(SQLITE_OMIT_CTE) */ Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -726,13 +726,14 @@ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ ){ int i; /* Index loop counter */ - int r1; /* Register holding an index key */ + int r1 = -1; /* Register holding an index key */ int iPartIdxLabel; /* Jump destination for skipping partial index entries */ Index *pIdx; /* Current index */ + Index *pPrior = 0; /* Prior index */ Vdbe *v; /* The prepared statement under construction */ Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ v = pParse->pVdbe; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); @@ -739,14 +740,16 @@ for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); - r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, &iPartIdxLabel); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3VdbeResolveLabel(v, iPartIdxLabel); + pPrior = pIdx; } } /* ** Generate code that will assemble an index key and stores it in register @@ -764,18 +767,31 @@ ** to that label if pIdx is a partial index that should be skipped. ** A partial index should be skipped if its WHERE clause evaluates ** to false or null. If pIdx is not a partial index, *piPartIdxLabel ** will be set to zero which is an empty label that is ignored by ** sqlite3VdbeResolveLabel(). +** +** The pPrior and regPrior parameters are used to implement a cache to +** avoid unnecessary register loads. If pPrior is not NULL, then it is +** a pointer to a different index for which an index key has just been +** computed into register regPrior. If the current pIdx index is generating +** its key into the same sequence of registers and if pPrior and pIdx share +** a column in common, then the register corresponding to that column already +** holds the correct value and the loading of that register is skipped. +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** on a table with multiple indices, and especially with the ROWID or +** PRIMARY KEY columns of the index. */ int sqlite3GenerateIndexKey( Parse *pParse, /* Parsing context */ Index *pIdx, /* The index for which to generate a key */ int iDataCur, /* Cursor number from which to take column data */ int regOut, /* Put the new key into this register if not 0 */ int prefixOnly, /* Compute only a unique prefix of the key */ - int *piPartIdxLabel /* OUT: Jump to this label to skip partial index */ + int *piPartIdxLabel, /* OUT: Jump to this label to skip partial index */ + Index *pPrior, /* Previously generated index key */ + int regPrior /* Register holding previous generated key */ ){ Vdbe *v = pParse->pVdbe; int j; Table *pTab = pIdx->pTable; int regBase; @@ -791,24 +807,24 @@ *piPartIdxLabel = 0; } } nCol = (prefixOnly && pIdx->uniqNotNull) ? pIdx->nKeyCol : pIdx->nColumn; regBase = sqlite3GetTempRange(pParse, nCol); + if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], regBase+j); + /* If the column affinity is REAL but the number is an integer, then it + ** might be stored in the table as an integer (using a compact + ** representation) then converted to REAL by an OP_RealAffinity opcode. + ** But we are getting ready to store this value back into an index, where + ** it should be converted by to INTEGER again. So omit the OP_RealAffinity + ** opcode if it is present */ + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); } if( regOut ){ - const char *zAff; - if( pTab->pSelect - || OptimizationDisabled(pParse->db, SQLITE_IdxRealAsInt) - ){ - zAff = 0; - }else{ - zAff = sqlite3IndexAffinityStr(v, pIdx); - } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); - sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); } sqlite3ReleaseTempRange(pParse, regBase, nCol); return regBase; } Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -521,20 +521,29 @@ } return p; } /* -** Return 1 if an expression must be FALSE in all cases and 0 if the -** expression might be true. This is an optimization. If is OK to -** return 0 here even if the expression really is always false (a -** false negative). But it is a bug to return 1 if the expression -** might be true in some rare circumstances (a false positive.) +** If the expression is always either TRUE or FALSE (respectively), +** then return 1. If one cannot determine the truth value of the +** expression at compile-time return 0. +** +** This is an optimization. If is OK to return 0 here even if +** the expression really is always false or false (a false negative). +** But it is a bug to return 1 if the expression might have different +** boolean values in different circumstances (a false positive.) ** ** Note that if the expression is part of conditional for a ** LEFT JOIN, then we cannot determine at compile-time whether or not ** is it true or false, so always return 0. */ +static int exprAlwaysTrue(Expr *p){ + int v = 0; + if( ExprHasProperty(p, EP_FromJoin) ) return 0; + if( !sqlite3ExprIsInteger(p, &v) ) return 0; + return v!=0; +} static int exprAlwaysFalse(Expr *p){ int v = 0; if( ExprHasProperty(p, EP_FromJoin) ) return 0; if( !sqlite3ExprIsInteger(p, &v) ) return 0; return v==0; @@ -884,10 +893,37 @@ } } return pNew; } +/* +** Create and return a deep copy of the object passed as the second +** argument. If an OOM condition is encountered, NULL is returned +** and the db->mallocFailed flag set. +*/ +#ifndef SQLITE_OMIT_CTE +static With *withDup(sqlite3 *db, With *p){ + With *pRet = 0; + if( p ){ + int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + pRet = sqlite3DbMallocZero(db, nByte); + if( pRet ){ + int i; + pRet->nCte = p->nCte; + for(i=0; inCte; i++){ + pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); + pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); + pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + } + } + } + return pRet; +} +#else +# define withDup(x,y) 0 +#endif + /* ** The following group of routines make deep copies of expressions, ** expression lists, ID lists, and select statements. The copies can ** be deleted (by being passed to their respective ...Delete() routines) ** without effecting the originals. @@ -964,10 +1000,11 @@ pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; pNewItem->isCorrelated = pOldItem->isCorrelated; pNewItem->viaCoroutine = pOldItem->viaCoroutine; + pNewItem->isRecursive = pOldItem->isRecursive; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ @@ -1025,10 +1062,11 @@ pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; pNew->pRightmost = 0; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; + pNew->pWith = withDup(db, p->pWith); return pNew; } #else Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ assert( p==0 ); @@ -3527,21 +3565,23 @@ op = pExpr->op; switch( op ){ case TK_AND: { int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); sqlite3ExprCachePop(pParse, 1); break; } case TK_OR: { testcase( jumpIfNull==0 ); sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse, 1); break; } case TK_NOT: { testcase( jumpIfNull==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); @@ -3612,14 +3652,20 @@ sqlite3VdbeResolveLabel(v, destIfFalse); break; } #endif default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); + if( exprAlwaysTrue(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysFalse(pExpr) ){ + /* No-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } break; } } sqlite3ReleaseTempReg(pParse, regFree1); sqlite3ReleaseTempReg(pParse, regFree2); @@ -3678,18 +3724,20 @@ switch( pExpr->op ){ case TK_AND: { testcase( jumpIfNull==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse, 1); break; } case TK_OR: { int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); sqlite3ExprCachePop(pParse, 1); break; } @@ -3757,14 +3805,20 @@ } break; } #endif default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); + if( exprAlwaysFalse(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysTrue(pExpr) ){ + /* no-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } break; } } sqlite3ReleaseTempReg(pParse, regFree1); sqlite3ReleaseTempReg(pParse, regFree2); Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -538,11 +538,10 @@ ** D: cleanup */ void sqlite3Insert( Parse *pParse, /* Parser context */ SrcList *pTabList, /* Name of table into which we are inserting */ - ExprList *pList, /* List of values to be inserted */ Select *pSelect, /* A SELECT statement to use as the data source */ IdList *pColumn, /* Column names corresponding to IDLIST. */ int onError /* How to handle constraint errors */ ){ sqlite3 *db; /* The main database structure */ @@ -566,10 +565,11 @@ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ Db *pDb; /* The database containing table being inserted into */ int appendFlag = 0; /* True if the insert is likely to be an append */ int withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + ExprList *pList = 0; /* List of VALUES() to be inserted */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ int regRowCount = 0; /* Memory cell used for the row counter */ @@ -588,10 +588,21 @@ db = pParse->db; memset(&dest, 0, sizeof(dest)); if( pParse->nErr || db->mallocFailed ){ goto insert_cleanup; } + + /* If the Select object is really just a simple VALUES() list with a + ** single row values (the common case) then keep that one row of values + ** and go ahead and discard the Select object + */ + if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ + pList = pSelect->pEList; + pSelect->pEList = 0; + sqlite3SelectDelete(db, pSelect); + pSelect = 0; + } /* Locate the table into which we will be inserting new information. */ assert( pTabList->nSrc==1 ); zTab = pTabList->a[0].zName; @@ -1231,10 +1242,11 @@ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ int ipkBottom = 0; /* Bottom of the rowid change constraint check */ u8 isUpdate; /* True if this is an UPDATE operation */ + int regRowid = -1; /* Register holding ROWID value */ isUpdate = regOldData!=0; db = pParse->db; v = sqlite3GetVdbe(pParse); assert( v!=0 ); @@ -1473,11 +1485,13 @@ regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn); for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; if( iField<0 || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; }else{ x = iField + regNewData + 1; } sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); @@ -1853,10 +1867,16 @@ int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ int regData, regRowid; /* Registers holding data and rowid */ if( pSelect==0 ){ return 0; /* Must be of the form INSERT INTO ... SELECT ... */ + } + if( pParse->pWith || pSelect->pWith ){ + /* Do not attempt to process this query if there are an WITH clauses + ** attached to it. Proceeding may generate a false "no such table: xxx" + ** error if pSelect reads from a CTE named "xxx". */ + return 0; } if( sqlite3TriggerList(pParse, pDest) ){ return 0; /* tab1 must not have triggers */ } #ifndef SQLITE_OMIT_VIRTUALTABLE Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -133,17 +133,10 @@ ** must be complete. So isInit must not be set until the very end ** of this routine. */ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; -#ifdef SQLITE_ENABLE_SQLLOG - { - extern void sqlite3_init_sqllog(void); - sqlite3_init_sqllog(); - } -#endif - /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, ** there is not much SQLite is going to be able to do. ** @@ -3120,11 +3113,11 @@ ** Reset the PRNG back to its uninitialized state. The next call ** to sqlite3_randomness() will reseed the PRNG using a single call ** to the xRandomness method of the default VFS. */ case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3PrngResetState(); + sqlite3_randomness(0,0); break; } /* ** sqlite3_test_control(BITVEC_TEST, size, program) Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -258,10 +258,16 @@ */ char aPadding[32]; #endif }; +/* This variable holds the process id (pid) from when the xRandomness() +** method was called. If xOpen() is called from a different process id, +** indicating that a fork() has occurred, the PRNG will be reset. +*/ +static int randomnessPid = 0; + /* ** Allowed values for the unixFile.ctrlFlags bitmask: */ #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ @@ -4840,14 +4846,14 @@ ** Or, if the third argument is NULL, then this function is being called ** to inform the VFS layer that, according to POSIX, any existing mapping ** may now be invalid and should be unmapped. */ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 unixFile *pFd = (unixFile *)fd; /* The underlying database file */ UNUSED_PARAMETER(iOff); -#if SQLITE_MAX_MMAP_SIZE>0 /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); @@ -4859,10 +4865,14 @@ }else{ unixUnmapfile(pFd); } assert( pFd->nFetchOut>=0 ); +#else + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(iOff); #endif return SQLITE_OK; } /* @@ -5648,10 +5658,20 @@ assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); + + /* Detect a pid change and reset the PRNG. There is a race condition + ** here such that two or more threads all trying to open databases at + ** the same instant might all reset the PRNG. But multiple resets + ** are harmless. + */ + if( randomnessPid!=getpid() ){ + randomnessPid = getpid(); + sqlite3_randomness(0,0); + } memset(p, 0, sizeof(unixFile)); if( eType==SQLITE_OPEN_MAIN_DB ){ UnixUnusedFd *pUnused; @@ -6036,22 +6056,22 @@ ** When testing, initializing zBuf[] to zero is all we do. That means ** that we always use the same random number sequence. This makes the ** tests repeatable. */ memset(zBuf, 0, nBuf); + randomnessPid = getpid(); #if !defined(SQLITE_TEST) { - int pid, fd, got; + int fd, got; fd = robust_open("/dev/urandom", O_RDONLY, 0); if( fd<0 ){ time_t t; time(&t); memcpy(zBuf, &t, sizeof(t)); - pid = getpid(); - memcpy(&zBuf[sizeof(t)], &pid, sizeof(pid)); - assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf ); - nBuf = sizeof(t) + sizeof(pid); + memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid)); + assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(randomnessPid); }else{ do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); robust_close(0, fd, __LINE__); } } Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3222,11 +3222,11 @@ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } static void winShmLeaveMutex(void){ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } -#ifdef SQLITE_DEBUG +#ifndef NDEBUG static int winShmMutexHeld(void) { return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } #endif Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -92,18 +92,10 @@ /* ** An instance of this structure holds the ATTACH key and the key type. */ struct AttachKey { int type; Token key; }; -/* -** One or more VALUES claues -*/ -struct ValueList { - ExprList *pList; - Select *pSelect; -}; - } // end %include // Input is a single SQL command input ::= cmdlist. cmdlist ::= cmdlist ecmd. @@ -200,24 +192,22 @@ // An IDENTIFIER can be a generic identifier, or one of several // keywords. Any non-standard keyword can also be an identifier. // -%type id {Token} -id(A) ::= ID(X). {A = X;} -id(A) ::= INDEXED(X). {A = X;} +%token_class id ID|INDEXED. // The following directive causes tokens ABORT, AFTER, ASC, etc. to // fallback to ID if they will not parse as their original value. // This obviates the need for the "id" nonterminal. // %fallback ID ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN - QUERY KEY OF OFFSET PRAGMA RAISE RELEASE REPLACE RESTRICT ROW ROLLBACK - SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITHOUT + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT %ifdef SQLITE_OMIT_COMPOUND_SELECT EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT REINDEX RENAME CTIME_KW IF . @@ -247,12 +237,11 @@ %left COLLATE. %right BITNOT. // And "ids" is an identifer-or-string. // -%type ids {Token} -ids(A) ::= ID|STRING(X). {A = X;} +%token_class ids ID|STRING. // The name of a column or table can be any of the following: // %type nm {Token} nm(A) ::= id(X). {A = X;} @@ -406,26 +395,37 @@ %endif SQLITE_OMIT_VIEW //////////////////////// The SELECT statement ///////////////////////////////// // cmd ::= select(X). { - SelectDest dest = {SRT_Output, 0, 0, 0, 0}; + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; sqlite3Select(pParse, X, &dest); sqlite3ExplainBegin(pParse->pVdbe); sqlite3ExplainSelect(pParse->pVdbe, X); sqlite3ExplainFinish(pParse->pVdbe); sqlite3SelectDelete(pParse->db, X); } %type select {Select*} %destructor select {sqlite3SelectDelete(pParse->db, $$);} +%type selectnowith {Select*} +%destructor selectnowith {sqlite3SelectDelete(pParse->db, $$);} %type oneselect {Select*} %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);} -select(A) ::= oneselect(X). {A = X;} +select(A) ::= with(W) selectnowith(X). { + if( X ){ + X->pWith = W; + }else{ + sqlite3WithDelete(pParse->db, W); + } + A = X; +} + +selectnowith(A) ::= oneselect(X). {A = X;} %ifndef SQLITE_OMIT_COMPOUND_SELECT -select(A) ::= select(X) multiselect_op(Y) oneselect(Z). { +selectnowith(A) ::= selectnowith(X) multiselect_op(Y) oneselect(Z). { if( Z ){ Z->op = (u8)Y; Z->pPrior = X; if( Y!=TK_ALL ) pParse->hasCompound = 1; }else{ @@ -439,10 +439,27 @@ multiselect_op(A) ::= EXCEPT|INTERSECT(OP). {A = @OP;} %endif SQLITE_OMIT_COMPOUND_SELECT oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). { A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); +} +oneselect(A) ::= values(X). {A = X;} + +%type values {Select*} +%destructor values {sqlite3SelectDelete(pParse->db, $$);} +values(A) ::= VALUES LP nexprlist(X) RP. { + A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0,0); +} +values(A) ::= values(X) COMMA LP exprlist(Y) RP. { + Select *pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values,0,0); + if( pRight ){ + pRight->op = TK_ALL; + pRight->pPrior = X; + A = pRight; + }else{ + A = X; + } } // The "distinct" nonterminal is true (1) if the DISTINCT keyword is // present and false (0) if it is not. // @@ -640,19 +657,21 @@ {A.pOffset = X.pExpr; A.pLimit = Y.pExpr;} /////////////////////////// The DELETE statement ///////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W) +cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE"); sqlite3DeleteFrom(pParse,X,W); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { +cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3DeleteFrom(pParse,X,W); } %endif @@ -663,21 +682,23 @@ where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} ////////////////////////// The UPDATE command //////////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) - orderby_opt(O) limit_opt(L). { +cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) + where_opt(W) orderby_opt(O) limit_opt(L). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); sqlite3Update(pParse,X,Y,W,R); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) +cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); sqlite3Update(pParse,X,Y,W,R); } %endif @@ -694,62 +715,24 @@ sqlite3ExprListSetName(pParse, A, &X, 1); } ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) valuelist(Y). - {sqlite3Insert(pParse, X, Y.pList, Y.pSelect, F, R);} -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). - {sqlite3Insert(pParse, X, 0, S, F, R);} -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. - {sqlite3Insert(pParse, X, 0, 0, F, R);} +cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). { + sqlite3WithPush(pParse, W, 1); + sqlite3Insert(pParse, X, S, F, R); +} +cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. +{ + sqlite3WithPush(pParse, W, 1); + sqlite3Insert(pParse, X, 0, F, R); +} %type insert_cmd {u8} insert_cmd(A) ::= INSERT orconf(R). {A = R;} insert_cmd(A) ::= REPLACE. {A = OE_Replace;} -// A ValueList is either a single VALUES clause or a comma-separated list -// of VALUES clauses. If it is a single VALUES clause then the -// ValueList.pList field points to the expression list of that clause. -// If it is a list of VALUES clauses, then those clauses are transformed -// into a set of SELECT statements without FROM clauses and connected by -// UNION ALL and the ValueList.pSelect points to the right-most SELECT in -// that compound. -%type valuelist {struct ValueList} -%destructor valuelist { - sqlite3ExprListDelete(pParse->db, $$.pList); - sqlite3SelectDelete(pParse->db, $$.pSelect); -} -valuelist(A) ::= VALUES LP nexprlist(X) RP. { - A.pList = X; - A.pSelect = 0; -} - -// Since a list of VALUEs is inplemented as a compound SELECT, we have -// to disable the value list option if compound SELECTs are disabled. -%ifndef SQLITE_OMIT_COMPOUND_SELECT -valuelist(A) ::= valuelist(X) COMMA LP exprlist(Y) RP. { - Select *pRight = sqlite3SelectNew(pParse, Y, 0, 0, 0, 0, 0, 0, 0, 0); - if( X.pList ){ - X.pSelect = sqlite3SelectNew(pParse, X.pList, 0, 0, 0, 0, 0, 0, 0, 0); - X.pList = 0; - } - A.pList = 0; - if( X.pSelect==0 || pRight==0 ){ - sqlite3SelectDelete(pParse->db, pRight); - sqlite3SelectDelete(pParse->db, X.pSelect); - A.pSelect = 0; - }else{ - pRight->op = TK_ALL; - pRight->pPrior = X.pSelect; - pRight->selFlags |= SF_Values; - pRight->pPrior->selFlags |= SF_Values; - A.pSelect = pRight; - } -} -%endif SQLITE_OMIT_COMPOUND_SELECT - %type inscollist_opt {IdList*} %destructor inscollist_opt {sqlite3IdListDelete(pParse->db, $$);} %type idlist {IdList*} %destructor idlist {sqlite3IdListDelete(pParse->db, $$);} @@ -808,28 +791,28 @@ A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); spanSet(&A,&X,&Z); } term(A) ::= INTEGER|FLOAT|BLOB(X). {spanExpr(&A, pParse, @X, &X);} term(A) ::= STRING(X). {spanExpr(&A, pParse, @X, &X);} -expr(A) ::= REGISTER(X). { - /* When doing a nested parse, one can include terms in an expression - ** that look like this: #1 #2 ... These terms refer to registers - ** in the virtual machine. #N is the N-th register. */ - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X); - A.pExpr = 0; +expr(A) ::= VARIABLE(X). { + if( X.n>=2 && X.z[0]=='#' && sqlite3Isdigit(X.z[1]) ){ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X); + A.pExpr = 0; + }else{ + A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X); + if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable); + } }else{ - A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X); - if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable); + spanExpr(&A, pParse, TK_VARIABLE, &X); + sqlite3ExprAssignVarNumber(pParse, A.pExpr); } spanSet(&A, &X, &X); } -expr(A) ::= VARIABLE(X). { - spanExpr(&A, pParse, TK_VARIABLE, &X); - sqlite3ExprAssignVarNumber(pParse, A.pExpr); - spanSet(&A, &X, &X); -} expr(A) ::= expr(E) COLLATE ids(C). { A.pExpr = sqlite3ExprAddCollateToken(pParse, E.pExpr, &C); A.zStart = E.zStart; A.zEnd = &C.z[C.n]; } @@ -837,21 +820,21 @@ expr(A) ::= CAST(X) LP expr(E) AS typetoken(T) RP(Y). { A.pExpr = sqlite3PExpr(pParse, TK_CAST, E.pExpr, 0, &T); spanSet(&A,&X,&Y); } %endif SQLITE_OMIT_CAST -expr(A) ::= ID(X) LP distinct(D) exprlist(Y) RP(E). { +expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP(E). { if( Y && Y->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &X); } A.pExpr = sqlite3ExprFunction(pParse, Y, &X); spanSet(&A,&X,&E); if( D && A.pExpr ){ A.pExpr->flags |= EP_Distinct; } } -expr(A) ::= ID(X) LP STAR RP(E). { +expr(A) ::= id(X) LP STAR RP(E). { A.pExpr = sqlite3ExprFunction(pParse, 0, &X); spanSet(&A,&X,&E); } term(A) ::= CTIME_KW(OP). { A.pExpr = sqlite3ExprFunction(pParse, 0, &OP); @@ -886,14 +869,12 @@ {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.bNot = 0;} -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.bNot = 0;} -likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.bNot = 1;} +likeop(A) ::= LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); @@ -1197,15 +1178,14 @@ nmnum(A) ::= nm(X). {A = X;} nmnum(A) ::= ON(X). {A = X;} nmnum(A) ::= DELETE(X). {A = X;} nmnum(A) ::= DEFAULT(X). {A = X;} %endif SQLITE_OMIT_PRAGMA +%token_class number INTEGER|FLOAT. plus_num(A) ::= PLUS number(X). {A = X;} plus_num(A) ::= number(X). {A = X;} minus_num(A) ::= MINUS number(X). {A = X;} -number(A) ::= INTEGER|FLOAT(X). {A = X;} - //////////////////////////// The CREATE TRIGGER command ///////////////////// %ifndef SQLITE_OMIT_TRIGGER cmd ::= createkw trigger_decl(A) BEGIN trigger_cmd_list(S) END(Z). { @@ -1293,16 +1273,12 @@ trigger_cmd(A) ::= UPDATE orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z). { A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R); } // INSERT -trigger_cmd(A) ::= - insert_cmd(R) INTO trnm(X) inscollist_opt(F) valuelist(Y). - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, Y.pList, Y.pSelect, R);} - trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S). - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, 0, S, R);} + {A = sqlite3TriggerInsertStep(pParse->db, &X, F, S, R);} // DELETE trigger_cmd(A) ::= DELETE FROM trnm(X) tridxby where_opt(Y). {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y);} @@ -1404,5 +1380,25 @@ lp ::= LP(X). {sqlite3VtabArgExtend(pParse,&X);} anylist ::= . anylist ::= anylist LP anylist RP. anylist ::= anylist ANY. %endif SQLITE_OMIT_VIRTUALTABLE + + +//////////////////////// COMMON TABLE EXPRESSIONS //////////////////////////// +%type with {With*} +%type wqlist {With*} +%destructor with {sqlite3WithDelete(pParse->db, $$);} +%destructor wqlist {sqlite3WithDelete(pParse->db, $$);} + +with(A) ::= . {A = 0;} +%ifndef SQLITE_OMIT_CTE +with(A) ::= WITH wqlist(W). { A = W; } +with(A) ::= WITH RECURSIVE wqlist(W). { A = W; } + +wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. { + A = sqlite3WithAdd(pParse, 0, &X, Y, Z); +} +wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. { + A = sqlite3WithAdd(pParse, W, &X, Y, Z); +} +%endif SQLITE_OMIT_CTE Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1879,12 +1879,14 @@ /* Make sure all the indices are constructed correctly. */ for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; + Index *pPrior = 0; int loopTop; int iDataCur, iIdxCur; + int r1 = -1; if( pTab->pIndex==0 ) continue; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); @@ -1899,13 +1901,14 @@ pParse->nMem = MAX(pParse->nMem, 8+j); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4; - int r1; if( pPk==pIdx ) continue; - r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, + pPrior, r1); + pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, 0, r1, pIdx->nColumn); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); Index: src/random.c ================================================================== --- src/random.c +++ src/random.c @@ -49,10 +49,16 @@ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); sqlite3_mutex_enter(mutex); #endif + + if( N<=0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not ** trying to do secure encryption or anything like that... @@ -77,19 +83,20 @@ wsdPrng.s[i] = t; } wsdPrng.isInit = 1; } - while( N-- ){ + assert( N>0 ); + do{ wsdPrng.i++; t = wsdPrng.s[wsdPrng.i]; wsdPrng.j += t; wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; wsdPrng.s[wsdPrng.j] = t; t += wsdPrng.s[wsdPrng.i]; *(zBuf++) = wsdPrng.s[t]; - } + }while( --N ); sqlite3_mutex_leave(mutex); } #ifndef SQLITE_OMIT_BUILTIN_TEST /* @@ -114,9 +121,6 @@ &GLOBAL(struct sqlite3PrngType, sqlite3Prng), &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), sizeof(sqlite3Prng) ); } -void sqlite3PrngResetState(void){ - GLOBAL(struct sqlite3PrngType, sqlite3Prng).isInit = 0; -} #endif /* SQLITE_OMIT_BUILTIN_TEST */ Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -27,10 +27,11 @@ sqlite3ExprDelete(db, p->pHaving); sqlite3ExprListDelete(db, p->pOrderBy); sqlite3SelectDelete(db, p->pPrior); sqlite3ExprDelete(db, p->pLimit); sqlite3ExprDelete(db, p->pOffset); + sqlite3WithDelete(db, p->pWith); } /* ** Initialize a SelectDest structure. */ @@ -459,17 +460,17 @@ /* ** Add code to implement the OFFSET */ static void codeOffset( Vdbe *v, /* Generate code into this VM */ - Select *p, /* The SELECT statement being coded */ + int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( p->iOffset && iContinue!=0 ){ + if( iOffset>0 && iContinue!=0 ){ int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); + sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); + addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); } } @@ -540,21 +541,20 @@ /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** -** If srcTab and nColumn are both zero, then the pEList expressions -** are evaluated in order to get the data for this row. If nColumn>0 -** then data is pulled from srcTab and pEList is used only to get the -** datatypes for each column. +** If srcTab is negative, then the pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and pEList is used only +** to get number columns and the datatype for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ int iBreak /* Jump here to break out of the inner loop */ @@ -566,61 +566,54 @@ int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ assert( v ); - if( NEVER(v==0) ) return; assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } /* Pull the requested columns. */ - if( nColumn>0 ){ - nResultCol = nColumn; - }else{ - nResultCol = pEList->nExpr; - } + nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ pDest->iSdst = pParse->nMem+1; pDest->nSdst = nResultCol; pParse->nMem += nResultCol; }else{ assert( pDest->nSdst==nResultCol ); } regResult = pDest->iSdst; - if( nColumn>0 ){ - for(i=0; i=0 ){ + for(i=0; ia[i].zName)); } }else if( eDest!=SRT_Exists ){ /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ sqlite3ExprCodeExprList(pParse, pEList, regResult, (eDest==SRT_Output)?SQLITE_ECEL_DUP:0); } - nColumn = nResultCol; /* If the DISTINCT keyword was present on the SELECT statement ** and this row has been seen before, then do not make this row ** part of the result. */ if( hasDistinct ){ - assert( pEList!=0 ); - assert( pEList->nExpr==nColumn ); switch( pDistinct->eTnctType ){ case WHERE_DISTINCT_ORDERED: { VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ int iJump; /* Jump destination */ int regPrev; /* Previous row content */ /* Allocate space for the previous row */ regPrev = pParse->nMem+1; - pParse->nMem += nColumn; + pParse->nMem += nResultCol; /* Change the OP_OpenEphemeral coded earlier to an OP_Null ** sets the MEM_Cleared bit on the first register of the ** previous value. This will cause the OP_Ne below to always ** fail on the first iteration of the loop even if the first @@ -630,23 +623,23 @@ pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); pOp->opcode = OP_Null; pOp->p1 = 1; pOp->p2 = regPrev; - iJump = sqlite3VdbeCurrentAddr(v) + nColumn; - for(i=0; ia[i].pExpr); - if( iaddrTnct); @@ -653,16 +646,16 @@ break; } default: { assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nColumn, regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); break; } } if( pOrderBy==0 ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } } switch( eDest ){ /* In this mode, write each query result to the key of the temporary @@ -670,11 +663,11 @@ */ #ifndef SQLITE_OMIT_COMPOUND_SELECT case SRT_Union: { int r1; r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; } @@ -681,23 +674,37 @@ /* Construct a record from the query result, but instead of ** saving that record, use it as a key to delete elements from ** the temporary table iParm. */ case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); break; } -#endif +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* Store the result as data using a unique key. */ + case SRT_DistTable: case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistTable ){ + /* If the destination is DistTable, then cursor (iParm+1) is open + ** on an ephemeral index. If the current row is already present + ** in the index, do not write it to the output. If not, add the + ** current row to the index and proceed with writing it to the + ** output table as well. */ + int addr = sqlite3VdbeCurrentAddr(v) + 4; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + assert( pOrderBy==0 ); + } +#endif if( pOrderBy ){ pushOntoSorter(pParse, pOrderBy, p, r1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -713,11 +720,11 @@ /* If we are creating a set for an "expr IN (SELECT ...)" construct, ** then there should be a single item on the stack. Write this ** item into the set table with bogus data. */ case SRT_Set: { - assert( nColumn==1 ); + assert( nResultCol==1 ); pDest->affSdst = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set @@ -745,11 +752,11 @@ /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out ** of the scan loop. */ case SRT_Mem: { - assert( nColumn==1 ); + assert( nResultCol==1 ); if( pOrderBy ){ pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -766,21 +773,67 @@ case SRT_Output: { testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pOrderBy ){ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); pushOntoSorter(pParse, pOrderBy, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ - sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); } break; } + +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. Only add this new value if it has never before + ** been added */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, r3, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + #if !defined(SQLITE_OMIT_TRIGGER) /* Discard the results. This is used for SELECT statements inside ** the body of a TRIGGER. The purpose of such selects is to call ** user-defined functions that have side effects. We do not care @@ -866,19 +919,19 @@ ** ** Space to hold the KeyInfo structure is obtain from malloc. The calling ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ +static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; sqlite3 *db = pParse->db; int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=0, pItem=pList->a; ipExpr); @@ -1015,17 +1068,17 @@ if( p->selFlags & SF_UseSorter ){ int regSortOut = ++pParse->nMem; int ptab2 = pParse->nTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -1200,11 +1253,11 @@ sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); } - }else if( ALWAYS(pTab->pSchema) ){ + }else if( pTab->pSchema ){ /* A real table */ assert( !pS ); if( iCol<0 ) iCol = pTab->iPKey; assert( iCol==-1 || (iCol>=0 && iColnCol) ); #ifdef SQLITE_ENABLE_COLUMN_METADATA @@ -1361,12 +1414,13 @@ sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); }else{ sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); } }else{ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, - sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); + const char *z = pEList->a[i].zSpan; + z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); } } generateColumnTypes(pParse, pTabList, pEList); } @@ -1584,12 +1638,17 @@ ** the limit and offset. If there is no limit and/or offset, then ** iLimit and iOffset are negative. ** ** This routine changes the values of iLimit and iOffset only if ** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values -** (usually but not always -1) prior to calling this routine. +** iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** ** Only if pLimit!=0 or pOffset!=0 do the limit registers get ** redefined. The UNION ALL operator uses this property to force ** the reuse of the same limit and offset registers across multiple ** SELECT statements. */ @@ -1609,11 +1668,11 @@ sqlite3ExprCacheClear(pParse); assert( p->pOffset==0 || p->pLimit!=0 ); if( p->pLimit ){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ + assert( v!=0 ); if( sqlite3ExprIsInteger(p->pLimit, &n) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); @@ -1666,11 +1725,169 @@ } return pRet; } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ -/* Forward reference */ +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from Queue is output to pDest. Then the single +** extracted row (now in the iCurrent table) becomes the content of the +** recursive-table for a recursive-query run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup = p->pPrior; /* The setup query */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent = 0; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Table; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + pOffset = p->pOffset; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = p->pOffset = 0; + p->iLimit = p->iOffset = 0; + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Detach the ORDER BY clause from the compound SELECT */ + pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistTable and SRT_DistQueue destinations to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = pOrderBy ? SRT_DistQueue : SRT_DistTable; + iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Table; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Store the results of the setup-query in Queue. */ + rc = sqlite3Select(pParse, pSetup, &destQueue); + if( rc ) goto end_of_recursive_query; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(v); + codeOffset(v, regOffset, addrCont); + selectInnerLoop(pParse, p, p->pEList, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + if( regLimit ) sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + sqlite3VdbeResolveLabel(v, addrCont); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + p->pOffset = pOffset; + return; +} +#endif + +/* Forward references */ static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ ); @@ -1718,18 +1935,19 @@ Vdbe *v; /* Generate code to this VDBE */ SelectDest dest; /* Alternative data destination */ Select *pDelete = 0; /* Chain of simple selects to delete */ sqlite3 *db; /* Database connection */ #ifndef SQLITE_OMIT_EXPLAIN - int iSub1; /* EQP id of left-hand query */ - int iSub2; /* EQP id of right-hand query */ + int iSub1 = 0; /* EQP id of left-hand query */ + int iSub2 = 0; /* EQP id of right-hand query */ #endif /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ assert( p && p->pPrior ); /* Calling function guarantees this much */ + assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); db = pParse->db; pPrior = p->pPrior; assert( pPrior->pRightmost!=pPrior ); assert( pPrior->pRightmost==p->pRightmost ); dest = *pDest; @@ -1770,16 +1988,22 @@ " do not have the same number of result columns", selectOpName(p->op)); } rc = 1; goto multi_select_end; } + +#ifndef SQLITE_OMIT_CTE + if( p->selFlags & SF_Recursive ){ + generateWithRecursiveQuery(pParse, p, &dest); + }else +#endif /* Compound SELECTs that have an ORDER BY clause are handled separately. */ if( p->pOrderBy ){ return multiSelectOrderBy(pParse, p, pDest); - } + }else /* Generate code for the left and right SELECT statements. */ switch( p->op ){ case TK_ALL: { @@ -1910,11 +2134,11 @@ iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, unionTab, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); @@ -1988,11 +2212,11 @@ sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, tab1, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); @@ -2110,11 +2334,11 @@ } if( pParse->db->mallocFailed ) return 0; /* Suppress the first OFFSET entries if there is an OFFSET clause */ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ case SRT_Table: @@ -2838,10 +3062,18 @@ ** appear as unmodified result columns in the outer query. But we ** have other optimizations in mind to deal with that case. ** ** (21) The subquery does not use LIMIT or the outer query is not ** DISTINCT. (See ticket [752e1646fc]). +** +** (22) The subquery is not a recursive CTE. +** +** (23) The parent is not a recursive CTE, or the sub-query is not a +** compound query. This restriction is because transforming the +** parent to a compound query confuses the code that handles +** recursive queries in multiSelect(). +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query ** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. ** @@ -2910,10 +3142,12 @@ if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } + if( pSub->selFlags & SF_Recursive ) return 0; /* Restriction (22) */ + if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0; /* (23) */ /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is ** not used as the right operand of an outer join. Examples of why this ** is not allowed: @@ -3391,10 +3625,201 @@ pNew->pLimit = 0; pNew->pOffset = 0; return WRC_Continue; } +#ifndef SQLITE_OMIT_CTE +/* +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) +** then return a pointer to the CTE definition for that table. Otherwise +** return NULL. +** +** If a non-NULL value is returned, set *ppContext to point to the With +** object that the returned CTE belongs to. +*/ +static struct Cte *searchWith( + With *pWith, /* Current outermost WITH clause */ + struct SrcList_item *pItem, /* FROM clause element to resolve */ + With **ppContext /* OUT: WITH clause return value belongs to */ +){ + const char *zName; + if( pItem->zDatabase==0 && (zName = pItem->zName)!=0 ){ + With *p; + for(p=pWith; p; p=p->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + *ppContext = p; + return &p->a[i]; + } + } + } + } + return 0; +} + +/* The code generator maintains a stack of active WITH clauses +** with the inner-most WITH clause being at the top of the stack. +** +** This routine pushes the WITH clause passed as the second argument +** onto the top of the stack. If argument bFree is true, then this +** WITH clause will never be popped from the stack. In this case it +** should be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT +** statement with which it is associated. +*/ +void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ + assert( bFree==0 || pParse->pWith==0 ); + if( pWith ){ + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + pParse->bFreeWith = bFree; + } +} + +/* +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser. And, +** if currently processing a CTE expression, if it is a recursive +** reference to the current CTE. +** +** If pFrom falls into either of the two categories above, pFrom->pTab +** and other fields are populated accordingly. The caller should check +** (pFrom->pTab!=0) to determine whether or not a successful match +** was found. +** +** Whether or not a match is found, SQLITE_OK is returned if no error +** occurs. If an error does occur, an error message is stored in the +** parser and some error code other than SQLITE_OK returned. +*/ +static int withExpand( + Walker *pWalker, + struct SrcList_item *pFrom +){ + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + struct Cte *pCte; /* Matched CTE (or NULL if no match) */ + With *pWith; /* WITH clause that pCte belongs to */ + + assert( pFrom->pTab==0 ); + + pCte = searchWith(pParse->pWith, pFrom, &pWith); + if( pCte ){ + Table *pTab; + ExprList *pEList; + Select *pSel; + Select *pLeft; /* Left-most SELECT statement */ + int bMayRecursive; /* True if compound joined by UNION [ALL] */ + With *pSavedWith; /* Initial value of pParse->pWith */ + + /* If pCte->zErr is non-NULL at this point, then this is an illegal + ** recursive reference to CTE pCte. Leave an error in pParse and return + ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** In this case, proceed. */ + if( pCte->zErr ){ + sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + return SQLITE_ERROR; + } + + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->nRef = 1; + pTab->zName = sqlite3DbStrDup(db, pCte->zName); + pTab->iPKey = -1; + pTab->nRowEst = 1048576; + pTab->tabFlags |= TF_Ephemeral; + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( db->mallocFailed ) return SQLITE_NOMEM; + assert( pFrom->pSelect ); + + /* Check if this is a recursive CTE. */ + pSel = pFrom->pSelect; + bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); + if( bMayRecursive ){ + int i; + SrcList *pSrc = pFrom->pSelect->pSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pSrc->a[i]; + if( pItem->zDatabase==0 + && pItem->zName!=0 + && 0==sqlite3StrICmp(pItem->zName, pCte->zName) + ){ + pItem->pTab = pTab; + pItem->isRecursive = 1; + pTab->nRef++; + pSel->selFlags |= SF_Recursive; + } + } + } + + /* Only one recursive reference is permitted. */ + if( pTab->nRef>2 ){ + sqlite3ErrorMsg( + pParse, "multiple references to recursive table: %s", pCte->zName + ); + return SQLITE_ERROR; + } + assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); + + pCte->zErr = "circular reference: %s"; + pSavedWith = pParse->pWith; + pParse->pWith = pWith; + sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; + if( pCte->pCols ){ + if( pEList->nExpr!=pCte->pCols->nExpr ){ + sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", + pCte->zName, pEList->nExpr, pCte->pCols->nExpr + ); + pParse->pWith = pSavedWith; + return SQLITE_ERROR; + } + pEList = pCte->pCols; + } + + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + if( bMayRecursive ){ + if( pSel->selFlags & SF_Recursive ){ + pCte->zErr = "multiple recursive references: %s"; + }else{ + pCte->zErr = "recursive reference in a subquery: %s"; + } + sqlite3WalkSelect(pWalker, pSel); + } + pCte->zErr = 0; + pParse->pWith = pSavedWith; + } + + return SQLITE_OK; +} +#endif + +#ifndef SQLITE_OMIT_CTE +/* +** If the SELECT passed as the second argument has an associated WITH +** clause, pop it from the stack stored as part of the Parse object. +** +** This function is used as the xSelectCallback2() callback by +** sqlite3SelectExpand() when walking a SELECT tree to resolve table +** names and other FROM clause elements. +*/ +static void selectPopWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + if( p->pWith ){ + assert( pParse->pWith==p->pWith ); + pParse->pWith = p->pWith->pOuter; + } +} +#else +#define selectPopWith 0 +#endif + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: ** ** (1) Make sure VDBE cursor numbers have been assigned to every @@ -3434,10 +3859,11 @@ if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } pTabList = p->pSrc; pEList = p->pEList; + sqlite3WithPush(pParse, p->pWith, 0); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. */ sqlite3SrcListAssignCursors(pParse, pTabList); @@ -3446,16 +3872,25 @@ ** an entry of the FROM clause is a subquery instead of a table or view, ** then create a transient table structure to describe the subquery. */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; + assert( pFrom->isRecursive==0 || pFrom->pTab ); + if( pFrom->isRecursive ) continue; if( pFrom->pTab!=0 ){ /* This statement has already been prepared. There is no need ** to go further. */ assert( i==0 ); +#ifndef SQLITE_OMIT_CTE + selectPopWith(pWalker, p); +#endif return WRC_Prune; } +#ifndef SQLITE_OMIT_CTE + if( withExpand(pWalker, pFrom) ) return WRC_Abort; + if( pFrom->pTab ) {} else +#endif if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY Select *pSel = pFrom->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); @@ -3714,10 +4149,11 @@ if( pParse->hasCompound ){ w.xSelectCallback = convertCompoundSelectToSubquery; sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; + w.xSelectCallback2 = selectPopWith; sqlite3WalkSelect(&w, pSelect); } #ifndef SQLITE_OMIT_SUBQUERY @@ -3732,11 +4168,11 @@ ** The Table structure that represents the result set was constructed ** by selectExpander() but the type and collation information was omitted ** at that point because identifiers had not yet been resolved. This ** routine is called after identifier resolution. */ -static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ +static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; int i; SrcList *pTabList; struct SrcList_item *pFrom; @@ -3748,17 +4184,17 @@ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ /* A sub-query in the FROM clause of a SELECT */ Select *pSel = pFrom->pSelect; - assert( pSel ); - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); + } } } } - return WRC_Continue; } #endif /* @@ -3770,14 +4206,13 @@ */ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; memset(&w, 0, sizeof(w)); - w.xSelectCallback = selectAddSubqueryTypeInfo; + w.xSelectCallback2 = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; - w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } @@ -3845,11 +4280,11 @@ if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } } } @@ -3978,54 +4413,12 @@ #endif /* ** Generate code for the SELECT statement given in the p argument. ** -** The results are distributed in various ways depending on the -** contents of the SelectDest structure pointed to by argument pDest -** as follows: -** -** pDest->eDest Result -** ------------ ------------------------------------------- -** SRT_Output Generate a row of output (using the OP_ResultRow -** opcode) for each row in the result set. -** -** SRT_Mem Only valid if the result is a single column. -** Store the first column of the first result row -** in register pDest->iSDParm then abandon the rest -** of the query. This destination implies "LIMIT 1". -** -** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iSDParm. -** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". -** -** SRT_Union Store results as a key in a temporary table -** identified by pDest->iSDParm. -** -** SRT_Except Remove results from the temporary table pDest->iSDParm. -** -** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. -** -** SRT_EphemTab Create an temporary table pDest->iSDParm and store -** the result there. The cursor is left open after -** returning. This is like SRT_Table except that -** this destination uses OP_OpenEphemeral to create -** the table first. -** -** SRT_Coroutine Generate a co-routine that returns a new row of -** results each time it is invoked. The entry point -** of the co-routine is stored in register pDest->iSDParm. -** -** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result -** set is not empty. -** -** SRT_Discard Throw the results away. This is used by SELECT -** statements within triggers whose only purpose is -** the side-effects of functions. +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. ** ** This routine returns the number of errors. If any errors are ** encountered, then an appropriate error message is left in ** pParse->zErrMsg. ** @@ -4296,11 +4689,11 @@ ** we figure out that the sorting index is not needed. The addrSortIndex ** variable is used to facilitate that change. */ if( pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); + pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 0); pOrderBy->iECursor = pParse->nTab++; p->addrOpenEphm[2] = addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pOrderBy->iECursor, pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -4328,11 +4721,11 @@ */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList), + (char*)keyInfoFromExprList(pParse, p->pEList, 0), P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ sDistinct.eTnctType = WHERE_DISTINCT_NOOP; @@ -4362,11 +4755,11 @@ sqlite3VdbeChangeToNoop(v, addrSortIndex); p->addrOpenEphm[2] = -1; } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, pOrderBy, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); /* End the database scan loop. */ @@ -4452,11 +4845,11 @@ ** implement it. Allocate that sorting index now. If it turns out ** that we do not need it after all, the OP_SorterOpen instruction ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); /* Initialize memory locations used by GROUP BY aggregate processing @@ -4634,11 +5027,11 @@ sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, + selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); VdbeComment((v, "end groupby result generator")); @@ -4777,11 +5170,11 @@ finalizeAggFunctions(pParse, &sAggInfo); } pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } sqlite3VdbeResolveLabel(v, addrEnd); Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -595,10 +595,11 @@ ** Output the given string with characters that are special to ** HTML escaped. */ static void output_html_string(FILE *out, const char *z){ int i; + if( z==0 ) z = ""; while( *z ){ for(i=0; z[i] && z[i]!='<' && z[i]!='&' && z[i]!='>' @@ -1174,11 +1175,11 @@ ** all opcodes that occur between the p2 jump destination and the opcode ** itself by 2 spaces. ** ** * For each "Goto", if the jump destination is earlier in the program ** and ends on one of: -** Yield SeekGt SeekLt RowSetRead +** Yield SeekGt SeekLt RowSetRead Rewind ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ const char *zSql; /* The text of the SQL statement */ @@ -1186,11 +1187,11 @@ int *abYield = 0; /* True if op is an OP_Yield */ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; - const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", 0 }; + const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; /* Try to figure out if this is really an EXPLAIN statement. If this ** cannot be verified, return early. */ zSql = sqlite3_sql(pSql); @@ -1223,11 +1224,11 @@ if( str_in_array(zOp, azNext) ){ for(i=p2op; iaiIndent[i] += 2; } if( str_in_array(zOp, azGoto) && p2opnIndent && abYield[p2op] ){ - for(i=p2op; iaiIndent[i] += 2; + for(i=p2op+1; iaiIndent[i] += 2; } } p->iIndent = 0; sqlite3_free(abYield); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -2393,15 +2393,17 @@ ** already uses the largest possible [ROWID]. The PRNG is also used for ** the build-in random() and randomblob() SQL functions. This interface allows ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. +** ^If N is less than one, then P can be a NULL pointer. ** -** ^The first time this routine is invoked (either internally or by -** the application) the PRNG is seeded using randomness obtained -** from the xRandomness method of the default [sqlite3_vfs] object. -** ^On all subsequent invocations, the pseudo-randomness is generated +** ^If this routine has not been previously called or if the previous +** call had N less than one, then the PRNG is seeded using randomness +** obtained from the xRandomness method of the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more then +** the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ void sqlite3_randomness(int N, void *P); @@ -2557,10 +2559,11 @@ #define SQLITE_CREATE_VTABLE 29 /* Table Name Module Name */ #define SQLITE_DROP_VTABLE 30 /* Table Name Module Name */ #define SQLITE_FUNCTION 31 /* NULL Function Name */ #define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ #define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ /* ** CAPI3REF: Tracing And Profiling Functions ** ** These routines register callback functions that can be used for Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -759,10 +759,11 @@ typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; +typedef struct With With; /* ** Defer sourcing vdbe.h and btree.h until after the "u8" and ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. @@ -1062,11 +1063,11 @@ */ #define SQLITE_QueryFlattener 0x0001 /* Query flattening */ #define SQLITE_ColumnCache 0x0002 /* Column cache */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ -#define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ +/* not used 0x0010 // Was: SQLITE_IdxRealAsInt */ #define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ #define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ @@ -1434,11 +1435,11 @@ Schema *pSchema; /* Schema that contains this table */ Table *pNextZombie; /* Next on the Parse.pZombieTab list */ }; /* -** Allowed values for Tabe.tabFlags. +** Allowed values for Table.tabFlags. */ #define TF_Readonly 0x01 /* Read-only system table */ #define TF_Ephemeral 0x02 /* An ephemeral table */ #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ @@ -2023,10 +2024,11 @@ int regReturn; /* Register holding return address of addrFillSub */ u8 jointype; /* Type of join between this able and the previous */ unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ @@ -2149,10 +2151,11 @@ Select *pPrior; /* Prior select in a compound select statement */ Select *pNext; /* Next select to the left in a compound */ Select *pRightmost; /* Right-most select in a compound select statement */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ Expr *pOffset; /* OFFSET expression. NULL means not used. */ + With *pWith; /* WITH clause attached to this select. Or NULL. */ }; /* ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". @@ -2166,15 +2169,74 @@ #define SF_UseSorter 0x0040 /* Sort using a sorter */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ #define SF_Materialize 0x0100 /* Force materialization of views */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ +#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ /* -** The results of a select can be distributed in several ways. The -** "SRT" prefix means "SELECT Result Type". +** The results of a SELECT can be distributed in several ways, as defined +** by one of the following macros. The "SRT" prefix means "SELECT Result +** Type". +** +** SRT_Union Store results as a key in a temporary index +** identified by pDest->iSDParm. +** +** SRT_Except Remove results from the temporary index pDest->iSDParm. +** +** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result +** set is not empty. +** +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. +** +** All of the above are free to ignore their ORDER BY clause. Those that +** follow must honor the ORDER BY clause. +** +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. +** +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iSDParm then abandon the rest +** of the query. This destination implies "LIMIT 1". +** +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iSDParm. +** Apply the affinity pDest->affSdst before storing +** results. Used to implement "IN (SELECT ...)". +** +** SRT_EphemTab Create an temporary table pDest->iSDParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. +** +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iSDParm +** and the result row is stored in pDest->nDest registers +** starting with pDest->iSdst. +** +** SRT_Table Store results in temporary table pDest->iSDParm. +** This is like SRT_EphemTab except that the table +** is assumed to already be open. +** +** SRT_DistTable Store results in a temporary table pDest->iSDParm. +** But also use temporary table pDest->iSDParm+1 as +** a record of all prior results and ignore any duplicate +** rows. Name means: "Distinct Table". +** +** SRT_Queue Store results in priority queue pDest->iSDParm (really +** an index). Append a sequence number so that all entries +** are distinct. +** +** SRT_DistQueue Store results in priority queue pDest->iSDParm only if +** the same record has never been stored before. The +** index at pDest->iSDParm+1 hold all prior stores. */ #define SRT_Union 1 /* Store result as keys in an index */ #define SRT_Except 2 /* Remove result from a UNION index */ #define SRT_Exists 3 /* Store 1 if the result is not empty */ #define SRT_Discard 4 /* Do not save the results anywhere */ @@ -2183,24 +2245,28 @@ #define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard) #define SRT_Output 5 /* Output each row of result */ #define SRT_Mem 6 /* Store result in a memory cell */ #define SRT_Set 7 /* Store results as keys in an index */ -#define SRT_Table 8 /* Store result as data with an automatic rowid */ -#define SRT_EphemTab 9 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 10 /* Generate a single row of result */ +#define SRT_EphemTab 8 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 9 /* Generate a single row of result */ +#define SRT_Table 10 /* Store result as data with an automatic rowid */ +#define SRT_DistTable 11 /* Like SRT_Table, but unique results only */ +#define SRT_Queue 12 /* Store result in an queue */ +#define SRT_DistQueue 13 /* Like SRT_Queue, but unique results only */ /* ** An instance of this object describes where to put of the results of ** a SELECT statement. */ struct SelectDest { - u8 eDest; /* How to dispose of the results. On of SRT_* above. */ - char affSdst; /* Affinity used when eDest==SRT_Set */ - int iSDParm; /* A parameter used by the eDest disposal method */ - int iSdst; /* Base register where results are written */ - int nSdst; /* Number of registers allocated */ + u8 eDest; /* How to dispose of the results. On of SRT_* above. */ + char affSdst; /* Affinity used when eDest==SRT_Set */ + int iSDParm; /* A parameter used by the eDest disposal method */ + int iSdst; /* Base register where results are written */ + int nSdst; /* Number of registers allocated */ + ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; /* ** During code generation of statements that do inserts into AUTOINCREMENT ** tables, the following information is attached to the Table.u.autoInc.p @@ -2299,10 +2365,11 @@ int nSet; /* Number of sets used so far */ int nOnce; /* Number of OP_Once instructions so far */ int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ + int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ int iPartIdxTab; /* Table corresponding to a partial index */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ struct yColCache { @@ -2369,10 +2436,12 @@ Token sArg; /* Complete text of a module argument */ Table **apVtabLock; /* Pointer to virtual tables needing locking */ #endif Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + With *pWith; /* Current WITH clause, or NULL */ + u8 bFreeWith; /* True if pWith should be freed with parser */ }; /* ** Return true if currently inside an sqlite3_declare_vtab() call. */ @@ -2492,11 +2561,11 @@ u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ Select *pSelect; /* SELECT statment or RHS of INSERT INTO .. SELECT ... */ Token target; /* Target table for DELETE, UPDATE, INSERT */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ - ExprList *pExprList; /* SET clause for UPDATE. VALUES clause for INSERT */ + ExprList *pExprList; /* SET clause for UPDATE. */ IdList *pIdList; /* Column names for INSERT */ TriggerStep *pNext; /* Next in the link-list */ TriggerStep *pLast; /* Last element in link-list. Valid for 1st elem only */ }; @@ -2614,13 +2683,13 @@ ** Context pointer passed down through the tree-walk. */ struct Walker { int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ + void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ - u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ @@ -2639,10 +2708,25 @@ ** callbacks. */ #define WRC_Continue 0 /* Continue down into children */ #define WRC_Prune 1 /* Omit children but continue walking siblings */ #define WRC_Abort 2 /* Abandon the tree walk */ + +/* +** An instance of this structure represents a set of one or more CTEs +** (common table expressions) created by a single WITH clause. +*/ +struct With { + int nCte; /* Number of CTEs in the WITH clause */ + With *pOuter; /* Containing WITH clause, or NULL */ + struct Cte { /* For each CTE in the WITH clause.... */ + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ + Select *pSelect; /* The definition of this CTE */ + const char *zErr; /* Error message for circular references */ + } a[1]; +}; /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. */ @@ -2907,11 +2991,11 @@ #else # define sqlite3AutoincrementBegin(X) # define sqlite3AutoincrementEnd(X) #endif int sqlite3CodeCoroutine(Parse*, Select*, SelectDest*); -void sqlite3Insert(Parse*, SrcList*, ExprList*, Select*, IdList*, int); +void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int); void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); int sqlite3IdListIndex(IdList*,const char*); SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int); SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*); @@ -2982,11 +3066,10 @@ void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); Vdbe *sqlite3GetVdbe(Parse*); void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); -void sqlite3PrngResetState(void); void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); void sqlite3CommitTransaction(Parse*); @@ -3002,11 +3085,11 @@ void sqlite3ExprCodeIsNullJump(Vdbe*, const Expr*, int, int); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); int sqlite3IsRowid(const char*); void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); -int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*); +int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*); void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); int sqlite3OpenTableAndIndices(Parse*, Table*, int, int, u8*, int*, int*); void sqlite3BeginWriteOperation(Parse*, int, int); @@ -3046,11 +3129,11 @@ void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, int, int, int); void sqliteViewTriggers(Parse*, Table*, Expr*, int, ExprList*); void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*); TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*, - ExprList*,Select*,u8); + Select*,u8); TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8); TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*); void sqlite3DeleteTrigger(sqlite3*, Trigger*); void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); @@ -3339,10 +3422,18 @@ const char *sqlite3JournalModename(int); #ifndef SQLITE_OMIT_WAL int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif +#ifndef SQLITE_OMIT_CTE + With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); + void sqlite3WithDelete(sqlite3*,With*); + void sqlite3WithPush(Parse*, With*, u8); +#else +#define sqlite3WithPush(x,y,z) +#define sqlite3WithDelete(x,y) +#endif /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign ** key functionality is available. If OMIT_TRIGGER is defined but ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -956,10 +956,11 @@ case SQLITE_ANALYZE : zCode="SQLITE_ANALYZE"; break; case SQLITE_CREATE_VTABLE : zCode="SQLITE_CREATE_VTABLE"; break; case SQLITE_DROP_VTABLE : zCode="SQLITE_DROP_VTABLE"; break; case SQLITE_FUNCTION : zCode="SQLITE_FUNCTION"; break; case SQLITE_SAVEPOINT : zCode="SQLITE_SAVEPOINT"; break; + case SQLITE_RECURSIVE : zCode="SQLITE_RECURSIVE"; break; default : zCode="????"; break; } Tcl_DStringInit(&str); Tcl_DStringAppend(&str, pDb->zAuth, -1); Tcl_DStringAppendElement(&str, zCode); Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -240,11 +240,32 @@ } #endif return TCL_OK; } - +/* +** Usage: clang_sanitize_address +** +** Returns true if the program was compiled using clang with the +** -fsanitize=address switch on the command line. False otherwise. +*/ +static int clang_sanitize_address( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + int res = 0; +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + res = 1; +# endif +#endif + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + return TCL_OK; +} + /* ** Usage: sqlite3_exec_printf DB FORMAT STRING ** ** Invoke the sqlite3_exec_printf() interface using the open database ** DB. The SQL is the string FORMAT. The format string should contain @@ -6158,11 +6179,10 @@ { "none", 0 }, { "query-flattener", SQLITE_QueryFlattener }, { "column-cache", SQLITE_ColumnCache }, { "groupby-order", SQLITE_GroupByOrder }, { "factor-constants", SQLITE_FactorOutConst }, - { "real-as-int", SQLITE_IdxRealAsInt }, { "distinct-opt", SQLITE_DistinctOpt }, { "cover-idx-scan", SQLITE_CoverIdxScan }, { "order-by-idx-join", SQLITE_OrderByIdxJoin }, { "transitive", SQLITE_Transitive }, { "subquery-coroutine", SQLITE_SubqCoroutine }, @@ -6322,10 +6342,11 @@ { "sqlite3_get_autocommit", (Tcl_CmdProc*)get_autocommit }, { "sqlite3_stack_used", (Tcl_CmdProc*)test_stack_used }, { "sqlite3_busy_timeout", (Tcl_CmdProc*)test_busy_timeout }, { "printf", (Tcl_CmdProc*)test_printf }, { "sqlite3IoTrace", (Tcl_CmdProc*)test_io_trace }, + { "clang_sanitize_address", (Tcl_CmdProc*)clang_sanitize_address }, }; static struct { char *zName; Tcl_ObjCmdProc *xProc; void *clientData; Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -228,10 +228,16 @@ #ifdef SQLITE_OMIT_CHECK Tcl_SetVar2(interp, "sqlite_options", "check", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "check", "1", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_OMIT_CTE + Tcl_SetVar2(interp, "sqlite_options", "cte", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "cte", "1", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_ENABLE_COLUMN_METADATA Tcl_SetVar2(interp, "sqlite_options", "columnmetadata", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "columnmetadata", "0", TCL_GLOBAL_ONLY); Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -301,28 +301,19 @@ case '?': { *tokenType = TK_VARIABLE; for(i=1; sqlite3Isdigit(z[i]); i++){} return i; } - case '#': { - for(i=1; sqlite3Isdigit(z[i]); i++){} - if( i>1 ){ - /* Parameters of the form #NNN (where NNN is a number) are used - ** internally by sqlite3NestedParse. */ - *tokenType = TK_REGISTER; - return i; - } - /* Fall through into the next case if the '#' is not followed by - ** a digit. Try to match #AAAA where AAAA is a parameter name. */ - } #ifndef SQLITE_OMIT_TCL_VARIABLE case '$': #endif case '@': /* For compatibility with MS SQL Server */ + case '#': case ':': { int n = 0; - testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; for(i=1; (c=z[i])!=0; i++){ if( IdChar(c) ){ n++; #ifndef SQLITE_OMIT_TCL_VARIABLE @@ -501,10 +492,11 @@ ** will take responsibility for freeing the Table structure. */ sqlite3DeleteTable(db, pParse->pNewTable); } + if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); sqlite3DeleteTrigger(db, pParse->pNewTrigger); for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); sqlite3DbFree(db, pParse->azVar); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -395,29 +395,25 @@ */ TriggerStep *sqlite3TriggerInsertStep( sqlite3 *db, /* The database connection */ Token *pTableName, /* Name of the table into which we insert */ IdList *pColumn, /* List of columns in pTableName to insert into */ - ExprList *pEList, /* The VALUE clause: a list of values to be inserted */ Select *pSelect, /* A SELECT statement that supplies values */ u8 orconf /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ ){ TriggerStep *pTriggerStep; - assert(pEList == 0 || pSelect == 0); - assert(pEList != 0 || pSelect != 0 || db->mallocFailed); + assert(pSelect != 0 || db->mallocFailed); pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName); if( pTriggerStep ){ pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); pTriggerStep->pIdList = pColumn; - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); pTriggerStep->orconf = orconf; }else{ sqlite3IdListDelete(db, pColumn); } - sqlite3ExprListDelete(db, pEList); sqlite3SelectDelete(db, pSelect); return pTriggerStep; } @@ -751,11 +747,10 @@ break; } case TK_INSERT: { sqlite3Insert(pParse, targetSrcList(pParse, pStep), - sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3SelectDup(db, pStep->pSelect, 0), sqlite3IdListDup(db, pStep->pIdList), pParse->eOrconf ); break; Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -4425,11 +4425,10 @@ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; - assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } break; } Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -175,10 +175,11 @@ void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); void sqlite3VdbeChangeP5(Vdbe*, u8 P5); void sqlite3VdbeJumpHere(Vdbe*, int addr); void sqlite3VdbeChangeToNoop(Vdbe*, int addr); +int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); void sqlite3VdbeUsesBtree(Vdbe*, int); VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); int sqlite3VdbeMakeLabel(Vdbe*); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -275,10 +275,11 @@ assert( v->magic==VDBE_MAGIC_INIT ); assert( jnLabel ); if( j>=0 && p->aLabel ){ p->aLabel[j] = v->nOp; } + p->iFixedOp = v->nOp - 1; } /* ** Mark the VDBE as one that can only be run one time. */ @@ -623,11 +624,12 @@ /* ** Change the P2 operand of instruction addr so that it points to ** the address of the next instruction to be coded. */ void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - if( ALWAYS(addr>=0) ) sqlite3VdbeChangeP2(p, addr, p->nOp); + sqlite3VdbeChangeP2(p, addr, p->nOp); + p->pParse->iFixedOp = p->nOp - 1; } /* ** If the input FuncDef structure is ephemeral, then free it. If @@ -724,10 +726,22 @@ memset(pOp, 0, sizeof(pOp[0])); pOp->opcode = OP_Noop; if( addr==p->nOp-1 ) p->nOp--; } } + +/* +** Remove the last opcode inserted +*/ +int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ + if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ + sqlite3VdbeChangeToNoop(p, p->nOp-1); + return 1; + }else{ + return 0; + } +} /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a ** static array using sqlite3VdbeAddOpList but we want to make a Index: src/walker.c ================================================================== --- src/walker.c +++ src/walker.c @@ -111,43 +111,44 @@ } /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. Invoke the xSelectCallback() -** either before or after the walk of expressions and FROM clause, depending -** on whether pWalker->bSelectDepthFirst is false or true, respectively. +** on the compound select chain, p->pPrior. +** +** If it is not NULL, the xSelectCallback() callback is invoked before +** the walk of the expressions and FROM clause. The xSelectCallback2() +** method, if it is not NULL, is invoked following the walk of the +** expressions and FROM clause. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. ** ** If the Walker does not have an xSelectCallback() then this routine ** is a no-op returning WRC_Continue. */ int sqlite3WalkSelect(Walker *pWalker, Select *p){ int rc; - if( p==0 || pWalker->xSelectCallback==0 ) return WRC_Continue; + if( p==0 || (pWalker->xSelectCallback==0 && pWalker->xSelectCallback2==0) ){ + return WRC_Continue; + } rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - if( !pWalker->bSelectDepthFirst ){ + if( pWalker->xSelectCallback ){ rc = pWalker->xSelectCallback(pWalker, p); if( rc ) break; } if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ pWalker->walkerDepth--; return WRC_Abort; } - if( pWalker->bSelectDepthFirst ){ - rc = pWalker->xSelectCallback(pWalker, p); - /* Depth-first search is currently only used for - ** selectAddSubqueryTypeInfo() and that routine always returns - ** WRC_Continue (0). So the following branch is never taken. */ - if( NEVER(rc) ) break; + if( pWalker->xSelectCallback2 ){ + pWalker->xSelectCallback2(pWalker, p); } p = p->pPrior; } pWalker->walkerDepth--; return rc & WRC_Abort; } Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -665,11 +665,11 @@ ** be the name of an indexed column with TEXT affinity. */ return 0; } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - pRight = pList->a[0].pExpr; + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); op = pRight->op; if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; int iCol = pRight->iColumn; pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_NONE); @@ -1708,11 +1708,11 @@ VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0); + sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); @@ -1749,11 +1749,12 @@ for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ if( pTerm->leftCursor != pSrc->iCursor ) continue; assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); testcase( pTerm->eOperator & WO_IN ); testcase( pTerm->eOperator & WO_ISNULL ); - if( pTerm->eOperator & (WO_ISNULL) ) continue; + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; } /* If the ORDER BY clause contains only columns in the current @@ -1801,11 +1802,12 @@ u8 op; if( pTerm->leftCursor != pSrc->iCursor ) continue; assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); testcase( pTerm->eOperator & WO_IN ); testcase( pTerm->eOperator & WO_ISNULL ); - if( pTerm->eOperator & (WO_ISNULL) ) continue; + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; op = (u8)pTerm->eOperator & WO_ALL; if( op==WO_IN ) op = WO_EQ; @@ -3406,14 +3408,20 @@ ** scan of the entire table. */ static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + if( pTabItem->isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } } /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. */ @@ -4194,10 +4202,11 @@ && pSrc->pIndex==0 && !pSrc->viaCoroutine && !pSrc->notIndexed && HasRowid(pTab) && !pSrc->isCorrelated + && !pSrc->isRecursive ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; WhereTerm *pWCEnd = pWC->a + pWC->nTerm; for(pTerm=pWC->a; rc==SQLITE_OK && pTermiBreak, SQLITE_JUMPIFNULL); - pWhere = 0; + for(ii=0; iinTerm; ii++){ + if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){ + sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak, + SQLITE_JUMPIFNULL); + sWLB.pWC->a[ii].wtFlags |= TERM_CODED; + } } /* Special case: No FROM clause */ if( nTabList==0 ){ Index: test/auth.test ================================================================== --- test/auth.test +++ test/auth.test @@ -2078,10 +2078,46 @@ set authargs } {main t5 {} {}} execsql {DROP TABLE t5} } ;# ifcapable altertable +ifcapable {cte} { + do_test auth-1.310 { + proc auth {code arg1 arg2 arg3 arg4} { + if {$code=="SQLITE_RECURSIVE"} { + return SQLITE_DENY + } + return SQLITE_OK + } + db eval { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,2),(3,4),(5,6); + } + } {} + do_catchsql_test auth-1.311 { + WITH + auth1311(x,y) AS (SELECT a+b, b-a FROM t1) + SELECT * FROM auth1311 ORDER BY x; + } {0 {3 1 7 1 11 1}} + do_catchsql_test auth-1.312 { + WITH RECURSIVE + auth1312(x,y) AS (SELECT a+b, b-a FROM t1) + SELECT x, y FROM auth1312 ORDER BY x; + } {0 {3 1 7 1 11 1}} + do_catchsql_test auth-1.313 { + WITH RECURSIVE + auth1313(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM auth1313 WHERE x<5) + SELECT * FROM t1; + } {0 {1 2 3 4 5 6}} + do_catchsql_test auth-1.314 { + WITH RECURSIVE + auth1314(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM auth1314 WHERE x<5) + SELECT * FROM t1 LEFT JOIN auth1314; + } {1 {not authorized}} +} ;# ifcapable cte + do_test auth-2.1 { proc auth {code arg1 arg2 arg3 arg4} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="x"} { return SQLITE_DENY } Index: test/capi3.test ================================================================== --- test/capi3.test +++ test/capi3.test @@ -177,20 +177,22 @@ sqlite3_errmsg $db2 } {unable to open database file} do_test capi3-3.5 { sqlite3_close $db2 } {SQLITE_OK} -do_test capi3-3.6.1-misuse { - sqlite3_close $db2 -} {SQLITE_MISUSE} -do_test capi3-3.6.2-misuse { - sqlite3_errmsg $db2 -} {library routine called out of sequence} -ifcapable {utf16} { - do_test capi3-3.6.3-misuse { - utf8 [sqlite3_errmsg16 $db2] +if {[clang_sanitize_address]==0} { + do_test capi3-3.6.1-misuse { + sqlite3_close $db2 + } {SQLITE_MISUSE} + do_test capi3-3.6.2-misuse { + sqlite3_errmsg $db2 } {library routine called out of sequence} + ifcapable {utf16} { + do_test capi3-3.6.3-misuse { + utf8 [sqlite3_errmsg16 $db2] + } {library routine called out of sequence} + } } do_test capi3-3.7 { set db2 [sqlite3_open] sqlite3_errcode $db2 @@ -659,14 +661,16 @@ #check_data $STMT capi3-6.3 {INTEGER} {1} {1.0} {1} do_test capi3-6.3 { sqlite3_finalize $STMT } {SQLITE_OK} -do_test capi3-6.4-misuse { - db cache flush - sqlite3_close $DB -} {SQLITE_OK} +if {[clang_sanitize_address]==0} { + do_test capi3-6.4-misuse { + db cache flush + sqlite3_close $DB + } {SQLITE_OK} +} db close # This procedure sets the value of the file-format in file 'test.db' # to $newval. Also, the schema cookie is incremented. # @@ -1058,15 +1062,17 @@ expr {$ms==80 || $ms==1000} } {1} } # Ticket #1219: Make sure binding APIs can handle a NULL pointer. -# -do_test capi3-14.1-misuse { - set rc [catch {sqlite3_bind_text 0 1 hello 5} msg] - lappend rc $msg -} {1 SQLITE_MISUSE} +# +if {[clang_sanitize_address]==0} { + do_test capi3-14.1-misuse { + set rc [catch {sqlite3_bind_text 0 1 hello 5} msg] + lappend rc $msg + } {1 SQLITE_MISUSE} +} # Ticket #1650: Honor the nBytes parameter to sqlite3_prepare. # do_test capi3-15.1 { set sql {SELECT * FROM t2} Index: test/capi3c.test ================================================================== --- test/capi3c.test +++ test/capi3c.test @@ -167,20 +167,22 @@ sqlite3_errmsg $db2 } {unable to open database file} do_test capi3c-3.5 { sqlite3_close $db2 } {SQLITE_OK} -do_test capi3c-3.6.1-misuse { - sqlite3_close $db2 -} {SQLITE_MISUSE} -do_test capi3c-3.6.2-misuse { - sqlite3_errmsg $db2 -} {library routine called out of sequence} -ifcapable {utf16} { - do_test capi3c-3.6.3-misuse { - utf8 [sqlite3_errmsg16 $db2] +if {[clang_sanitize_address]==0} { + do_test capi3c-3.6.1-misuse { + sqlite3_close $db2 + } {SQLITE_MISUSE} + do_test capi3c-3.6.2-misuse { + sqlite3_errmsg $db2 } {library routine called out of sequence} + ifcapable {utf16} { + do_test capi3c-3.6.3-misuse { + utf8 [sqlite3_errmsg16 $db2] + } {library routine called out of sequence} + } } # rename sqlite3_open "" # rename sqlite3_open_old sqlite3_open @@ -625,17 +627,21 @@ } {SQLITE_ROW} check_data $STMT capi3c-6.3 {INTEGER} {1} {1.0} {1} do_test capi3c-6.3 { sqlite3_finalize $STMT } {SQLITE_OK} -do_test capi3c-6.4 { - db cache flush - sqlite3_close $DB -} {SQLITE_OK} -do_test capi3c-6.99-misuse { +if {[clang_sanitize_address]==0} { + do_test capi3c-6.4 { + db cache flush + sqlite3_close $DB + } {SQLITE_OK} + do_test capi3c-6.99-misuse { + db close + } {} +} else { db close -} {} +} # This procedure sets the value of the file-format in file 'test.db' # to $newval. Also, the schema cookie is incremented. # proc set_file_format {newval} { ADDED test/corruptH.test Index: test/corruptH.test ================================================================== --- test/corruptH.test +++ test/corruptH.test @@ -0,0 +1,150 @@ +# 2014-01-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix corruptH + +# Do not use a codec for tests in this file, as the database file is +# manipulated directly using tcl scripts (using the [hexio_write] command). +# +do_not_use_codec +database_may_be_corrupt + +# Initialize the database. +# +do_execsql_test 1.1 { + PRAGMA page_size=1024; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(randomblob(200)); + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; +} {} + +# Corrupt the file so that the root page of t1 is also linked into t2 as +# a leaf page. +# +do_test 1.2 { + db eval { SELECT name, rootpage FROM sqlite_master } { + set r($name) $rootpage + } + db close + hexio_write test.db [expr {($r(t2)-1)*1024 + 11}] [format %.2X $r(t1)] + sqlite3 db test.db +} {} + +do_test 1.3 { + db eval { PRAGMA secure_delete=1 } + list [catch { + db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { + db eval { DELETE FROM t2 } + } + } msg] $msg +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db + +# Initialize the database. +# +do_execsql_test 2.1 { + PRAGMA page_size=1024; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + + CREATE TABLE t3(x); + + CREATE TABLE t2(x PRIMARY KEY) WITHOUT ROWID; + INSERT INTO t2 VALUES(randomblob(100)); + + DROP TABLE t3; +} {} + +do_test 2.2 { + db eval { SELECT name, rootpage FROM sqlite_master } { + set r($name) $rootpage + } + db close + set fl [hexio_get_int [hexio_read test.db 32 4]] + + hexio_write test.db [expr {($fl-1) * 1024 + 0}] 00000000 + hexio_write test.db [expr {($fl-1) * 1024 + 4}] 00000001 + hexio_write test.db [expr {($fl-1) * 1024 + 8}] [format %.8X $r(t1)] + hexio_write test.db 36 00000002 + + sqlite3 db test.db +} {} + +do_test 2.3 { + list [catch { + db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { + db eval { + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + } + } + } msg] $msg +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db + +# Initialize the database. +# +do_execsql_test 3.1 { + PRAGMA page_size=1024; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + + CREATE TABLE t2(c INTEGER PRAGMA KEY, d); + INSERT INTO t2 VALUES(1, randomblob(1100)); +} {} + +do_test 3.2 { + db eval { SELECT name, rootpage FROM sqlite_master } { + set r($name) $rootpage + } + db close + + hexio_write test.db [expr {($r(t2)-1) * 1024 + 1020}] 00000002 + + sqlite3 db test.db +} {} + +do_test 3.3 { + list [catch { + db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { + db eval { + DELETE FROM t2 WHERE c=1; + } + } + } msg] $msg +} {1 {database disk image is malformed}} + +finish_test + Index: test/e_fkey.test ================================================================== --- test/e_fkey.test +++ test/e_fkey.test @@ -2944,42 +2944,49 @@ UPDATE t0 SET a = 'yyy'; SELECT NOT (a='yyy') FROM t$limit; " } -do_test e_fkey-63.1.1 { - test_on_delete_recursion $SQLITE_MAX_TRIGGER_DEPTH -} {0 0} -do_test e_fkey-63.1.2 { - test_on_delete_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.1.3 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 - test_on_delete_recursion 5 -} {0 0} -do_test e_fkey-63.1.4 { - test_on_delete_recursion 6 -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.1.5 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 -} {5} -do_test e_fkey-63.2.1 { - test_on_update_recursion $SQLITE_MAX_TRIGGER_DEPTH -} {0 0} -do_test e_fkey-63.2.2 { - test_on_update_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.2.3 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 - test_on_update_recursion 5 -} {0 0} -do_test e_fkey-63.2.4 { - test_on_update_recursion 6 -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.2.5 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 -} {5} +# If the current build was created using clang with the -fsanitize=address +# switch, then the library uses considerably more stack space than usual. +# So much more, that some of the following tests cause stack overflows +# if they are run under this configuration. +# +if {[clang_sanitize_address]==0} { + do_test e_fkey-63.1.1 { + test_on_delete_recursion $SQLITE_MAX_TRIGGER_DEPTH + } {0 0} + do_test e_fkey-63.1.2 { + test_on_delete_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.1.3 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 + test_on_delete_recursion 5 + } {0 0} + do_test e_fkey-63.1.4 { + test_on_delete_recursion 6 + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.1.5 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 + } {5} + do_test e_fkey-63.2.1 { + test_on_update_recursion $SQLITE_MAX_TRIGGER_DEPTH + } {0 0} + do_test e_fkey-63.2.2 { + test_on_update_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.2.3 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 + test_on_update_recursion 5 + } {0 0} + do_test e_fkey-63.2.4 { + test_on_update_recursion 6 + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.2.5 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 + } {5} +} #------------------------------------------------------------------------- # The setting of the recursive_triggers pragma does not affect foreign # key actions. # ADDED test/fts3join.test Index: test/fts3join.test ================================================================== --- test/fts3join.test +++ test/fts3join.test @@ -0,0 +1,66 @@ +# 2014 January 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts3join + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts4(x); + INSERT INTO ft1 VALUES('aaa aaa'); + INSERT INTO ft1 VALUES('aaa bbb'); + INSERT INTO ft1 VALUES('bbb aaa'); + INSERT INTO ft1 VALUES('bbb bbb'); + + CREATE TABLE t1(id, y); + INSERT INTO t1 VALUES(1, 'aaa'); + INSERT INTO t1 VALUES(2, 'bbb'); +} + +do_execsql_test 1.1 { + SELECT docid FROM ft1, t1 WHERE ft1 MATCH y AND id=1; +} {1 2 3} + +do_execsql_test 1.2 { + SELECT docid FROM ft1, t1 WHERE ft1 MATCH y AND id=1 ORDER BY docid; +} {1 2 3} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft2 USING fts4(x); + CREATE VIRTUAL TABLE ft3 USING fts4(y); + + INSERT INTO ft2 VALUES('abc'); + INSERT INTO ft2 VALUES('def'); + INSERT INTO ft3 VALUES('ghi'); + INSERT INTO ft3 VALUES('abc'); +} + +do_execsql_test 2.1 { SELECT * FROM ft2, ft3 WHERE x MATCH y; } {abc abc} +do_execsql_test 2.2 { SELECT * FROM ft2, ft3 WHERE y MATCH x; } {abc abc} +do_execsql_test 2.3 { SELECT * FROM ft3, ft2 WHERE x MATCH y; } {abc abc} +do_execsql_test 2.4 { SELECT * FROM ft3, ft2 WHERE y MATCH x; } {abc abc} + +do_catchsql_test 2.5 { + SELECT * FROM ft3, ft2 WHERE y MATCH x AND x MATCH y; +} {1 {unable to use function MATCH in the requested context}} + +finish_test + + Index: test/keyword1.test ================================================================== --- test/keyword1.test +++ test/keyword1.test @@ -63,10 +63,11 @@ offset plan pragma query raise + recursive regexp reindex release rename replace @@ -78,10 +79,12 @@ temporary trigger vacuum view virtual + with + without }; set exprkw { cast current_date current_time Index: test/like.test ================================================================== --- test/like.test +++ test/like.test @@ -890,8 +890,63 @@ do_test like-11.10 { queryplan { SELECT c FROM t11 WHERE c GLOB 'abc*' ORDER BY +a; } } {abc abcd sort {} t11cb} + +# A COLLATE clause on the pattern does not change the result of a +# LIKE operator. +# +do_execsql_test like-12.1 { + CREATE TABLE t12nc(id INTEGER, x TEXT UNIQUE COLLATE nocase); + INSERT INTO t12nc VALUES(1,'abcde'),(2,'uvwxy'),(3,'ABCDEF'); + CREATE TABLE t12b(id INTEGER, x TEXT UNIQUE COLLATE binary); + INSERT INTO t12b VALUES(1,'abcde'),(2,'uvwxy'),(3,'ABCDEF'); + SELECT id FROM t12nc WHERE x LIKE 'abc%' ORDER BY +id; +} {1 3} +do_execsql_test like-12.2 { + SELECT id FROM t12b WHERE x LIKE 'abc%' ORDER BY +id; +} {1 3} +do_execsql_test like-12.3 { + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {1 3} +do_execsql_test like-12.4 { + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {1 3} +do_execsql_test like-12.5 { + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {1 3} +do_execsql_test like-12.6 { + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {1 3} + +# Adding a COLLATE clause to the pattern of a LIKE operator does nothing +# to change the suitability of using an index to satisfy that LIKE +# operator. +# +do_execsql_test like-12.11 { + EXPLAIN QUERY PLAN + SELECT id FROM t12nc WHERE x LIKE 'abc%' ORDER BY +id; +} {/SEARCH/} +do_execsql_test like-12.12 { + EXPLAIN QUERY PLAN + SELECT id FROM t12b WHERE x LIKE 'abc%' ORDER BY +id; +} {/SCAN/} +do_execsql_test like-12.13 { + EXPLAIN QUERY PLAN + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {/SEARCH/} +do_execsql_test like-12.14 { + EXPLAIN QUERY PLAN + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {/SCAN/} +do_execsql_test like-12.15 { + EXPLAIN QUERY PLAN + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {/SEARCH/} +do_execsql_test like-12.16 { + EXPLAIN QUERY PLAN + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {/SCAN/} finish_test Index: test/misc1.test ================================================================== --- test/misc1.test +++ test/misc1.test @@ -589,7 +589,20 @@ do_test misc1-18.1 { set n [sqlite3_sleep 100] expr {$n>=100} } {1} + +# 2014-01-10: In a CREATE TABLE AS, if one or more of the column names +# are an empty string, that is still OK. +# +do_execsql_test misc1-19.1 { + CREATE TABLE t19 AS SELECT 1, 2 AS '', 3; + SELECT * FROM t19; +} {1 2 3} +do_execsql_test misc1-19.2 { + CREATE TABLE t19b AS SELECT 4 AS '', 5 AS '', 6 AS ''; + SELECT * FROM t19b; +} {4 5 6} + finish_test Index: test/misc7.test ================================================================== --- test/misc7.test +++ test/misc7.test @@ -13,13 +13,15 @@ # $Id: misc7.test,v 1.29 2009/07/16 18:21:18 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl -do_test misc7-1-misuse { - c_misuse_test -} {} +if {[clang_sanitize_address]==0} { + do_test misc7-1-misuse { + c_misuse_test + } {} +} do_test misc7-2 { c_realloc_test } {} Index: test/misuse.test ================================================================== --- test/misuse.test +++ test/misuse.test @@ -169,39 +169,42 @@ set r [sqlite3_close $::DB] } } msg] lappend v $msg $r } {0 {} SQLITE_BUSY} -do_test misuse-4.4 { + +if {[clang_sanitize_address]==0} { + do_test misuse-4.4 { # Flush the TCL statement cache here, otherwise the sqlite3_close() will # fail because there are still un-finalized() VDBEs. - db cache flush - sqlite3_close $::DB - catchsql2 {SELECT * FROM t1} -} {1 {library routine called out of sequence}} -do_test misuse-4.5 { - catchsql { - SELECT * FROM t1 - } -} {1 {library routine called out of sequence}} + db cache flush + sqlite3_close $::DB + catchsql2 {SELECT * FROM t1} + } {1 {library routine called out of sequence}} + do_test misuse-4.5 { + catchsql { + SELECT * FROM t1 + } + } {1 {library routine called out of sequence}} -# Attempt to use a database after it has been closed. -# -do_test misuse-5.1 { - db close - sqlite3 db test2.db; set ::DB [sqlite3_connection_pointer db] - execsql { - SELECT * FROM t1 - } -} {1 2} -do_test misuse-5.2 { - catchsql2 {SELECT * FROM t1} -} {0 {a b 1 2}} -do_test misuse-5.3 { - db close - set r [catch { - sqlite3_prepare $::DB {SELECT * FROM t1} -1 TAIL - } msg] - lappend r $msg -} {1 {(21) library routine called out of sequence}} + # Attempt to use a database after it has been closed. + # + do_test misuse-5.1 { + db close + sqlite3 db test2.db; set ::DB [sqlite3_connection_pointer db] + execsql { + SELECT * FROM t1 + } + } {1 2} + do_test misuse-5.2 { + catchsql2 {SELECT * FROM t1} + } {0 {a b 1 2}} + do_test misuse-5.3 { + db close + set r [catch { + sqlite3_prepare $::DB {SELECT * FROM t1} -1 TAIL + } msg] + lappend r $msg + } {1 {(21) library routine called out of sequence}} +} finish_test Index: test/select7.test ================================================================== --- test/select7.test +++ test/select7.test @@ -136,25 +136,27 @@ } # Verify that an error occurs if you have too many terms on a # compound select statement. # -ifcapable compound { - if {$SQLITE_MAX_COMPOUND_SELECT>0} { - set sql {SELECT 0} - set result 0 - for {set i 1} {$i<$SQLITE_MAX_COMPOUND_SELECT} {incr i} { - append sql " UNION ALL SELECT $i" - lappend result $i +if {[clang_sanitize_address]==0} { + ifcapable compound { + if {$SQLITE_MAX_COMPOUND_SELECT>0} { + set sql {SELECT 0} + set result 0 + for {set i 1} {$i<$SQLITE_MAX_COMPOUND_SELECT} {incr i} { + append sql " UNION ALL SELECT $i" + lappend result $i + } + do_test select7-6.1 { + catchsql $sql + } [list 0 $result] + append sql { UNION ALL SELECT 99999999} + do_test select7-6.2 { + catchsql $sql + } {1 {too many terms in compound SELECT}} } - do_test select7-6.1 { - catchsql $sql - } [list 0 $result] - append sql { UNION ALL SELECT 99999999} - do_test select7-6.2 { - catchsql $sql - } {1 {too many terms in compound SELECT}} } } # This block of tests verifies that bug aa92c76cd4 is fixed. # Index: test/spellfix.test ================================================================== --- test/spellfix.test +++ test/spellfix.test @@ -218,9 +218,54 @@ do_execsql_test 5.1.$tn { SELECT word, distance FROM t3 WHERE word MATCH $word ORDER BY score, word LIMIT 1 } $res } + +#------------------------------------------------------------------------- +# Try some queries by rowid. +# +do_execsql_test 6.1.1 { + SELECT word FROM t3 WHERE rowid = 10; +} {keener} +do_execsql_test 6.1.2 { + SELECT word, distance FROM t3 WHERE rowid = 10; +} {keener {}} +do_execsql_test 6.1.3 { + SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner'; +} {keener 300} + +proc trace_callback {sql} { + if {[string range $sql 0 2] == "-- "} { + lappend ::trace [string range $sql 3 end] + } +} + +proc do_tracesql_test {tn sql {res {}}} { + set ::trace [list] + uplevel [list do_test $tn [subst -nocommands { + set vals [execsql {$sql}] + concat [set vals] [set ::trace] + }] [list {*}$res]] +} + +db trace trace_callback +do_tracesql_test 6.2.1 { + SELECT word FROM t3 WHERE rowid = 10; +} {keener + {SELECT word, rank, NULL, langid, id FROM "main"."t3_vocab" WHERE rowid=?} +} +do_tracesql_test 6.2.2 { + SELECT word, distance FROM t3 WHERE rowid = 10; +} {keener {} + {SELECT word, rank, NULL, langid, id FROM "main"."t3_vocab" WHERE rowid=?} +} +do_tracesql_test 6.2.3 { + SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner'; +} {keener 300 + {SELECT id, word, rank, k1 FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k20 + AND NOT EXISTS ( + SELECT 1 + FROM digits AS lp + WHERE z.z = substr(s, ((ind-1)/9)*9 + lp, 1) + OR z.z = substr(s, ((ind-1)%9) + (lp-1)*9 + 1, 1) + OR z.z = substr(s, (((ind-1)/3) % 3) * 3 + + ((ind-1)/27) * 27 + lp + + ((lp-1) / 3) * 6, 1) + ) + ) + SELECT s FROM x WHERE ind=0; +} {534678912672195348198342567859761423426853791713924856961537284287419635345286179} + + +# Test cases to illustrate on the ORDER BY clause on a recursive query can be +# used to control depth-first versus breath-first search in a tree. +# +do_execsql_test 9.1 { + CREATE TABLE org( + name TEXT PRIMARY KEY, + boss TEXT REFERENCES org + ) WITHOUT ROWID; + INSERT INTO org VALUES('Alice',NULL); + INSERT INTO org VALUES('Bob','Alice'); + INSERT INTO org VALUES('Cindy','Alice'); + INSERT INTO org VALUES('Dave','Bob'); + INSERT INTO org VALUES('Emma','Bob'); + INSERT INTO org VALUES('Fred','Cindy'); + INSERT INTO org VALUES('Gail','Cindy'); + INSERT INTO org VALUES('Harry','Dave'); + INSERT INTO org VALUES('Ingrid','Dave'); + INSERT INTO org VALUES('Jim','Emma'); + INSERT INTO org VALUES('Kate','Emma'); + INSERT INTO org VALUES('Lanny','Fred'); + INSERT INTO org VALUES('Mary','Fred'); + INSERT INTO org VALUES('Noland','Gail'); + INSERT INTO org VALUES('Olivia','Gail'); + -- The above are all under Alice. Add a few more records for people + -- not in Alice's group, just to prove that they won't be selected. + INSERT INTO org VALUES('Xaviar',NULL); + INSERT INTO org VALUES('Xia','Xaviar'); + INSERT INTO org VALUES('Xerxes','Xaviar'); + INSERT INTO org VALUES('Xena','Xia'); + -- Find all members of Alice's group, breath-first order + WITH RECURSIVE + under_alice(name,level) AS ( + VALUES('Alice','0') + UNION ALL + SELECT org.name, under_alice.level+1 + FROM org, under_alice + WHERE org.boss=under_alice.name + ORDER BY 2 + ) + SELECT group_concat(substr('...............',1,level*3) || name,x'0a') + FROM under_alice; +} {{Alice +...Bob +...Cindy +......Dave +......Emma +......Fred +......Gail +.........Harry +.........Ingrid +.........Jim +.........Kate +.........Lanny +.........Mary +.........Noland +.........Olivia}} + +# The previous query used "ORDER BY level" to yield a breath-first search. +# Change that to "ORDER BY level DESC" for a depth-first search. +# +do_execsql_test 9.2 { + WITH RECURSIVE + under_alice(name,level) AS ( + VALUES('Alice','0') + UNION ALL + SELECT org.name, under_alice.level+1 + FROM org, under_alice + WHERE org.boss=under_alice.name + ORDER BY 2 DESC + ) + SELECT group_concat(substr('...............',1,level*3) || name,x'0a') + FROM under_alice; +} {{Alice +...Bob +......Dave +.........Harry +.........Ingrid +......Emma +.........Jim +.........Kate +...Cindy +......Fred +.........Lanny +.........Mary +......Gail +.........Noland +.........Olivia}} + +# Without an ORDER BY clause, the recursive query should use a FIFO, +# resulting in a breath-first search. +# +do_execsql_test 9.3 { + WITH RECURSIVE + under_alice(name,level) AS ( + VALUES('Alice','0') + UNION ALL + SELECT org.name, under_alice.level+1 + FROM org, under_alice + WHERE org.boss=under_alice.name + ) + SELECT group_concat(substr('...............',1,level*3) || name,x'0a') + FROM under_alice; +} {{Alice +...Bob +...Cindy +......Dave +......Emma +......Fred +......Gail +.........Harry +.........Ingrid +.........Jim +.........Kate +.........Lanny +.........Mary +.........Noland +.........Olivia}} + +finish_test ADDED test/with2.test Index: test/with2.test ================================================================== --- test/with2.test +++ test/with2.test @@ -0,0 +1,391 @@ +# 2014 January 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix with2 + +ifcapable {!cte} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); +} + +do_execsql_test 1.1 { + WITH x1 AS (SELECT * FROM t1) + SELECT sum(a) FROM x1; +} {3} + +do_execsql_test 1.2 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1); +} {3} + +do_execsql_test 1.3 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1); +} {3} + +do_execsql_test 1.4 { + CREATE TABLE t2(i); + INSERT INTO t2 VALUES(2); + INSERT INTO t2 VALUES(3); + INSERT INTO t2 VALUES(5); + + WITH x1 AS (SELECT i FROM t2), + i(a) AS ( + SELECT min(i)-1 FROM x1 UNION SELECT a+1 FROM i WHERE a<10 + ) + SELECT a FROM i WHERE a NOT IN x1 +} {1 4 6 7 8 9 10} + +do_execsql_test 1.5 { + WITH x1 AS (SELECT a FROM t1), + x2 AS (SELECT i FROM t2), + x3 AS (SELECT * FROM x1, x2 WHERE x1.a IN x2 AND x2.i IN x1) + SELECT * FROM x3 +} {2 2} + +do_execsql_test 1.6 { + CREATE TABLE t3 AS SELECT 3 AS x; + CREATE TABLE t4 AS SELECT 4 AS x; + + WITH x1 AS (SELECT * FROM t3), + x2 AS ( + WITH t3 AS (SELECT * FROM t4) + SELECT * FROM x1 + ) + SELECT * FROM x2; +} {3} + +do_execsql_test 1.7 { + WITH x2 AS ( + WITH t3 AS (SELECT * FROM t4) + SELECT * FROM t3 + ) + SELECT * FROM x2; +} {4} + +do_execsql_test 1.8 { + WITH x2 AS ( + WITH t3 AS (SELECT * FROM t4) + SELECT * FROM main.t3 + ) + SELECT * FROM x2; +} {3} + +do_execsql_test 1.9 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1), (SELECT max(a) FROM x1); +} {3 2} + +do_execsql_test 1.10 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1), (SELECT max(a) FROM x1), a FROM x1; +} {3 2 1 3 2 2} + +do_execsql_test 1.11 { + WITH + i(x) AS ( + WITH + j(x) AS ( SELECT * FROM i ), + i(x) AS ( SELECT * FROM t1 ) + SELECT * FROM j + ) + SELECT * FROM i; +} {1 2} + +do_execsql_test 1.12 { + WITH r(i) AS ( + VALUES('.') + UNION ALL + SELECT i || '.' FROM r, ( + SELECT x FROM x INTERSECT SELECT y FROM y + ) WHERE length(i) < 10 + ), + x(x) AS ( VALUES(1) UNION ALL VALUES(2) UNION ALL VALUES(3) ), + y(y) AS ( VALUES(2) UNION ALL VALUES(4) UNION ALL VALUES(6) ) + + SELECT * FROM r; +} {. .. ... .... ..... ...... ....... ........ ......... ..........} + +do_execsql_test 1.13 { + WITH r(i) AS ( + VALUES('.') + UNION ALL + SELECT i || '.' FROM r, ( SELECT x FROM x WHERE x=2 ) WHERE length(i) < 10 + ), + x(x) AS ( VALUES(1) UNION ALL VALUES(2) UNION ALL VALUES(3) ) + + SELECT * FROM r ORDER BY length(i) DESC; +} {.......... ......... ........ ....... ...... ..... .... ... .. .} + +do_execsql_test 1.14 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM t4 WHERE x<10 + ) + SELECT * FROM t4; +} {4 5 6 7 8 9 10} + +do_execsql_test 1.15 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM main.t4 WHERE x<10 + ) + SELECT * FROM t4; +} {4 5} + +do_catchsql_test 1.16 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM t4, main.t4, t4 WHERE x<10 + ) + SELECT * FROM t4; +} {1 {multiple references to recursive table: t4}} + + +#--------------------------------------------------------------------------- +# Check that variables can be used in CTEs. +# +set ::min [expr 3] +set ::max [expr 9] +do_execsql_test 2.1 { + WITH i(x) AS ( + VALUES($min) UNION ALL SELECT x+1 FROM i WHERE x < $max + ) + SELECT * FROM i; +} {3 4 5 6 7 8 9} + +do_execsql_test 2.2 { + WITH i(x) AS ( + VALUES($min) UNION ALL SELECT x+1 FROM i WHERE x < $max + ) + SELECT x FROM i JOIN i AS j USING (x); +} {3 4 5 6 7 8 9} + +#--------------------------------------------------------------------------- +# Check that circular references are rejected. +# +do_catchsql_test 3.1 { + WITH i(x, y) AS ( VALUES(1, (SELECT x FROM i)) ) + SELECT * FROM i; +} {1 {circular reference: i}} + +do_catchsql_test 3.2 { + WITH + i(x) AS ( SELECT * FROM j ), + j(x) AS ( SELECT * FROM k ), + k(x) AS ( SELECT * FROM i ) + SELECT * FROM i; +} {1 {circular reference: i}} + +do_catchsql_test 3.3 { + WITH + i(x) AS ( SELECT * FROM (SELECT * FROM j) ), + j(x) AS ( SELECT * FROM (SELECT * FROM i) ) + SELECT * FROM i; +} {1 {circular reference: i}} + +do_catchsql_test 3.4 { + WITH + i(x) AS ( SELECT * FROM (SELECT * FROM j) ), + j(x) AS ( SELECT * FROM (SELECT * FROM i) ) + SELECT * FROM j; +} {1 {circular reference: j}} + +do_catchsql_test 3.5 { + WITH + i(x) AS ( + WITH j(x) AS ( SELECT * FROM i ) + SELECT * FROM j + ) + SELECT * FROM i; +} {1 {circular reference: i}} + +#--------------------------------------------------------------------------- +# Try empty and very long column lists. +# +do_catchsql_test 4.1 { + WITH x() AS ( SELECT 1,2,3 ) + SELECT * FROM x; +} {1 {near ")": syntax error}} + +proc genstmt {n} { + for {set i 1} {$i<=$n} {incr i} { + lappend cols "c$i" + lappend vals $i + } + return " + WITH x([join $cols ,]) AS (SELECT [join $vals ,]) + SELECT (c$n == $n) FROM x + " +} + +do_execsql_test 4.2 [genstmt 10] 1 +do_execsql_test 4.3 [genstmt 100] 1 +do_execsql_test 4.4 [genstmt 255] 1 +set nLimit [sqlite3_limit db SQLITE_LIMIT_COLUMN -1] +do_execsql_test 4.5 [genstmt [expr $nLimit-1]] 1 +do_execsql_test 4.6 [genstmt $nLimit] 1 +do_catchsql_test 4.7 [genstmt [expr $nLimit+1]] {1 {too many columns in index}} + +#--------------------------------------------------------------------------- +# Check that adding a WITH clause to an INSERT disables the xfer +# optimization. +# +proc do_xfer_test {tn bXfer sql {res {}}} { + set ::sqlite3_xferopt_count 0 + uplevel [list do_test $tn [subst -nocommands { + set dres [db eval {$sql}] + list [set ::sqlite3_xferopt_count] [set dres] + }] [list $bXfer $res]] +} + +do_execsql_test 5.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); +} + +do_xfer_test 5.2 1 { INSERT INTO t1 SELECT * FROM t2 } +do_xfer_test 5.3 0 { INSERT INTO t1 SELECT a, b FROM t2 } +do_xfer_test 5.4 0 { INSERT INTO t1 SELECT b, a FROM t2 } +do_xfer_test 5.5 0 { + WITH x AS (SELECT a, b FROM t2) INSERT INTO t1 SELECT * FROM x +} +do_xfer_test 5.6 0 { + WITH x AS (SELECT a, b FROM t2) INSERT INTO t1 SELECT * FROM t2 +} +do_xfer_test 5.7 0 { + INSERT INTO t1 WITH x AS ( SELECT * FROM t2 ) SELECT * FROM x +} +do_xfer_test 5.8 0 { + INSERT INTO t1 WITH x(a,b) AS ( SELECT * FROM t2 ) SELECT * FROM x +} + +#--------------------------------------------------------------------------- +# Check that syntax (and other) errors in statements with WITH clauses +# attached to them do not cause problems (e.g. memory leaks). +# +do_execsql_test 6.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); +} + +do_catchsql_test 6.2 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 VALUES(1, 2,); +} {1 {near ")": syntax error}} + +do_catchsql_test 6.3 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 SELECT a, b, FROM t1; +} {1 {near "FROM": syntax error}} + +do_catchsql_test 6.3 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 SELECT a, b FROM abc; +} {1 {no such table: abc}} + +do_catchsql_test 6.4 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 SELECT a, b, FROM t1 a a a; +} {1 {near "FROM": syntax error}} + +do_catchsql_test 6.5 { + WITH x AS (SELECT * FROM t1) + DELETE FROM t2 WHERE; +} {1 {near ";": syntax error}} + +do_catchsql_test 6.6 { + WITH x AS (SELECT * FROM t1) DELETE FROM t2 WHERE +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.7 { + WITH x AS (SELECT * FROM t1) DELETE FROM t2 WHRE 1; +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.8 { + WITH x AS (SELECT * FROM t1) UPDATE t2 SET a = 10, b = ; +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.9 { + WITH x AS (SELECT * FROM t1) UPDATE t2 SET a = 10, b = 1 WHERE a===b; +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.10 { + WITH x(a,b) AS ( + SELECT 1, 1 + UNION ALL + SELECT a*b,a+b FROM x WHERE c=2 + ) + SELECT * FROM x +} {1 {no such column: c}} + +#------------------------------------------------------------------------- +# Recursive queries in IN(...) expressions. +# +do_execsql_test 7.1 { + CREATE TABLE t5(x INTEGER); + CREATE TABLE t6(y INTEGER); + + WITH s(x) AS ( VALUES(7) UNION ALL SELECT x+7 FROM s WHERE x<49 ) + INSERT INTO t5 + SELECT * FROM s; + + INSERT INTO t6 + WITH s(x) AS ( VALUES(2) UNION ALL SELECT x+2 FROM s WHERE x<49 ) + SELECT * FROM s; +} + +do_execsql_test 7.2 { + SELECT * FROM t6 WHERE y IN (SELECT x FROM t5) +} {14 28 42} + +do_execsql_test 7.3 { + WITH ss AS (SELECT x FROM t5) + SELECT * FROM t6 WHERE y IN (SELECT x FROM ss) +} {14 28 42} + +do_execsql_test 7.4 { + WITH ss(x) AS ( VALUES(7) UNION ALL SELECT x+7 FROM ss WHERE x<49 ) + SELECT * FROM t6 WHERE y IN (SELECT x FROM ss) +} {14 28 42} + +do_execsql_test 7.5 { + SELECT * FROM t6 WHERE y IN ( + WITH ss(x) AS ( VALUES(7) UNION ALL SELECT x+7 FROM ss WHERE x<49 ) + SELECT x FROM ss + ) +} {14 28 42} + + + +finish_test + ADDED test/withM.test Index: test/withM.test ================================================================== --- test/withM.test +++ test/withM.test @@ -0,0 +1,77 @@ +# 2014 January 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix withM + +ifcapable {!cte} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(x INTEGER, y INTEGER); + INSERT INTO t1 VALUES(123, 456); +} + +do_faultsim_test withM-1.1 -prep { + sqlite3 db test.db +} -body { + execsql { + WITH tmp AS ( SELECT * FROM t1 ) + SELECT * FROM tmp; + } +} -test { + faultsim_test_result {0 {123 456}} + db close +} + +do_faultsim_test withM-1.2 -prep { + sqlite3 db test.db +} -body { + execsql { + WITH w1 AS ( SELECT * FROM t1 ), + w2 AS ( + WITH w3 AS ( SELECT * FROM w1 ) + SELECT * FROM w3 + ) + SELECT * FROM w2; + } +} -test { + faultsim_test_result {0 {123 456}} + db close +} + +do_faultsim_test withM-1.3 -prep { + sqlite3 db test.db +} -body { + execsql { + WITH w1(a,b) AS ( + SELECT 1, 1 + UNION ALL + SELECT a+1, b + 2*a + 1 FROM w1 + ) + SELECT * FROM w1 LIMIT 5; + } +} -test { + faultsim_test_result {0 {1 1 2 4 3 9 4 16 5 25}} + db close +} + +finish_test + + + Index: tool/lemon.c ================================================================== --- tool/lemon.c +++ tool/lemon.c @@ -47,10 +47,111 @@ ** Compilers are getting increasingly pedantic about type conversions ** as C evolves ever closer to Ada.... To work around the latest problems ** we have to define the following variant of strlen(). */ #define lemonStrlen(X) ((int)strlen(X)) + +/* +** Compilers are starting to complain about the use of sprintf() and strcpy(), +** saying they are unsafe. So we define our own versions of those routines too. +** +** There are three routines here: lemon_sprintf(), lemon_vsprintf(), and +** lemon_addtext(). The first two are replacements for sprintf() and vsprintf(). +** The third is a helper routine for vsnprintf() that adds texts to the end of a +** buffer, making sure the buffer is always zero-terminated. +** +** The string formatter is a minimal subset of stdlib sprintf() supporting only +** a few simply conversions: +** +** %d +** %s +** %.*s +** +*/ +static void lemon_addtext( + char *zBuf, /* The buffer to which text is added */ + int *pnUsed, /* Slots of the buffer used so far */ + const char *zIn, /* Text to add */ + int nIn, /* Bytes of text to add. -1 to use strlen() */ + int iWidth /* Field width. Negative to left justify */ +){ + if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){} + while( iWidth>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth--; } + if( nIn==0 ) return; + memcpy(&zBuf[*pnUsed], zIn, nIn); + *pnUsed += nIn; + while( (-iWidth)>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth++; } + zBuf[*pnUsed] = 0; +} +static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ + int i, j, k, c; + int nUsed = 0; + const char *z; + char zTemp[50]; + str[0] = 0; + for(i=j=0; (c = zFormat[i])!=0; i++){ + if( c=='%' ){ + int iWidth = 0; + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + c = zFormat[++i]; + if( isdigit(c) || (c=='-' && isdigit(zFormat[i+1])) ){ + if( c=='-' ) i++; + while( isdigit(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0'; + if( c=='-' ) iWidth = -iWidth; + c = zFormat[i]; + } + if( c=='d' ){ + int v = va_arg(ap, int); + if( v<0 ){ + lemon_addtext(str, &nUsed, "-", 1, iWidth); + v = -v; + }else if( v==0 ){ + lemon_addtext(str, &nUsed, "0", 1, iWidth); + } + k = 0; + while( v>0 ){ + k++; + zTemp[sizeof(zTemp)-k] = (v%10) + '0'; + v /= 10; + } + lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth); + }else if( c=='s' ){ + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, -1, iWidth); + }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){ + i += 2; + k = va_arg(ap, int); + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, k, iWidth); + }else if( c=='%' ){ + lemon_addtext(str, &nUsed, "%", 1, 0); + }else{ + fprintf(stderr, "illegal format\n"); + exit(1); + } + j = i+1; + } + } + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); + return nUsed; +} +static int lemon_sprintf(char *str, const char *format, ...){ + va_list ap; + int rc; + va_start(ap, format); + rc = lemon_vsprintf(str, format, ap); + va_end(ap); + return rc; +} +static void lemon_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} +static void lemon_strcat(char *dest, const char *src){ + while( *dest ) dest++; + lemon_strcpy(dest, src); +} + /* a few forward declarations... */ struct rule; struct lemon; struct action; @@ -1365,11 +1466,11 @@ *paz = (char *) malloc( lemonStrlen(z)+1 ); if( *paz==0 ){ fprintf(stderr,"out of memory\n"); exit(1); } - strcpy(*paz, z); + lemon_strcpy(*paz, z); for(z=*paz; *z && *z!='='; z++){} *z = 0; } static char *user_templatename = NULL; @@ -1376,11 +1477,11 @@ static void handle_T_option(char *z){ user_templatename = (char *) malloc( lemonStrlen(z)+1 ); if( user_templatename==0 ){ memory_error(); } - strcpy(user_templatename, z); + lemon_strcpy(user_templatename, z); } /* The main program. Parse the command line and do it... */ int main(int argc, char **argv) { @@ -1445,16 +1546,19 @@ fprintf(stderr,"Empty grammar.\n"); exit(1); } /* Count and index the symbols of the grammar */ - lem.nsymbol = Symbol_count(); Symbol_new("{default}"); + lem.nsymbol = Symbol_count(); lem.symbols = Symbol_arrayof(); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; - qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; + for(i=0; iindex = i; + qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp); + for(i=0; iindex = i; + while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } + assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); + lem.nsymbol = i - 1; for(i=1; isupper(lem.symbols[i]->name[0]); i++); lem.nterminal = i; /* Generate a reprint of the grammar, if requested on the command line */ if( rpflag ){ @@ -1938,20 +2042,23 @@ RESYNC_AFTER_RULE_ERROR, RESYNC_AFTER_DECL_ERROR, WAITING_FOR_DESTRUCTOR_SYMBOL, WAITING_FOR_DATATYPE_SYMBOL, WAITING_FOR_FALLBACK_ID, - WAITING_FOR_WILDCARD_ID + WAITING_FOR_WILDCARD_ID, + WAITING_FOR_CLASS_ID, + WAITING_FOR_CLASS_TOKEN }; struct pstate { char *filename; /* Name of the input file */ int tokenlineno; /* Linenumber at which current token starts */ int errorcnt; /* Number of errors so far */ char *tokenstart; /* Text of current token */ struct lemon *gp; /* Global state vector */ enum e_state state; /* The state of the parser */ struct symbol *fallback; /* The fallback token */ + struct symbol *tkclass; /* Token class symbol */ struct symbol *lhs; /* Left-hand side of current rule */ const char *lhsalias; /* Alias for the LHS */ int nrhs; /* Number of right-hand side symbols seen */ struct symbol *rhs[MAXRHS]; /* RHS symbols */ const char *alias[MAXRHS]; /* Aliases for each RHS symbol (or NULL) */ @@ -2252,10 +2359,12 @@ }else if( strcmp(x,"fallback")==0 ){ psp->fallback = 0; psp->state = WAITING_FOR_FALLBACK_ID; }else if( strcmp(x,"wildcard")==0 ){ psp->state = WAITING_FOR_WILDCARD_ID; + }else if( strcmp(x,"token_class")==0 ){ + psp->state = WAITING_FOR_CLASS_ID; }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Unknown declaration keyword: \"%%%s\".",x); psp->errorcnt++; psp->state = RESYNC_AFTER_DECL_ERROR; @@ -2345,11 +2454,11 @@ (psp->decllinenoslot==0 || psp->decllinenoslot[0]!=0); if( addLineMacro ){ for(z=psp->filename, nBack=0; *z; z++){ if( *z=='\\' ) nBack++; } - sprintf(zLine, "#line %d ", psp->tokenlineno); + lemon_sprintf(zLine, "#line %d ", psp->tokenlineno); nLine = lemonStrlen(zLine); n += nLine + lemonStrlen(psp->filename) + nBack; } *psp->declargslot = (char *) realloc(*psp->declargslot, n); zBuf = *psp->declargslot + nOld; @@ -2419,10 +2528,44 @@ ErrorMsg(psp->filename, psp->tokenlineno, "Extra wildcard to token: %s", x); psp->errorcnt++; } } + break; + case WAITING_FOR_CLASS_ID: + if( !islower(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class must be followed by an identifier: ", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else if( Symbol_find(x) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "Symbol \"%s\" already used", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + psp->tkclass = Symbol_new(x); + psp->tkclass->type = MULTITERMINAL; + psp->state = WAITING_FOR_CLASS_TOKEN; + } + break; + case WAITING_FOR_CLASS_TOKEN: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){ + struct symbol *msp = psp->tkclass; + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + if( !isupper(x[0]) ) x++; + msp->subsym[msp->nsubsym-1] = Symbol_new(x); + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class argument \"%s\" should be a token", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } break; case RESYNC_AFTER_RULE_ERROR: /* if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; ** break; */ case RESYNC_AFTER_DECL_ERROR: @@ -2514,13 +2657,12 @@ } fseek(fp,0,2); filesize = ftell(fp); rewind(fp); filebuf = (char *)malloc( filesize+1 ); - if( filebuf==0 ){ - ErrorMsg(ps.filename,0,"Can't allocate %d of memory to hold this file.", - filesize+1); + if( filesize>100000000 || filebuf==0 ){ + ErrorMsg(ps.filename,0,"Input file too large."); gp->errorcnt++; fclose(fp); return; } if( fread(filebuf,1,filesize,fp)!=filesize ){ @@ -2714,14 +2856,14 @@ name = (char*)malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5 ); if( name==0 ){ fprintf(stderr,"Can't allocate space for a filename.\n"); exit(1); } - strcpy(name,lemp->filename); + lemon_strcpy(name,lemp->filename); cp = strrchr(name,'.'); if( cp ) *cp = 0; - strcat(name,suffix); + lemon_strcat(name,suffix); return name; } /* Open a file with a name based on the name of the input file, ** but with a different (specified) suffix, and return a pointer @@ -2774,15 +2916,17 @@ printf("%s",rp->lhs->name); /* if( rp->lhsalias ) printf("(%s)",rp->lhsalias); */ printf(" ::="); for(i=0; inrhs; i++){ sp = rp->rhs[i]; - printf(" %s", sp->name); if( sp->type==MULTITERMINAL ){ + printf(" %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ printf("|%s", sp->subsym[j]->name); } + }else{ + printf(" %s", sp->name); } /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */ } printf("."); if( rp->precsym ) printf(" [%s]",rp->precsym->name); @@ -2800,15 +2944,17 @@ fprintf(fp,"%s ::=",rp->lhs->name); for(i=0; i<=rp->nrhs; i++){ if( i==cfp->dot ) fprintf(fp," *"); if( i==rp->nrhs ) break; sp = rp->rhs[i]; - fprintf(fp," %s", sp->name); if( sp->type==MULTITERMINAL ){ + fprintf(fp," %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ fprintf(fp,"|%s",sp->subsym[j]->name); } + }else{ + fprintf(fp," %s", sp->name); } } } /* #define TEST */ @@ -2914,11 +3060,11 @@ if( lemp->basisflag ) cfp=stp->bp; else cfp=stp->cfp; while( cfp ){ char buf[20]; if( cfp->dot==cfp->rp->nrhs ){ - sprintf(buf,"(%d)",cfp->rp->index); + lemon_sprintf(buf,"(%d)",cfp->rp->index); fprintf(fp," %5s ",buf); }else{ fprintf(fp," "); } ConfigPrint(fp,cfp); @@ -2979,26 +3125,26 @@ #endif if( cp ){ c = *cp; *cp = 0; path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 ); - if( path ) sprintf(path,"%s/%s",argv0,name); + if( path ) lemon_sprintf(path,"%s/%s",argv0,name); *cp = c; }else{ pathlist = getenv("PATH"); if( pathlist==0 ) pathlist = ".:/bin:/usr/bin"; pathbuf = (char *) malloc( lemonStrlen(pathlist) + 1 ); path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); if( (pathbuf != 0) && (path!=0) ){ pathbufptr = pathbuf; - strcpy(pathbuf, pathlist); + lemon_strcpy(pathbuf, pathlist); while( *pathbuf ){ cp = strchr(pathbuf,':'); if( cp==0 ) cp = &pathbuf[lemonStrlen(pathbuf)]; c = *cp; *cp = 0; - sprintf(path,"%s/%s",pathbuf,name); + lemon_sprintf(path,"%s/%s",pathbuf,name); *cp = c; if( c==0 ) pathbuf[0] = 0; else pathbuf = &cp[1]; if( access(path,modemask)==0 ) break; } @@ -3085,13 +3231,13 @@ return in; } cp = strrchr(lemp->filename,'.'); if( cp ){ - sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); + lemon_sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); }else{ - sprintf(buf,"%s.lt",lemp->filename); + lemon_sprintf(buf,"%s.lt",lemp->filename); } if( access(buf,004)==0 ){ tpltname = buf; }else if( access(templatename,004)==0 ){ tpltname = templatename; @@ -3238,13 +3384,13 @@ } if( z==0 ) return empty; while( n-- > 0 ){ c = *(zText++); if( c=='%' && n>0 && zText[0]=='d' ){ - sprintf(zInt, "%d", p1); + lemon_sprintf(zInt, "%d", p1); p1 = p2; - strcpy(&z[used], zInt); + lemon_strcpy(&z[used], zInt); used += lemonStrlen(&z[used]); zText++; n--; }else{ z[used++] = c; @@ -3465,11 +3611,11 @@ types[hash] = (char*)malloc( lemonStrlen(stddt)+1 ); if( types[hash]==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); } - strcpy(types[hash],stddt); + lemon_strcpy(types[hash],stddt); } } /* Print out the definition of YYTOKENTYPE and YYMINORTYPE */ name = lemp->name ? lemp->name : "Parse"; @@ -3551,13 +3697,15 @@ static void writeRuleText(FILE *out, struct rule *rp){ int j; fprintf(out,"%s ::=", rp->lhs->name); for(j=0; jnrhs; j++){ struct symbol *sp = rp->rhs[j]; - fprintf(out," %s", sp->name); - if( sp->type==MULTITERMINAL ){ + if( sp->type!=MULTITERMINAL ){ + fprintf(out," %s", sp->name); + }else{ int k; + fprintf(out," %s", sp->subsym[0]->name); for(k=1; knsubsym; k++){ fprintf(out,"|%s",sp->subsym[k]->name); } } } @@ -3854,11 +4002,11 @@ tplt_xfer(lemp->name, in, out, &lineno); /* Generate a table containing the symbolic name of every symbol */ for(i=0; insymbol; i++){ - sprintf(line,"\"%s\",",lemp->symbols[i]->name); + lemon_sprintf(line,"\"%s\",",lemp->symbols[i]->name); fprintf(out," %-15s",line); if( (i&3)==3 ){ fprintf(out,"\n"); lineno++; } } if( (i&3)!=0 ){ fprintf(out,"\n"); lineno++; } tplt_xfer(lemp->name,in,out,&lineno); @@ -4021,11 +4169,12 @@ else prefix = ""; in = file_open(lemp,".h","rb"); if( in ){ int nextChar; for(i=1; interminal && fgets(line,LINESIZE,in); i++){ - sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + lemon_sprintf(pattern,"#define %s%-30s %3d\n", + prefix,lemp->symbols[i]->name,i); if( strcmp(line,pattern) ) break; } nextChar = fgetc(in); fclose(in); if( i==lemp->nterminal && nextChar==EOF ){ @@ -4034,11 +4183,11 @@ } } out = file_open(lemp,".h","wb"); if( out ){ for(i=1; interminal; i++){ - fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); } fclose(out); } return; } @@ -4251,11 +4400,11 @@ char *cpy; if( y==0 ) return 0; z = Strsafe_find(y); if( z==0 && (cpy=(char *)malloc( lemonStrlen(y)+1 ))!=0 ){ - strcpy(cpy,y); + lemon_strcpy(cpy,y); z = cpy; Strsafe_insert(z); } MemoryCheck(z); return z; @@ -4290,12 +4439,11 @@ if( x1a ) return; x1a = (struct s_x1*)malloc( sizeof(struct s_x1) ); if( x1a ){ x1a->size = 1024; x1a->count = 0; - x1a->tbl = (x1node*)malloc( - (sizeof(x1node) + sizeof(x1node*))*1024 ); + x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*)); if( x1a->tbl==0 ){ free(x1a); x1a = 0; }else{ int i; @@ -4328,12 +4476,11 @@ /* Need to make the hash table bigger */ int i,size; struct s_x1 array; array.size = size = x1a->size*2; array.count = x1a->count; - array.tbl = (x1node*)malloc( - (sizeof(x1node) + sizeof(x1node*))*size ); + array.tbl = (x1node*)calloc(size, sizeof(x1node) + sizeof(x1node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x1node**)&(array.tbl[size]); for(i=0; icount; i++){ x1node *oldnp, *newnp; @@ -4404,28 +4551,31 @@ } sp->useCnt++; return sp; } -/* Compare two symbols for working purposes +/* Compare two symbols for sorting purposes. Return negative, +** zero, or positive if a is less then, equal to, or greater +** than b. ** ** Symbols that begin with upper case letters (terminals or tokens) ** must sort before symbols that begin with lower case letters -** (non-terminals). Other than that, the order does not matter. +** (non-terminals). And MULTITERMINAL symbols (created using the +** %token_class directive) must sort at the very end. Other than +** that, the order does not matter. ** ** We find experimentally that leaving the symbols in their original ** order (the order they appeared in the grammar file) gives the ** smallest parser tables in SQLite. */ int Symbolcmpp(const void *_a, const void *_b) { - const struct symbol **a = (const struct symbol **) _a; - const struct symbol **b = (const struct symbol **) _b; - int i1 = (**a).index + 10000000*((**a).name[0]>'Z'); - int i2 = (**b).index + 10000000*((**b).name[0]>'Z'); - assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 ); - return i1-i2; + const struct symbol *a = *(const struct symbol **) _a; + const struct symbol *b = *(const struct symbol **) _b; + int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1; + int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1; + return i1==i2 ? a->index - b->index : i1 - i2; } /* There is one instance of the following structure for each ** associative array of type "x2". */ @@ -4456,12 +4606,11 @@ if( x2a ) return; x2a = (struct s_x2*)malloc( sizeof(struct s_x2) ); if( x2a ){ x2a->size = 128; x2a->count = 0; - x2a->tbl = (x2node*)malloc( - (sizeof(x2node) + sizeof(x2node*))*128 ); + x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*)); if( x2a->tbl==0 ){ free(x2a); x2a = 0; }else{ int i; @@ -4494,12 +4643,11 @@ /* Need to make the hash table bigger */ int i,size; struct s_x2 array; array.size = size = x2a->size*2; array.count = x2a->count; - array.tbl = (x2node*)malloc( - (sizeof(x2node) + sizeof(x2node*))*size ); + array.tbl = (x2node*)calloc(size, sizeof(x2node) + sizeof(x2node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x2node**)&(array.tbl[size]); for(i=0; icount; i++){ x2node *oldnp, *newnp; @@ -4655,12 +4803,11 @@ if( x3a ) return; x3a = (struct s_x3*)malloc( sizeof(struct s_x3) ); if( x3a ){ x3a->size = 128; x3a->count = 0; - x3a->tbl = (x3node*)malloc( - (sizeof(x3node) + sizeof(x3node*))*128 ); + x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*)); if( x3a->tbl==0 ){ free(x3a); x3a = 0; }else{ int i; @@ -4693,12 +4840,11 @@ /* Need to make the hash table bigger */ int i,size; struct s_x3 array; array.size = size = x3a->size*2; array.count = x3a->count; - array.tbl = (x3node*)malloc( - (sizeof(x3node) + sizeof(x3node*))*size ); + array.tbl = (x3node*)calloc(size, sizeof(x3node) + sizeof(x3node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x3node**)&(array.tbl[size]); for(i=0; icount; i++){ x3node *oldnp, *newnp; @@ -4751,11 +4897,11 @@ { struct state **array; int i,size; if( x3a==0 ) return 0; size = x3a->count; - array = (struct state **)malloc( sizeof(struct state *)*size ); + array = (struct state **)calloc(size, sizeof(struct state *)); if( array ){ for(i=0; itbl[i].data; } return array; } @@ -4797,12 +4943,11 @@ if( x4a ) return; x4a = (struct s_x4*)malloc( sizeof(struct s_x4) ); if( x4a ){ x4a->size = 64; x4a->count = 0; - x4a->tbl = (x4node*)malloc( - (sizeof(x4node) + sizeof(x4node*))*64 ); + x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*)); if( x4a->tbl==0 ){ free(x4a); x4a = 0; }else{ int i; @@ -4835,12 +4980,11 @@ /* Need to make the hash table bigger */ int i,size; struct s_x4 array; array.size = size = x4a->size*2; array.count = x4a->count; - array.tbl = (x4node*)malloc( - (sizeof(x4node) + sizeof(x4node*))*size ); + array.tbl = (x4node*)calloc(size, sizeof(x4node) + sizeof(x4node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x4node**)&(array.tbl[size]); for(i=0; icount; i++){ x4node *oldnp, *newnp; Index: tool/mkkeywordhash.c ================================================================== --- tool/mkkeywordhash.c +++ tool/mkkeywordhash.c @@ -136,10 +136,15 @@ #ifdef SQLITE_OMIT_AUTOVACUUM # define AUTOVACUUM 0 #else # define AUTOVACUUM 0x00020000 #endif +#ifdef SQLITE_OMIT_CTE +# define CTE 0 +#else +# define CTE 0x00040000 +#endif /* ** These are the keywords */ static Keyword aKeywordTable[] = { @@ -232,10 +237,11 @@ { "PLAN", "TK_PLAN", EXPLAIN }, { "PRAGMA", "TK_PRAGMA", PRAGMA }, { "PRIMARY", "TK_PRIMARY", ALWAYS }, { "QUERY", "TK_QUERY", EXPLAIN }, { "RAISE", "TK_RAISE", TRIGGER }, + { "RECURSIVE", "TK_RECURSIVE", CTE }, { "REFERENCES", "TK_REFERENCES", FKEY }, { "REGEXP", "TK_LIKE_KW", ALWAYS }, { "REINDEX", "TK_REINDEX", REINDEX }, { "RELEASE", "TK_RELEASE", ALWAYS }, { "RENAME", "TK_RENAME", ALTER }, @@ -260,10 +266,11 @@ { "USING", "TK_USING", ALWAYS }, { "VACUUM", "TK_VACUUM", VACUUM }, { "VALUES", "TK_VALUES", ALWAYS }, { "VIEW", "TK_VIEW", VIEW }, { "VIRTUAL", "TK_VIRTUAL", VTAB }, + { "WITH", "TK_WITH", CTE }, { "WITHOUT", "TK_WITHOUT", ALWAYS }, { "WHEN", "TK_WHEN", ALWAYS }, { "WHERE", "TK_WHERE", ALWAYS }, };