Index: ext/session/sessionwor.test ================================================================== --- ext/session/sessionwor.test +++ ext/session/sessionwor.test @@ -67,11 +67,13 @@ 1 "" 2 "WITHOUT ROWID" } { reset_db - do_execsql_test 2.$tn.0 "CREATE TABLE t1(a INTEGER PRIMARY KEY, b) $wo ;" + do_execsql_test 2.$tn.0.1 "CREATE TABLE t1(a INTEGER PRIMARY KEY, b) $wo ;" + do_execsql_test 2.$tn.0.2 "CREATE TABLE t2(a INTEGER PRIMARY KEY, b) $wo ;" + do_execsql_test 2.$tn.0.3 "CREATE TABLE t3(a INTEGER PRIMARY KEY, b) $wo ;" do_iterator_test 1.1 t1 { INSERT INTO t1 VALUES(1, 'two'); } { {INSERT t1 0 X. {} {i 1 t two}} @@ -92,9 +94,30 @@ do_iterator_test 2.$tn.4 t1 { DELETE FROM t1; } { {DELETE t1 0 X. {i 1 t four} {}} } + + do_execsql_test 2.$tn.5 { + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + } + + do_iterator_test 2.$tn.6 t2 { + INSERT INTO t2 SELECT a, b FROM t1 + } { + {INSERT t2 0 X. {} {i 1 t one}} + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t2 0 X. {} {i 3 t three}} + } + do_iterator_test 2.$tn.7 t3 { + INSERT INTO t3 SELECT * FROM t1 + } { + {INSERT t3 0 X. {} {i 1 t one}} + {INSERT t3 0 X. {} {i 2 t two}} + {INSERT t3 0 X. {} {i 3 t three}} + } } finish_test Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -47,44 +47,49 @@ ** bTemp is not true, database "temp", can still be parsed. This is ** called at the end of the generation of an ALTER TABLE ... RENAME ... ** statement to ensure that the operation has not rendered any schema ** objects unusable. */ -static void renameTestSchema(Parse *pParse, const char *zDb, int bTemp){ +static void renameTestSchema( + Parse *pParse, /* Parse context */ + const char *zDb, /* Name of db to verify schema of */ + int bTemp, /* True if this is the temp db */ + const char *zWhen /* "when" part of error message */ +){ pParse->colNamesSet = 1; sqlite3NestedParse(pParse, "SELECT 1 " "FROM \"%w\"." DFLT_SCHEMA_TABLE " " "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" " AND sql NOT LIKE 'create virtual%%'" - " AND sqlite_rename_test(%Q, sql, type, name, %d)=NULL ", + " AND sqlite_rename_test(%Q, sql, type, name, %d, %Q)=NULL ", zDb, - zDb, bTemp + zDb, bTemp, zWhen ); if( bTemp==0 ){ sqlite3NestedParse(pParse, "SELECT 1 " "FROM temp." DFLT_SCHEMA_TABLE " " "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" " AND sql NOT LIKE 'create virtual%%'" - " AND sqlite_rename_test(%Q, sql, type, name, 1)=NULL ", - zDb + " AND sqlite_rename_test(%Q, sql, type, name, 1, %Q)=NULL ", + zDb, zWhen ); } } /* ** Generate code to reload the schema for database iDb. And, if iDb!=1, for ** the temp database as well. */ -static void renameReloadSchema(Parse *pParse, int iDb){ +static void renameReloadSchema(Parse *pParse, int iDb, u16 p5){ Vdbe *v = pParse->pVdbe; if( v ){ sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, iDb, 0); - if( iDb!=1 ) sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, 1, 0); + sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, iDb, 0, p5); + if( iDb!=1 ) sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, 1, 0, p5); } } /* ** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" @@ -229,11 +234,11 @@ sqlite3NestedParse(pParse, "UPDATE sqlite_temp_schema SET " "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, 1), " "tbl_name = " "CASE WHEN tbl_name=%Q COLLATE nocase AND " - " sqlite_rename_test(%Q, sql, type, name, 1) " + " sqlite_rename_test(%Q, sql, type, name, 1, 'after rename') " "THEN %Q ELSE tbl_name END " "WHERE type IN ('view', 'trigger')" , zDb, zTabName, zName, zTabName, zDb, zName); } @@ -248,12 +253,12 @@ sqlite3VdbeLoadString(v, i, zName); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); } #endif - renameReloadSchema(pParse, iDb); - renameTestSchema(pParse, zDb, iDb==1); + renameReloadSchema(pParse, iDb, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iDb==1, "after rename"); exit_rename_table: sqlite3SrcListDelete(db, pSrc); sqlite3DbFree(db, zName); db->mDbFlags = savedDbFlags; @@ -411,11 +416,11 @@ sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); sqlite3ReleaseTempReg(pParse, r1); } /* Reload the table definition */ - renameReloadSchema(pParse, iDb); + renameReloadSchema(pParse, iDb, INITFLAG_AlterRename); } /* ** This function is called by the parser after the table-name in ** an "ALTER TABLE ADD" statement is parsed. Argument @@ -511,11 +516,11 @@ ** it loads an error message into pParse and returns non-zero. ** ** Or, if pTab is not a view or virtual table, zero is returned. */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) -static int isRealTable(Parse *pParse, Table *pTab){ +static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ const char *zType = 0; #ifndef SQLITE_OMIT_VIEW if( pTab->pSelect ){ zType = "view"; } @@ -524,19 +529,20 @@ if( IsVirtual(pTab) ){ zType = "virtual table"; } #endif if( zType ){ - sqlite3ErrorMsg( - pParse, "cannot rename columns of %s \"%s\"", zType, pTab->zName + sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", + (bDrop ? "drop column from" : "rename columns of"), + zType, pTab->zName ); return 1; } return 0; } #else /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ -# define isRealTable(x,y) (0) +# define isRealTable(x,y,z) (0) #endif /* ** Handles the following parser reduction: ** @@ -561,11 +567,11 @@ pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_column; /* Cannot alter a system table */ if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_rename_column; - if( SQLITE_OK!=isRealTable(pParse, pTab) ) goto exit_rename_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 0) ) goto exit_rename_column; /* Which schema holds the table to be altered */ iSchema = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iSchema>=0 ); zDb = db->aDb[iSchema].zDbSName; @@ -615,12 +621,12 @@ "WHERE type IN ('trigger', 'view')", zDb, pTab->zName, iCol, zNew, bQuote ); /* Drop and reload the database schema. */ - renameReloadSchema(pParse, iSchema); - renameTestSchema(pParse, zDb, iSchema==1); + renameReloadSchema(pParse, iSchema, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iSchema==1, "after rename"); exit_rename_column: sqlite3SrcListDelete(db, pSrc); sqlite3DbFree(db, zOld); sqlite3DbFree(db, zNew); @@ -868,27 +874,37 @@ } } /* ** Search the Parse object passed as the first argument for a RenameToken -** object associated with parse tree element pPtr. If found, remove it -** from the Parse object and add it to the list maintained by the -** RenameCtx object passed as the second argument. +** object associated with parse tree element pPtr. If found, return a pointer +** to it. Otherwise, return NULL. +** +** If the second argument passed to this function is not NULL and a matching +** RenameToken object is found, remove it from the Parse object and add it to +** the list maintained by the RenameCtx object. */ -static void renameTokenFind(Parse *pParse, struct RenameCtx *pCtx, void *pPtr){ +static RenameToken *renameTokenFind( + Parse *pParse, + struct RenameCtx *pCtx, + void *pPtr +){ RenameToken **pp; assert( pPtr!=0 ); for(pp=&pParse->pRename; (*pp); pp=&(*pp)->pNext){ if( (*pp)->p==pPtr ){ RenameToken *pToken = *pp; - *pp = pToken->pNext; - pToken->pNext = pCtx->pList; - pCtx->pList = pToken; - pCtx->nList++; - break; + if( pCtx ){ + *pp = pToken->pNext; + pToken->pNext = pCtx->pList; + pCtx->pList = pToken; + pCtx->nList++; + } + return pToken; } } + return 0; } /* ** This is a Walker select callback. It does nothing. It is only required ** because without a dummy callback, sqlite3WalkExpr() and similar do not @@ -955,21 +971,21 @@ ** sub-routine is currently stored in pParse->zErrMsg. This function ** adds context to the error message and then stores it in pCtx. */ static void renameColumnParseError( sqlite3_context *pCtx, - int bPost, + const char *zWhen, sqlite3_value *pType, sqlite3_value *pObject, Parse *pParse ){ const char *zT = (const char*)sqlite3_value_text(pType); const char *zN = (const char*)sqlite3_value_text(pObject); char *zErr; - zErr = sqlite3_mprintf("error in %s %s%s: %s", - zT, zN, (bPost ? " after rename" : ""), + zErr = sqlite3_mprintf("error in %s %s%s%s: %s", + zT, zN, (zWhen[0] ? " " : ""), zWhen, pParse->zErrMsg ); sqlite3_result_error(pCtx, zErr, -1); sqlite3_free(zErr); } @@ -1044,11 +1060,11 @@ ** trigger object, the database must be corrupt. */ memset(p, 0, sizeof(Parse)); p->eParseMode = PARSE_MODE_RENAME; p->db = db; p->nQueryLoop = 1; - rc = sqlite3RunParser(p, zSql, &zErr); + rc = zSql ? sqlite3RunParser(p, zSql, &zErr) : SQLITE_NOMEM; assert( p->zErrMsg==0 ); assert( rc!=SQLITE_OK || zErr==0 ); p->zErrMsg = zErr; if( db->mallocFailed ) rc = SQLITE_NOMEM; if( rc==SQLITE_OK @@ -1471,11 +1487,11 @@ rc = renameEditSql(context, &sCtx, zSql, zNew, bQuote); renameColumnFunc_done: if( rc!=SQLITE_OK ){ if( sParse.zErrMsg ){ - renameColumnParseError(context, 0, argv[1], argv[2], &sParse); + renameColumnParseError(context, "", argv[1], argv[2], &sParse); }else{ sqlite3_result_error_code(context, rc); } } @@ -1660,11 +1676,11 @@ if( rc==SQLITE_OK ){ rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote); } if( rc!=SQLITE_OK ){ if( sParse.zErrMsg ){ - renameColumnParseError(context, 0, argv[1], argv[2], &sParse); + renameColumnParseError(context, "", argv[1], argv[2], &sParse); }else{ sqlite3_result_error_code(context, rc); } } @@ -1689,10 +1705,11 @@ ** 0: Database name ("main", "temp" etc.). ** 1: SQL statement. ** 2: Object type ("view", "table", "trigger" or "index"). ** 3: Object name. ** 4: True if object is from temp schema. +** 5: "when" part of error message. ** ** Unless it finds an error, this function normally returns NULL. However, it ** returns integer value 1 if: ** ** * the SQL argument creates a trigger, and @@ -1706,10 +1723,11 @@ sqlite3 *db = sqlite3_context_db_handle(context); char const *zDb = (const char*)sqlite3_value_text(argv[0]); char const *zInput = (const char*)sqlite3_value_text(argv[1]); int bTemp = sqlite3_value_int(argv[4]); int isLegacy = (db->flags & SQLITE_LegacyAlter); + char const *zWhen = (const char*)sqlite3_value_text(argv[5]); #ifndef SQLITE_OMIT_AUTHORIZATION sqlite3_xauth xAuth = db->xAuth; db->xAuth = 0; #endif @@ -1738,28 +1756,219 @@ if( i1==i2 ) sqlite3_result_int(context, 1); } } } - if( rc!=SQLITE_OK ){ - renameColumnParseError(context, 1, argv[2], argv[3], &sParse); + if( rc!=SQLITE_OK && zWhen ){ + renameColumnParseError(context, zWhen, argv[2], argv[3],&sParse); } renameParseCleanup(&sParse); } #ifndef SQLITE_OMIT_AUTHORIZATION db->xAuth = xAuth; #endif } + +/* +** The implementation of internal UDF sqlite_drop_column(). +** +** Arguments: +** +** argv[0]: An integer - the index of the schema containing the table +** argv[1]: CREATE TABLE statement to modify. +** argv[2]: An integer - the index of the column to remove. +** +** The value returned is a string containing the CREATE TABLE statement +** with column argv[2] removed. +*/ +static void dropColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + int iSchema = sqlite3_value_int(argv[0]); + const char *zSql = (const char*)sqlite3_value_text(argv[1]); + int iCol = sqlite3_value_int(argv[2]); + const char *zDb = db->aDb[iSchema].zDbSName; + int rc; + Parse sParse; + RenameToken *pCol; + Table *pTab; + const char *zEnd; + char *zNew = 0; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + rc = renameParseSql(&sParse, zDb, db, zSql, iSchema==1); + if( rc!=SQLITE_OK ) goto drop_column_done; + pTab = sParse.pNewTable; + if( pTab->nCol==1 || iCol>=pTab->nCol ){ + /* This can happen if the sqlite_schema table is corrupt */ + rc = SQLITE_CORRUPT_BKPT; + goto drop_column_done; + } + + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zName); + if( iColnCol-1 ){ + RenameToken *pEnd; + pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zName); + zEnd = (const char*)pEnd->t.z; + }else{ + zEnd = (const char*)&zSql[pTab->addColOffset]; + while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; + } + + zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); + sqlite3_result_text(context, zNew, -1, SQLITE_TRANSIENT); + sqlite3_free(zNew); + +drop_column_done: + renameParseCleanup(&sParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(context, rc); + } +} + +/* +** This function is called by the parser upon parsing an +** +** ALTER TABLE pSrc DROP COLUMN pName +** +** statement. Argument pSrc contains the possibly qualified name of the +** table being edited, and token pName the name of the column to drop. +*/ +void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, Token *pName){ + sqlite3 *db = pParse->db; /* Database handle */ + Table *pTab; /* Table to modify */ + int iDb; /* Index of db containing pTab in aDb[] */ + const char *zDb; /* Database containing pTab ("main" etc.) */ + char *zCol = 0; /* Name of column to drop */ + int iCol; /* Index of column zCol in pTab->aCol[] */ + + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( NEVER(db->mallocFailed) ) goto exit_drop_column; + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_drop_column; + + /* Make sure this is not an attempt to ALTER a view, virtual table or + ** system table. */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_drop_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 1) ) goto exit_drop_column; + + /* Find the index of the column being dropped. */ + zCol = sqlite3NameFromToken(db, pName); + if( zCol==0 ){ + assert( db->mallocFailed ); + goto exit_drop_column; + } + iCol = sqlite3ColumnIndex(pTab, zCol); + if( iCol<0 ){ + sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zCol); + goto exit_drop_column; + } + + /* Do not allow the user to drop a PRIMARY KEY column or a column + ** constrained by a UNIQUE constraint. */ + if( pTab->aCol[iCol].colFlags & (COLFLAG_PRIMKEY|COLFLAG_UNIQUE) ){ + sqlite3ErrorMsg(pParse, "cannot drop %s column: \"%s\"", + (pTab->aCol[iCol].colFlags&COLFLAG_PRIMKEY) ? "PRIMARY KEY" : "UNIQUE", + zCol + ); + goto exit_drop_column; + } + + /* Do not allow the number of columns to go to zero */ + if( pTab->nCol<=1 ){ + sqlite3ErrorMsg(pParse, "cannot drop column \"%s\": no other columns exist",zCol); + goto exit_drop_column; + } + + /* Edit the sqlite_schema table */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + zDb = db->aDb[iDb].zDbSName; + renameTestSchema(pParse, zDb, iDb==1, ""); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_drop_column(%d, sql, %d) " + "WHERE (type=='table' AND tbl_name=%Q COLLATE nocase)" + , zDb, iDb, iCol, pTab->zName + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDrop); + renameTestSchema(pParse, zDb, iDb==1, "after drop column"); + + /* Edit rows of table on disk */ + if( pParse->nErr==0 && (pTab->aCol[iCol].colFlags & COLFLAG_VIRTUAL)==0 ){ + int i; + int addr; + int reg; + int regRec; + Index *pPk = 0; + int nField = 0; /* Number of non-virtual columns after drop */ + int iCur; + Vdbe *v = sqlite3GetVdbe(pParse); + iCur = pParse->nTab++; + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenWrite); + addr = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + reg = ++pParse->nMem; + pParse->nMem += pTab->nCol; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, reg); + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + } + for(i=0; inCol; i++){ + if( i!=iCol && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + int regOut; + if( pPk ){ + int iPos = sqlite3TableColumnToIndex(pPk, i); + int iColPos = sqlite3TableColumnToIndex(pPk, iCol); + regOut = reg+1+iPos-(iPos>iColPos); + }else{ + regOut = reg+1+nField; + } + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + nField++; + } + } + regRec = reg + pTab->nCol; + sqlite3VdbeAddOp3(v, OP_MakeRecord, reg+1, nField, regRec); + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iCur, regRec, reg+1, pPk->nKeyCol); + }else{ + sqlite3VdbeAddOp3(v, OP_Insert, iCur, regRec, reg); + } + + sqlite3VdbeAddOp2(v, OP_Next, iCur, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + } + +exit_drop_column: + sqlite3DbFree(db, zCol); + sqlite3SrcListDelete(db, pSrc); +} /* ** Register built-in functions used to help implement ALTER TABLE */ void sqlite3AlterFunctions(void){ static FuncDef aAlterTableFuncs[] = { - INTERNAL_FUNCTION(sqlite_rename_column, 9, renameColumnFunc), - INTERNAL_FUNCTION(sqlite_rename_table, 7, renameTableFunc), - INTERNAL_FUNCTION(sqlite_rename_test, 5, renameTableTest), + INTERNAL_FUNCTION(sqlite_rename_column, 9, renameColumnFunc), + INTERNAL_FUNCTION(sqlite_rename_table, 7, renameTableFunc), + INTERNAL_FUNCTION(sqlite_rename_test, 6, renameTableTest), + INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), }; sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); } #endif /* SQLITE_ALTER_TABLE */ Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -161,16 +161,18 @@ int i; int reg; addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur); + VdbeCoverage(v); reg = pReturning->iRetReg; for(i=0; inRetCol; i++){ sqlite3VdbeAddOp3(v, OP_Column, pReturning->iRetCur, i, reg+i); } sqlite3VdbeAddOp2(v, OP_ResultRow, reg, i); sqlite3VdbeAddOp2(v, OP_Next, pReturning->iRetCur, addrRewind+1); + VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrRewind); } sqlite3VdbeAddOp0(v, OP_Halt); #if SQLITE_USER_AUTHENTICATION @@ -1260,10 +1262,11 @@ */ #if SQLITE_ENABLE_HIDDEN_COLUMNS void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){ pCol->colFlags |= COLFLAG_HIDDEN; + if( pTab ) pTab->tabFlags |= TF_HasHidden; }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){ pTab->tabFlags |= TF_OOOHidden; } } #endif @@ -2636,11 +2639,11 @@ } #endif /* Reparse everything to update our internal data structures */ sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName)); + sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName),0); } /* Add the table to the in-memory representation of the database. */ if( db->init.busy ){ @@ -2653,24 +2656,21 @@ sqlite3OomFault(db); return; } pParse->pNewTable = 0; db->mDbFlags |= DBFLAG_SchemaChange; + } #ifndef SQLITE_OMIT_ALTERTABLE - if( !p->pSelect ){ - const char *zName = (const char *)pParse->sNameToken.z; - int nName; - assert( !pSelect && pCons && pEnd ); - if( pCons->z==0 ){ - pCons = pEnd; - } - nName = (int)((const char *)pCons->z - zName); - p->addColOffset = 13 + nName; - } -#endif - } + if( !pSelect && !p->pSelect ){ + assert( pCons && pEnd ); + if( pCons->z==0 ){ + pCons = pEnd; + } + p->addColOffset = 13 + (int)(pCons->z - pParse->sNameToken.z); + } +#endif } #ifndef SQLITE_OMIT_VIEW /* ** The parser calls this routine in order to create a new VIEW @@ -4124,11 +4124,11 @@ */ if( pTblName ){ sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), 0); sqlite3VdbeAddOp2(v, OP_Expire, 0, 1); } sqlite3VdbeJumpHere(v, (int)pIndex->tnum); } @@ -5210,17 +5210,18 @@ Cte *sqlite3CteNew( Parse *pParse, /* Parsing context */ Token *pName, /* Name of the common-table */ ExprList *pArglist, /* Optional column name list for the table */ Select *pQuery, /* Query used to initialize the table */ - int eMaterialized /* Force or prohibit materialization */ + int eMaterialize /* Force or prohibit materialization */ ){ Cte *pNew; sqlite3 *db = pParse->db; - assert( eMaterialized==MAT_NotSpec || eMaterialized==MAT_Yes - || eMaterialized==MAT_No ); + assert( eMaterialize==Materialize_Any + || eMaterialize==Materialize_Yes + || eMaterialize==Materialize_No ); pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); assert( pNew!=0 || db->mallocFailed ); if( db->mallocFailed ){ sqlite3ExprListDelete(db, pArglist); @@ -5228,30 +5229,69 @@ }else{ pNew->pSelect = pQuery; pNew->pCols = pArglist; pNew->zName = sqlite3NameFromToken(pParse->db, pName); pNew->zCteErr = 0; - pNew->eMaterialized = (u8)eMaterialized; + pNew->eMaterialize = (u8)eMaterialize; } return pNew; } + +#ifdef SQLITE_DEBUG +/* +** Validate the back-reference from a SrcList_item back to its Cte object. +** Raise an assertion fault if anything is wrong. This routine is for +** sanity and design checking and does not appear in release builds. +*/ +void sqlite3AssertValidCteBackRef(struct SrcList_item *pItem){ + Cte *pCte; + if( !pItem->fg.isCte ) return; /* Nothing to check here */ + assert( pItem->pSelect!=0 ); + pCte = pItem->u2.pCteSrc; + if( pCte==0 ) return; + assert( pCte->eCteMagic==CTE_MAGIC ); + assert( pItem->pTab!=0 ); + assert( sqlite3_stricmp(pCte->zName, pItem->pTab->zName)==0 ); + assert( pCte->nRefCte>=1 ); +} +#endif /* SQLITE_DEBUG */ + +/* +** Clear information from a Cte object, but do not deallocate storage. +*/ +static void cteClear(sqlite3 *db, Cte *pCte){ + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); +#ifdef SQLITE_DEBUG + if( db==0 || db->pnBytesFreed==0 ){ + pCte->zName = 0; + pCte->eCteMagic = 0; + } +#endif +} /* ** Free the contents of the CTE object passed as the second argument. */ void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ if( pCte ){ - sqlite3ExprListDelete(db, pCte->pCols); - sqlite3SelectDelete(db, pCte->pSelect); - sqlite3DbFree(db, pCte->zName); + cteClear(db, pCte); sqlite3DbFree(db, pCte); } } /* ** This routine is invoked once per CTE by the parser while parsing a -** WITH clause. +** WITH clause. Construct a new Cte object and append it to the +** input With object, reallocating the With object as needed. If the +** pWith input is NULL, then create a new With object. +** +** The returned With object is unowned. The caller needs to assign +** ownership. Note, however, that ownership should not be assigned +** until the object stops growing. The input pWith parameter must be +** either NULL or a pointer to an unowned With object. */ With *sqlite3WithAdd( Parse *pParse, /* Parsing context */ With *pWith, /* Existing WITH clause, or NULL */ Cte *pCte /* The CTE to add to the WITH clause */ @@ -5258,10 +5298,12 @@ ){ sqlite3 *db = pParse->db; With *pNew; char *zName; + /* The input WITH clause is yet unowned */ + assert( pWith==0 || pWith->mOwner==0 ); if( pCte==0 ){ return pWith; } @@ -5295,19 +5337,61 @@ return pNew; } /* ** Free the contents of the With object passed as the second argument. +** +** The sqlite3WithReleaseBySelect() causes the Select-object ownership +** to be released, and sqlite3WithReleaseByParse() causes the Parse-object +** ownership to be released. The With object is only deallocated after +** both owners release it. */ void sqlite3WithDelete(sqlite3 *db, With *pWith){ - if( pWith ){ - int i; - for(i=0; inCte; i++){ - struct Cte *pCte = &pWith->a[i]; - sqlite3ExprListDelete(db, pCte->pCols); - sqlite3SelectDelete(db, pCte->pSelect); - sqlite3DbFree(db, pCte->zName); - } - sqlite3DbFree(db, pWith); - } -} + int i; + for(i=0; inCte; i++){ + cteClear(db, &pWith->a[i]); + } + sqlite3DbFree(db, pWith); +} +void sqlite3WithReleaseBySelect(sqlite3 *db, With *pWith){ + if( pWith==0 ) return; + if( db && db->pnBytesFreed ){ + if( pWith->mOwner==WithOwnedBySelect ) sqlite3WithDelete(db, pWith); + }else{ + pWith->mOwner &= ~WithOwnedBySelect; + if( pWith->mOwner==0 ) sqlite3WithDelete(db, pWith); + } +} +void sqlite3WithReleaseByParse(sqlite3 *db, With *pWith){ + if( pWith==0 ) return; + assert( db==0 || db->pnBytesFreed==0 ); + pWith->mOwner &= ~WithOwnedByParse; + if( pWith->mOwner==0 ) sqlite3WithDelete(db, pWith); +} + +/* +** Add Select-object or Parse-object ownership to a With object. If +** the With object is already owned appropriately, then these routines +** are no-ops. +** +** The first time that a With becomes owned by a Parse-object, make +** arrangements to automatically release the ownership when the Parse +** object is destroyed using a call to sqlite3ParserAddCleanup(). +** This involves a memory allocation. If that memory allocation fails, +** the With object might be destroyed immediately. In that +** case, return a NULL pointer. But usually, return a copy of the pWith +** pointer. +*/ +void sqlite3WithClaimedBySelect(With *pWith){ + pWith->mOwner |= WithOwnedBySelect; +} +With *sqlite3WithClaimedByParse(Parse *pParse, With *pWith){ + if( (pWith->mOwner & WithOwnedByParse)==0 ){ + pWith->mOwner |= WithOwnedByParse; + pWith = (With*)sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3WithReleaseByParse, + (void*)pWith); + } + return pWith; +} + #endif /* !defined(SQLITE_OMIT_CTE) */ Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -207,11 +207,11 @@ /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ pSrc->a[0].pTab = 0; pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); pSrc->a[0].pTab = pTab; - pSrc->a[0].pIBIndex = 0; + pSrc->a[0].u2.pIBIndex = 0; assert( !pSrc->a[0].fg.isCte ); /* generate the SELECT expression tree. */ pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, pOrderBy,0,pLimit ); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -1399,10 +1399,11 @@ sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; pRet->nCte = p->nCte; + pRet->mOwner = WithOwnedBySelect; for(i=0; inCte; i++){ pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); } @@ -1542,11 +1543,11 @@ pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); } - pNewItem->pIBIndex = pOldItem->pIBIndex; + pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isTabFunc ){ pNewItem->u1.pFuncArg = sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); } pTab = pNewItem->pTab = pOldItem->pTab; Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -928,23 +928,28 @@ ipkColumn--; } } } #endif - } - - /* Make sure the number of columns in the source data matches the number - ** of columns to be inserted into the table. - */ - for(i=0; inCol; i++){ - if( pTab->aCol[i].colFlags & COLFLAG_NOINSERT ) nHidden++; - } - if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ - sqlite3ErrorMsg(pParse, - "table %S has %d columns but %d values were supplied", - pTabList, 0, pTab->nCol-nHidden, nColumn); - goto insert_cleanup; + + /* Make sure the number of columns in the source data matches the number + ** of columns to be inserted into the table. + */ + assert( TF_HasHidden==COLFLAG_HIDDEN ); + assert( TF_HasGenerated==COLFLAG_GENERATED ); + assert( COLFLAG_NOINSERT==(COLFLAG_GENERATED|COLFLAG_HIDDEN) ); + if( (pTab->tabFlags & (TF_HasGenerated|TF_HasHidden))!=0 ){ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_NOINSERT ) nHidden++; + } + } + if( nColumn!=(pTab->nCol-nHidden) ){ + sqlite3ErrorMsg(pParse, + "table %S has %d columns but %d values were supplied", + pTabList, 0, pTab->nCol-nHidden, nColumn); + goto insert_cleanup; + } } if( pColumn!=0 && nColumn!=pColumn->nId ){ sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); goto insert_cleanup; } @@ -2403,10 +2408,36 @@ } sqlite3VdbeChangeP5(v, i+1); } #endif +/* +** Table pTab is a WITHOUT ROWID table that is being written to. The cursor +** number is iCur, and register regData contains the new record for the +** PK index. This function adds code to invoke the pre-update hook, +** if one is registered. +*/ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +static void codeWithoutRowidPreupdate( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated */ + int iCur, /* Cursor number for table */ + int regData /* Data containing new record */ +){ + Vdbe *v = pParse->pVdbe; + int r = sqlite3GetTempReg(pParse); + assert( !HasRowid(pTab) ); + assert( 0==(pParse->db->mDbFlags & DBFLAG_Vacuum) ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r); + sqlite3VdbeAddOp4(v, OP_Insert, iCur, regData, r, (char*)pTab, P4_TABLE); + sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + sqlite3ReleaseTempReg(pParse, r); +} +#else +# define codeWithoutRowidPreupdate(a,b,c,d) +#endif + /* ** This routine generates code to finish the INSERT or UPDATE operation ** that was started by a prior call to sqlite3GenerateConstraintChecks. ** A consecutive range of registers starting at regNewData contains the ** rowid and the content to be inserted. @@ -2451,21 +2482,13 @@ pik_flags = (useSeekResult ? OPFLAG_USESEEKRESULT : 0); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK if( update_flags==0 ){ - int r = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Integer, 0, r); - sqlite3VdbeAddOp4(v, OP_Insert, - iIdxCur+i, aRegIdx[i], r, (char*)pTab, P4_TABLE - ); - sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); - sqlite3ReleaseTempReg(pParse, r); - } -#endif + codeWithoutRowidPreupdate(pParse, pTab, iIdxCur+i, aRegIdx[i]); + } } sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], aRegIdx[i]+1, pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); sqlite3VdbeChangeP5(v, pik_flags); @@ -2934,20 +2957,22 @@ insFlags = OPFLAG_APPEND|OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT; }else{ insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND|OPFLAG_PREFORMAT; } #ifdef SQLITE_ENABLE_PREUPDATE_HOOK - if( db->xPreUpdateCallback ){ + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); insFlags &= ~OPFLAG_PREFORMAT; }else #endif { sqlite3VdbeAddOp3(v, OP_RowCell, iDest, iSrc, regRowid); } - sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, - (char*)pDest, P4_TABLE); + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeChangeP4(v, -1, (char*)pDest, P4_TABLE); + } sqlite3VdbeChangeP5(v, insFlags); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -2996,10 +3021,16 @@ }else if( !HasRowid(pSrc) && pDestIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ idxInsFlags |= OPFLAG_NCHANGE; } if( idxInsFlags!=(OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT) ){ sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && !HasRowid(pDest) + && IsPrimaryKeyIndex(pDestIdx) + ){ + codeWithoutRowidPreupdate(pParse, pDest, iDest, regData); + } } sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); sqlite3VdbeChangeP5(v, idxInsFlags|OPFLAG_APPEND); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -511,33 +511,33 @@ ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } } } -} +#ifndef SQLITE_OMIT_CTE + /* Link a With object to a Select object. + ** Both Select- and Parse-ownership is assigned to the With object + */ + static Select *linkWithToSelect(Parse *pParse, Select *pSelect, With *pWith){ + assert( pWith->mOwner==0 ); + if( pSelect ){ + pSelect->pWith = pWith; + sqlite3WithClaimedBySelect(pWith); + parserDoubleLinkSelect(pParse, pSelect); + }else{ + sqlite3WithClaimedByParse(pParse,pWith); + } + return pSelect; + } +#endif /* SQLITE_OMIT_CTE */ +} // end %include + %ifndef SQLITE_OMIT_CTE -select(A) ::= WITH wqlist(W) selectnowith(X). { - Select *p = X; - if( p ){ - p->pWith = W; - parserDoubleLinkSelect(pParse, p); - }else{ - sqlite3WithDelete(pParse->db, W); - } - A = p; -} -select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X). { - Select *p = X; - if( p ){ - p->pWith = W; - parserDoubleLinkSelect(pParse, p); - }else{ - sqlite3WithDelete(pParse->db, W); - } - A = p; -} +select(A) ::= WITH wqlist(W) selectnowith(X). { A = linkWithToSelect(pParse,X,W); } +select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X). + { A = linkWithToSelect(pParse,X,W); } %endif /* SQLITE_OMIT_CTE */ select(A) ::= selectnowith(X). { Select *p = X; if( p ){ parserDoubleLinkSelect(pParse, p); @@ -1622,10 +1622,14 @@ cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname(Y) carglist. { Y.n = (int)(pParse->sLastToken.z-Y.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &Y); } +cmd ::= ALTER TABLE fullname(X) DROP kwcolumn_opt nm(Y). { + sqlite3AlterDropColumn(pParse, X, &Y); +} + add_column_fullname ::= fullname(X). { disableLookaside(pParse); sqlite3AlterBeginAddColumn(pParse, X); } cmd ::= ALTER TABLE fullname(X) RENAME kwcolumn_opt nm(Y) TO nm(Z). { @@ -1658,23 +1662,23 @@ %endif SQLITE_OMIT_VIRTUALTABLE //////////////////////// COMMON TABLE EXPRESSIONS //////////////////////////// %type wqlist {With*} -%destructor wqlist {sqlite3WithDelete(pParse->db, $$);} +%destructor wqlist {if($$)sqlite3WithDelete(pParse->db, $$);} %type wqitem {Cte*} %destructor wqitem {sqlite3CteDelete(pParse->db, $$);} %type wqas {int} with ::= . %ifndef SQLITE_OMIT_CTE -with ::= WITH wqlist(W). { sqlite3WithPush(pParse, W, 1); } -with ::= WITH RECURSIVE wqlist(W). { sqlite3WithPush(pParse, W, 1); } +with ::= WITH wqlist(W). { sqlite3WithPush(pParse, W); } +with ::= WITH RECURSIVE wqlist(W). { sqlite3WithPush(pParse, W); } -wqas(A) ::= AS. {A=MAT_NotSpec;} -wqas(A) ::= AS MATERIALIZED. {A=MAT_Yes;} -//wqas(A) ::= AS NOT MATERIALIZED. {A=MAT_No;} +wqas(A) ::= AS. {A = Materialize_Any;} +wqas(A) ::= AS MATERIALIZED. {A = Materialize_Yes;} +wqas(A) ::= AS NOT MATERIALIZED. {A = Materialize_No;} wqitem(A) ::= nm(X) eidlist_opt(Y) wqas(F) LP select(Z) RP. { A = sqlite3CteNew(pParse, &X, Y, Z, F); /*A-overwrites-X*/ } wqlist(A) ::= wqitem(X). { A = sqlite3WithAdd(pParse, 0, X); /*A-overwrites-X*/ Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -19,26 +19,30 @@ ** Fill the InitData structure with an error message that indicates ** that the database is corrupt. */ static void corruptSchema( InitData *pData, /* Initialization context */ - const char *zObj, /* Object being parsed at the point of error */ + char **azObj, /* Type and name of object being parsed */ const char *zExtra /* Error information */ ){ sqlite3 *db = pData->db; if( db->mallocFailed ){ pData->rc = SQLITE_NOMEM_BKPT; }else if( pData->pzErrMsg[0]!=0 ){ /* A error message has already been generated. Do not overwrite it */ - }else if( pData->mInitFlags & INITFLAG_AlterTable ){ - *pData->pzErrMsg = sqlite3DbStrDup(db, zExtra); + }else if( pData->mInitFlags & (INITFLAG_AlterRename|INITFLAG_AlterDrop) ){ + *pData->pzErrMsg = sqlite3MPrintf(db, + "error in %s %s after %s: %s", azObj[0], azObj[1], + (pData->mInitFlags & INITFLAG_AlterRename) ? "rename" : "drop column", + zExtra + ); pData->rc = SQLITE_ERROR; }else if( db->flags & SQLITE_WriteSchema ){ pData->rc = SQLITE_CORRUPT_BKPT; }else{ char *z; - if( zObj==0 ) zObj = "?"; + const char *zObj = azObj[1] ? azObj[1] : "?"; z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); if( zExtra && zExtra[0] ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); *pData->pzErrMsg = z; pData->rc = SQLITE_CORRUPT_BKPT; } @@ -92,18 +96,18 @@ UNUSED_PARAMETER2(NotUsed, argc); assert( sqlite3_mutex_held(db->mutex) ); db->mDbFlags |= DBFLAG_EncodingFixed; pData->nInitRow++; if( db->mallocFailed ){ - corruptSchema(pData, argv[1], 0); + corruptSchema(pData, argv, 0); return 1; } assert( iDb>=0 && iDbnDb ); if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ if( argv[3]==0 ){ - corruptSchema(pData, argv[1], 0); + corruptSchema(pData, argv, 0); }else if( argv[4] && 'c'==sqlite3UpperToLower[(unsigned char)argv[4][0]] && 'r'==sqlite3UpperToLower[(unsigned char)argv[4][1]] ){ /* Call the parser to process a CREATE TABLE, INDEX or VIEW. ** But because db->init.busy is set to 1, no VDBE code is generated @@ -124,11 +128,11 @@ db->init.iDb = iDb; if( sqlite3GetUInt32(argv[3], &db->init.newTnum)==0 || (db->init.newTnum>pData->mxPage && pData->mxPage>0) ){ if( sqlite3Config.bExtraSchemaChecks ){ - corruptSchema(pData, argv[1], "invalid rootpage"); + corruptSchema(pData, argv, "invalid rootpage"); } } db->init.orphanTrigger = 0; db->init.azInit = argv; pStmt = 0; @@ -143,17 +147,17 @@ }else{ if( rc > pData->rc ) pData->rc = rc; if( rc==SQLITE_NOMEM ){ sqlite3OomFault(db); }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ - corruptSchema(pData, argv[1], sqlite3_errmsg(db)); + corruptSchema(pData, argv, sqlite3_errmsg(db)); } } } sqlite3_finalize(pStmt); }else if( argv[1]==0 || (argv[4]!=0 && argv[4][0]!=0) ){ - corruptSchema(pData, argv[1], 0); + corruptSchema(pData, argv, 0); }else{ /* If the SQL column is blank it means this is an index that ** was created to be the PRIMARY KEY or to fulfill a UNIQUE ** constraint for a CREATE TABLE. The index should have already ** been created when we processed the CREATE TABLE. All we have @@ -160,19 +164,19 @@ ** to do here is record the root page number for that index. */ Index *pIndex; pIndex = sqlite3FindIndex(db, argv[1], db->aDb[iDb].zDbSName); if( pIndex==0 ){ - corruptSchema(pData, argv[1], "orphan index"); + corruptSchema(pData, argv, "orphan index"); }else if( sqlite3GetUInt32(argv[3],&pIndex->tnum)==0 || pIndex->tnum<2 || pIndex->tnum>pData->mxPage || sqlite3IndexHasDuplicateRootPage(pIndex) ){ if( sqlite3Config.bExtraSchemaChecks ){ - corruptSchema(pData, argv[1], "invalid rootpage"); + corruptSchema(pData, argv, "invalid rootpage"); } } } return 0; } @@ -548,35 +552,20 @@ assert( i>=0 && inDb ); } return i; } -/* -** Deallocate a single AggInfo object -*/ -static void agginfoFree(sqlite3 *db, AggInfo *p){ - sqlite3DbFree(db, p->aCol); - sqlite3DbFree(db, p->aFunc); - sqlite3DbFree(db, p); -} - /* ** Free all memory allocations in the pParse object */ void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; - AggInfo *pThis = pParse->pAggList; - while( pThis ){ - AggInfo *pNext = pThis->pNext; - agginfoFree(db, pThis); - pThis = pNext; - } while( pParse->pCleanup ){ ParseCleanup *pCleanup = pParse->pCleanup; pParse->pCleanup = pCleanup->pNext; pCleanup->xCleanup(db, pCleanup->pPtr); - sqlite3DbFree(db, pCleanup); + sqlite3DbFreeNN(db, pCleanup); } sqlite3DbFree(db, pParse->aLabel); if( pParse->pConstExpr ){ sqlite3ExprListDelete(db, pParse->pConstExpr); } @@ -606,11 +595,11 @@ ** use-after-free errors following an OOM. The preferred way to do this is ** to immediately follow the call to this routine with: ** ** testcase( pParse->earlyCleanup ); */ -void sqlite3ParserAddCleanup( +void *sqlite3ParserAddCleanup( Parse *pParse, /* Destroy when this Parser finishes */ void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ ){ ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); @@ -619,14 +608,16 @@ pParse->pCleanup = pCleanup; pCleanup->pPtr = pPtr; pCleanup->xCleanup = xCleanup; }else{ xCleanup(pParse->db, pPtr); + pPtr = 0; #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) pParse->earlyCleanup = 1; #endif } + return pPtr; } /* ** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -68,11 +68,10 @@ static void resolveAlias( Parse *pParse, /* Parsing context */ ExprList *pEList, /* A result set */ int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ Expr *pExpr, /* Transform this into an alias to the result set */ - const char *zType, /* "GROUP" or "ORDER" or "" */ int nSubquery /* Number of subqueries that the label is moving */ ){ Expr *pOrig; /* The iCol-th column of the result set */ Expr *pDup; /* Copy of pOrig */ sqlite3 *db; /* The database connection */ @@ -81,11 +80,11 @@ pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup!=0 ){ - if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); + incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This @@ -522,11 +521,11 @@ } if( sqlite3ExprVectorSize(pOrig)!=1 ){ sqlite3ErrorMsg(pParse, "row value misused"); return WRC_Abort; } - resolveAlias(pParse, pEList, j, pExpr, "", nSubquery); + resolveAlias(pParse, pEList, j, pExpr, nSubquery); cnt = 1; pMatch = 0; assert( zTab==0 && zDb==0 ); if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); @@ -624,14 +623,16 @@ pMatch->colUsed |= sqlite3ExprColUsed(pExpr); } /* Clean up and return */ - sqlite3ExprDelete(db, pExpr->pLeft); - pExpr->pLeft = 0; - sqlite3ExprDelete(db, pExpr->pRight); - pExpr->pRight = 0; + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + sqlite3ExprDelete(db, pExpr->pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + } pExpr->op = eNewExprOp; ExprSetProperty(pExpr, EP_Leaf); lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); @@ -1399,12 +1400,11 @@ if( pItem->u.x.iOrderByCol ){ if( pItem->u.x.iOrderByCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); return 1; } - resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, - zType,0); + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr,0); } } return 0; } Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -88,11 +88,11 @@ #ifndef SQLITE_OMIT_WINDOWFUNC if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){ sqlite3WindowListDelete(db, p->pWinDefn); } #endif - if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); + if( p->pWith ) sqlite3WithReleaseBySelect(db, p->pWith); if( bFree ) sqlite3DbFreeNN(db, p); p = pPrior; bFree = 1; } } @@ -260,11 +260,11 @@ /* ** Return the index of a column in a table. Return -1 if the column ** is not contained in the table. */ -static int columnIndex(Table *pTab, const char *zCol){ +int sqlite3ColumnIndex(Table *pTab, const char *zCol){ int i; u8 h = sqlite3StrIHash(zCol); Column *pCol; for(pCol=pTab->aCol, i=0; inCol; pCol++, i++){ if( pCol->hName==h && sqlite3StrICmp(pCol->zName, zCol)==0 ) return i; @@ -292,11 +292,11 @@ int i; /* For looping over tables in pSrc */ int iCol; /* Index of column matching zCol */ assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ for(i=0; ia[i].pTab, zCol); + iCol = sqlite3ColumnIndex(pSrc->a[i].pTab, zCol); if( iCol>=0 && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) ){ if( piTab ){ *piTab = i; @@ -502,11 +502,11 @@ int iLeft; /* Table on the left with matching column name */ int iLeftCol; /* Column number of matching column on the left */ int iRightCol; /* Column number of matching column on the right */ zName = pList->a[j].zName; - iRightCol = columnIndex(pRightTab, zName); + iRightCol = sqlite3ColumnIndex(pRightTab, zName); if( iRightCol<0 || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 0) ){ sqlite3ErrorMsg(pParse, "cannot join using column %s - column " "not present in both tables", zName); @@ -2084,10 +2084,11 @@ sNC.pSrcList = pSelect->pSrc; a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; int n, m; + pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; zType = columnType(&sNC, p, 0, 0, 0); /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); if( zType ){ @@ -4721,11 +4722,13 @@ if( !pIdx ){ sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); pParse->checkSchema = 1; return SQLITE_ERROR; } - pFrom->pIBIndex = pIdx; + assert( pFrom->fg.isCte==0 ); + assert( pFrom->pSelect==0 ); + pFrom->u2.pIBIndex = pIdx; } return SQLITE_OK; } /* ** Detect compound SELECT statements that use an ORDER BY clause with @@ -4853,29 +4856,17 @@ return 0; } /* The code generator maintains a stack of active WITH clauses ** with the inner-most WITH clause being at the top of the stack. -** -** This routine pushes the WITH clause passed as the second argument -** onto the top of the stack. If argument bFree is true, then this -** WITH clause will never be popped from the stack. In this case it -** should be freed along with the Parse object. In other cases, when -** bFree==0, the With object will be freed along with the SELECT -** statement with which it is associated. */ -void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ +void sqlite3WithPush(Parse *pParse, With *pWith){ if( pWith ){ assert( pParse->pWith!=pWith ); pWith->pOuter = pParse->pWith; pParse->pWith = pWith; - if( bFree ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3WithDelete, - pWith); - testcase( pParse->earlyCleanup ); - } + sqlite3WithClaimedByParse(pParse, pWith); } } /* ** This function checks if argument pFrom refers to a CTE declared by @@ -4936,13 +4927,21 @@ pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; assert( pFrom->pSelect ); - if( pCte->eMaterialized==MAT_Yes ){ + if( pCte->eMaterialize==Materialize_Yes ){ pFrom->pSelect->selFlags |= SF_OptBarrier; } + assert( !pFrom->fg.isIndexedBy ); + pFrom->fg.isCte = 1; + assert( (pCte->eCteMagic==0 && pCte->nRefCte==0) + || (pCte->eCteMagic==CTE_MAGIC && pCte->nRefCte>0) ); + pFrom->u2.pCteSrc = pCte; + sqlite3WithClaimedByParse(pParse, pWith); + pCte->nRefCte++; + VVA_ONLY( pCte->eCteMagic = CTE_MAGIC ); /* Check if this is a recursive CTE. */ pRecTerm = pSel = pFrom->pSelect; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); while( bMayRecursive && pRecTerm->op==pSel->op ){ @@ -5119,11 +5118,11 @@ /* Renumber selId because it has been copied from a view */ p->selId = ++pParse->nSelect; } pTabList = p->pSrc; pEList = p->pEList; - sqlite3WithPush(pParse, p->pWith, 0); + sqlite3WithPush(pParse, p->pWith); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. */ sqlite3SrcListAssignCursors(pParse, pTabList); @@ -5758,21 +5757,27 @@ } #endif } /* -** Check to see if the pThis entry of pTabList is a self-join of a prior view. -** If it is, then return the SrcList_item for the prior view. If it is not, -** then return 0. +** Check to see if the pThis entry of pTabList is a self-join of a prior view, +** or any reuse of a CTE that is not necessarily a self-join. +** If it is, then return the SrcList_item for the prior view or CTE. +** If it is not, then return 0. */ static struct SrcList_item *isSelfJoinView( SrcList *pTabList, /* Search for self-joins in this FROM clause */ struct SrcList_item *pThis /* Search for prior reference to this subquery */ ){ struct SrcList_item *pItem; assert( pThis->pSelect!=0 ); if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + if( pThis->fg.isCte ){ + Cte *pCte = pThis->u2.pCteSrc; + sqlite3AssertValidCteBackRef(pThis); + return pCte->pCteMat; + } for(pItem = pTabList->a; pItempSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; @@ -5793,10 +5798,19 @@ } return pItem; } return 0; } + +/* +** Deallocate a single AggInfo object +*/ +static void agginfoFree(sqlite3 *db, AggInfo *p){ + sqlite3DbFree(db, p->aCol); + sqlite3DbFree(db, p->aFunc); + sqlite3DbFreeNN(db, p); +} #ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION /* ** Attempt to transform a query of the form ** @@ -6139,10 +6153,11 @@ ** (1) Authorized unreferenced tables ** (2) Generate code for all sub-queries */ for(i=0; inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; + struct SrcList_item *pPrior; SelectDest dest; Select *pSub; #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) const char *zSavedAuthContext; #endif @@ -6216,17 +6231,20 @@ SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); } zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; + + /* Verify that the CTE back-reference information is valid */ + sqlite3AssertValidCteBackRef(pItem); /* Generate code to implement the subquery ** ** The subquery is implemented as a co-routine if (1) the subquery is ** guaranteed to be the outer loop (so that it does not need to be - ** computed more than once) and if (2) the subquery is not optimization - ** barrier. + ** computed more than once) and if (2) the subquery is a CTE that is + ** used only this one time. ** ** TODO: Are there other reasons beside (1) and (2) to use a co-routine ** implementation? ** ** TODO: We might should allow a subquery that is an optimization barrier @@ -6234,11 +6252,11 @@ ** only use of the subquery. */ if( i==0 && (pTabList->nSrc==1 || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ - && (pSub->selFlags & SF_OptBarrier)==0 /* (2) */ + && (pItem->fg.isCte && pItem->u2.pCteSrc->nRefCte==1) /* (2) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; @@ -6254,20 +6272,24 @@ pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeEndCoroutine(v, pItem->regReturn); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); + }else if( (pPrior = isSelfJoinView(pTabList,pItem))!=0 ){ + /* This view has been previously materialized. Use the + ** prior materialization */ + if( pPrior->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + } + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); + assert( pPrior->pSelect!=0 ); + pSub->nSelectRow = pPrior->pSelect->nSelectRow; }else{ - /* Generate a subroutine that will fill an ephemeral table with - ** the content of this subquery. pItem->addrFillSub will point - ** to the address of the generated subroutine. pItem->regReturn - ** is a register allocated to hold the subroutine return address - */ + /* Materialize the view */ int topAddr; int onceAddr = 0; int retAddr; - struct SrcList_item *pPrior; testcase( pItem->addrFillSub==0 ); /* Ticket c52b09c7f38903b1311 */ pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; @@ -6278,19 +6300,16 @@ onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); }else{ VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); } - pPrior = isSelfJoinView(pTabList, pItem); - if( pPrior ){ - sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); - assert( pPrior->pSelect!=0 ); - pSub->nSelectRow = pPrior->pSelect->nSelectRow; - }else{ - sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE %u", pSub->selId)); - sqlite3Select(pParse, pSub, &dest); + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + ExplainQueryPlan((pParse, 1, "MATERIALIZE %u", pSub->selId)); + sqlite3Select(pParse, pSub, &dest); + if( pItem->fg.isCte ){ + sqlite3AssertValidCteBackRef(pItem); + pItem->u2.pCteSrc->pCteMat = pItem; } pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); @@ -6438,10 +6457,11 @@ sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } } + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ @@ -6476,10 +6496,11 @@ sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); /* End the database scan loop. */ + SELECTTRACE(1,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); } }else{ /* This case when there exist aggregate functions or a GROUP BY clause ** or both */ @@ -6546,15 +6567,17 @@ /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the ** SELECT statement. */ pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); - if( pAggInfo==0 ){ + if( pAggInfo ){ + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))agginfoFree, pAggInfo); + } + if( db->mallocFailed ){ goto select_end; } - pAggInfo->pNext = pParse->pAggList; - pParse->pAggList = pAggInfo; pAggInfo->selId = p->selId; memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; sNC.pSrcList = pTabList; sNC.uNC.pAggInfo = pAggInfo; @@ -6669,10 +6692,11 @@ SELECTTRACE(1,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 ); if( pWInfo==0 ) goto select_end; + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo */ @@ -6717,10 +6741,11 @@ regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); + SELECTTRACE(1,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); @@ -6794,10 +6819,11 @@ */ if( groupBySort ){ sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ + SELECTTRACE(1,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); } /* Output the final row of result @@ -6949,15 +6975,17 @@ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, 0, minMaxFlag, 0); if( pWInfo==0 ){ goto select_end; } + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); updateAccumulator(pParse, regAcc, pAggInfo); if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( minMaxFlag ){ sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); } + SELECTTRACE(1,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); finalizeAggFunctions(pParse, pAggInfo); } sSort.pOrderBy = 0; Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -8751,11 +8751,11 @@ if( (c=='o' && (strncmp(azArg[0], "output", n)==0||strncmp(azArg[0], "once", n)==0)) || (c=='e' && n==5 && strcmp(azArg[0],"excel")==0) ){ - const char *zFile = 0; + char *zFile = 0; int bTxtMode = 0; int i; int eMode = 0; int bBOM = 0; int bOnce = 0; /* 0: .output, 1: .once, 2: .excel */ @@ -8781,21 +8781,26 @@ azArg[i]); showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; } - }else if( zFile==0 ){ - zFile = z; + }else if( zFile==0 && eMode!='e' && eMode!='x' ){ + zFile = sqlite3_mprintf("%s", z); + if( zFile[0]=='|' ){ + while( i+1out,"ERROR: extra parameter: \"%s\". Usage:\n", azArg[i]); showHelp(p->out, azArg[0]); rc = 1; + sqlite3_free(zFile); goto meta_command_exit; } } - if( zFile==0 ) zFile = "stdout"; + if( zFile==0 ) zFile = sqlite3_mprintf("stdout"); if( bOnce ){ p->outCount = 2; }else{ p->outCount = 0; } @@ -8814,11 +8819,12 @@ }else{ /* text editor mode */ newTempFile(p, "txt"); bTxtMode = 1; } - zFile = p->zTempFile; + sqlite3_free(zFile); + zFile = sqlite3_mprintf("%s", p->zTempFile); } #endif /* SQLITE_NOHAVE_SYSTEM */ if( zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN raw_printf(stderr, "Error: pipes are not supported in this OS\n"); @@ -8846,10 +8852,11 @@ } else { if( bBOM ) fprintf(p->out,"\357\273\277"); sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); } } + sqlite3_free(zFile); }else if( c=='p' && n>=3 && strncmp(azArg[0], "parameter", n)==0 ){ open_db(p,0); if( nArg<=1 ) goto parameter_syntax_error; Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -2014,11 +2014,16 @@ u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ u8 hName; /* Column name hash for faster lookup */ u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; -/* Allowed values for Column.colFlags: +/* Allowed values for Column.colFlags. +** +** Constraints: +** TF_HasVirtual == COLFLAG_VIRTUAL +** TF_HasStored == COLFLAG_STORED +** TF_HasHidden == COLFLAG_HIDDEN */ #define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ #define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ #define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ @@ -2203,15 +2208,16 @@ ** the TF_OOOHidden attribute would apply in this case. Such tables require ** special handling during INSERT processing. The "OOO" means "Out Of Order". ** ** Constraints: ** -** TF_HasVirtual == COLFLAG_Virtual -** TF_HasStored == COLFLAG_Stored +** TF_HasVirtual == COLFLAG_VIRTUAL +** TF_HasStored == COLFLAG_STORED +** TF_HasHidden == COLFLAG_HIDDEN */ #define TF_Readonly 0x0001 /* Read-only system table */ -#define TF_Ephemeral 0x0002 /* An ephemeral table */ +#define TF_HasHidden 0x0002 /* Has one or more hidden columns */ #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ #define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ #define TF_HasVirtual 0x0020 /* Has one or more VIRTUAL columns */ #define TF_HasStored 0x0040 /* Has one or more STORED columns */ @@ -2222,10 +2228,11 @@ #define TF_NoVisibleRowid 0x0200 /* No user-visible "rowid" column */ #define TF_OOOHidden 0x0400 /* Out-of-Order hidden columns */ #define TF_HasNotNull 0x0800 /* Contains NOT NULL constraints */ #define TF_Shadow 0x1000 /* True for a shadow table */ #define TF_HasStat4 0x2000 /* STAT4 info available for this table */ +#define TF_Ephemeral 0x4000 /* An ephemeral table */ /* ** Test to see whether or not a table is a virtual table. This is ** done as a macro so that it will be optimized out when virtual ** table support is omitted from the build. @@ -2590,11 +2597,10 @@ int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ u32 selId; /* Select to which this AggInfo belongs */ - AggInfo *pNext; /* Next in list of them all */ }; /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater @@ -2950,20 +2956,24 @@ unsigned isTabFunc :1; /* True if table-valued-function syntax */ unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned viaCoroutine :1; /* Implemented as a co-routine */ unsigned isRecursive :1; /* True for recursive reference in WITH */ unsigned fromDDL :1; /* Comes from sqlite_schema */ + unsigned isCte :1; /* True if table is a CTE */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<" clause */ ExprList *pFuncArg; /* Arguments to table-valued-function */ } u1; - Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ + union { + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ + Cte *pCteSrc; /* The original CTE, if fg.isCte is true */ + } u2; } a[1]; /* One entry for each identifier on the list */ }; /* ** Permitted values of the SrcList.a.jointype field @@ -3427,11 +3437,10 @@ #endif AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ Parse *pParentParse; /* Parent parser if this parser is nested */ - AggInfo *pAggList; /* List of all AggInfo objects */ union { int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ Returning *pReturning; /* The RETURNING clause */ } u1; u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ @@ -3711,11 +3720,12 @@ } InitData; /* ** Allowed values for mInitFlags */ -#define INITFLAG_AlterTable 0x0001 /* This is a reparse after ALTER TABLE */ +#define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */ +#define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */ /* ** Structure containing global configuration data for the SQLite library. ** ** This structure also contains some state information. @@ -3868,33 +3878,59 @@ #define WRC_Continue 0 /* Continue down into children */ #define WRC_Prune 1 /* Omit children but continue walking siblings */ #define WRC_Abort 2 /* Abandon the tree walk */ /* -** Allowed values for Cte.eMAterialized +** Allowed values for Cte.eMaterialize */ -#define MAT_NotSpec 0 /* Not specified */ -#define MAT_Yes 1 /* AS MATERIALIZED ... */ -#define MAT_No 2 /* AS NOT MATERIALIZED. Not currently used */ +#define Materialize_Any 0 /* Not specified */ +#define Materialize_Yes 1 /* AS MATERIALIZED ... */ +#define Materialize_No 2 /* AS NOT MATERIALIZED. Not currently used */ /* ** An instance of this structure represents a set of one or more CTEs ** (common table expressions) created by a single WITH clause. +** +** A With object describes an entire WITH clause, which is composed of +** one or more Cte objects. During parsing, the Cte objects are created +** separately for each common-table-expression, then copied into the +** With object container. +** +** DUAL OWNERSHIP +** The With object can be owned by a Select object (via the Select.pWith +** field), or it can be owned by the Parse object of the current parse, +** or by both. The With.mOwner bitmask determines who owns each With object. +** The object is not deallocated until all owners have released it. */ struct Cte { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ const char *zCteErr; /* Error message for circular references */ - u8 eMaterialized; /* One of the MAT_* values */ + u8 eMaterialize; /* One of the Materialize_* options */ + u16 eCteMagic; /* Magic number for sanity checking */ + u32 nRefCte; /* Number of times used */ + struct SrcList_item *pCteMat; /* The materialization of this CTE */ }; struct With { int nCte; /* Number of CTEs in the WITH clause */ + int mOwner; /* Mask of bits to discribe ownership */ With *pOuter; /* Containing WITH clause, or NULL */ Cte a[1]; /* The CTEs of this WITH clause */ }; +/* +** The correct value for Cte.eCteMagic if the Cte object is valid +*/ +#define CTE_MAGIC 0x3f96 /* Magic number for valid Cte objects */ + +/* +** Ownership flags for With. +*/ +#define WithOwnedBySelect 0x01 /* Owned by a Select object */ +#define WithOwnedByParse 0x10 /* Owned by the Parse object */ + #ifdef SQLITE_DEBUG /* ** An instance of the TreeView object is used for printing the content of ** data structures on sqlite3DebugPrintf() using a tree-like view. */ @@ -4540,10 +4576,11 @@ # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 # define sqlite3TriggerStepSrc(A,B) 0 #endif int sqlite3JoinType(Parse*, Token*, Token*, Token*); +int sqlite3ColumnIndex(Table *pTab, const char *zCol); void sqlite3SetJoinExpr(Expr*,int); void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); void sqlite3DeferForeignKey(Parse*, int); #ifndef SQLITE_OMIT_AUTHORIZATION void sqlite3AuthRead(Parse*,Expr*,Schema*,SrcList*); @@ -4724,10 +4761,11 @@ int sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); void sqlite3ColumnDefault(Vdbe *, Table *, int, int); void sqlite3AlterFinishAddColumn(Parse *, Token *); void sqlite3AlterBeginAddColumn(Parse *, SrcList *); +void sqlite3AlterDropColumn(Parse*, SrcList*, Token*); void *sqlite3RenameTokenMap(Parse*, void*, Token*); void sqlite3RenameTokenRemap(Parse*, void *pTo, void *pFrom); void sqlite3RenameExprUnmap(Parse*, Expr*); void sqlite3RenameExprlistUnmap(Parse*, ExprList*); CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); @@ -4877,11 +4915,11 @@ FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); void sqlite3ParserReset(Parse*); -void sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*); +void *sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*); #ifdef SQLITE_ENABLE_NORMALIZE char *sqlite3Normalize(Vdbe*, const char*); #endif int sqlite3Reprepare(Vdbe*); void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); @@ -4896,15 +4934,29 @@ #ifndef SQLITE_OMIT_CTE Cte *sqlite3CteNew(Parse*,Token*,ExprList*,Select*,int); With *sqlite3WithAdd(Parse*,With*,Cte*); void sqlite3CteDelete(sqlite3*,Cte*); void sqlite3WithDelete(sqlite3*,With*); - void sqlite3WithPush(Parse*, With*, u8); -#else -#define sqlite3WithPush(x,y,z) -#define sqlite3WithDelete(x,y) -#endif + void sqlite3WithReleaseBySelect(sqlite3*,With*); + void sqlite3WithReleaseByParse(sqlite3*,With*); + void sqlite3WithClaimedBySelect(With*); + With *sqlite3WithClaimedByParse(Parse*,With*); + void sqlite3WithPush(Parse*, With*); +# ifdef SQLITE_DEBUG + void sqlite3AssertValidCteBackRef(struct SrcList_item*); +# else +# define sqlite3AssertValidCteBackRef(X) +# endif +#else /* if defined(SQLITE_OMIT_CTE) */ +# define sqlite3WithPush(PARSE,W) +# deifne sqlite3WithDelete(Db,W) +# define sqlite3WithReleaseBySelect(DB,W) +# define sqlite3WithReleaseByParse(DB,W) +# define sqlite3WithClaimedBySelect(W) +# define sqlite3WithClaimedByParse(PARSE,W); +# define sqlite3AssertValidCteBackRef(X) +#endif /* SQLITE_OMIT_CTE */ #ifndef SQLITE_OMIT_UPSERT Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); void sqlite3UpsertDelete(sqlite3*,Upsert*); Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*); Index: src/treeview.c ================================================================== --- src/treeview.c +++ src/treeview.c @@ -88,11 +88,12 @@ void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){ int i; if( pWith==0 ) return; if( pWith->nCte==0 ) return; if( pWith->pOuter ){ - sqlite3TreeViewLine(pView, "WITH (0x%p, pOuter=0x%p)",pWith,pWith->pOuter); + sqlite3TreeViewLine(pView, "WITH (0x%p, pOuter=0x%p)", + pWith,pWith->pOuter); }else{ sqlite3TreeViewLine(pView, "WITH (0x%p)", pWith); } if( pWith->nCte>0 ){ pView = sqlite3TreeViewPush(pView, 1); @@ -99,11 +100,14 @@ for(i=0; inCte; i++){ StrAccum x; char zLine[1000]; const struct Cte *pCte = &pWith->a[i]; sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3_str_appendf(&x, "%s", pCte->zName); + sqlite3_str_appendf(&x, "%s%s (0x%p, n=%d)", pCte->zName, + pCte->eMaterialize==Materialize_Any ? "" : + pCte->eMaterialize==Materialize_Yes ? " MATERIALIZED" : + " NOT MATERIALIZED", pCte, pCte->nRefCte); if( pCte->pCols && pCte->pCols->nExpr>0 ){ char cSep = '('; int j; for(j=0; jpCols->nExpr; j++){ sqlite3_str_appendf(&x, "%c%s", cSep, pCte->pCols->a[j].zEName); @@ -148,10 +152,13 @@ sqlite3_str_appendf(&x, " LEFT-JOIN"); } if( pItem->fg.fromDDL ){ sqlite3_str_appendf(&x, " DDL"); } + if( pItem->fg.isCte && pItem->u2.pCteSrc ){ + sqlite3_str_appendf(&x, " CTE=0x%p", pItem->u2.pCteSrc); + } sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inSrc-1); if( pItem->pSelect ){ sqlite3TreeViewSelect(pView, pItem->pSelect, 0); } Index: src/trigger.c ================================================================== --- src/trigger.c +++ src/trigger.c @@ -354,11 +354,11 @@ db->aDb[iDb].zDbSName, zName, pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName), 0); } if( db->init.busy ){ Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -6328,11 +6328,11 @@ #ifndef SQLITE_OMIT_ALTERTABLE if( pOp->p4.z==0 ){ sqlite3SchemaClear(db->aDb[iDb].pSchema); db->mDbFlags &= ~DBFLAG_SchemaKnownOk; - rc = sqlite3InitOne(db, iDb, &p->zErrMsg, INITFLAG_AlterTable); + rc = sqlite3InitOne(db, iDb, &p->zErrMsg, pOp->p5); db->mDbFlags |= DBFLAG_SchemaChange; p->expired = 0; }else #endif { Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -221,11 +221,11 @@ #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) void sqlite3ExplainBreakpoint(const char*,const char*); #else # define sqlite3ExplainBreakpoint(A,B) /*no-op*/ #endif -void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); +void sqlite3VdbeAddParseSchemaOp(Vdbe*, int, char*, u16); void sqlite3VdbeChangeOpcode(Vdbe*, int addr, u8); void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); void sqlite3VdbeChangeP5(Vdbe*, u16 P5); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -469,13 +469,14 @@ ** as having been used. ** ** The zWhere string must have been obtained from sqlite3_malloc(). ** This routine will take ownership of the allocated memory. */ -void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ +void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere, u16 p5){ int j; sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeChangeP5(p, p5); for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); sqlite3MayAbort(p->pParse); } /* Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -487,11 +487,11 @@ v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp0(v, OP_Expire); zWhere = sqlite3MPrintf(db, "name=%Q AND sql=%Q", pTab->zName, zStmt); - sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere, 0); sqlite3DbFree(db, zStmt); iReg = ++pParse->nMem; sqlite3VdbeLoadString(v, iReg, pTab->zName); sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); @@ -658,10 +658,11 @@ if( zType[i]=='\0' && i>0 ){ assert(zType[i-1]==' '); zType[i-1] = '\0'; } pTab->aCol[iCol].colFlags |= COLFLAG_HIDDEN; + pTab->tabFlags |= TF_HasHidden; oooHidden = TF_OOOHidden; }else{ pTab->tabFlags |= oooHidden; } } Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -2952,13 +2952,13 @@ pSrc = pTabList->a + pNew->iTab; pTab = pSrc->pTab; pWC = pBuilder->pWC; assert( !IsVirtual(pSrc->pTab) ); - if( pSrc->pIBIndex ){ + if( pSrc->fg.isIndexedBy ){ /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIBIndex; + pProbe = pSrc->u2.pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ /* There is no INDEXED BY clause. Create a fake Index object in local ** variable sPk to represent the rowid primary key index. Make this @@ -2990,11 +2990,11 @@ #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ if( !pBuilder->pOrSet /* Not part of an OR optimization */ && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ && !pSrc->fg.isCorrelated /* Not a correlated subquery */ && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ ){ @@ -3040,11 +3040,11 @@ #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ /* Loop over all indices. If there was an INDEXED BY clause, then only ** consider index pProbe. */ for(; rc==SQLITE_OK && pProbe; - pProbe=(pSrc->pIBIndex ? 0 : pProbe->pNext), iSortIdx++ + pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ ){ int isLeft = (pSrc->fg.jointype & JT_OUTER)!=0; if( pProbe->pPartIdxWhere!=0 && !whereUsablePartialIndex(pSrc->iCursor, isLeft, pWC, pProbe->pPartIdxWhere) Index: test/alterauth2.test ================================================================== --- test/alterauth2.test +++ test/alterauth2.test @@ -92,7 +92,24 @@ {SQLITE_READ sqlite_temp_master type temp {}} {SQLITE_SELECT {} {} {} {}} {SQLITE_UPDATE sqlite_master sql main {}} {SQLITE_UPDATE sqlite_temp_master sql temp {}} } + +do_auth_test 1.3 { + ALTER TABLE t2 DROP COLUMN c; +} { + {SQLITE_FUNCTION {} like {} {}} + {SQLITE_FUNCTION {} sqlite_drop_column {} {}} + {SQLITE_FUNCTION {} sqlite_rename_test {} {}} + {SQLITE_READ sqlite_master name main {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_READ sqlite_temp_master name temp {}} + {SQLITE_READ sqlite_temp_master sql temp {}} + {SQLITE_READ sqlite_temp_master type temp {}} + {SQLITE_SELECT {} {} {} {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} finish_test ADDED test/altercorrupt.test Index: test/altercorrupt.test ================================================================== --- /dev/null +++ test/altercorrupt.test @@ -0,0 +1,177 @@ +# 2019-01-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix altercorrupt + +database_may_be_corrupt + +#-------------------------------------------------------------------------- +reset_db +do_test 1.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-685346d89b5e5f.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 63 00 00 05 f0 00 00 00 00 04 10 00 00 04 ..c............. +| 48: 00 00 00 00 00 00 0f f0 00 00 00 01 00 00 00 00 ................ +| 64: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f f8 00 05 0e cf 00 0f 79 0f d3 .............y.. +| 112: 0f 2e 0e f3 0e cf 00 00 00 00 00 00 00 00 00 00 ................ +| 3776: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3792: 05 06 17 11 11 01 31 74 61 62 6c 65 74 34 74 34 ......1tablet4t4 +| 3808: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 74 34 .CREATE TABLE t4 +| 3824: 28 7a 29 39 04 06 17 11 11 01 5f 74 61 62 6c 65 (z)9......_table +| 3840: 74 33 74 33 05 43 52 45 41 54 45 20 54 41 42 4c t3t3.CREATE TABL +| 3856: 45 20 74 33 28 78 20 49 4e 54 45 47 45 52 20 50 E t3(x INTEGER P +| 3872: 52 49 4d 41 52 59 20 4b 45 59 2c 20 79 29 49 03 RIMARY KEY, y)I. +| 3888: 06 17 11 11 01 7f 74 61 62 6c 65 74 32 74 32 04 ......tablet2t2. +| 3904: 43 52 45 41 54 45 20 54 41 42 4c 45 20 74 32 28 CREATE TABLE t2( +| 3920: 61 2c 62 2c 63 20 50 52 49 4d 41 52 59 20 4b 45 a,b,c PRIMARY KE +| 3936: 59 2c 20 64 2c 20 65 2c 20 66 29 20 57 49 54 48 Y, d, e, f) WITH +| 3952: 4f 55 54 20 52 4f 57 49 44 58 03 07 17 11 11 01 OUT ROWIDX...... +| 3968: 81 1b 74 61 62 6c 65 74 31 74 31 02 43 52 45 41 ..tablet1t1.CREA +| 3984: 54 45 20 54 41 42 4c 45 20 74 31 28 61 2c 62 2c TE TABLE t1(a,b, +| 4000: 63 20 41 53 20 28 2d 62 29 20 56 49 52 54 55 41 c AS (-b) VIRTUA +| 4016: 4c 2c 64 20 43 48 45 43 4b 28 64 3e 35 29 2c 65 L,d CHECK(d>5),e +| 4032: 20 55 4e 49 51 55 45 2c 20 66 20 41 53 20 28 2b UNIQUE, f AS (+ +| 4048: 62 29 29 23 02 06 17 37 11 01 00 69 6e 64 65 78 b))#...7...index +| 4064: 73 71 6c 69 74 65 5f 61 75 74 6f 69 6e 64 65 78 sqlite_autoindex +| 4080: 5f 74 31 5f 31 74 31 03 00 00 00 08 00 00 00 00 _t1_1t1......... +| page 2 offset 4096 +| 0: 0d 00 00 00 0a 0f 93 00 0f f6 0f eb 0f e0 0f d5 ................ +| 16: 0f ca 0f 8f 0f b4 0f a9 0f 9e 0f 93 00 00 00 00 ................ +| 3984: 00 00 00 09 0a 05 01 01 01 01 0a 64 6e 14 09 09 ...........dn... +| 4000: 05 01 01 01 01 09 5a 6d 12 09 08 05 01 01 01 01 ......Zm........ +| 4016: 08 50 6c 10 09 07 05 01 01 01 01 07 46 6b 0e 09 .Pl.........Fk.. +| 4032: 06 05 01 01 01 01 06 3c 6a 0c 09 05 05 01 01 01 .......5),e +| 4032: 20 55 4e 49 51 55 45 2c 20 66 20 41 53 20 28 2b UNIQUE, f AS (+ +| 4048: 62 29 29 23 02 06 17 37 11 01 00 69 6e 64 65 78 b))#...7...index +| 4064: 73 71 6c 69 74 65 5f 61 75 74 6f 69 6e 64 65 78 sqlite_autoindex +| 4080: 5f 74 31 5f 31 84 31 03 01 00 00 08 00 00 00 00 _t1_1.1......... +| page 2 offset 4096 +| 0: 0d 00 00 00 0a 0f 93 00 0f f6 0f eb 0f e0 0f d5 ................ +| 16: 0f ca 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3984: 00 00 00 09 0a 05 01 01 01 01 0a 64 6e 14 09 09 ...........dn... +| 4000: 05 01 01 01 01 09 5a 6d 12 09 08 05 01 00 f1 01 ......Zm........ +| 4016: 08 50 6c 10 09 07 05 01 01 01 01 07 46 6b 0e 09 .Pl.........Fk.. +| 4032: 06 05 01 00 f1 01 06 3c 6a 0c 09 05 05 01 01 01 .......10)); + CREATE TABLE t13(a, b, c CHECK(c>10)); +} +do_catchsql_test 3.1 { + ALTER TABLE t12 DROP COLUMN c; +} {1 {error in table t12 after drop column: no such column: c}} + +do_catchsql_test 3.2 { + ALTER TABLE t13 DROP COLUMN c; +} {0 {}} + +#------------------------------------------------------------------------- +# Test that generated columns can be dropped. And that other columns from +# tables that contain generated columns can be dropped. +# +foreach {tn wo vs} { + 1 "" "" + 2 "" VIRTUAL + 3 "" STORED + 4 "WITHOUT ROWID" STORED + 5 "WITHOUT ROWID" VIRTUAL +} { + reset_db + + do_execsql_test 4.$tn.0 " + CREATE TABLE 'my table'(a, b PRIMARY KEY, c AS (a+b) $vs, d) $wo + " + do_execsql_test 4.$tn.1 { + INSERT INTO "my table"(a, b, d) VALUES(1, 2, 'hello'); + INSERT INTO "my table"(a, b, d) VALUES(3, 4, 'world'); + + SELECT * FROM "my table" + } { + 1 2 3 hello + 3 4 7 world + } + + do_execsql_test 4.$tn.2 { + ALTER TABLE "my table" DROP COLUMN c; + } + do_execsql_test 4.$tn.3 { + SELECT * FROM "my table" + } { + 1 2 hello + 3 4 world + } + + do_execsql_test 4.$tn.4 " + CREATE TABLE x1(a, b, c PRIMARY KEY, d AS (b+c) $vs, e) $wo + " + do_execsql_test 4.$tn.5 { + INSERT INTO x1(a, b, c, e) VALUES(1, 2, 3, 4); + INSERT INTO x1(a, b, c, e) VALUES(5, 6, 7, 8); + INSERT INTO x1(a, b, c, e) VALUES(9, 10, 11, 12); + SELECT * FROM x1; + } { + 1 2 3 5 4 + 5 6 7 13 8 + 9 10 11 21 12 + } + + do_execsql_test 4.$tn.6 { + ALTER TABLE x1 DROP COLUMN a + } + do_execsql_test 4.$tn.7 { + SELECT * FROM x1 + } { + 2 3 5 4 + 6 7 13 8 + 10 11 21 12 + } + do_execsql_test 4.$tn.8 { + ALTER TABLE x1 DROP COLUMN e + } + do_execsql_test 4.$tn.9 { + SELECT * FROM x1 + } { + 2 3 5 + 6 7 13 + 10 11 21 + } +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE p1(a PRIMARY KEY, b UNIQUE); + CREATE TABLE c1(x, y, z REFERENCES p1(c)); + CREATE TABLE c2(x, y, z, w REFERENCES p1(b)); +} +do_execsql_test 5.1 { + ALTER TABLE c1 DROP COLUMN z; + ALTER TABLE c2 DROP COLUMN z; + SELECT sql FROM sqlite_schema WHERE name IN ('c1', 'c2'); +} { + {CREATE TABLE c1(x, y)} + {CREATE TABLE c2(x, y, w REFERENCES p1(b))} +} + +do_execsql_test 5.2.1 { + CREATE VIEW v1 AS SELECT d, e FROM p1 +} +do_catchsql_test 5.2.2 { + ALTER TABLE c1 DROP COLUMN x +} {1 {error in view v1: no such column: d}} +do_execsql_test 5.3.1 { + DROP VIEW v1; + CREATE VIEW v1 AS SELECT x, y FROM c1; +} +do_catchsql_test 5.3.2 { + ALTER TABLE c1 DROP COLUMN x +} {1 {error in view v1 after drop column: no such column: x}} + +do_execsql_test 5.4.1 { + CREATE TRIGGER tr AFTER INSERT ON c1 BEGIN + INSERT INTO p1 VALUES(new.y, new.xyz); + END; +} +do_catchsql_test 5.4.2 { + ALTER TABLE c1 DROP COLUMN y +} {1 {error in trigger tr: no such column: new.xyz}} +do_execsql_test 5.5.1 { + DROP TRIGGER tr; + CREATE TRIGGER tr AFTER INSERT ON c1 BEGIN + INSERT INTO p1 VALUES(new.y, new.z); + END; +} +do_catchsql_test 5.5.2 { + ALTER TABLE c1 DROP COLUMN y +} {1 {error in trigger tr: no such column: new.z}} + +finish_test ADDED test/alterdropcol2.test Index: test/alterdropcol2.test ================================================================== --- /dev/null +++ test/alterdropcol2.test @@ -0,0 +1,214 @@ +# 2021 February 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix alterdropcol2 + +# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. +ifcapable !altertable { + finish_test + return +} + +# EVIDENCE-OF: R-58318-35349 The DROP COLUMN syntax is used to remove an +# existing column from a table. +do_execsql_test 1.0 { + CREATE TABLE t1(c, b, a, PRIMARY KEY(b, a)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3), (4, 5, 6); +} +do_execsql_test 1.1 { + ALTER TABLE t1 DROP c; +} + +# EVIDENCE-OF: The DROP COLUMN command removes the named column from the table, +# and also rewrites the entire table to purge the data associated with that +# column. +do_execsql_test 1.2.1 { + SELECT * FROM t1; +} {2 3 5 6} + +do_execsql_test 1.2.2 { + SELECT sql FROM sqlite_schema; +} { + {CREATE TABLE t1(b, a, PRIMARY KEY(b, a)) WITHOUT ROWID} +} + +proc do_atdc_error_test {tn schema atdc error} { + reset_db + execsql $schema + uplevel [list do_catchsql_test $tn $atdc [list 1 [string trim $error]]] +} + +#------------------------------------------------------------------------- +# Test cases 2.* attempt to verify the following: +# +# EVIDENCE-OF: R-24098-10282 The DROP COLUMN command only works if the column +# is not referenced by any other parts of the schema and is not a PRIMARY KEY +# and does not have a UNIQUE constraint. +# + +# EVIDENCE-OF: R-05184-13006 The column is a PRIMARY KEY. +# +do_atdc_error_test 2.1.1 { + CREATE TABLE x1(a PRIMARY KEY, b, c); +} { + ALTER TABLE x1 DROP COLUMN a +} { + cannot drop PRIMARY KEY column: "a" +} + +# EVIDENCE-OF: R-43412-16016 The column has a UNIQUE constraint. +# +do_atdc_error_test 2.2.1 { + CREATE TABLE x1(a PRIMARY KEY, b, c UNIQUE); +} { + ALTER TABLE x1 DROP COLUMN c +} { + cannot drop UNIQUE column: "c" +} +do_atdc_error_test 2.2.2 { + CREATE TABLE x1(a PRIMARY KEY, b, c, UNIQUE(b, c)); +} { + ALTER TABLE x1 DROP COLUMN c +} { + error in table x1 after drop column: no such column: c +} + +# EVIDENCE-OF: R-46731-08965: The column is indexed. +# +do_atdc_error_test 2.3.1 { + CREATE TABLE 'one two'('x y', 'z 1', 'a b'); + CREATE INDEX idx ON 'one two'('z 1'); +} { + ALTER TABLE 'one two' DROP COLUMN 'z 1' +} { + error in index idx after drop column: no such column: z 1 +} +do_atdc_error_test 2.3.2 { + CREATE TABLE x1(a, b, c); + CREATE INDEX idx ON x1(a); +} { + ALTER TABLE x1 DROP COLUMN a; +} { + error in index idx after drop column: no such column: a +} + +# EVIDENCE-OF: R-46731-08965: The column is indexed. +# +do_atdc_error_test 2.4.1 { + CREATE TABLE x1234(a, b, c PRIMARY KEY) WITHOUT ROWID; + CREATE INDEX i1 ON x1234(b) WHERE ((a+5) % 10)==0; +} { + ALTER TABLE x1234 DROP a +} { + error in index i1 after drop column: no such column: a +} + +# EVIDENCE-OF: R-18825-17786 The column appears in a table CHECK constraint. +# +do_atdc_error_test 2.5.1 { + CREATE TABLE x1234(a, b, c PRIMARY KEY, CHECK(((a+5)%10)!=0)) WITHOUT ROWID; +} { + ALTER TABLE x1234 DROP a +} { + error in table x1234 after drop column: no such column: a +} + +# EVIDENCE-OF: R-55640-01652 The column is used in a foreign key constraint. +# +do_atdc_error_test 2.6.1 { + CREATE TABLE p1(x, y UNIQUE); + CREATE TABLE c1(u, v, FOREIGN KEY (v) REFERENCES p1(y)) +} { + ALTER TABLE c1 DROP v +} { + error in table c1 after drop column: unknown column "v" in foreign key definition +} + +# EVIDENCE-OF: R-20795-39479 The column is used in the expression of a +# generated column. +do_atdc_error_test 2.7.1 { + CREATE TABLE c1(u, v, w AS (u+v)); +} { + ALTER TABLE c1 DROP v +} { + error in table c1 after drop column: no such column: v +} +do_atdc_error_test 2.7.2 { + CREATE TABLE c1(u, v, w AS (u+v) STORED); +} { + ALTER TABLE c1 DROP u +} { + error in table c1 after drop column: no such column: u +} + +# EVIDENCE-OF: R-01515-49025 The column appears in a trigger or view. +# +do_atdc_error_test 2.8.1 { + CREATE TABLE log(l); + CREATE TABLE c1(u, v, w); + CREATE TRIGGER tr1 AFTER INSERT ON c1 BEGIN + INSERT INTO log VALUES(new.w); + END; +} { + ALTER TABLE c1 DROP w +} { + error in trigger tr1 after drop column: no such column: new.w +} +do_atdc_error_test 2.8.2 { + CREATE TABLE c1(u, v, w); + CREATE VIEW v1 AS SELECT u, v, w FROM c1; +} { + ALTER TABLE c1 DROP w +} { + error in view v1 after drop column: no such column: w +} +do_atdc_error_test 2.8.3 { + CREATE TABLE c1(u, v, w); + CREATE VIEW v1 AS SELECT * FROM c1 WHERE w IS NOT NULL; +} { + ALTER TABLE c1 DROP w +} { + error in view v1 after drop column: no such column: w +} + +#------------------------------------------------------------------------- +# Verify that a column that is part of a CHECK constraint may be dropped +# if the CHECK constraint was specified as part of the column definition. +# + +# EVIDENCE-OF: R-60924-11170 However, the column being deleted can be used in a +# column CHECK constraint because the column CHECK constraint is dropped +# together with the column itself. +do_execsql_test 3.0 { + CREATE TABLE yyy(q, w, e CHECK (e > 0), r); + INSERT INTO yyy VALUES(1,1,1,1), (2,2,2,2); + + CREATE TABLE zzz(q, w, e, r, CHECK (e > 0)); + INSERT INTO zzz VALUES(1,1,1,1), (2,2,2,2); +} +do_catchsql_test 3.1.1 { + INSERT INTO yyy VALUES(0,0,0,0); +} {1 {CHECK constraint failed: e > 0}} +do_catchsql_test 3.1.2 { + INSERT INTO yyy VALUES(0,0,0,0); +} {1 {CHECK constraint failed: e > 0}} + +do_execsql_test 3.2.1 { + ALTER TABLE yyy DROP e; +} +do_catchsql_test 3.2.2 { + ALTER TABLE zzz DROP e; +} {1 {error in table zzz after drop column: no such column: e}} + +finish_test Index: test/alterlegacy.test ================================================================== --- test/alterlegacy.test +++ test/alterlegacy.test @@ -38,11 +38,11 @@ # Legacy behavior is to corrupt the schema in this case, as the table name in # the CHECK constraint is incorrect after "t1" is renamed. This version is # slightly different - it rejects the change and rolls back the transaction. do_catchsql_test 1.2 { ALTER TABLE t1 RENAME TO t1new; -} {1 {no such column: t1.a}} +} {1 {error in table t1new after rename: no such column: t1.a}} do_execsql_test 1.3 { CREATE TABLE t3(c, d); ALTER TABLE t3 RENAME TO t3new; DROP TABLE t3new; @@ -57,11 +57,11 @@ } do_catchsql_test 1.3 { ALTER TABLE t2 RENAME TO t2new; -} {1 {no such column: t2.b}} +} {1 {error in index t2expr after rename: no such column: t2.b}} do_execsql_test 1.4 { SELECT sql FROM sqlite_master } { {CREATE TABLE t1(a, b, CHECK(t1.a != t1.b))} {CREATE TABLE t2(a, b)} ADDED test/altermalloc3.test Index: test/altermalloc3.test ================================================================== --- /dev/null +++ test/altermalloc3.test @@ -0,0 +1,40 @@ +# 2021 February 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix altermalloc3 + +# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. +ifcapable !altertable { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, d, PRIMARY KEY(d, b)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3, 4); +} +faultsim_save_and_close + +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen +} -body { + execsql { ALTER TABLE t1 DROP COLUMN c } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test + Index: test/fts4rename.test ================================================================== --- test/fts4rename.test +++ test/fts4rename.test @@ -27,11 +27,11 @@ CREATE TABLE t2(x); } {} do_catchsql_test 1.1 { ALTER TABLE t1_content RENAME c0a TO docid; -} {1 {duplicate column name: docid}} +} {1 {error in table t1_content after rename: duplicate column name: docid}} do_catchsql_test 1.2 { UPDATE t1 SET Col0 = 1 ; } {1 {no such column: Col0}} Index: test/hook.test ================================================================== --- test/hook.test +++ test/hook.test @@ -954,7 +954,65 @@ preupdate {INSERT main sqlite_stat1 1 1} preupdate {INSERT main sqlite_stat1 2 2} }] } +#------------------------------------------------------------------------- +# Test that the pre-update hook is fired for INSERT statements that use +# the xfer optimization on without rowid tables. +# +reset_db +do_execsql_test 12.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b) WITHOUT ROWID; + + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES(5, 6); + INSERT INTO t2 VALUES(7, 8); + + CREATE TABLE t3 (a INTEGER PRIMARY KEY, b) WITHOUT ROWID; +} + +db preupdate hook preupdate_cb +db update_hook update_cb + +proc preupdate_cb {args} { lappend ::res "preupdate" $args } +proc update_cb {args} { lappend ::res "update" $args } + +set ::res [list] +do_test 12.2 { + execsql VACUUM + set ::res +} {} + +do_test 12.3 { + set ::res [list] + execsql { INSERT INTO t3 SELECT a, b FROM t2 } + set ::res +} {preupdate {INSERT main t3 0 0} preupdate {INSERT main t3 0 0}} + +do_test 12.4 { + execsql { DELETE FROM t3 } + set ::res [list] + execsql { INSERT INTO t3 SELECT * FROM t2 } + set ::res +} {preupdate {INSERT main t3 0 0} preupdate {INSERT main t3 0 0}} + +do_execsql_test 12.5 { + CREATE TABLE t4(a COLLATE nocase PRIMARY KEY, b) WITHOUT ROWID; + INSERT INTO t4 VALUES('abc', 1); + INSERT INTO t4 VALUES('DEF', 2); +} + +set ::res [list] +do_test 12.6 { + execsql VACUUM + set ::res +} {} + +do_catchsql_test 12.6 { + INSERT INTO t4 VALUES('def', 3); +} {1 {UNIQUE constraint failed: t4.a}} finish_test +