Index: src/alter.c ================================================================== --- src/alter.c +++ src/alter.c @@ -296,18 +296,10 @@ if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ return; } #endif - /* If the default value for the new column was specified with a - ** literal NULL, then set pDflt to 0. This simplifies checking - ** for an SQL NULL default below. - */ - assert( pDflt==0 || pDflt->op==TK_SPAN ); - if( pDflt && pDflt->pLeft->op==TK_NULL ){ - pDflt = 0; - } /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. ** If there is a NOT NULL constraint, then the default value for the ** column must not be NULL. */ @@ -317,39 +309,53 @@ } if( pNew->pIndex ){ sqlite3ErrorMsg(pParse, "Cannot add a UNIQUE column"); return; } - if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a REFERENCES column with non-NULL default value"); - return; - } - if( pCol->notNull && !pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a NOT NULL column with default value NULL"); - return; - } - - /* Ensure the default expression is something that sqlite3ValueFromExpr() - ** can handle (i.e. not CURRENT_TIME etc.) - */ - if( pDflt ){ - sqlite3_value *pVal = 0; - int rc; - rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - if( rc!=SQLITE_OK ){ - assert( db->mallocFailed == 1 ); - return; - } - if( !pVal ){ - sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default"); - return; - } - sqlite3ValueFree(pVal); - } + if( (pCol->colFlags & COLFLAG_GENERATED)==0 ){ + /* If the default value for the new column was specified with a + ** literal NULL, then set pDflt to 0. This simplifies checking + ** for an SQL NULL default below. + */ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ + pDflt = 0; + } + if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ + sqlite3ErrorMsg(pParse, + "Cannot add a REFERENCES column with non-NULL default value"); + return; + } + if( pCol->notNull && !pDflt ){ + sqlite3ErrorMsg(pParse, + "Cannot add a NOT NULL column with default value NULL"); + return; + } + + /* Ensure the default expression is something that sqlite3ValueFromExpr() + ** can handle (i.e. not CURRENT_TIME etc.) + */ + if( pDflt ){ + sqlite3_value *pVal = 0; + int rc; + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc!=SQLITE_OK ){ + assert( db->mallocFailed == 1 ); + return; + } + if( !pVal ){ + sqlite3ErrorMsg(pParse,"Cannot add a column with non-constant default"); + return; + } + sqlite3ValueFree(pVal); + } + }else if( pCol->colFlags & COLFLAG_STORED ){ + sqlite3ErrorMsg(pParse, "cannot add a STORED column"); + return; + } + /* Modify the CREATE TABLE statement. */ zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); if( zCol ){ char *zEnd = &zCol[pColDef->n-1]; @@ -1329,10 +1335,15 @@ } for(pIdx=sParse.pNewIndex; pIdx; pIdx=pIdx->pNext){ sqlite3WalkExprList(&sWalker, pIdx->aColExpr); } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + for(i=0; inCol; i++){ + sqlite3WalkExpr(&sWalker, sParse.pNewTable->aCol[i].pDflt); + } +#endif for(pFKey=sParse.pNewTable->pFKey; pFKey; pFKey=pFKey->pNextFrom){ for(i=0; inCol; i++){ if( bFKOnly==0 && pFKey->aCol[i].iFrom==iCol ){ renameTokenFind(&sParse, &sCtx, (void*)&pFKey->aCol[i]); Index: src/analyze.c ================================================================== --- src/analyze.c +++ src/analyze.c @@ -1180,11 +1180,11 @@ }else{ Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); int j, k, regKey; regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); assert( k>=0 && knColumn ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -875,20 +875,98 @@ for(p=pTab->pIndex; p && !IsPrimaryKeyIndex(p); p=p->pNext){} return p; } /* -** Return the column of index pIdx that corresponds to table -** column iCol. Return -1 if not found. +** Convert an table column number into a index column number. That is, +** for the column iCol in the table (as defined by the CREATE TABLE statement) +** find the (first) offset of that column in index pIdx. Or return -1 +** if column iCol is not used in index pIdx. */ -i16 sqlite3ColumnOfIndex(Index *pIdx, i16 iCol){ +i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ int i; for(i=0; inColumn; i++){ if( iCol==pIdx->aiColumn[i] ) return i; } return -1; } + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a storage column number into a table column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. The true column number +** is the index (0,1,2,...) of the column in the CREATE TABLE statement. +** +** The storage column number is less than the table column number if +** and only there are VIRTUAL columns to the left. +** +** If SQLITE_OMIT_GENERATED_COLUMNS, this routine is a no-op macro. +*/ +i16 sqlite3StorageColumnToTable(Table *pTab, i16 iCol){ + if( pTab->tabFlags & TF_HasVirtual ){ + int i; + for(i=0; i<=iCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) iCol++; + } + } + return iCol; +} +#endif + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a table column number into a storage column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. Or, if the input column is +** the N-th virtual column (zero-based) then the storage number is +** the number of non-virtual columns in the table plus N. +** +** The true column number is the index (0,1,2,...) of the column in +** the CREATE TABLE statement. +** +** If the input column is a VIRTUAL column, then it should not appear +** in storage. But the value sometimes is cached in registers that +** follow the range of registers used to construct storage. This +** avoids computing the same VIRTUAL column multiple times, and provides +** values for use by OP_Param opcodes in triggers. Hence, if the +** input column is a VIRTUAL table, put it after all the other columns. +** +** In the following, N means "normal column", S means STORED, and +** V means VIRTUAL. Suppose the CREATE TABLE has columns like this: +** +** CREATE TABLE ex(N,S,V,N,S,V,N,S,V); +** -- 0 1 2 3 4 5 6 7 8 +** +** Then the mapping from this function is as follows: +** +** INPUTS: 0 1 2 3 4 5 6 7 8 +** OUTPUTS: 0 1 6 2 3 7 4 5 8 +** +** So, in other words, this routine shifts all the virtual columns to +** the end. +** +** If SQLITE_OMIT_GENERATED_COLUMNS then there are no virtual columns and +** this routine is a no-op macro. +*/ +i16 sqlite3TableColumnToStorage(Table *pTab, i16 iCol){ + int i; + i16 n; + assert( iColnCol ); + if( (pTab->tabFlags & TF_HasVirtual)==0 ) return iCol; + for(i=0, n=0; iaCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) n++; + } + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ){ + /* iCol is a virtual column itself */ + return pTab->nNVCol + i - n; + }else{ + /* iCol is a normal or stored column */ + return n; + } +} +#endif /* ** Begin constructing a new table representation in memory. This is ** the first of several action routines that get called in response ** to a CREATE TABLE statement. In particular, this routine is called @@ -1176,10 +1254,11 @@ sqlite3Dequote(zType); pCol->affinity = sqlite3AffinityType(zType, pCol); pCol->colFlags |= COLFLAG_HASTYPE; } p->nCol++; + p->nNVCol++; pParse->constraintName.n = 0; } /* ** This routine is called by the parser while in the middle of @@ -1324,10 +1403,16 @@ if( p!=0 ){ pCol = &(p->aCol[p->nCol-1]); if( !sqlite3ExprIsConstantOrFunction(pExpr, db->init.busy) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", pCol->zName); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, "cannot use DEFAULT on a generated column"); +#endif }else{ /* A copy of pExpr is used instead of the original, as pExpr contains ** tokens that point to volatile memory. */ Expr x; @@ -1368,10 +1453,25 @@ p->op = TK_ID; }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ p->pLeft->op = TK_ID; } } + +/* +** Tag the given column as being part of the PRIMARY KEY +*/ +static void makeColumnPartOfPrimaryKey(Parse *pParse, Column *pCol){ + pCol->colFlags |= COLFLAG_PRIMKEY; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "generated columns cannot be part of the PRIMARY KEY"); + } +#endif +} /* ** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the ** most recently added column of the table is the primary key. @@ -1408,11 +1508,11 @@ } pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; + makeColumnPartOfPrimaryKey(pParse, pCol); nTerm = 1; }else{ nTerm = pList->nExpr; for(i=0; ia[i].pExpr); @@ -1421,11 +1521,11 @@ if( pCExpr->op==TK_ID ){ const char *zCName = pCExpr->u.zToken; for(iCol=0; iColnCol; iCol++){ if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; + makeColumnPartOfPrimaryKey(pParse, pCol); break; } } } } @@ -1517,10 +1617,56 @@ } }else{ sqlite3DbFree(db, zColl); } } + +/* Change the most recently parsed column to be a GENERATED ALWAYS AS +** column. +*/ +void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + u8 eType = COLFLAG_VIRTUAL; + Table *pTab = pParse->pNewTable; + Column *pCol; + if( NEVER(pTab==0) ) goto generated_done; + pCol = &(pTab->aCol[pTab->nCol-1]); + if( IN_DECLARE_VTAB ){ + sqlite3ErrorMsg(pParse, "virtual tables cannot use computed columns"); + goto generated_done; + } + if( pCol->pDflt ) goto generated_error; + if( pType ){ + if( pType->n==7 && sqlite3StrNICmp("virtual",pType->z,7)==0 ){ + /* no-op */ + }else if( pType->n==6 && sqlite3StrNICmp("stored",pType->z,6)==0 ){ + eType = COLFLAG_STORED; + }else{ + goto generated_error; + } + } + if( eType==COLFLAG_VIRTUAL ) pTab->nNVCol--; + pCol->colFlags |= eType; + assert( TF_HasVirtual==COLFLAG_VIRTUAL ); + assert( TF_HasStored==COLFLAG_STORED ); + pTab->tabFlags |= eType; + pCol->pDflt = pExpr; + pExpr = 0; + goto generated_done; + +generated_error: + sqlite3ErrorMsg(pParse, "error in generated column \"%s\"", + pCol->zName); +generated_done: + sqlite3ExprDelete(pParse->db, pExpr); +#else + /* Throw and error for the GENERATED ALWAYS AS clause if the + ** SQLITE_OMIT_GENERATED_COLUMNS compile-time option is used. */ + sqlite3ErrorMsg(pParse, "generated columns not supported"); + sqlite3ExprDelete(pParse->db, pExpr); +#endif +} /* ** Generate code that will increment the schema cookie. ** ** The schema cookie is used to determine when the schema for the @@ -1774,20 +1920,29 @@ ** ** colNotIdxed is a bitmask that has a 0 bit representing each indexed ** columns that are within the first 63 columns of the table. The ** high-order bit of colNotIdxed is always 1. All unindexed columns ** of the table have a 1. +** +** 2019-10-24: For the purpose of this computation, virtual columns are +** not considered to be covered by the index, even if they are in the +** index, because we do not trust the logic in whereIndexExprTrans() to be +** able to find all instances of a reference to the indexed table column +** and convert them into references to the index. Hence we always want +** the actual table at hand in order to recompute the virtual column, if +** necessary. ** ** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask ** to determine if the index is covering index. */ static void recomputeColumnsNotIndexed(Index *pIdx){ Bitmask m = 0; int j; + Table *pTab = pIdx->pTable; for(j=pIdx->nColumn-1; j>=0; j--){ int x = pIdx->aiColumn[j]; - if( x>=0 ){ + if( x>=0 && (pTab->aCol[x].colFlags & COLFLAG_VIRTUAL)==0 ){ testcase( x==BMS-1 ); testcase( x==BMS-2 ); if( xnCol; i++){ - if( !hasColumn(pPk->aiColumn, nPk, i) ) nExtra++; + if( !hasColumn(pPk->aiColumn, nPk, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; } if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; for(i=0, j=nPk; inCol; i++){ - if( !hasColumn(pPk->aiColumn, j, i) ){ + if( !hasColumn(pPk->aiColumn, j, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 + ){ assert( jnColumn ); pPk->aiColumn[j] = i; pPk->azColl[j] = sqlite3StrBINARY; j++; } } assert( pPk->nColumn==j ); - assert( pTab->nCol<=j ); + assert( pTab->nNVCol<=j ); recomputeColumnsNotIndexed(pPk); } #ifndef SQLITE_OMIT_VIRTUALTABLE /* @@ -2062,25 +2220,40 @@ "AUTOINCREMENT not allowed on WITHOUT ROWID tables"); return; } if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); - }else{ - p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; - convertToWithoutRowidTable(pParse, p); + return; } + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; + convertToWithoutRowidTable(pParse, p); } - iDb = sqlite3SchemaToIndex(db, p->pSchema); #ifndef SQLITE_OMIT_CHECK /* Resolve names in all CHECK constraint expressions. */ if( p->pCheck ){ sqlite3ResolveSelfReference(pParse, p, NC_IsCheck, 0, p->pCheck); } #endif /* !defined(SQLITE_OMIT_CHECK) */ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( p->tabFlags & TF_HasGenerated ){ + int ii; + testcase( p->tabFlags & TF_HasVirtual ); + testcase( p->tabFlags & TF_HasStored ); + for(ii=0; iinCol; ii++){ + u32 colFlags = p->aCol[ii].colFlags; + if( (colFlags & COLFLAG_GENERATED)!=0 ){ + testcase( colFlags & COLFLAG_VIRTUAL ); + testcase( colFlags & COLFLAG_STORED ); + sqlite3ResolveSelfReference(pParse, p, NC_GenCol, + p->aCol[ii].pDflt, 0); + } + } + } +#endif /* Estimate the average row size for the table and for all implied indices */ estimateTableWidth(p); for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ estimateIndexWidth(pIdx); @@ -2153,11 +2326,11 @@ sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect, SQLITE_AFF_BLOB); if( pSelTab==0 ) return; assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; + p->nCol = p->nNVCol = pSelTab->nCol; p->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; sqlite3DeleteTable(db, pSelTab); sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); @@ -2441,11 +2614,11 @@ }else if( pSelTab ){ /* CREATE VIEW name AS... without an argument list. Construct ** the column names from the SELECT statement that defines the view. */ assert( pTable->aCol==0 ); - pTable->nCol = pSelTab->nCol; + pTable->nCol = pTable->nNVCol = pSelTab->nCol; pTable->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); }else{ @@ -3451,12 +3624,17 @@ }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); if( j<0 ){ j = pTab->iPKey; - }else if( pTab->aCol[j].notNull==0 ){ - pIndex->uniqNotNull = 0; + }else{ + if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ + pIndex->bHasVCol = 1; + } } pIndex->aiColumn[i] = (i16)j; } zColl = 0; if( pListItem->pExpr->op==TK_COLLATE ){ @@ -3507,17 +3685,17 @@ if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); /* If this index contains every column of its table, then mark ** it as a covering index */ assert( HasRowid(pTab) - || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + || pTab->iPKey<0 || sqlite3TableColumnToIndex(pIndex, pTab->iPKey)>=0 ); recomputeColumnsNotIndexed(pIndex); if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ pIndex->isCovering = 1; for(j=0; jnCol; j++){ if( j==pTab->iPKey ) continue; - if( sqlite3ColumnOfIndex(pIndex,j)>=0 ) continue; + if( sqlite3TableColumnToIndex(pIndex,j)>=0 ) continue; pIndex->isCovering = 0; break; } } Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -3397,35 +3397,77 @@ sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, iTabCol, regOut); } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** Generate code that will compute the value of generated column pCol +** and store the result in register regOut +*/ +void sqlite3ExprCodeGeneratedColumn( + Parse *pParse, + Column *pCol, + int regOut +){ + sqlite3ExprCode(pParse, pCol->pDflt, regOut); + if( pCol->affinity>=SQLITE_AFF_TEXT ){ + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Affinity, regOut, 1, 0, + &pCol->affinity, 1); + } +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + /* ** Generate code to extract the value of the iCol-th column of a table. */ void sqlite3ExprCodeGetColumnOfTable( - Vdbe *v, /* The VDBE under construction */ + Vdbe *v, /* Parsing context */ Table *pTab, /* The table containing the value */ int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ int iCol, /* Index of the column to extract */ int regOut /* Extract the value into this register */ ){ + Column *pCol; + assert( v!=0 ); if( pTab==0 ){ sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); return; } if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); }else{ - int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; - int x = iCol; - if( !HasRowid(pTab) && !IsVirtual(pTab) ){ - x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + int op; + int x; + if( IsVirtual(pTab) ){ + op = OP_VColumn; + x = iCol; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( (pCol = &pTab->aCol[iCol])->colFlags & COLFLAG_VIRTUAL ){ + Parse *pParse = sqlite3VdbeParser(v); + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", pCol->zName); + }else{ + int savedSelfTab = pParse->iSelfTab; + pCol->colFlags |= COLFLAG_BUSY; + pParse->iSelfTab = iTabCur+1; + sqlite3ExprCodeGeneratedColumn(pParse, pCol, regOut); + pParse->iSelfTab = savedSelfTab; + pCol->colFlags &= ~COLFLAG_BUSY; + } + return; +#endif + }else if( !HasRowid(pTab) ){ + testcase( iCol!=sqlite3TableColumnToStorage(pTab, iCol) ); + x = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + op = OP_Column; + }else{ + x = sqlite3TableColumnToStorage(pTab,iCol); + testcase( x!=iCol ); + op = OP_Column; } sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); - } - if( iCol>=0 ){ sqlite3ColumnDefault(v, pTab, iCol, regOut); } } /* @@ -3441,15 +3483,14 @@ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ int iReg, /* Store results here */ u8 p5 /* P5 value for OP_Column + FLAGS */ ){ - Vdbe *v = pParse->pVdbe; - assert( v!=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); + assert( pParse->pVdbe!=0 ); + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ - sqlite3VdbeChangeP5(v, p5); + sqlite3VdbeChangeP5(pParse->pVdbe, p5); } return iReg; } /* @@ -3581,23 +3622,50 @@ } return iReg; } if( iTab<0 ){ if( pParse->iSelfTab<0 ){ - /* Generating CHECK constraints or inserting into partial index */ - assert( pExpr->y.pTab!=0 ); - assert( pExpr->iColumn>=XN_ROWID ); - assert( pExpr->iColumny.pTab->nCol ); - if( pExpr->iColumn>=0 - && pExpr->y.pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL - ){ - sqlite3VdbeAddOp2(v, OP_SCopy, pExpr->iColumn - pParse->iSelfTab, - target); + /* Other columns in the same row for CHECK constraints or + ** generated columns or for inserting into partial index. + ** The row is unpacked into registers beginning at + ** 0-(pParse->iSelfTab). The rowid (if any) is in a register + ** immediately prior to the first column. + */ + Column *pCol; + Table *pTab = pExpr->y.pTab; + int iSrc; + int iCol = pExpr->iColumn; + assert( pTab!=0 ); + assert( iCol>=XN_ROWID ); + assert( iColy.pTab->nCol ); + if( iCol<0 ){ + return -1-pParse->iSelfTab; + } + pCol = pTab->aCol + iCol; + testcase( iCol!=sqlite3TableColumnToStorage(pTab,iCol) ); + iSrc = sqlite3TableColumnToStorage(pTab, iCol) - pParse->iSelfTab; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", + pCol->zName); + return 0; + } + pCol->colFlags |= COLFLAG_BUSY; + if( pCol->colFlags & COLFLAG_NOTAVAIL ){ + sqlite3ExprCodeGeneratedColumn(pParse, pCol, iSrc); + } + pCol->colFlags &= ~(COLFLAG_BUSY|COLFLAG_NOTAVAIL); + return iSrc; + }else +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + if( pCol->affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp2(v, OP_SCopy, iSrc, target); sqlite3VdbeAddOp1(v, OP_RealAffinity, target); return target; }else{ - return pExpr->iColumn - pParse->iSelfTab; + return iSrc; } }else{ /* Coding an expression that is part of an index where column names ** in the index refer to the table to which the index belongs */ iTab = pParse->iSelfTab - 1; @@ -4061,32 +4129,32 @@ ** p1==0 -> old.rowid p1==3 -> new.rowid ** p1==1 -> old.a p1==4 -> new.a ** p1==2 -> old.b p1==5 -> new.b */ Table *pTab = pExpr->y.pTab; - int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + pExpr->iColumn; + int iCol = pExpr->iColumn; + int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + + (iCol>=0 ? sqlite3TableColumnToStorage(pTab, iCol) : -1); assert( pExpr->iTable==0 || pExpr->iTable==1 ); - assert( pExpr->iColumn>=-1 && pExpr->iColumnnCol ); - assert( pTab->iPKey<0 || pExpr->iColumn!=pTab->iPKey ); + assert( iCol>=-1 && iColnCol ); + assert( pTab->iPKey<0 || iCol!=pTab->iPKey ); assert( p1>=0 && p1<(pTab->nCol*2+2) ); sqlite3VdbeAddOp2(v, OP_Param, p1, target); VdbeComment((v, "r[%d]=%s.%s", target, (pExpr->iTable ? "new" : "old"), - (pExpr->iColumn<0 ? "rowid" : pExpr->y.pTab->aCol[pExpr->iColumn].zName) + (pExpr->iColumn<0 ? "rowid" : pExpr->y.pTab->aCol[iCol].zName) )); #ifndef SQLITE_OMIT_FLOATING_POINT /* If the column has REAL affinity, it may currently be stored as an ** integer. Use OP_RealAffinity to make sure it is really real. ** ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to ** floating point when extracting it from the record. */ - if( pExpr->iColumn>=0 - && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL - ){ + if( iCol>=0 && pTab->aCol[iCol].affinity==SQLITE_AFF_REAL ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, target); } #endif break; } @@ -4315,18 +4383,14 @@ */ void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ int inReg; assert( target>0 && target<=pParse->nMem ); - if( pExpr && pExpr->op==TK_REGISTER ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); - }else{ - inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); - if( inReg!=target && pParse->pVdbe ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); - } + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); } } /* ** Make a transient copy of expression pExpr and then code it using @@ -4352,34 +4416,10 @@ }else{ sqlite3ExprCode(pParse, pExpr, target); } } -/* -** Generate code that evaluates the given expression and puts the result -** in register target. -** -** Also make a copy of the expression results into another "cache" register -** and modify the expression so that the next time it is evaluated, -** the result is a copy of the cache register. -** -** This routine is used for expressions that are used multiple -** times. They are evaluated once and the results of the expression -** are reused. -*/ -void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ - Vdbe *v = pParse->pVdbe; - int iMem; - - assert( target>0 ); - assert( pExpr->op!=TK_REGISTER ); - sqlite3ExprCode(pParse, pExpr, target); - iMem = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Copy, target, iMem); - exprToRegister(pExpr, iMem); -} - /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. ** ** Return the number of elements evaluated. The number returned will @@ -5298,11 +5338,11 @@ ** pWalker->u.pIdxCover->pIdx. */ static int exprIdxCover(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_COLUMN && pExpr->iTable==pWalker->u.pIdxCover->iCur - && sqlite3ColumnOfIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + && sqlite3TableColumnToIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 ){ pWalker->eCode = 1; return WRC_Abort; } return WRC_Continue; Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -347,11 +347,11 @@ if( nIncr<0 ){ sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); VdbeCoverage(v); } for(i=0; inCol; i++){ - int iReg = aiCol[i] + regData + 1; + int iReg = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + regData + 1; sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); } if( isIgnore==0 ){ if( pIdx==0 ){ @@ -363,11 +363,12 @@ /* Invoke MustBeInt to coerce the child key value to an integer (i.e. ** apply the affinity of the parent key). If this fails, then there ** is no matching parent key. Before using MustBeInt, make a copy of ** the value. Otherwise, the value inserted into the child key column ** will have INTEGER affinity applied to it, which may not be correct. */ - sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); + sqlite3VdbeAddOp2(v, OP_SCopy, + sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[0])+1+regData, regTemp); iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); VdbeCoverage(v); /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), @@ -390,11 +391,13 @@ int regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); for(i=0; ipFrom, aiCol[i])+1+regData, + regTemp+i); } /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not @@ -406,12 +409,15 @@ ** none of the child key values are). */ if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + int iChild = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + +1+regData; + int iParent = 1+regData; + iParent += sqlite3TableColumnToStorage(pIdx->pTable, + pIdx->aiColumn[i]); assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ /* The parent key is a composite key that includes the IPK column */ iParent = regData; Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -35,11 +35,11 @@ v = sqlite3GetVdbe(pParse); assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); sqlite3TableLock(pParse, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); if( HasRowid(pTab) ){ - sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nCol); + sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); VdbeComment((v, "%s", pTab->zName)); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); assert( pPk->tnum==pTab->tnum ); @@ -127,27 +127,29 @@ ** 'C' NUMERIC ** 'D' INTEGER ** 'E' REAL */ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ - int i; + int i, j; char *zColAff = pTab->zColAff; if( zColAff==0 ){ sqlite3 *db = sqlite3VdbeDb(v); zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ sqlite3OomFault(db); return; } - for(i=0; inCol; i++){ + for(i=j=0; inCol; i++){ assert( pTab->aCol[i].affinity!=0 ); - zColAff[i] = pTab->aCol[i].affinity; + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; + } } do{ - zColAff[i--] = 0; - }while( i>=0 && zColAff[i]<=SQLITE_AFF_BLOB ); + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } assert( zColAff!=0 ); i = sqlite3Strlen30NN(zColAff); if( i ){ @@ -196,10 +198,65 @@ } #endif } return 0; } + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** All regular columns for table pTab have been puts into registers +** starting with iRegStore. The registers that correspond to STORED +** or VIRTUAL columns have not yet been initialized. This routine goes +** back and computes the values for those columns based on the previously +** computed normal columns. +*/ +void sqlite3ComputeGeneratedColumns( + Parse *pParse, /* Parsing context */ + int iRegStore, /* Register holding the first column */ + Table *pTab /* The table */ +){ + int i; + int nv; + /* Because there can be multiple generated columns that refer to one another, + ** this is a two-pass algorithm. On the first pass, mark all generated + ** columns as "not available". + */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + pTab->aCol[i].colFlags |= COLFLAG_NOTAVAIL; + } + } + /* On the second pass, compute the value of each NOT-AVAILABLE column. + ** Companion code in the TK_COLUMN case of sqlite3ExprCodeTarget() will + ** compute dependencies and mark remove the COLSPAN_NOTAVAIL mark, as + ** they are needed. + */ + pParse->iSelfTab = -iRegStore; + for(i=nv=0; inCol; i++){ + u32 colFlags = pTab->aCol[i].colFlags; + if( (colFlags & COLFLAG_NOTAVAIL)!=0 ){ + assert( colFlags & COLFLAG_GENERATED ); + if( colFlags & COLFLAG_VIRTUAL ){ + /* Virtual columns go at the end */ + assert( pTab->nNVCol+nv == sqlite3TableColumnToStorage(pTab,i) ); + sqlite3ExprCodeGeneratedColumn(pParse, &pTab->aCol[i], + iRegStore+pTab->nNVCol+nv); + }else{ + /* Stored columns go in column order */ + assert( i-nv == sqlite3TableColumnToStorage(pTab,i) ); + sqlite3ExprCodeGeneratedColumn(pParse, &pTab->aCol[i], iRegStore+i-nv); + } + pTab->aCol[i].colFlags &= ~COLFLAG_NOTAVAIL; + } + if( (colFlags & COLFLAG_VIRTUAL)!=0 ) nv++; + } + pParse->iSelfTab = 0; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + #ifndef SQLITE_OMIT_AUTOINCREMENT /* ** Locate or create an AutoincInfo structure associated with table pTab ** which is in database iDb. Return the register number for the register @@ -504,11 +561,11 @@ */ void sqlite3Insert( Parse *pParse, /* Parser context */ SrcList *pTabList, /* Name of table into which we are inserting */ Select *pSelect, /* A SELECT statement to use as the data source */ - IdList *pColumn, /* Column names corresponding to IDLIST. */ + IdList *pColumn, /* Column names corresponding to IDLIST, or NULL. */ int onError, /* How to handle constraint errors */ Upsert *pUpsert /* ON CONFLICT clauses for upsert, or NULL */ ){ sqlite3 *db; /* The main database structure */ Table *pTab; /* The table to insert into. aka TABLE */ @@ -529,10 +586,11 @@ u8 useTempTable = 0; /* Store SELECT results in intermediate table */ u8 appendFlag = 0; /* True if the insert is likely to be an append */ u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ u8 bIdListInOrder; /* True if IDLIST is in table order */ ExprList *pList = 0; /* List of VALUES() to be inserted */ + int iRegStore; /* Register in which to store next column */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ int regRowCount = 0; /* Memory cell used for the row counter */ @@ -636,12 +694,12 @@ /* If this is an AUTOINCREMENT table, look up the sequence number in the ** sqlite_sequence table and store it in memory cell regAutoinc. */ regAutoinc = autoIncBegin(pParse, iDb, pTab); - /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assembled row record. + /* Allocate a block registers to hold the rowid and the values + ** for all columns of the new row. */ regRowid = regIns = pParse->nMem+1; pParse->nMem += pTab->nCol + 1; if( IsVirtual(pTab) ){ regRowid++; @@ -656,13 +714,21 @@ ** If the table has an INTEGER PRIMARY KEY column and that column ** is named in the IDLIST, then record in the ipkColumn variable ** the index into IDLIST of the primary key column. ipkColumn is ** the index of the primary key as it appears in IDLIST, not as ** is appears in the original table. (The index of the INTEGER - ** PRIMARY KEY in the original table is pTab->iPKey.) + ** PRIMARY KEY in the original table is pTab->iPKey.) After this + ** loop, if ipkColumn==(-1), that means that integer primary key + ** is unspecified, and hence the table is either WITHOUT ROWID or + ** it will automatically generated an integer primary key. + ** + ** bIdListInOrder is true if the columns in IDLIST are in storage + ** order. This enables an optimization that avoids shuffling the + ** columns into storage order. False negatives are harmless, + ** but false positives will cause database corruption. */ - bIdListInOrder = (pTab->tabFlags & TF_OOOHidden)==0; + bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; if( pColumn ){ for(i=0; inId; i++){ pColumn->a[i].idx = -1; } for(i=0; inId; i++){ @@ -671,10 +737,18 @@ pColumn->a[i].idx = j; if( i!=j ) bIdListInOrder = 0; if( j==pTab->iPKey ){ ipkColumn = i; assert( !withoutRowid ); } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zName); + goto insert_cleanup; + } +#endif break; } } if( j>=pTab->nCol ){ if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ @@ -780,17 +854,30 @@ ** key, the set the ipkColumn variable to the integer primary key ** column index in the original table definition. */ if( pColumn==0 && nColumn>0 ){ ipkColumn = pTab->iPKey; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( ipkColumn>=0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + for(i=ipkColumn-1; i>=0; i--){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + ipkColumn--; + } + } + } +#endif } /* Make sure the number of columns in the source data matches the number ** of columns to be inserted into the table. */ for(i=0; inCol; i++){ - nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); + if( pTab->aCol[i].colFlags & COLFLAG_NOINSERT ) nHidden++; } if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ sqlite3ErrorMsg(pParse, "table %S has %d columns but %d values were supplied", pTabList, 0, pTab->nCol-nHidden, nColumn); @@ -871,11 +958,91 @@ ** goto C ** D: ... */ addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); + if( ipkColumn>=0 ){ + /* tag-20191021-001: If the INTEGER PRIMARY KEY is being generated by the + ** SELECT, go ahead and copy the value into the rowid slot now, so that + ** the value does not get overwritten by a NULL at tag-20191021-002. */ + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + } + } + + /* Compute data for ordinary columns of the new entry. Values + ** are written in storage order into registers starting with regData. + ** Only ordinary columns are computed in this loop. The rowid + ** (if there is one) is computed later and generated columns are + ** computed after the rowid since they might depend on the value + ** of the rowid. + */ + nHidden = 0; + iRegStore = regData; assert( regData==regRowid+1 ); + for(i=0; inCol; i++, iRegStore++){ + int k; + u32 colFlags; + assert( i>=nHidden ); + if( i==pTab->iPKey ){ + /* tag-20191021-002: References to the INTEGER PRIMARY KEY are filled + ** using the rowid. So put a NULL in the IPK slot of the record to avoid + ** using excess space. The file format definition requires this extra + ** NULL - we cannot optimize further by skipping the column completely */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + continue; + } + if( ((colFlags = pTab->aCol[i].colFlags) & COLFLAG_NOINSERT)!=0 ){ + nHidden++; + if( (colFlags & COLFLAG_VIRTUAL)!=0 ){ + /* Virtual columns do not participate in OP_MakeRecord. So back up + ** iRegStore by one slot to compensate for the iRegStore++ in the + ** outer for() loop */ + iRegStore--; + continue; + }else if( (colFlags & COLFLAG_STORED)!=0 ){ + /* Stored columns are computed later. But if there are BEFORE + ** triggers, the slots used for stored columns will be OP_Copy-ed + ** to a second block of registers, so the register needs to be + ** initialized to NULL to avoid an uninitialized register read */ + if( tmask & TRIGGER_BEFORE ){ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + } + continue; + }else if( pColumn==0 ){ + /* Hidden columns that are not explicitly named in the INSERT + ** get there default value */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + } + } + if( pColumn ){ + for(j=0; jnId && pColumn->a[j].idx!=i; j++){} + if( j>=pColumn->nId ){ + /* A column not named in the insert column list gets its + ** default value */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + } + k = j; + }else if( nColumn==0 ){ + /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + }else{ + k = i - nHidden; + } + + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, k, iRegStore); + }else if( pSelect ){ + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); + } + }else{ + sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + } } + /* Run the BEFORE and INSTEAD OF triggers, if there are any */ endOfLoop = sqlite3VdbeMakeLabel(pParse); if( tmask & TRIGGER_BEFORE ){ @@ -907,29 +1074,25 @@ /* Cannot have triggers on a virtual table. If it were possible, ** this block would have to account for hidden column. */ assert( !IsVirtual(pTab) ); - /* Create the new column data - */ - for(i=j=0; inCol; i++){ - if( pColumn ){ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) - || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); - }else{ - assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); - } - if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++; - } + /* Copy the new data already generated. */ + assert( pTab->nNVCol>0 ); + sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** refers to the ROWID. */ + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regCols+1, pTab); + } +#endif /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, ** do not attempt any conversions before assembling the record. ** If this is a real table, attempt conversions as required by the ** table column affinities. @@ -943,23 +1106,21 @@ pTab, regCols-pTab->nCol-1, onError, endOfLoop); sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); } - /* Compute the content of the next row to insert into a range of - ** registers beginning at regIns. - */ if( !isView ){ if( IsVirtual(pTab) ){ /* The row that the VUpdate opcode will delete: none */ sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); } if( ipkColumn>=0 ){ + /* Compute the new rowid */ if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + /* Rowid already initialized at tag-20191021-001 */ }else{ Expr *pIpk = pList->a[ipkColumn].pExpr; if( pIpk->op==TK_NULL && !IsVirtual(pTab) ){ sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); appendFlag = 1; @@ -988,49 +1149,21 @@ sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); appendFlag = 1; } autoIncStep(pParse, regAutoinc, regRowid); - /* Compute data for all columns of the new entry, beginning - ** with the first column. - */ - nHidden = 0; - for(i=0; inCol; i++){ - int iRegStore = regRowid+1+i; - if( i==pTab->iPKey ){ - /* The value of the INTEGER PRIMARY KEY column is always a NULL. - ** Whenever this column is read, the rowid will be substituted - ** in its place. Hence, fill this column with a NULL to avoid - ** taking up data space with information that will never be used. - ** As there may be shallow copies of this value, make it a soft-NULL */ - sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); - continue; - } - if( pColumn==0 ){ - if( IsHiddenColumn(&pTab->aCol[i]) ){ - j = -1; - nHidden++; - }else{ - j = i - nHidden; - } - }else{ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); - }else if( pSelect ){ - if( regFromSelect!=regData ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); - } - }else{ - sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); - } - } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** refers to the ROWID. */ + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regRowid+1, pTab); + } +#endif /* Generate code to check constraints and generate index keys and ** do the insertion. */ #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -1344,10 +1477,11 @@ iDataCur, iIdxCur, regNewData, regOldData, pkChng)); /* Test all NOT NULL constraints. */ for(i=0; iiPKey ){ continue; /* ROWID is never NULL */ } if( aiChng && aiChng[i]<0 ){ /* Don't bother checking for NOT NULL on columns that do not change */ @@ -1364,18 +1498,22 @@ onError = OE_Abort; } assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail || onError==OE_Ignore || onError==OE_Replace ); addr1 = 0; + testcase( i!=sqlite3TableColumnToStorage(pTab, i) ); + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + iReg = sqlite3TableColumnToStorage(pTab, i) + regNewData + 1; switch( onError ){ case OE_Replace: { assert( onError==OE_Replace ); addr1 = sqlite3VdbeMakeLabel(pParse); - sqlite3VdbeAddOp2(v, OP_NotNull, regNewData+1+i, addr1); + sqlite3VdbeAddOp2(v, OP_NotNull, iReg, addr1); VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeAddOp2(v, OP_NotNull, regNewData+1+i, addr1); + sqlite3VdbeAddOp2(v, OP_NotNull, iReg, addr1); VdbeCoverage(v); onError = OE_Abort; /* Fall through into the OE_Abort case to generate code that runs ** if both the input and the default value are NULL */ } @@ -1385,20 +1523,20 @@ case OE_Rollback: case OE_Fail: { char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, pTab->aCol[i].zName); sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, - regNewData+1+i); + iReg); sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); VdbeCoverage(v); if( addr1 ) sqlite3VdbeResolveLabel(v, addr1); break; } default: { assert( onError==OE_Ignore ); - sqlite3VdbeAddOp2(v, OP_IsNull, regNewData+1+i, ignoreDest); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, ignoreDest); VdbeCoverage(v); break; } } } @@ -1669,18 +1807,26 @@ if( iField==XN_EXPR ){ pParse->iSelfTab = -(regNewData+1); sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); pParse->iSelfTab = 0; VdbeComment((v, "%s column %d", pIdx->zName, i)); - }else{ - if( iField==XN_ROWID || iField==pTab->iPKey ){ - x = regNewData; - }else{ - x = iField + regNewData + 1; - } - sqlite3VdbeAddOp2(v, iField<0 ? OP_IntCopy : OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); + }else if( iField==XN_ROWID || iField==pTab->iPKey ){ + x = regNewData; + sqlite3VdbeAddOp2(v, OP_IntCopy, x, regIdx+i); + VdbeComment((v, "rowid")); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( pTab->aCol[iField].colFlags & COLFLAG_VIRTUAL ){ + pParse->iSelfTab = -(regNewData+1); + sqlite3ExprCodeCopy(pParse, pTab->aCol[iField].pDflt, regIdx+i); + pParse->iSelfTab = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); +#endif + }else{ + testcase( sqlite3TableColumnToStorage(pTab, iField)!=iField ); + x = sqlite3TableColumnToStorage(pTab, iField) + regNewData + 1; + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", pTab->aCol[iField].zName)); } } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); #ifdef SQLITE_ENABLE_NULL_TRIM @@ -1765,11 +1911,11 @@ /* Extract the PRIMARY KEY from the end of the index entry and ** store it in registers regR..regR+nPk-1 */ if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ assert( pPk->aiColumn[i]>=0 ); - x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); + x = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, pTab->aCol[pPk->aiColumn[i]].zName)); } } @@ -1868,11 +2014,11 @@ } /* Generate the table record */ if( HasRowid(pTab) ){ int regRec = aRegIdx[ix]; - sqlite3VdbeAddOp3(v, OP_MakeRecord, regNewData+1, pTab->nCol, regRec); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regNewData+1, pTab->nNVCol, regRec); sqlite3SetMakeRecordP5(v, pTab); if( !bAffinityDone ){ sqlite3TableAffinity(v, pTab, 0); } } @@ -2262,10 +2408,43 @@ if( (db->mDbFlags & DBFLAG_Vacuum)==0 && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN ){ return 0; /* Neither table may have __hidden__ columns */ } +#endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Even if tables t1 and t2 have identical schemas, if they contain + ** generated columns, then this statement is semantically incorrect: + ** + ** INSERT INTO t2 SELECT * FROM t1; + ** + ** The reason is that generated column values are returned by the + ** the SELECT statement on the right but the INSERT statement on the + ** left wants them to be omitted. + ** + ** Nevertheless, this is a useful notational shorthand to tell SQLite + ** to do a bulk transfer all of the content from t1 over to t2. + ** + ** We could, in theory, disable this (except for internal use by the + ** VACUUM command where it is actually needed). But why do that? It + ** seems harmless enough, and provides a useful service. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED) != + (pSrcCol->colFlags & COLFLAG_GENERATED) ){ + return 0; /* Both columns have the same generated-column type */ + } + /* But the transfer is only allowed if both the source and destination + ** tables have the exact same expressions for generated columns. + ** This requirement could be relaxed for VIRTUAL columns, I suppose. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED)!=0 ){ + if( sqlite3ExprCompare(0, pSrcCol->pDflt, pDestCol->pDflt, -1)!=0 ){ + testcase( pDestCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pDestCol->colFlags & COLFLAG_STORED ); + return 0; /* Different generator expressions */ + } + } #endif if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } if( sqlite3_stricmp(pDestCol->zColl, pSrcCol->zColl)!=0 ){ @@ -2273,11 +2452,11 @@ } if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } /* Default values for second and subsequent columns need to match. */ - if( i>0 ){ + if( (pDestCol->colFlags & COLFLAG_GENERATED)==0 && i>0 ){ assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) || (pDestCol->pDflt && strcmp(pDestCol->pDflt->u.zToken, pSrcCol->pDflt->u.zToken)!=0) Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -345,10 +345,14 @@ ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X);} ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R). {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} ccons ::= defer_subclause(D). {sqlite3DeferForeignKey(pParse,D);} ccons ::= COLLATE ids(C). {sqlite3AddCollateType(pParse, &C);} +ccons ::= GENERATED ALWAYS AS generated. +ccons ::= AS generated. +generated ::= LP expr(E) RP. {sqlite3AddGenerated(pParse,E,0);} +generated ::= LP expr(E) RP ID(TYPE). {sqlite3AddGenerated(pParse,E,&TYPE);} // The optional AUTOINCREMENT keyword %type autoinc {int} autoinc(X) ::= . {X = 0;} autoinc(X) ::= AUTOINCR. {X = 1;} Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1098,29 +1098,38 @@ Index *pPk = sqlite3PrimaryKeyIndex(pTab); pParse->nMem = 7; sqlite3CodeVerifySchema(pParse, iTabDb); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ - int isHidden = IsHiddenColumn(pCol); - if( isHidden && pPragma->iArg==0 ){ - nHidden++; - continue; + int isHidden = 0; + if( pCol->colFlags & COLFLAG_NOINSERT ){ + if( pPragma->iArg==0 ){ + nHidden++; + continue; + } + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + isHidden = 2; /* GENERATED ALWAYS AS ... VIRTUAL */ + }else if( pCol->colFlags & COLFLAG_STORED ){ + isHidden = 3; /* GENERATED ALWAYS AS ... STORED */ + }else{ assert( pCol->colFlags & COLFLAG_HIDDEN ); + isHidden = 1; /* HIDDEN */ + } } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; }else if( pPk==0 ){ k = 1; }else{ for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } - assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); + assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN || isHidden>=2 ); sqlite3VdbeMultiLoad(v, 1, pPragma->iArg ? "issisii" : "issisi", i-nHidden, pCol->zName, sqlite3ColumnType(pCol,""), pCol->notNull ? 1 : 0, - pCol->pDflt ? pCol->pDflt->u.zToken : 0, + pCol->pDflt && isHidden<2 ? pCol->pDflt->u.zToken : 0, k, isHidden); } } } @@ -1575,11 +1584,11 @@ assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); if( !isQuick ){ /* Sanity check on record header decoding */ - sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nCol-1, 3); + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); } /* Verify that all NOT NULL columns really are NOT NULL */ for(j=0; jnCol; j++){ char *zErr; Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -412,11 +412,11 @@ ** Perhaps the name is a reference to the ROWID */ if( cnt==0 && cntTab==1 && pMatch - && (pNC->ncFlags & NC_IdxExpr)==0 + && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) && VisibleRowid(pMatch->pTab) ){ cnt = 1; pExpr->iColumn = -1; @@ -623,17 +623,20 @@ Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ const char *zMsg, /* Type of error */ int validMask /* Set of contexts for which prohibited */ ){ - assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); + assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol))==0 ); if( (pNC->ncFlags & validMask)!=0 ){ const char *zIn = "partial index WHERE clauses"; if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; #endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pNC->ncFlags & NC_GenCol ) zIn = "generated columns"; +#endif sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); } } /* @@ -721,11 +724,11 @@ zDb = 0; zTable = 0; zColumn = pExpr->u.zToken; }else{ Expr *pLeft = pExpr->pLeft; - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr|NC_GenCol); pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; }else{ assert( pRight->op==TK_DOT ); @@ -818,11 +821,11 @@ if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ /* Date/time functions that use 'now', and other functions like ** sqlite_version() that might change over time cannot be used ** in an index. */ notValid(pParse, pNC, "non-deterministic functions", - NC_IdxExpr|NC_PartIdx); + NC_IdxExpr|NC_PartIdx|NC_GenCol); } if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 && pParse->nested==0 && sqlite3Config.bInternalFunctions==0 ){ @@ -962,11 +965,12 @@ #endif case TK_IN: { testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); + notValid(pParse, pNC, "subqueries", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); pNC->ncFlags |= NC_VarSelect; @@ -973,11 +977,12 @@ } } break; } case TK_VARIABLE: { - notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); + notValid(pParse, pNC, "parameters", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol); break; } case TK_IS: case TK_ISNOT: { Expr *pRight = sqlite3ExprSkipCollateAndLikely(pExpr->pRight); @@ -1786,30 +1791,32 @@ ** ** (1) CHECK constraints ** (2) WHERE clauses on partial indices ** (3) Expressions in indexes on expressions ** (4) Expression arguments to VACUUM INTO. +** (5) GENERATED ALWAYS as expressions ** ** In all cases except (4), the Expr.iTable value for Expr.op==TK_COLUMN ** nodes of the expression is set to -1 and the Expr.iColumn value is ** set to the column number. In case (4), TK_COLUMN nodes cause an error. ** ** Any errors cause an error message to be set in pParse. */ int sqlite3ResolveSelfReference( - Parse *pParse, /* Parsing context */ - Table *pTab, /* The table being referenced, or NULL */ - int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr, or 0 */ - Expr *pExpr, /* Expression to resolve. May be NULL. */ - ExprList *pList /* Expression list to resolve. May be NULL. */ + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table being referenced, or NULL */ + int type, /* NC_IsCheck, NC_PartIdx, NC_IdxExpr, NC_GenCol, or 0 */ + Expr *pExpr, /* Expression to resolve. May be NULL. */ + ExprList *pList /* Expression list to resolve. May be NULL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ int rc; assert( type==0 || pTab!=0 ); - assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr || pTab==0 ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr + || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); if( pTab ){ sSrc.nSrc = 1; sSrc.a[0].zName = pTab->zName; Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1816,30 +1816,49 @@ void (*xDestroy)(void *); /* Module destructor function */ Table *pEpoTab; /* Eponymous table for this module */ }; /* -** information about each column of an SQL table is held in an instance -** of this structure. +** Information about each column of an SQL table is held in an instance +** of the Column structure, in the Table.aCol[] array. +** +** Definitions: +** +** "table column index" This is the index of the column in the +** Table.aCol[] array, and also the index of +** the column in the original CREATE TABLE stmt. +** +** "storage column index" This is the index of the column in the +** record BLOB generated by the OP_MakeRecord +** opcode. The storage column index is less than +** or equal to the table column index. It is +** equal if and only if there are no VIRTUAL +** columns to the left. */ struct Column { char *zName; /* Name of this column, \000, then the type */ - Expr *pDflt; /* Default value of this column */ + Expr *pDflt; /* Default value or GENERATED ALWAYS AS value */ char *zColl; /* Collating sequence. If NULL, use the default */ u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ - u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ + u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; /* Allowed values for Column.colFlags: */ -#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ -#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ -#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ -#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ +#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ #define COLFLAG_SORTERREF 0x0010 /* Use sorter-refs with this column */ +#define COLFLAG_VIRTUAL 0x0020 /* GENERATED ALWAYS AS ... VIRTUAL */ +#define COLFLAG_STORED 0x0040 /* GENERATED ALWAYS AS ... STORED */ +#define COLFLAG_BUSY 0x0080 /* Blocks recursion on GENERATED columns */ +#define COLFLAG_NOTAVAIL 0x0100 /* STORED column not yet calculated */ +#define COLFLAG_GENERATED 0x0060 /* Combo: _STORED, _VIRTUAL */ +#define COLFLAG_NOINSERT 0x0062 /* Combo: _HIDDEN, _STORED, _VIRTUAL */ /* ** A "Collating Sequence" is defined by an instance of the following ** structure. Conceptually, a collating sequence consists of a name and ** a comparison routine that defines the order of that sequence. @@ -1975,10 +1994,11 @@ int tnum; /* Root BTree page for this table */ u32 nTabRef; /* Number of pointers to this Table */ u32 tabFlags; /* Mask of TF_* values */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ + i16 nNVCol; /* Number of columns that are not VIRTUAL */ LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ #ifdef SQLITE_ENABLE_COSTMULT LogEst costMult; /* Cost multiplier for using this table */ #endif @@ -2001,24 +2021,32 @@ ** ** TF_OOOHidden applies to tables or view that have hidden columns that are ** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING ** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, ** the TF_OOOHidden attribute would apply in this case. Such tables require -** special handling during INSERT processing. +** special handling during INSERT processing. The "OOO" means "Out Of Order". +** +** Constraints: +** +** TF_HasVirtual == COLFLAG_Virtual +** TF_HasStored == COLFLAG_Stored */ #define TF_Readonly 0x0001 /* Read-only system table */ #define TF_Ephemeral 0x0002 /* An ephemeral table */ #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ #define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ -#define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ +#define TF_HasVirtual 0x0020 /* Has one or more VIRTUAL columns */ +#define TF_HasStored 0x0040 /* Has one or more STORED columns */ +#define TF_HasGenerated 0x0060 /* Combo: HasVirtual + HasStored */ +#define TF_WithoutRowid 0x0080 /* No rowid. PRIMARY KEY is the key */ #define TF_StatsUsed 0x0100 /* Query planner decisions affected by ** Index.aiRowLogEst[] values */ -#define TF_HasNotNull 0x0200 /* Contains NOT NULL constraints */ -#define TF_Shadow 0x0400 /* True for a shadow table */ +#define TF_NoVisibleRowid 0x0200 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x0400 /* Out-of-Order hidden columns */ +#define TF_HasNotNull 0x0800 /* Contains NOT NULL constraints */ +#define TF_Shadow 0x1000 /* True for a shadow table */ /* ** Test to see whether or not a table is a virtual table. This is ** done as a macro so that it will be optimized out when virtual ** table support is omitted from the build. @@ -2265,10 +2293,11 @@ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ + unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ @@ -2806,25 +2835,26 @@ ** NC_HasAgg == SF_HasAgg == EP_Agg ** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX ** NC_HasWin == EP_Win ** */ -#define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ -#define NC_PartIdx 0x0002 /* True if resolving a partial index WHERE */ -#define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ -#define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ -#define NC_HasAgg 0x0010 /* One or more aggregate functions seen */ -#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ -#define NC_VarSelect 0x0040 /* A correlated subquery has been seen */ -#define NC_UEList 0x0080 /* True if uNC.pEList is used */ -#define NC_UAggInfo 0x0100 /* True if uNC.pAggInfo is used */ -#define NC_UUpsert 0x0200 /* True if uNC.pUpsert is used */ -#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ -#define NC_Complex 0x2000 /* True if a function or subquery seen */ -#define NC_AllowWin 0x4000 /* Window functions are allowed here */ -#define NC_HasWin 0x8000 /* One or more window functions seen */ -#define NC_IsDDL 0x10000 /* Resolving names in a CREATE statement */ +#define NC_AllowAgg 0x00001 /* Aggregate functions are allowed here */ +#define NC_PartIdx 0x00002 /* True if resolving a partial index WHERE */ +#define NC_IsCheck 0x00004 /* True if resolving a CHECK constraint */ +#define NC_InAggFunc 0x00008 /* True if analyzing arguments to an agg func */ +#define NC_HasAgg 0x00010 /* One or more aggregate functions seen */ +#define NC_IdxExpr 0x00020 /* True if resolving columns of CREATE INDEX */ +#define NC_VarSelect 0x00040 /* A correlated subquery has been seen */ +#define NC_UEList 0x00080 /* True if uNC.pEList is used */ +#define NC_UAggInfo 0x00100 /* True if uNC.pAggInfo is used */ +#define NC_UUpsert 0x00200 /* True if uNC.pUpsert is used */ +#define NC_MinMaxAgg 0x01000 /* min/max aggregates seen. See note above */ +#define NC_Complex 0x02000 /* True if a function or subquery seen */ +#define NC_AllowWin 0x04000 /* Window functions are allowed here */ +#define NC_HasWin 0x08000 /* One or more window functions seen */ +#define NC_IsDDL 0x10000 /* Resolving names in a CREATE statement */ +#define NC_GenCol 0x20000 /* True for a GENERATED ALWAYS AS clause */ /* ** An instance of the following object describes a single ON CONFLICT ** clause in an upsert. ** @@ -3935,11 +3965,18 @@ int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); void sqlite3OpenMasterTable(Parse *, int); Index *sqlite3PrimaryKeyIndex(Table*); -i16 sqlite3ColumnOfIndex(Index*, i16); +i16 sqlite3TableColumnToIndex(Index*, i16); +#ifdef SQLITE_OMIT_GENERATED_COLUMNS +# define sqlite3TableColumnToStorage(T,X) (X) /* No-op pass-through */ +# define sqlite3StorageColumnToTable(T,X) (X) /* No-op pass-through */ +#else + i16 sqlite3TableColumnToStorage(Table*, i16); + i16 sqlite3StorageColumnToTable(Table*, i16); +#endif void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); #if SQLITE_ENABLE_HIDDEN_COLUMNS void sqlite3ColumnPropertiesFromName(Table*, Column*); #else # define sqlite3ColumnPropertiesFromName(T,C) /* no-op */ @@ -3948,10 +3985,11 @@ void sqlite3AddNotNull(Parse*, int); void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); void sqlite3AddCheckConstraint(Parse*, Expr*); void sqlite3AddDefaultValue(Parse*,Expr*,const char*,const char*); void sqlite3AddCollateType(Parse*, Token*); +void sqlite3AddGenerated(Parse*,Expr*,Token*); void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); #ifdef SQLITE_HAS_CODEC int sqlite3CodecQueryParameters(sqlite3*,const char*,const char*); @@ -4005,10 +4043,13 @@ #else # define sqlite3AutoincrementBegin(X) # define sqlite3AutoincrementEnd(X) #endif void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + void sqlite3ComputeGeneratedColumns(Parse*, int, Table*); +#endif void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); IdList *sqlite3IdListAppend(Parse*, IdList*, Token*); int sqlite3IdListIndex(IdList*,const char*); SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); @@ -4054,16 +4095,18 @@ void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCode(Parse*, Expr*, int); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +void sqlite3ExprCodeGeneratedColumn(Parse*, Column*, int); +#endif void sqlite3ExprCodeCopy(Parse*, Expr*, int); void sqlite3ExprCodeFactorable(Parse*, Expr*, int); int sqlite3ExprCodeAtInit(Parse*, Expr*, int); int sqlite3ExprCodeTemp(Parse*, Expr*, int*); int sqlite3ExprCodeTarget(Parse*, Expr*, int); -void sqlite3ExprCodeAndCache(Parse*, Expr*, int); int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ #define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ #define SQLITE_ECEL_OMITREF 0x08 /* Omit if ExprList.u.x.iOrderByCol */ Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -145,11 +145,11 @@ int onError, /* How to handle constraint errors */ ExprList *pOrderBy, /* ORDER BY clause. May be null */ Expr *pLimit, /* LIMIT clause. May be null */ Upsert *pUpsert /* ON CONFLICT clause, or null */ ){ - int i, j; /* Loop counters */ + int i, j, k; /* Loop counters */ Table *pTab; /* The table to be updated */ int addrTop = 0; /* VDBE instruction address of the start of the loop */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Vdbe *v; /* The virtual database engine */ Index *pIdx; /* For looping over indices */ @@ -310,10 +310,20 @@ chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ chngPk = 1; } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "cannot UPDATE generated column \"%s\"", + pTab->aCol[j].zName); + goto update_cleanup; + } +#endif aXRef[j] = i; break; } } if( j>=pTab->nCol ){ @@ -343,10 +353,37 @@ } assert( (chngRowid & chngPk)==0 ); assert( chngRowid==0 || chngRowid==1 ); assert( chngPk==0 || chngPk==1 ); chngKey = chngRowid + chngPk; + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Mark generated columns as changing if their generator expressions + ** reference any changing column. The actual aXRef[] value for + ** generated expressions is not used, other than to check to see that it + ** is non-negative, so the value of aXRef[] for generated columns can be + ** set to any non-negative number. We use 99999 so that the value is + ** obvious when looking at aXRef[] in a symbolic debugger. + */ + if( pTab->tabFlags & TF_HasGenerated ){ + int bProgress; + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + do{ + bProgress = 0; + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ) continue; + if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ) continue; + if( sqlite3ExprReferencesUpdatedColumn(pTab->aCol[i].pDflt, + aXRef, chngRowid) ){ + aXRef[i] = 99999; + bProgress = 1; + } + } + }while( bProgress ); + } +#endif /* The SET expressions are not actually used inside the WHERE loop. ** So reset the colUsed mask. Unless this is a virtual table. In that ** case, set all bits of the colUsed mask (to ensure that the virtual ** table implementation makes all columns available). @@ -541,11 +578,12 @@ ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table ** is not required) and leave the PK fields in the array of registers. */ for(i=0; iaiColumn[i]>=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, + pPk->aiColumn[i], iPk+i); } if( eOnePass ){ if( addrOpen ) sqlite3VdbeChangeToNoop(v, addrOpen); nKey = nPk; regKey = iPk; @@ -621,19 +659,24 @@ if( chngPk || hasFK || pTrigger ){ u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); oldmask |= sqlite3TriggerColmask(pParse, pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError ); - for(i=0; inCol; i++){ + for(i=0, k=regOld; inCol; i++, k++){ + u32 colFlags = pTab->aCol[i].colFlags; + if( colFlags & COLFLAG_VIRTUAL ){ + k--; + continue; + } if( oldmask==0xffffffff || (i<32 && (oldmask & MASKBIT32(i))!=0) - || (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 + || (colFlags & COLFLAG_PRIMKEY)!=0 ){ testcase( oldmask!=0xffffffff && i==31 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regOld+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, regOld+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); } } if( chngRowid==0 && pPk==0 ){ sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } @@ -653,31 +696,40 @@ ** be used eliminates some redundant opcodes. */ newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - for(i=0; inCol; i++){ + for(i=0, k=regNew; inCol; i++, k++){ if( i==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); + }else if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)!=0 ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; }else{ j = aXRef[i]; if( j>=0 ){ - sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); + sqlite3ExprCode(pParse, pChanges->a[j].pExpr, k); }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask & MASKBIT32(i)) ){ /* This branch loads the value of a column that will not be changed ** into a register. This is done if there are no BEFORE triggers, or ** if there are one or more BEFORE triggers that use this value via ** a new.* reference in a trigger program. */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); } } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif /* Fire any BEFORE UPDATE triggers. This happens before constraints are ** verified. One could argue that this is wrong. */ if( tmask&TRIGGER_BEFORE ){ @@ -706,15 +758,24 @@ ** in case this has happened. Only unmodified columns are reloaded. ** The values computed for modified columns use the values before the ** BEFORE trigger runs. See test case trigger1-18.0 (added 2018-04-26) ** for an example. */ - for(i=0; inCol; i++){ - if( aXRef[i]<0 && i!=pTab->iPKey ){ - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + for(i=0, k=regNew; inCol; i++, k++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; + }else if( aXRef[i]<0 && i!=pTab->iPKey ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif } if( !isView ){ /* Do constraint checks. */ assert( regOldRowid>0 ); @@ -916,10 +977,11 @@ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); if( pWInfo==0 ) return; /* Populate the argument registers. */ for(i=0; inCol; i++){ + assert( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ); if( aXRef[i]>=0 ){ sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); }else{ sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG);/* Enable sqlite3_vtab_nochange() */ Index: src/upsert.c ================================================================== --- src/upsert.c +++ src/upsert.c @@ -224,11 +224,11 @@ int iPk = pParse->nMem+1; pParse->nMem += nPk; for(i=0; iaiColumn[i]>=0 ); - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iCur, k, iPk+i); VdbeComment((v, "%s.%s", pIdx->zName, pTab->aCol[pPk->aiColumn[i]].zName)); } sqlite3VdbeVerifyAbortable(v, OE_Abort); Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -3373,11 +3373,10 @@ p->pc = (int)(pOp - aOp); db->autoCommit = (u8)(1-desiredAutoCommit); p->rc = rc = SQLITE_BUSY; goto vdbe_return; } - assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); if( p->rc==SQLITE_OK ){ rc = SQLITE_DONE; }else{ rc = SQLITE_ERROR; Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -177,10 +177,11 @@ /* ** Prototypes for the VDBE interface. See comments on the implementation ** for a description of what each of these routines does. */ Vdbe *sqlite3VdbeCreate(Parse*); +Parse *sqlite3VdbeParser(Vdbe*); int sqlite3VdbeAddOp0(Vdbe*,int); int sqlite3VdbeAddOp1(Vdbe*,int,int); int sqlite3VdbeAddOp2(Vdbe*,int,int,int); int sqlite3VdbeGoto(Vdbe*,int); int sqlite3VdbeLoadString(Vdbe*,int,const char*); Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -1829,11 +1829,11 @@ if( !p || p->op==SQLITE_INSERT ){ rc = SQLITE_MISUSE_BKPT; goto preupdate_old_out; } if( p->pPk ){ - iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_old_out; } @@ -1919,11 +1919,11 @@ if( !p || p->op==SQLITE_DELETE ){ rc = SQLITE_MISUSE_BKPT; goto preupdate_new_out; } if( p->pPk && p->op!=SQLITE_UPDATE ){ - iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_new_out; } Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -39,10 +39,17 @@ assert( p->nOpAlloc==0 ); assert( pParse->szOpAlloc==0 ); sqlite3VdbeAddOp2(p, OP_Init, 0, 1); return p; } + +/* +** Return the Parse object that owns a Vdbe object. +*/ +Parse *sqlite3VdbeParser(Vdbe *p){ + return p->pParse; +} /* ** Change the error string stored in Vdbe.zErrMsg */ void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -622,11 +622,11 @@ *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ int iCol; - u8 oooHidden = 0; + u16 oooHidden = 0; /* If everything went according to plan, link the new VTable structure ** into the linked list headed by pTab->pVTable. Then loop through the ** columns of the table to see if any of them contain the token "hidden". ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from ** the type string. */ Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -5374,12 +5374,15 @@ assert( pIdx->pTable==pTab ); if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; assert( x>=0 ); + }else{ + testcase( x!=sqlite3StorageColumnToTable(pTab,x) ); + x = sqlite3StorageColumnToTable(pTab,x); } - x = sqlite3ColumnOfIndex(pIdx, x); + x = sqlite3TableColumnToIndex(pIdx, x); if( x>=0 ){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; OpcodeRewriteTrace(db, k, pOp); } Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -821,11 +821,11 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ struct CCurHint *pHint = pWalker->u.pCCurHint; assert( pHint->pIdx!=0 ); if( pExpr->op==TK_COLUMN && pExpr->iTable==pHint->iTabCur - && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0 + && sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn)<0 ){ pWalker->eCode = 1; } return WRC_Continue; } @@ -889,11 +889,11 @@ sqlite3ExprCode(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ pExpr->iTable = pHint->iIdxCur; - pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn); + pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } }else if( pExpr->op==TK_AGG_FUNCTION ){ /* An aggregate function in the WHERE clause of a query means this must ** be a correlated sub-query, and expression pExpr is an aggregate from @@ -1104,10 +1104,11 @@ typedef struct IdxExprTrans { Expr *pIdxExpr; /* The index expression */ int iTabCur; /* The cursor of the corresponding table */ int iIdxCur; /* The cursor for the index */ int iIdxCol; /* The column for the index */ + int iTabCol; /* The column for the table */ } IdxExprTrans; /* The walker node callback used to transform matching expressions into ** a reference to an index column for an index on an expression. ** @@ -1126,37 +1127,73 @@ }else{ return WRC_Continue; } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* A walker node callback that translates a column reference to a table +** into a corresponding column reference of an index. +*/ +static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + IdxExprTrans *pX = p->u.pIdxTrans; + if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){ + pExpr->iTable = pX->iIdxCur; + pExpr->iColumn = pX->iIdxCol; + pExpr->y.pTab = 0; + } + } + return WRC_Continue; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + /* ** For an indexes on expression X, locate every instance of expression X ** in pExpr and change that subexpression into a reference to the appropriate ** column of the index. +** +** 2019-10-24: Updated to also translate references to a VIRTUAL column in +** the table into references to the corresponding (stored) column of the +** index. */ static void whereIndexExprTrans( Index *pIdx, /* The Index */ int iTabCur, /* Cursor of the table that is being indexed */ int iIdxCur, /* Cursor of the index itself */ WhereInfo *pWInfo /* Transform expressions in this WHERE clause */ ){ int iIdxCol; /* Column number of the index */ ExprList *aColExpr; /* Expressions that are indexed */ + Table *pTab; Walker w; IdxExprTrans x; aColExpr = pIdx->aColExpr; - if( aColExpr==0 ) return; /* Not an index on expressions */ + if( aColExpr==0 && !pIdx->bHasVCol ){ + /* The index does not reference any expressions or virtual columns + ** so no translations are needed. */ + return; + } + pTab = pIdx->pTable; memset(&w, 0, sizeof(w)); - w.xExprCallback = whereIndexExprTransNode; w.u.pIdxTrans = &x; x.iTabCur = iTabCur; x.iIdxCur = iIdxCur; - for(iIdxCol=0; iIdxColnExpr; iIdxCol++){ - if( pIdx->aiColumn[iIdxCol]!=XN_EXPR ) continue; - assert( aColExpr->a[iIdxCol].pExpr!=0 ); + for(iIdxCol=0; iIdxColnColumn; iIdxCol++){ + i16 iRef = pIdx->aiColumn[iIdxCol]; + if( iRef==XN_EXPR ){ + assert( aColExpr->a[iIdxCol].pExpr!=0 ); + x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; + w.xExprCallback = whereIndexExprTransNode; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( iRef>=0 && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 ){ + x.iTabCol = iRef; + w.xExprCallback = whereIndexExprTransColumn; +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + }else{ + continue; + } x.iIdxCol = iIdxCol; - x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; sqlite3WalkExpr(&w, pWInfo->pWhere); sqlite3WalkExprList(&w, pWInfo->pOrderBy); sqlite3WalkExprList(&w, pWInfo->pResultSet); } } @@ -1824,20 +1861,22 @@ } }else if( iCur!=iIdxCur ){ Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); } sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } /* If pIdx is an index on one or more expressions, then look through ** all the expressions in pWInfo and try to transform matching expressions - ** into reference to index columns. + ** into reference to index columns. Also attempt to translate references + ** to virtual columns in the table into references to (stored) columns + ** of the index. ** ** Do not do this for the RHS of a LEFT JOIN. This is because the ** expression may be evaluated after OP_NullRow has been executed on ** the cursor. In this case it is important to do the full evaluation, ** as the result of the expression may not be NULL, even if all table @@ -2084,11 +2123,11 @@ /* Read the PK into an array of temp registers. */ r = sqlite3GetTempRange(pParse, nPk); for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol, r+iPk); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+iPk); } /* Check if the temp table already contains this key. If so, ** the row has already been included in the result set and ** can be ignored (by jumping past the Gosub below). Otherwise, Index: tool/mkkeywordhash.c ================================================================== --- tool/mkkeywordhash.c +++ tool/mkkeywordhash.c @@ -162,10 +162,11 @@ { "ACTION", "TK_ACTION", FKEY }, { "ADD", "TK_ADD", ALTER }, { "AFTER", "TK_AFTER", TRIGGER }, { "ALL", "TK_ALL", ALWAYS }, { "ALTER", "TK_ALTER", ALTER }, + { "ALWAYS", "TK_ALWAYS", ALWAYS }, { "ANALYZE", "TK_ANALYZE", ANALYZE }, { "AND", "TK_AND", ALWAYS }, { "AS", "TK_AS", ALWAYS }, { "ASC", "TK_ASC", ALWAYS }, { "ATTACH", "TK_ATTACH", ATTACH }, @@ -214,10 +215,11 @@ { "FOLLOWING", "TK_FOLLOWING", WINDOWFUNC }, { "FOR", "TK_FOR", TRIGGER }, { "FOREIGN", "TK_FOREIGN", FKEY }, { "FROM", "TK_FROM", ALWAYS }, { "FULL", "TK_JOIN_KW", ALWAYS }, + { "GENERATED", "TK_GENERATED", ALWAYS }, { "GLOB", "TK_LIKE_KW", ALWAYS }, { "GROUP", "TK_GROUP", ALWAYS }, { "GROUPS", "TK_GROUPS", WINDOWFUNC }, { "HAVING", "TK_HAVING", ALWAYS }, { "IF", "TK_IF", ALWAYS },