Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -19,11 +19,11 @@ # <> # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN -USE_FULLWARN = 0 +USE_FULLWARN = 1 !ENDIF # Set this non-0 to enable treating warnings as errors (-WX, etc) when # compiling. # Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -370,12 +370,12 @@ *v = b; return (int)(p - pStart); } /* -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a -** 32-bit integer before it is returned. +** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to +** a non-negative 32-bit integer before it is returned. */ int sqlite3Fts3GetVarint32(const char *p, int *pi){ u32 a; #ifndef fts3GetVarint32 @@ -387,11 +387,13 @@ GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); a = (a & 0x0FFFFFFF ); - *pi = (int)(a | ((u32)(*p & 0x0F) << 28)); + *pi = (int)(a | ((u32)(*p & 0x07) << 28)); + assert( 0==(a & 0x80000000) ); + assert( *pi>=0 ); return 5; } /* ** Return the number of bytes required to encode v as a varint @@ -1217,69 +1219,70 @@ struct Fts4Option *pOp = &aFts4Opt[iOpt]; if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ break; } } - if( iOpt==SizeofArray(aFts4Opt) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); - rc = SQLITE_ERROR; - }else{ - switch( iOpt ){ - case 0: /* MATCHINFO */ - if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - bNoDocsize = 1; - break; - - case 1: /* PREFIX */ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; - break; - - case 2: /* COMPRESS */ - sqlite3_free(zCompress); - zCompress = zVal; - zVal = 0; - break; - - case 3: /* UNCOMPRESS */ - sqlite3_free(zUncompress); - zUncompress = zVal; - zVal = 0; - break; - - case 4: /* ORDER */ - if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) - ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); - rc = SQLITE_ERROR; - } - bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); - break; - - case 5: /* CONTENT */ - sqlite3_free(zContent); - zContent = zVal; - zVal = 0; - break; - - case 6: /* LANGUAGEID */ - assert( iOpt==6 ); - sqlite3_free(zLanguageid); - zLanguageid = zVal; - zVal = 0; - break; - - case 7: /* NOTINDEXED */ - azNotindexed[nNotindexed++] = zVal; - zVal = 0; - break; - } + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + + default: + assert( iOpt==SizeofArray(aFts4Opt) ); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + break; } sqlite3_free(zVal); } } @@ -1844,11 +1847,12 @@ zCsr += fts3GetVarint32(zCsr, &nPrefix); } isFirstTerm = 0; zCsr += fts3GetVarint32(zCsr, &nSuffix); - if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){ + assert( nPrefix>=0 && nSuffix>=0 ); + if( &zCsr[nSuffix]>zEnd ){ rc = FTS_CORRUPT_VTAB; goto finish_scan; } if( nPrefix+nSuffix>nAlloc ){ char *zNew; @@ -2654,11 +2658,11 @@ bWritten = 1; } fts3ColumnlistCopy(0, &p); } - while( ppVtab; /* The column value supplied by SQLite must be in range. */ assert( iCol>=0 && iCol<=p->nColumn+2 ); - if( iCol==p->nColumn+1 ){ - /* This call is a request for the "docid" column. Since "docid" is an - ** alias for "rowid", use the xRowid() method to obtain the value. - */ - sqlite3_result_int64(pCtx, pCsr->iPrevId); - }else if( iCol==p->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. */ - sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ - sqlite3_result_int64(pCtx, pCsr->iLangid); - }else{ - /* The requested column is either a user column (one that contains - ** indexed data), or the language-id column. */ - rc = fts3CursorSeek(0, pCsr); - - if( rc==SQLITE_OK ){ - if( iCol==p->nColumn+2 ){ - int iLangid = 0; - if( p->zLanguageid ){ - iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); - } - sqlite3_result_int(pCtx, iLangid); - }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + switch( iCol-p->nColumn ){ + case 0: + /* The special 'table-name' column */ + sqlite3_result_blob(pCtx, &pCsr, sizeof(Fts3Cursor*), SQLITE_TRANSIENT); + sqlite3_result_subtype(pCtx, SQLITE_BLOB); + break; + + case 1: + /* The docid column */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + break; + + case 2: + if( pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + break; + }else if( p->zLanguageid==0 ){ + sqlite3_result_int(pCtx, 0); + break; + }else{ + iCol = p->nColumn; + /* fall-through */ + } + + default: + /* A user column. Or, if this is a full-table scan, possibly the + ** language-id column. Seek the cursor. */ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){ sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); } - } + break; } assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -3440,21 +3449,15 @@ ** if an error occurs. */ static int fts3SetHasStat(Fts3Table *p){ int rc = SQLITE_OK; if( p->bHasStat==2 ){ - const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; - char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); - if( zSql ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); - rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat; - } - sqlite3_free(zSql); + char *zTbl = sqlite3_mprintf("%s_stat", p->zName); + if( zTbl ){ + int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0); + sqlite3_free(zTbl); + p->bHasStat = (res==SQLITE_OK); }else{ rc = SQLITE_NOMEM; } } return rc; @@ -3557,22 +3560,20 @@ sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ - Fts3Cursor *pRet; - if( sqlite3_value_type(pVal)!=SQLITE_BLOB - || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) - ){ + int rc = SQLITE_OK; + if( sqlite3_value_subtype(pVal)==SQLITE_BLOB ){ + *ppCsr = *(Fts3Cursor**)sqlite3_value_blob(pVal); + }else{ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); sqlite3_result_error(pContext, zErr, -1); sqlite3_free(zErr); - return SQLITE_ERROR; + rc = SQLITE_ERROR; } - memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); - *ppCsr = pRet; - return SQLITE_OK; + return rc; } /* ** Implementation of the snippet() function for FTS3 */ @@ -3955,11 +3956,11 @@ rc = sqlite3Fts3ExprInitTestInterface(db); } #endif /* Create the virtual table wrapper around the hash-table and overload - ** the two scalar functions. If this is successful, register the + ** the four scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) @@ -4538,11 +4539,11 @@ /* This is only called if it is guaranteed that the phrase has at least ** one incremental token. In which case the bIncr flag is set. */ assert( p->bIncr==1 ); - if( p->nToken==1 && p->bIncr ){ + if( p->nToken==1 ){ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList ); if( pDL->pList==0 ) bEof = 1; }else{ @@ -4771,10 +4772,11 @@ ** of data that will fit on a single leaf page of an intkey table in ** this database, then the average docsize is 1. Otherwise, it is 1 plus ** the number of overflow pages consumed by a record B bytes in size. */ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + int rc = SQLITE_OK; if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost ** of each doclist, has not yet been determined. Read the required ** data from the %_stat table to calculate it. ** @@ -4810,15 +4812,14 @@ pCsr->nDoc = nDoc; pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); assert( pCsr->nRowAvg>0 ); rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; } *pnPage = pCsr->nRowAvg; - return SQLITE_OK; + return rc; } /* ** This function is called to select the tokens (if any) that will be ** deferred. The array aTC[] has already been populated when this is @@ -5164,11 +5165,12 @@ } } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ - if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ + assert( pRight->eType==FTSQUERY_PHRASE ); + if( pRight->pPhrase->doclist.aAll ){ Fts3Doclist *pDl = &pRight->pPhrase->doclist; while( *pRc==SQLITE_OK && pRight->bEof==0 ){ memset(pDl->pList, 0, pDl->nList); fts3EvalNextRow(pCsr, pRight, pRc); } @@ -5193,11 +5195,11 @@ assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNextRow(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + }else if( pLeft->bEof || iCmp>0 ){ fts3EvalNextRow(pCsr, pRight, pRc); }else{ fts3EvalNextRow(pCsr, pLeft, pRc); fts3EvalNextRow(pCsr, pRight, pRc); } @@ -5285,55 +5287,51 @@ ** left-hand child may be either a phrase or a NEAR node. There are ** no exceptions to this - it's the way the parser in fts3_expr.c works. */ if( *pRc==SQLITE_OK && pExpr->eType==FTSQUERY_NEAR - && pExpr->bEof==0 && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) ){ Fts3Expr *p; int nTmp = 0; /* Bytes of temp space */ char *aTmp; /* Temp space for PoslistNearMerge() */ /* Allocate temporary working space. */ for(p=pExpr; p->pLeft; p=p->pLeft){ + assert( p->pRight->pPhrase->doclist.nList>0 ); nTmp += p->pRight->pPhrase->doclist.nList; } nTmp += p->pPhrase->doclist.nList; - if( nTmp==0 ){ + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; res = 0; }else{ - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; - res = 0; - }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; - - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear; - Fts3Phrase *pPhrase; - assert( p->pParent && p->pParent->pLeft==p ); - nNear = p->pParent->nNear; - pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } + } - sqlite3_free(aTmp); - } + sqlite3_free(aTmp); } return res; } Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -652,10 +652,11 @@ }; /* Parse a MATCH expression. */ int sqlite3Fts5ExprNew( Fts5Config *pConfig, + int iCol, /* Column on LHS of MATCH operator */ const char *zExpr, Fts5Expr **ppNew, char **pzErr ); @@ -736,11 +737,11 @@ void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*); void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); -void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*); Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); /* Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -211,10 +211,11 @@ static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); } static void fts5ParseFree(void *p){ sqlite3_free(p); } int sqlite3Fts5ExprNew( Fts5Config *pConfig, /* FTS5 Configuration */ + int iCol, const char *zExpr, /* Expression text */ Fts5Expr **ppNew, char **pzErr ){ Fts5Parse sParse; @@ -234,10 +235,22 @@ do { t = fts5ExprGetToken(&sParse, &z, &token); sqlite3Fts5Parser(pEngine, t, token, &sParse); }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + + /* If the LHS of the MATCH expression was a user column, apply the + ** implicit column-filter. */ + if( iColnCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ + int n = sizeof(Fts5Colset); + Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); + if( pColset ){ + pColset->nCol = 1; + pColset->aiCol[0] = iCol; + sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset); + } + } assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); if( pNew==0 ){ @@ -1884,29 +1897,114 @@ } return pRet; } +/* +** If argument pOrig is NULL, or if (*pRc) is set to anything other than +** SQLITE_OK when this function is called, NULL is returned. +** +** Otherwise, a copy of (*pOrig) is made into memory obtained from +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned. +*/ +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ + Fts5Colset *pRet; + if( pOrig ){ + int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); + if( pRet ){ + memcpy(pRet, pOrig, nByte); + } + }else{ + pRet = 0; + } + return pRet; +} + +/* +** Remove from colset pColset any columns that are not also in colset pMerge. +*/ +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){ + int iIn = 0; /* Next input in pColset */ + int iMerge = 0; /* Next input in pMerge */ + int iOut = 0; /* Next output slot in pColset */ + + while( iInnCol && iMergenCol ){ + int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge]; + if( iDiff==0 ){ + pColset->aiCol[iOut++] = pMerge->aiCol[iMerge]; + iMerge++; + iIn++; + }else if( iDiff>0 ){ + iMerge++; + }else{ + iIn++; + } + } + pColset->nCol = iOut; +} + +/* +** Recursively apply colset pColset to expression node pNode and all of +** its decendents. If (*ppFree) is not NULL, it contains a spare copy +** of pColset. This function may use the spare copy and set (*ppFree) to +** zero, or it may create copies of pColset using fts5CloneColset(). +*/ +static void fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pNode, + Fts5Colset *pColset, + Fts5Colset **ppFree +){ + if( pParse->rc==SQLITE_OK ){ + assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING + || pNode->eType==FTS5_AND || pNode->eType==FTS5_OR + || pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF + ); + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->pColset ){ + fts5MergeColset(pNear->pColset, pColset); + if( pNear->pColset->nCol==0 ){ + pNode->eType = FTS5_EOF; + pNode->xNext = 0; + } + }else if( *ppFree ){ + pNear->pColset = pColset; + *ppFree = 0; + }else{ + pNear->pColset = fts5CloneColset(&pParse->rc, pColset); + } + }else{ + int i; + assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 ); + for(i=0; inChild; i++){ + fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree); + } + } + } +} + +/* +** Apply colset pColset to expression node pExpr and all of its descendents. +*/ void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, - Fts5ExprNearset *pNear, + Fts5ExprNode *pExpr, Fts5Colset *pColset ){ + Fts5Colset *pFree = pColset; if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ pParse->rc = SQLITE_ERROR; pParse->zErr = sqlite3_mprintf( "fts5: column queries are not supported (detail=none)" ); - sqlite3_free(pColset); - return; - } - - if( pNear ){ - pNear->pColset = pColset; }else{ - sqlite3_free(pColset); + fts5ParseSetColset(pParse, pExpr, pColset, &pFree); } + sqlite3_free(pFree); } static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ switch( pNode->eType ){ case FTS5_STRING: { @@ -2356,11 +2454,11 @@ zExpr = (const char*)sqlite3_value_text(apVal[0]); rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr); + rc = sqlite3Fts5ExprNew(pConfig, pConfig->nCol, zExpr, &pExpr, &zErr); } if( rc==SQLITE_OK ){ char *zText; if( pExpr->pRoot->xNext==0 ){ zText = sqlite3_mprintf(""); Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -2876,11 +2876,12 @@ Fts5Iter *pIter, int *pbNewTerm /* OUT: True if *might* be new term */ ){ assert( pIter->bSkipEmpty ); if( p->rc==SQLITE_OK ){ - do { + *pbNewTerm = 0; + do{ int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; assert( p->rc==SQLITE_OK ); @@ -2889,12 +2890,10 @@ || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); *pbNewTerm = 1; - }else{ - *pbNewTerm = 0; } fts5AssertMultiIterSetup(p, pIter); }while( fts5MultiIterIsEmpty(p, pIter) ); } @@ -3156,27 +3155,27 @@ } return p - (*pa); } -static int fts5IndexExtractColset ( +static void fts5IndexExtractColset( + int *pRc, Fts5Colset *pColset, /* Colset to filter on */ const u8 *pPos, int nPos, /* Position list */ Fts5Buffer *pBuf /* Output buffer */ ){ - int rc = SQLITE_OK; - int i; - - fts5BufferZero(pBuf); - for(i=0; inCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + if( *pRc==SQLITE_OK ){ + int i; + fts5BufferZero(pBuf); + for(i=0; inCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(pRc, pBuf, nSub, pSub); + } } } - return rc; } /* ** xSetOutputs callback used by detail=none tables. */ @@ -3296,12 +3295,13 @@ const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; if( pColset->nCol==1 ){ pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); pIter->base.pData = a; }else{ + int *pRc = &pIter->pIndex->rc; fts5BufferZero(&pIter->poslist); - fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); + fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist); pIter->base.pData = pIter->poslist.p; pIter->base.nData = pIter->poslist.n; } }else{ /* The data is distributed over two or more pages. Copy it into the @@ -3842,13 +3842,10 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; -static int nCall = 0; -nCall++; - assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); Index: ext/fts5/fts5_main.c ================================================================== --- ext/fts5/fts5_main.c +++ ext/fts5/fts5_main.c @@ -504,10 +504,11 @@ ** Costs are not modified by the ORDER BY clause. */ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts5Table *pTab = (Fts5Table*)pVTab; Fts5Config *pConfig = pTab->pConfig; + const int nCol = pConfig->nCol; int idxFlags = 0; /* Parameter passed through to xFilter() */ int bHasMatch; int iNext; int i; @@ -529,28 +530,38 @@ FTS5_BI_ROWID_GE, 0, 0, -1}, }; int aColMap[3]; aColMap[0] = -1; - aColMap[1] = pConfig->nCol; - aColMap[2] = pConfig->nCol+1; + aColMap[1] = nCol; + aColMap[2] = nCol+1; /* Set idxFlags flags for all WHERE clause terms that will be used. */ for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; - int j; - for(j=0; jiColumn==aColMap[pC->iCol] && p->op & pC->op ){ - if( p->usable ){ + int iCol = p->iColumn; + + if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol) + || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol) + ){ + /* A MATCH operator or equivalent */ + if( p->usable ){ + idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16); + aConstraint[0].iConsIndex = i; + }else{ + /* As there exists an unusable MATCH constraint this is an + ** unusable plan. Set a prohibitively high cost. */ + pInfo->estimatedCost = 1e50; + return SQLITE_OK; + } + }else{ + int j; + for(j=1; jiCol] && p->op & pC->op && p->usable ){ pC->iConsIndex = i; idxFlags |= pC->fts5op; - }else if( j==0 ){ - /* As there exists an unusable MATCH constraint this is an - ** unusable plan. Set a prohibitively high cost. */ - pInfo->estimatedCost = 1e50; - return SQLITE_OK; } } } } @@ -1121,10 +1132,11 @@ sqlite3_value *pMatch = 0; /* MATCH ? expression (or NULL) */ sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ + int iCol; /* Column on LHS of MATCH operator */ char **pzErrmsg = pConfig->pzErrmsg; UNUSED_PARAM(zUnused); UNUSED_PARAM(nVal); @@ -1151,10 +1163,12 @@ if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++]; if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++]; if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++]; if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++]; + iCol = (idxNum>>16); + assert( iCol>=0 && iCol<=pConfig->nCol ); assert( iVal==nVal ); bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); /* Set the cursor upper and lower rowid limits. Only some strategies @@ -1197,11 +1211,11 @@ ** indicates that the MATCH expression is not a full text query, ** but a request for an internal parameter. */ rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); }else{ char **pzErr = &pTab->base.zErrMsg; - rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr); + rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr); if( rc==SQLITE_OK ){ if( bOrderByRank ){ pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); }else{ Index: ext/fts5/fts5parse.y ================================================================== --- ext/fts5/fts5parse.y +++ ext/fts5/fts5parse.y @@ -87,36 +87,10 @@ %type exprlist {Fts5ExprNode*} %destructor cnearset { sqlite3Fts5ParseNodeFree($$); } %destructor expr { sqlite3Fts5ParseNodeFree($$); } %destructor exprlist { sqlite3Fts5ParseNodeFree($$); } -expr(A) ::= expr(X) AND expr(Y). { - A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0); -} -expr(A) ::= expr(X) OR expr(Y). { - A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0); -} -expr(A) ::= expr(X) NOT expr(Y). { - A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0); -} - -expr(A) ::= LP expr(X) RP. {A = X;} -expr(A) ::= exprlist(X). {A = X;} - -exprlist(A) ::= cnearset(X). {A = X;} -exprlist(A) ::= exprlist(X) cnearset(Y). { - A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y); -} - -cnearset(A) ::= nearset(X). { - A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); -} -cnearset(A) ::= colset(X) COLON nearset(Y). { - sqlite3Fts5ParseSetColset(pParse, Y, X); - A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); -} - %type colset {Fts5Colset*} %destructor colset { sqlite3_free($$); } %type colsetlist {Fts5Colset*} %destructor colsetlist { sqlite3_free($$); } @@ -135,10 +109,41 @@ colsetlist(A) ::= colsetlist(Y) STRING(X). { A = sqlite3Fts5ParseColset(pParse, Y, &X); } colsetlist(A) ::= STRING(X). { A = sqlite3Fts5ParseColset(pParse, 0, &X); } + +expr(A) ::= expr(X) AND expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0); +} +expr(A) ::= expr(X) OR expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0); +} +expr(A) ::= expr(X) NOT expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0); +} + +expr(A) ::= colset(X) COLON LP expr(Y) RP. { + sqlite3Fts5ParseSetColset(pParse, Y, X); + A = Y; +} +expr(A) ::= LP expr(X) RP. {A = X;} +expr(A) ::= exprlist(X). {A = X;} + +exprlist(A) ::= cnearset(X). {A = X;} +exprlist(A) ::= exprlist(X) cnearset(Y). { + A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y); +} + +cnearset(A) ::= nearset(X). { + A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); +} +cnearset(A) ::= colset(X) COLON nearset(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); + sqlite3Fts5ParseSetColset(pParse, A, X); +} + %type nearset {Fts5ExprNearset*} %type nearphrases {Fts5ExprNearset*} %destructor nearset { sqlite3Fts5ParseNearsetFree($$); } %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); } Index: ext/fts5/test/fts5aa.test ================================================================== --- ext/fts5/test/fts5aa.test +++ ext/fts5/test/fts5aa.test @@ -574,12 +574,26 @@ BEGIN; INSERT INTO ft VALUES('a b c'); DROP TABLE t8; COMMIT; } + +do_execsql_test 22.0 { + CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%); + INSERT INTO t9(rowid, x) VALUES(2, 'bbb'); + BEGIN; + INSERT INTO t9(rowid, x) VALUES(1, 'aaa'); + DELETE FROM t9 WHERE rowid = 2; + INSERT INTO t9(rowid, x) VALUES(3, 'bbb'); + COMMIT; +} + +do_execsql_test 22.1 { + SELECT rowid FROM t9('a*') +} {1} } finish_test Index: ext/fts5/test/fts5colset.test ================================================================== --- ext/fts5/test/fts5colset.test +++ ext/fts5/test/fts5colset.test @@ -42,18 +42,47 @@ 7 "-{\"a\"} : b" {1 2 3} 8 "- c : a" {1 2 4} 9 "-c : a" {1 2 4} 10 "-\"c\" : a" {1 2 4} } { - breakpoint do_execsql_test 1.$tn { SELECT rowid FROM t1($q) } $res } + foreach {tn q res} { + 0 {{a} : (a AND ":")} {} + 1 "{a b c} : (a AND d)" {2 3} + 2 "{a b c} : (a AND b:d)" {3} + 3 "{a b c} : (a AND d:d)" {} + 4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4} + 5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3} + 6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {} + 7 "{a b c} : (b:a AND c:b)" {2} + } { + do_execsql_test 2.$tn { + SELECT rowid FROM t1($q) + } $res + } + + foreach {tn w res} { + 0 "a MATCH 'a'" {1} + 1 "b MATCH 'a'" {2} + 2 "b MATCH '{a b c} : a'" {2} + 3 "b MATCH 'a OR b'" {1 2} + 4 "b MATCH 'a OR a:b'" {2} + 5 "b MATCH 'a OR b:b'" {1 2} + } { + do_execsql_test 3.$tn " + SELECT rowid FROM t1 WHERE $w + " $res + } + do_catchsql_test 4.1 { + SELECT * FROM t1 WHERE rowid MATCH 'a' + } {1 {unable to use function MATCH in the requested context}} } finish_test Index: ext/fts5/test/fts5faultB.test ================================================================== --- ext/fts5/test/fts5faultB.test +++ ext/fts5/test/fts5faultB.test @@ -104,8 +104,32 @@ SELECT rowid FROM x1('c') WHERE rowid>1; } } -test { faultsim_test_result {0 {2 3}} } + +#------------------------------------------------------------------------- +# Test OOM injection with nested colsets. +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d); + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 +} +do_faultsim_test 4.1 -faults oom* -body { + execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); } +} -test { + faultsim_test_result {0 2} +} + +do_faultsim_test 4.2 -faults oom* -body { + execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') } +} -test { + faultsim_test_result {0 {2 3}} +} + finish_test Index: ext/fts5/test/fts5plan.test ================================================================== --- ext/fts5/test/fts5plan.test +++ ext/fts5/test/fts5plan.test @@ -28,11 +28,11 @@ do_eqp_test 1.1 { SELECT * FROM t1, f1 WHERE f1 MATCH t1.x } { 0 0 0 {SCAN TABLE t1} - 0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:} } do_eqp_test 1.2 { SELECT * FROM t1, f1 WHERE f1 > t1.x } { @@ -41,11 +41,11 @@ } do_eqp_test 1.3 { SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff } { - 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_eqp_test 1.4 { SELECT * FROM f1 ORDER BY rank ADDED ext/misc/anycollseq.c Index: ext/misc/anycollseq.c ================================================================== --- ext/misc/anycollseq.c +++ ext/misc/anycollseq.c @@ -0,0 +1,58 @@ +/* +** 2017-04-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a run-time loadable extension to SQLite that +** registers a sqlite3_collation_needed() callback to register a fake +** collating function for any unknown collating sequence. The fake +** collating function works like BINARY. +** +** This extension can be used to load schemas that contain one or more +** unknown collating sequences. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +static int anyCollFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1xCallback(z, p->pArg); sqlite3_free(z); } /* -** Output the given string as a quoted string using SQL quoting conventions. +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. ** -** The "\n" and "\r" characters are converted to char(10) and char(13) -** to prevent them from being transformed by end-of-line translators. +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *unused_string( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** Additionallly , escape the "\n" and "\r" characters so that they do not +** get corrupted by end-of-line translation facilities in some operating +** systems. */ -static void output_quoted_string(DState *p, const unsigned char *z){ +static void output_quoted_escaped_string(DState *p, const char *z){ int i; char c; - int inQuote = 0; - int bStarted = 0; - for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} if( c==0 ){ - output_formatted(p, "'%s'", z); - return; - } - while( *z ){ - for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} - if( c=='\'' ) i++; - if( i ){ - if( !inQuote ){ - if( bStarted ) p->xCallback("||", p->pArg); + output_formatted(p,"'%s'",z); + }else{ + const char *zNL = 0; + const char *zCR = 0; + int nNL = 0; + int nCR = 0; + char zBuf1[20], zBuf2[20]; + for(i=0; z[i]; i++){ + if( z[i]=='\n' ) nNL++; + if( z[i]=='\r' ) nCR++; + } + if( nNL ){ + p->xCallback("replace(", p->pArg); + zNL = unused_string(z, "\\n", "\\012", zBuf1); + } + if( nCR ){ + p->xCallback("replace(", p->pArg); + zCR = unused_string(z, "\\r", "\\015", zBuf2); + } + p->xCallback("'", p->pArg); + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + output_formatted(p, "%.*s", i, z); + z += i; + } + if( c=='\'' ){ p->xCallback("'", p->pArg); - inQuote = 1; + continue; + } + if( c==0 ){ + break; + } + z++; + if( c=='\n' ){ + p->xCallback(zNL, p->pArg); + continue; } - output_formatted(p, "%.*s", i, z); - z += i; - bStarted = 1; + p->xCallback(zCR, p->pArg); } - if( c=='\'' ){ - p->xCallback("'", p->pArg); - continue; + p->xCallback("'", p->pArg); + if( nCR ){ + output_formatted(p, ",'%s',char(13))", zCR); } - if( inQuote ){ - p->xCallback("'", p->pArg); - inQuote = 0; + if( nNL ){ + output_formatted(p, ",'%s',char(10))", zNL); } - if( c==0 ){ - break; - } - for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ - if( bStarted ) p->xCallback("||", p->pArg); - output_formatted(p, "char(%d)", c); - bStarted = 1; - } - z += i; } - if( inQuote ) p->xCallback("'", p->pArg); } /* ** This is an sqlite3_exec callback routine used for dumping the database. ** Each row received by this callback consists of a table name, @@ -493,11 +525,12 @@ case SQLITE_NULL: { p->xCallback("NULL", p->pArg); break; } case SQLITE_TEXT: { - output_quoted_string(p, sqlite3_column_text(pStmt,i)); + output_quoted_escaped_string(p, + (const char*)sqlite3_column_text(pStmt,i)); break; } case SQLITE_BLOB: { int nByte = sqlite3_column_bytes(pStmt,i); unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i); Index: ext/misc/json1.c ================================================================== --- ext/misc/json1.c +++ ext/misc/json1.c @@ -88,10 +88,11 @@ #ifndef SQLITE_AMALGAMATION /* Unsigned integer types. These are already defined in the sqliteInt.h, ** but the definitions need to be repeated for separate compilation. */ typedef sqlite3_uint64 u64; typedef unsigned int u32; + typedef unsigned short int u16; typedef unsigned char u8; #endif /* Objects */ typedef struct JsonString JsonString; @@ -167,12 +168,22 @@ JsonNode *aNode; /* Array of nodes containing the parse */ const char *zJson; /* Original JSON string */ u32 *aUp; /* Index of parent of each node */ u8 oom; /* Set to true if out of memory */ u8 nErr; /* Number of errors seen */ + u16 iDepth; /* Nesting depth */ }; +/* +** Maximum nesting depth of JSON for this implementation. +** +** This limit is needed to avoid a stack overflow in the recursive +** descent parser. A depth of 2000 is far deeper than any sane JSON +** should go. +*/ +#define JSON_MAX_DEPTH 2000 + /************************************************************************** ** Utility routines for dealing with JsonString objects **************************************************************************/ /* Set the JsonString object to an empty string @@ -733,12 +744,14 @@ /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; for(j=i+1;;j++){ while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); if( x<0 ){ + pParse->iDepth--; if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; return -1; } if( pParse->oom ) return -1; pNode = &pParse->aNode[pParse->nNode-1]; @@ -747,10 +760,11 @@ j = x; while( safe_isspace(z[j]) ){ j++; } if( z[j]!=':' ) return -1; j++; x = jsonParseValue(pParse, j); + pParse->iDepth--; if( x<0 ) return -1; j = x; while( safe_isspace(z[j]) ){ j++; } c = z[j]; if( c==',' ) continue; @@ -763,11 +777,13 @@ /* Parse array */ iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); if( iThis<0 ) return -1; for(j=i+1;;j++){ while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); + pParse->iDepth--; if( x<0 ){ if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; return -1; } j = x; @@ -783,11 +799,14 @@ /* Parse string */ u8 jnFlags = 0; j = i+1; for(;;){ c = z[j]; - if( c==0 ) return -1; + if( (c & ~0x1f)==0 ){ + /* Control characters are not allowed in strings */ + return -1; + } if( c=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' || (c=='u' && jsonIs4Hex(z+j+1)) ){ @@ -883,10 +902,11 @@ if( zJson==0 ) return 1; pParse->zJson = zJson; i = jsonParseValue(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ + assert( pParse->iDepth==0 ); while( safe_isspace(zJson[i]) ) i++; if( zJson[i] ) i = -1; } if( i<=0 ){ if( pCtx!=0 ){ @@ -1383,11 +1403,11 @@ /* This is the RFC 7396 MergePatch algorithm. */ static JsonNode *jsonMergePatch( JsonParse *pParse, /* The JSON parser that contains the TARGET */ - int iTarget, /* Node of the TARGET in pParse */ + u32 iTarget, /* Node of the TARGET in pParse */ JsonNode *pPatch /* The PATCH */ ){ u32 i, j; u32 iRoot; JsonNode *pTarget; Index: ext/rbu/rbu.c ================================================================== --- ext/rbu/rbu.c +++ ext/rbu/rbu.c @@ -80,14 +80,14 @@ sqlite3rbu *pRbu; /* RBU handle */ int nStep = 0; /* Maximum number of step() calls */ int bVacuum = 0; int rc; sqlite3_int64 nProgress = 0; - int nArg = argc-2; + int nArgc = argc-2; if( argc<3 ) usage(argv[0]); - for(i=1; i1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){ bVacuum = 1; }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && icurFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){ loc = 0; - }else if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0 - && pCur->info.nKey==pX->nKey-1 ){ - loc = -1; }else if( loc==0 ){ rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc); if( rc ) return rc; } }else if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){ Index: src/btree.h ================================================================== --- src/btree.h +++ src/btree.h @@ -274,11 +274,11 @@ */ struct BtreePayload { const void *pKey; /* Key content for indexes. NULL for tables */ sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ const void *pData; /* Data for tables. NULL for indexes */ - struct Mem *aMem; /* First of nMem value in the unpacked pKey */ + sqlite3_value *aMem; /* First of nMem value in the unpacked pKey */ u16 nMem; /* Number of aMem[] value. Might be zero */ int nData; /* Size of pData. 0 if none. */ int nZero; /* Extra zero data appended after pData,nData */ }; Index: src/date.c ================================================================== --- src/date.c +++ src/date.c @@ -421,12 +421,14 @@ if( p->validYMD ) return; if( !p->validJD ){ p->Y = 2000; p->M = 1; p->D = 1; + }else if( !validJulianDay(p->iJD) ){ + datetimeError(p); + return; }else{ - assert( validJulianDay(p->iJD) ); Z = (int)((p->iJD + 43200000)/86400000); A = (int)((Z - 1867216.25)/36524.25); A = Z + 1 + A - (A/4); B = A + 1524; C = (int)((B - 122.1)/365.25); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -56,11 +56,11 @@ if( op==TK_CAST ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); return sqlite3AffinityType(pExpr->u.zToken, 0); } #endif - if( op==TK_AGG_COLUMN || op==TK_COLUMN ){ + if( (op==TK_AGG_COLUMN || op==TK_COLUMN) && pExpr->pTab ){ return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn); } if( op==TK_SELECT_COLUMN ){ assert( pExpr->pLeft->flags&EP_xIsSelect ); return sqlite3ExprAffinity( @@ -954,11 +954,11 @@ if( pExpr==0 ) return; assert( !ExprHasProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); z = pExpr->u.zToken; assert( z!=0 ); assert( z[0]!=0 ); - assert( n==sqlite3Strlen30(z) ); + assert( n==(u32)sqlite3Strlen30(z) ); if( z[1]==0 ){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); x = (ynVar)(++pParse->nVar); }else{ @@ -3190,10 +3190,14 @@ Table *pTab, /* The table containing the value */ int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ int iCol, /* Index of the column to extract */ int regOut /* Extract the value into this register */ ){ + if( pTab==0 ){ + sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); + return; + } if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); }else{ int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; int x = iCol; @@ -3882,10 +3886,21 @@ case TK_VECTOR: { sqlite3ErrorMsg(pParse, "row value misused"); break; } + + case TK_IF_NULL_ROW: { + int addrINR; + addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); + sqlite3ExprCachePush(pParse); + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + sqlite3ExprCachePop(pParse); + sqlite3VdbeJumpHere(v, addrINR); + sqlite3VdbeChangeP3(v, addrINR, inReg); + break; + } /* ** Form A: ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END ** Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -1085,42 +1085,57 @@ ** entry in the aChange[] array is set to -1. If the column is modified, ** the value is 0 or greater. Parameter chngRowid is set to true if the ** UPDATE statement modifies the rowid fields of the table. ** ** If any foreign key processing will be required, this function returns -** true. If there is no foreign key related processing, this function -** returns false. +** non-zero. If there is no foreign key related processing, this function +** returns zero. +** +** For an UPDATE, this function returns 2 if: +** +** * There are any FKs for which pTab is the child and the parent table, or +** * the UPDATE modifies one or more parent keys for which the action is +** not "NO ACTION" (i.e. is CASCADE, SET DEFAULT or SET NULL). +** +** Or, assuming some other foreign key processing is required, 1. */ int sqlite3FkRequired( Parse *pParse, /* Parse context */ Table *pTab, /* Table being modified */ int *aChange, /* Non-NULL for UPDATE operations */ int chngRowid /* True for UPDATE that affects rowid */ ){ + int eRet = 0; if( pParse->db->flags&SQLITE_ForeignKeys ){ if( !aChange ){ /* A DELETE operation. Foreign key processing is required if the ** table in question is either the child or parent table for any ** foreign key constraint. */ - return (sqlite3FkReferences(pTab) || pTab->pFKey); + eRet = (sqlite3FkReferences(pTab) || pTab->pFKey); }else{ /* This is an UPDATE. Foreign key processing is only required if the ** operation modifies one or more child or parent key columns. */ FKey *p; /* Check if any child key columns are being modified. */ for(p=pTab->pFKey; p; p=p->pNextFrom){ - if( fkChildIsModified(pTab, p, aChange, chngRowid) ) return 1; + if( 0==sqlite3_stricmp(pTab->zName, p->zTo) ) return 2; + if( fkChildIsModified(pTab, p, aChange, chngRowid) ){ + eRet = 1; + } } /* Check if any parent key columns are being modified. */ for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - if( fkParentIsModified(pTab, p, aChange, chngRowid) ) return 1; + if( fkParentIsModified(pTab, p, aChange, chngRowid) ){ + if( p->aAction[1]!=OE_None ) return 2; + eRet = 1; + } } } } - return 0; + return eRet; } /* ** This function is called when an UPDATE or DELETE operation is being ** compiled on table pTab, which is the parent table of foreign-key pFKey. Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -190,10 +190,27 @@ } } columnlist ::= columnlist COMMA columnname carglist. columnlist ::= columnname carglist. columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} + +// The following directive causes tokens ABORT, AFTER, ASC, etc. to +// fallback to ID if they will not parse as their original value. +// This obviates the need for the "id" nonterminal. +// +%fallback ID + ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW + CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR + IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT +%ifdef SQLITE_OMIT_COMPOUND_SELECT + EXCEPT INTERSECT UNION +%endif SQLITE_OMIT_COMPOUND_SELECT + REINDEX RENAME CTIME_KW IF + . +%wildcard ANY. // Define operator precedence early so that this is the first occurrence // of the operator tokens in the grammer. Keeping the operators together // causes them to be assigned integer values that are close together, // which keeps parser tables smaller. @@ -220,27 +237,10 @@ // An IDENTIFIER can be a generic identifier, or one of several // keywords. Any non-standard keyword can also be an identifier. // %token_class id ID|INDEXED. -// The following directive causes tokens ABORT, AFTER, ASC, etc. to -// fallback to ID if they will not parse as their original value. -// This obviates the need for the "id" nonterminal. -// -%fallback ID - ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW - CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR - IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN - QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW - ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT -%ifdef SQLITE_OMIT_COMPOUND_SELECT - EXCEPT INTERSECT UNION -%endif SQLITE_OMIT_COMPOUND_SELECT - REINDEX RENAME CTIME_KW IF - . -%wildcard ANY. - // And "ids" is an identifer-or-string. // %token_class ids ID|STRING. Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1327,37 +1327,41 @@ if( pParent ){ x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); assert( x==0 ); } addrOk = sqlite3VdbeMakeLabel(v); - if( pParent && pIdx==0 ){ - int iKey = pFK->aCol[0].iFrom; - assert( iKey>=0 && iKeynCol ); - if( iKey!=pTab->iPKey ){ - sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); - sqlite3ColumnDefault(v, pTab, iKey, regRow); - sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); - }else{ - sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); - } - sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v); + + /* Generate code to read the child key values into registers + ** regRow..regRow+n. If any of the child key values are NULL, this + ** row cannot cause an FK violation. Jump directly to addrOk in + ** this case. */ + for(j=0; jnCol; j++){ + int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); + } + + /* Generate code to query the parent index for a matching parent + ** key. If a match is found, jump to addrOk. */ + if( pIdx ){ + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + VdbeCoverage(v); + }else if( pParent ){ + int jmp = sqlite3VdbeCurrentAddr(v)+2; + sqlite3VdbeAddOp3(v, OP_SeekRowid, i, jmp, regRow); VdbeCoverage(v); sqlite3VdbeGoto(v, addrOk); - sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + assert( pFK->nCol==1 ); + } + + /* Generate code to report an FK violation to the caller. */ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); }else{ - for(j=0; jnCol; j++){ - sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, - aiCols ? aiCols[j] : pFK->aCol[j].iFrom, regRow+j); - sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); - } - if( pParent ){ - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); - sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); - VdbeCoverage(v); - } + sqlite3VdbeAddOp2(v, OP_Null, 0, regResult+1); } - sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1); sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); } Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -110,18 +110,17 @@ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ Select *pNew; Select standin; - sqlite3 *db = pParse->db; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) ); if( pNew==0 ){ - assert( db->mallocFailed ); + assert( pParse->db->mallocFailed ); pNew = &standin; } if( pEList==0 ){ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(pParse->db,TK_ASTERISK,0)); } pNew->pEList = pEList; pNew->op = TK_SELECT; pNew->selFlags = selFlags; pNew->iLimit = 0; @@ -130,11 +129,11 @@ pNew->zSelName[0] = 0; #endif pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; - if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; pNew->pHaving = pHaving; pNew->pOrderBy = pOrderBy; @@ -141,13 +140,13 @@ pNew->pPrior = 0; pNew->pNext = 0; pNew->pLimit = pLimit; pNew->pOffset = pOffset; pNew->pWith = 0; - assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 ); - if( db->mallocFailed ) { - clearSelect(db, pNew, pNew!=&standin); + assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || pParse->db->mallocFailed!=0 ); + if( pParse->db->mallocFailed ) { + clearSelect(pParse->db, pNew, pNew!=&standin); pNew = 0; }else{ assert( pNew->pSrc!=0 || pParse->nErr>0 ); } assert( pNew!=&standin ); @@ -1528,10 +1527,11 @@ Vdbe *v = pParse->pVdbe; int i; NameContext sNC; sNC.pSrcList = pTabList; sNC.pParse = pParse; + sNC.pNext = 0; for(i=0; inExpr; i++){ Expr *p = pEList->a[i].pExpr; const char *zType; #ifdef SQLITE_ENABLE_COLUMN_METADATA const char *zOrigDb = 0; @@ -1551,10 +1551,23 @@ #endif sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } #endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } + +/* +** Return the Table objecct in the SrcList that has cursor iCursor. +** Or return NULL if no such Table object exists in the SrcList. +*/ +static Table *tableWithCursor(SrcList *pList, int iCursor){ + int j; + for(j=0; jnSrc; j++){ + if( pList->a[j].iCursor==iCursor ) return pList->a[j].pTab; + } + return 0; +} + /* ** Generate code that will tell the VDBE the names of columns ** in the result set. This information is used to provide the ** azCol[] values in the callback. @@ -1563,11 +1576,12 @@ Parse *pParse, /* Parser context */ SrcList *pTabList, /* List of tables */ ExprList *pEList /* Expressions defining the result set */ ){ Vdbe *v = pParse->pVdbe; - int i, j; + int i; + Table *pTab; sqlite3 *db = pParse->db; int fullNames, shortNames; #ifndef SQLITE_OMIT_EXPLAIN /* If this is an EXPLAIN, skip this step */ @@ -1588,19 +1602,15 @@ p = pEList->a[i].pExpr; if( NEVER(p==0) ) continue; if( pEList->a[i].zName ){ char *zName = pEList->a[i].zName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); - }else if( p->op==TK_COLUMN || p->op==TK_AGG_COLUMN ){ - Table *pTab; + }else if( (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) + && (pTab = tableWithCursor(pTabList, p->iTable))!=0 + ){ char *zCol; int iCol = p->iColumn; - for(j=0; ALWAYS(jnSrc); j++){ - if( pTabList->a[j].iCursor==p->iTable ) break; - } - assert( jnSrc ); - pTab = pTabList->a[j].pTab; if( iCol<0 ) iCol = pTab->iPKey; assert( iCol==-1 || (iCol>=0 && iColnCol) ); if( iCol<0 ){ zCol = "rowid"; }else{ @@ -1678,11 +1688,11 @@ Table *pTab; /* Table associated with this expression */ while( pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } - if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ + if( pColExpr->op==TK_COLUMN && pColExpr->pTab!=0 ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; pTab = pColExpr->pTab; if( iCol<0 ) iCol = pTab->iPKey; zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; @@ -3132,13 +3142,28 @@ return pParse->nErr!=0; } #endif #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + +/* An instance of the SubstContext object describes an substitution edit +** to be performed on a parse tree. +** +** All references to columns in table iTable are to be replaced by corresponding +** expressions in pEList. +*/ +typedef struct SubstContext { + Parse *pParse; /* The parsing context */ + int iTable; /* Replace references to this table */ + int iNewTable; /* New table number */ + int isLeftJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + ExprList *pEList; /* Replacement expressions */ +} SubstContext; + /* Forward Declarations */ -static void substExprList(Parse*, ExprList*, int, ExprList*); -static void substSelect(Parse*, Select *, int, ExprList*, int); +static void substExprList(SubstContext*, ExprList*); +static void substSelect(SubstContext*, Select*, int); /* ** Scan through the expression pExpr. Replace every reference to ** a column in table number iTable with a copy of the iColumn-th ** entry in pEList. (But leave references to the ROWID column @@ -3145,33 +3170,42 @@ ** unchanged.) ** ** This routine is part of the flattening procedure. A subquery ** whose result set is defined by pEList appears as entry in the ** FROM clause of a SELECT such that the VDBE cursor assigned to that -** FORM clause entry is iTable. This routine make the necessary +** FORM clause entry is iTable. This routine makes the necessary ** changes to pExpr so that it refers directly to the source table ** of the subquery rather the result set of the subquery. */ static Expr *substExpr( - Parse *pParse, /* Report errors here */ - Expr *pExpr, /* Expr in which substitution occurs */ - int iTable, /* Table to be substituted */ - ExprList *pEList /* Substitute expressions */ + SubstContext *pSubst, /* Description of the substitution */ + Expr *pExpr /* Expr in which substitution occurs */ ){ - sqlite3 *db = pParse->db; if( pExpr==0 ) return 0; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){ + if( ExprHasProperty(pExpr, EP_FromJoin) && pExpr->iRightJoinTable==pSubst->iTable ){ + pExpr->iRightJoinTable = pSubst->iNewTable; + } + if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable ){ if( pExpr->iColumn<0 ){ pExpr->op = TK_NULL; }else{ Expr *pNew; - Expr *pCopy = pEList->a[pExpr->iColumn].pExpr; - assert( pEList!=0 && pExpr->iColumnnExpr ); + Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + Expr ifNullRow; + assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); assert( pExpr->pLeft==0 && pExpr->pRight==0 ); if( sqlite3ExprIsVector(pCopy) ){ - sqlite3VectorErrorMsg(pParse, pCopy); + sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ + sqlite3 *db = pSubst->pParse->db; + if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){ + memset(&ifNullRow, 0, sizeof(ifNullRow)); + ifNullRow.op = TK_IF_NULL_ROW; + ifNullRow.pLeft = pCopy; + ifNullRow.iTable = pSubst->iNewTable; + pCopy = &ifNullRow; + } pNew = sqlite3ExprDup(db, pCopy, 0); if( pNew && (pExpr->flags & EP_FromJoin) ){ pNew->iRightJoinTable = pExpr->iRightJoinTable; pNew->flags |= EP_FromJoin; } @@ -3178,55 +3212,51 @@ sqlite3ExprDelete(db, pExpr); pExpr = pNew; } } }else{ - pExpr->pLeft = substExpr(pParse, pExpr->pLeft, iTable, pEList); - pExpr->pRight = substExpr(pParse, pExpr->pRight, iTable, pEList); + pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); + pExpr->pRight = substExpr(pSubst, pExpr->pRight); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(pParse, pExpr->x.pSelect, iTable, pEList, 1); + substSelect(pSubst, pExpr->x.pSelect, 1); }else{ - substExprList(pParse, pExpr->x.pList, iTable, pEList); + substExprList(pSubst, pExpr->x.pList); } } return pExpr; } static void substExprList( - Parse *pParse, /* Report errors here */ - ExprList *pList, /* List to scan and in which to make substitutes */ - int iTable, /* Table to be substituted */ - ExprList *pEList /* Substitute values */ + SubstContext *pSubst, /* Description of the substitution */ + ExprList *pList /* List to scan and in which to make substitutes */ ){ int i; if( pList==0 ) return; for(i=0; inExpr; i++){ - pList->a[i].pExpr = substExpr(pParse, pList->a[i].pExpr, iTable, pEList); + pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr); } } static void substSelect( - Parse *pParse, /* Report errors here */ - Select *p, /* SELECT statement in which to make substitutions */ - int iTable, /* Table to be replaced */ - ExprList *pEList, /* Substitute values */ - int doPrior /* Do substitutes on p->pPrior too */ + SubstContext *pSubst, /* Description of the substitution */ + Select *p, /* SELECT statement in which to make substitutions */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; do{ - substExprList(pParse, p->pEList, iTable, pEList); - substExprList(pParse, p->pGroupBy, iTable, pEList); - substExprList(pParse, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(pParse, p->pHaving, iTable, pEList); - p->pWhere = substExpr(pParse, p->pWhere, iTable, pEList); + substExprList(pSubst, p->pEList); + substExprList(pSubst, p->pGroupBy); + substExprList(pSubst, p->pOrderBy); + p->pHaving = substExpr(pSubst, p->pHaving); + p->pWhere = substExpr(pSubst, p->pWhere); pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(pParse, pItem->pSelect, iTable, pEList, 1); + substSelect(pSubst, pItem->pSelect, 1); if( pItem->fg.isTabFunc ){ - substExprList(pParse, pItem->u1.pFuncArg, iTable, pEList); + substExprList(pSubst, pItem->u1.pFuncArg); } } }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -3265,12 +3295,12 @@ ** (2) The subquery is not an aggregate or (2a) the outer query is not a join ** and (2b) the outer query does not use subqueries other than the one ** FROM-clause subquery that is a candidate for flattening. (2b is ** due to ticket [2f7170d73bf9abf80] from 2015-02-09.) ** -** (3) The subquery is not the right operand of a left outer join -** (Originally ticket #306. Strengthened by ticket #3300) +** (3) The subquery is not the right operand of a LEFT JOIN +** or the subquery is not itself a join. ** ** (4) The subquery is not DISTINCT. ** ** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT ** sub-queries that were excluded from this optimization. Restriction @@ -3278,11 +3308,11 @@ ** ** (6) The subquery does not use aggregates or the outer query is not ** DISTINCT. ** ** (7) The subquery has a FROM clause. TODO: For subqueries without -** A FROM clause, consider adding a FROM close with the special +** A FROM clause, consider adding a FROM clause with the special ** table sqlite_once that consists of a single row containing a ** single NULL. ** ** (8) The subquery does not use LIMIT or the outer query is not a join. ** @@ -3384,10 +3414,12 @@ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ SrcList *pSubSrc; /* The FROM clause of the subquery */ ExprList *pList; /* The result set of the outer query */ int iParent; /* VDBE cursor number of the pSub result set temp table */ + int iNewParent = -1;/* Replacement table for iParent */ + int isLeftJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ int i; /* Loop counter */ Expr *pWhere; /* The WHERE clause */ struct SrcList_item *pSubitem; /* The subquery */ sqlite3 *db = pParse->db; @@ -3410,11 +3442,11 @@ || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0 ){ return 0; /* Restriction (2b) */ } } - + pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET ** because they could be computed at compile-time. But when LIMIT and OFFSET @@ -3448,44 +3480,29 @@ } if( (p->selFlags & SF_Recursive) && pSub->pPrior ){ return 0; /* Restriction (23) */ } - /* OBSOLETE COMMENT 1: - ** Restriction 3: If the subquery is a join, make sure the subquery is - ** not used as the right operand of an outer join. Examples of why this - ** is not allowed: + /* + ** If the subquery is the right operand of a LEFT JOIN, then the + ** subquery may not be a join itself. Example of why this is not allowed: ** ** t1 LEFT OUTER JOIN (t2 JOIN t3) ** ** If we flatten the above, we would get ** ** (t1 LEFT OUTER JOIN t2) JOIN t3 ** ** which is not at all the same thing. ** - ** OBSOLETE COMMENT 2: - ** Restriction 12: If the subquery is the right operand of a left outer - ** join, make sure the subquery has no WHERE clause. - ** An examples of why this is not allowed: - ** - ** t1 LEFT OUTER JOIN (SELECT * FROM t2 WHERE t2.x>0) - ** - ** If we flatten the above, we would get - ** - ** (t1 LEFT OUTER JOIN t2) WHERE t2.x>0 - ** - ** But the t2.x>0 test will always fail on a NULL row of t2, which - ** effectively converts the OUTER JOIN into an INNER JOIN. - ** - ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE: - ** Ticket #3300 shows that flattening the right term of a LEFT JOIN - ** is fraught with danger. Best to avoid the whole thing. If the - ** subquery is the right term of a LEFT JOIN, then do not flatten. + ** See also tickets #306, #350, and #3300. */ if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ - return 0; + isLeftJoin = 1; + if( pSubSrc->nSrc>1 ){ + return 0; /* Restriction (3) */ + } } /* Restriction 17: If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries ** that make up the compound SELECT are allowed to be aggregate or distinct @@ -3690,10 +3707,11 @@ */ for(i=0; ia[i+iFrom].pUsing); assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); pSrc->a[i+iFrom] = pSubSrc->a[i]; + iNewParent = pSubSrc->a[i].iCursor; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } pSrc->a[iFrom].fg.jointype = jointype; /* Now begin substituting subquery result set expressions for @@ -3735,10 +3753,13 @@ assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; } pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); + if( isLeftJoin ){ + setJoinExpr(pWhere, iNewParent); + } if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; pParent->pHaving = sqlite3ExprAnd(db, @@ -3748,11 +3769,17 @@ pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); } if( db->mallocFailed==0 ){ - substSelect(pParse, pParent, iParent, pSub->pEList, 0); + SubstContext x; + x.pParse = pParse; + x.iTable = iParent; + x.iNewTable = iNewParent; + x.isLeftJoin = isLeftJoin; + x.pEList = pSub->pEList; + substSelect(&x, pParent, 0); } /* The flattened query is distinct if either the inner or the ** outer query is distinct. */ @@ -3851,12 +3878,18 @@ } if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ nChng++; while( pSubq ){ + SubstContext x; pNew = sqlite3ExprDup(pParse->db, pWhere, 0); - pNew = substExpr(pParse, pNew, iCursor, pSubq->pEList); + x.pParse = pParse; + x.iTable = iCursor; + x.iNewTable = iCursor; + x.isLeftJoin = 0; + x.pEList = pSubq->pEList; + pNew = substExpr(&x, pNew); pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); pSubq = pSubq->pPrior; } } return nChng; Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -425,10 +425,40 @@ utf8_printf(iotrace, "%s", z); sqlite3_free(z); } #endif +/* +** Output string zUtf to stream pOut as w characters. If w is negative, +** then right-justify the text. W is the width in UTF-8 characters, not +** in bytes. This is different from the %*.*s specification in printf +** since with %*.*s the width is measured in bytes, not characters. +*/ +static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ + int i; + int n; + int aw = w<0 ? -w : w; + char zBuf[1000]; + if( aw>sizeof(zBuf)/3 ) aw = sizeof(zBuf)/3; + for(i=n=0; zUtf[i]; i++){ + if( (zUtf[i]&0xc0)!=0x80 ){ + n++; + if( n==aw ){ + do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); + break; + } + } + } + if( n>=aw ){ + utf8_printf(pOut, "%.*s", i, zUtf); + }else if( w<0 ){ + utf8_printf(pOut, "%*s%s", aw-n, "", zUtf); + }else{ + utf8_printf(pOut, "%s%*s", zUtf, aw-n, ""); + } +} + /* ** Determines if a string is a number of not. */ static int isNumber(const char *z, int *realnum){ @@ -1876,17 +1906,12 @@ } if( iactualWidth) ){ p->actualWidth[i] = w; } if( showHdr ){ - if( w<0 ){ - utf8_printf(p->out,"%*.*s%s",-w,-w,azCol[i], - i==nArg-1 ? rowSep : " "); - }else{ - utf8_printf(p->out,"%-*.*s%s",w,w,azCol[i], - i==nArg-1 ? rowSep : " "); - } + utf8_width_print(p->out, w, azCol[i]); + utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : " "); } } if( showHdr ){ for(i=0; iiIndentnIndent ){ utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - if( w<0 ){ - utf8_printf(p->out,"%*.*s%s",-w,-w, - azArg[i] ? azArg[i] : p->nullValue, - i==nArg-1 ? rowSep : " "); - }else{ - utf8_printf(p->out,"%-*.*s%s",w,w, - azArg[i] ? azArg[i] : p->nullValue, - i==nArg-1 ? rowSep : " "); - } + utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue); + utf8_printf(p->out, "%s", i==nArg-1 ? rowSep : " "); } break; } case MODE_Semi: { /* .schema and .fullschema output */ printSchemaLine(p->out, azArg[0], ";\n"); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -3701,11 +3701,11 @@ ** Unprotected sqlite3_value objects may only be used with ** [sqlite3_result_value()] and [sqlite3_bind_value()]. ** The [sqlite3_value_blob | sqlite3_value_type()] family of ** interfaces require protected sqlite3_value objects. */ -typedef struct Mem sqlite3_value; +typedef struct sqlite3_value sqlite3_value; /* ** CAPI3REF: SQL Function Context Object ** ** The context in which an SQL function executes is stored in an Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -3322,10 +3322,11 @@ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ int *aiCol; /* array of column indexes */ struct IdxCover *pIdxCover; /* Check for index coverage */ + struct IdxExprTrans *pIdxTrans; /* Convert indexed expr to column */ } u; }; /* Forward declarations */ int sqlite3WalkExpr(Walker*, Expr*); Index: src/test_delete.c ================================================================== --- src/test_delete.c +++ src/test_delete.c @@ -17,18 +17,16 @@ ** * The SQLITE_ENABLE_8_3_NAMES version of the db, journal or wal files. ** * Files created by the test_multiplex.c module to extend any of the ** above. */ -#if SQLITE_OS_WIN -# include -# define F_OK 0 -#else +#ifndef SQLITE_OS_WIN # include +# include #endif #include -#include +#include #include "sqlite3.h" /* The following #defines are copied from test_multiplex.c */ #ifndef MX_CHUNK_NUMBER # define MX_CHUNK_NUMBER 299 @@ -55,27 +53,41 @@ /* ** zFile is a filename. Assuming no error occurs, if this file exists, ** set *pbExists to true and unlink it. Or, if the file does not exist, ** set *pbExists to false before returning. ** -** If an error occurs, the value of errno is returned. Or, if no error -** occurs, zero is returned. +** If an error occurs, non-zero is returned. Or, if no error occurs, zero. */ -static int sqlite3DeleteUnlinkIfExists(const char *zFile, int *pbExists){ - int rc; +static int sqlite3DeleteUnlinkIfExists( + sqlite3_vfs *pVfs, + const char *zFile, + int *pbExists +){ + int rc = SQLITE_ERROR; +#if SQLITE_OS_WIN + if( pVfs ){ + if( pbExists ) *pbExists = 1; + rc = pVfs->xDelete(pVfs, zFile, 0); + if( rc==SQLITE_IOERR_DELETE_NOENT ){ + if( pbExists ) *pbExists = 0; + rc = SQLITE_OK; + } + } +#else + assert( pVfs==0 ); rc = access(zFile, F_OK); if( rc ){ if( errno==ENOENT ){ if( pbExists ) *pbExists = 0; - return 0; + rc = SQLITE_OK; } - return errno; + }else{ + if( pbExists ) *pbExists = 1; + rc = unlink(zFile); } - if( pbExists ) *pbExists = 1; - rc = unlink(zFile); - if( rc ) return errno; - return 0; +#endif + return rc; } /* ** Delete the database file identified by the string argument passed to this ** function. The string must contain a filename, not an SQLite URI. @@ -100,10 +112,16 @@ { "%s-wal%03d", 0, 0 }, { "%s%03d", 0, 1 }, { "%s-journal%03d", SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET, 1 }, { "%s-wal%03d", SQLITE_MULTIPLEX_WAL_8_3_OFFSET, 1 }, }; + +#ifdef SQLITE_OS_WIN + sqlite3_vfs *pVfs = sqlite3_vfs_find("win32"); +#else + sqlite3_vfs *pVfs = 0; +#endif /* Allocate a buffer large enough for any of the files that need to be ** deleted. */ nBuf = (int)strlen(zFile) + 100; zBuf = (char*)sqlite3_malloc(nBuf); @@ -111,14 +129,14 @@ /* Delete both the regular and 8.3 filenames versions of the database, ** journal, wal and shm files. */ for(i=0; rc==0 && izFmt, zFile, iChunk+p->iOffset); if( p->b83 ) sqlite3Delete83Name(zBuf); - rc = sqlite3DeleteUnlinkIfExists(zBuf, &bExists); + rc = sqlite3DeleteUnlinkIfExists(pVfs, zBuf, &bExists); if( bExists==0 || rc!=0 ) break; } } sqlite3_free(zBuf); return (rc ? SQLITE_ERROR : SQLITE_OK); } Index: src/treeview.c ================================================================== --- src/treeview.c +++ src/treeview.c @@ -236,19 +236,24 @@ ** Generate a human-readable explanation of an expression tree. */ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ const char *zBinOp = 0; /* Binary operator */ const char *zUniOp = 0; /* Unary operator */ - char zFlgs[30]; + char zFlgs[60]; pView = sqlite3TreeViewPush(pView, moreToFollow); if( pExpr==0 ){ sqlite3TreeViewLine(pView, "nil"); sqlite3TreeViewPop(pView); return; } if( pExpr->flags ){ - sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x iRJT=%d", + pExpr->flags, pExpr->iRightJoinTable); + }else{ + sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + } }else{ zFlgs[0] = 0; } switch( pExpr->op ){ case TK_AGG_COLUMN: { @@ -462,10 +467,15 @@ } case TK_SELECT_COLUMN: { sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn); sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0); break; + } + case TK_IF_NULL_ROW: { + sqlite3TreeViewLine(pView, "IF-NULL-ROW %d", pExpr->iTable); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; } default: { sqlite3TreeViewLine(pView, "op=%d", pExpr->op); break; } Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -283,11 +283,11 @@ ** ** FIXME: Be smarter about omitting indexes that use expressions. */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; - if( chngKey || hasFK || pIdx->pPartIdxWhere || pIdx==pPk ){ + if( chngKey || hasFK>1 || pIdx->pPartIdxWhere || pIdx==pPk ){ reg = ++pParse->nMem; pParse->nMem += pIdx->nColumn; }else{ reg = 0; for(i=0; inKeyCol; i++){ @@ -638,11 +638,11 @@ ** is the column index supplied by the user. */ assert( regNew==regNewRowid+1 ); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK sqlite3VdbeAddOp3(v, OP_Delete, iDataCur, - OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP), + OPFLAG_ISUPDATE | ((hasFK>1 || chngKey) ? 0 : OPFLAG_ISNOOP), regNewRowid ); if( eOnePass==ONEPASS_MULTI ){ assert( hasFK==0 && chngKey==0 ); sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); @@ -649,11 +649,11 @@ } if( !pParse->nested ){ sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } #else - if( hasFK || chngKey ){ + if( hasFK>1 || chngKey ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } #endif if( bReplace || chngKey ){ sqlite3VdbeJumpHere(v, addr1); Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -484,10 +484,11 @@ } static void registerTrace(int iReg, Mem *p){ printf("REG[%d] = ", iReg); memTracePrint(p); printf("\n"); + sqlite3VdbeCheckMemInvariants(p); } #endif #ifdef SQLITE_DEBUG # define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) @@ -1150,11 +1151,11 @@ ** previously copied using OP_SCopy, the copies will continue to be valid. */ case OP_SoftNull: { assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pOut = &aMem[pOp->p1]; - pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; + pOut->flags = (pOut->flags&~(MEM_Undefined|MEM_AffMask))|MEM_Null; break; } /* Opcode: Blob P1 P2 * P4 * ** Synopsis: r[P2]=P4 (len=P1) @@ -1493,11 +1494,10 @@ type1 = numericType(pIn1); pIn2 = &aMem[pOp->p2]; type2 = numericType(pIn2); pOut = &aMem[pOp->p3]; flags = pIn1->flags | pIn2->flags; - if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; if( (type1 & type2 & MEM_Int)!=0 ){ iA = pIn1->u.i; iB = pIn2->u.i; bIntint = 1; switch( pOp->opcode ){ @@ -1517,10 +1517,12 @@ break; } } pOut->u.i = iB; MemSetTypeFlag(pOut, MEM_Int); + }else if( (flags & MEM_Null)!=0 ){ + goto arithmetic_result_is_null; }else{ bIntint = 0; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -2427,10 +2429,27 @@ if( (pIn1->flags & MEM_Null)==0 ){ goto jump_to_p2; } break; } + +/* Opcode: IfNullRow P1 P2 P3 * * +** Synopsis: if P1.nullRow then r[P3]=NULL, goto P2 +** +** Check the cursor P1 to see if it is currently pointing at a NULL row. +** If it is, then set register P3 to NULL and jump immediately to P2. +** If P1 is not on a NULL row, then fall through without making any +** changes. +*/ +case OP_IfNullRow: { /* jump */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + if( p->apCsr[pOp->p1]->nullRow ){ + sqlite3VdbeMemSetNull(aMem + pOp->p3); + goto jump_to_p2; + } + break; +} /* Opcode: Column P1 P2 P3 P4 P5 ** Synopsis: r[P3]=PX ** ** Interpret the data that cursor P1 points to as a structure built using Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -28,11 +28,11 @@ /* ** The names of the following types declared in vdbeInt.h are required ** for the VdbeOp definition. */ -typedef struct Mem Mem; +typedef struct sqlite3_value Mem; typedef struct SubProgram SubProgram; /* ** A single instruction of the virtual machine has an opcode ** and as many as three operands. The instruction is recorded Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -183,11 +183,11 @@ /* ** Internally, the vdbe manipulates nearly all SQL values as Mem ** structures. Each Mem struct may cache multiple representations (string, ** integer etc.) of the same value. */ -struct Mem { +struct sqlite3_value { union MemValue { double r; /* Real value used when MEM_Real is set in flags */ i64 i; /* Integer value used when MEM_Int is set in flags */ int nZero; /* Used when bit MEM_Zero is set in flags */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -877,11 +877,11 @@ ** nOp entries. */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ if( aOp ){ Op *pOp; - for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ + for(pOp=&aOp[nOp-1]; pOp>=aOp; pOp--){ if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); #endif } Index: src/vdbemem.c ================================================================== --- src/vdbemem.c +++ src/vdbemem.c @@ -38,10 +38,14 @@ assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); /* Cannot be both MEM_Int and MEM_Real at the same time */ assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + /* Cannot be both MEM_Null and some other type */ + assert( (p->flags & MEM_Null)==0 || + (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob))==0 ); + /* The szMalloc field holds the correct memory allocation size */ assert( p->szMalloc==0 || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) ); /* If p holds a string or blob, the Mem.z must point to exactly @@ -123,30 +127,28 @@ assert( bPreserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); testcase( bPreserve && pMem->z==0 ); assert( pMem->szMalloc==0 || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); - if( pMem->szMallocszMalloc>0 && pMem->z==pMem->zMalloc ){ - pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); - bPreserve = 0; - }else{ - if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); - pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); - } - if( pMem->zMalloc==0 ){ - sqlite3VdbeMemSetNull(pMem); - pMem->z = 0; - pMem->szMalloc = 0; - return SQLITE_NOMEM_BKPT; - }else{ - pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); - } + if( n<32 ) n = 32; + if( bPreserve && pMem->szMalloc>0 && pMem->z==pMem->zMalloc ){ + pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + bPreserve = 0; + }else{ + if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); + pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); + } + if( pMem->zMalloc==0 ){ + sqlite3VdbeMemSetNull(pMem); + pMem->z = 0; + pMem->szMalloc = 0; + return SQLITE_NOMEM_BKPT; + }else{ + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); } - if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){ + if( bPreserve && pMem->z && ALWAYS(pMem->z!=pMem->zMalloc) ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( (pMem->flags&MEM_Dyn)!=0 ){ assert( pMem->xDel!=0 && pMem->xDel!=SQLITE_DYNAMIC ); pMem->xDel((void *)(pMem->z)); Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -1051,11 +1051,11 @@ /* Check to see the left operand is a column in a virtual table */ if( NEVER(pExpr==0) ) return pDef; if( pExpr->op!=TK_COLUMN ) return pDef; pTab = pExpr->pTab; - if( NEVER(pTab==0) ) return pDef; + if( pTab==0 ) return pDef; if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); assert( pVtab->pModule!=0 ); pMod = (sqlite3_module *)pVtab->pModule; Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -3435,11 +3435,11 @@ whereLoopClear(db, pNew); return rc; } /* -** Examine a WherePath (with the addition of the extra WhereLoop of the 5th +** Examine a WherePath (with the addition of the extra WhereLoop of the 6th ** parameters) to see if it outputs rows in the requested ORDER BY ** (or GROUP BY) without requiring a separate sort operation. Return N: ** ** N>0: N terms of the ORDER BY clause are satisfied ** N==0: No terms of the ORDER BY clause are satisfied @@ -3530,10 +3530,12 @@ pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ if( pLoop->u.vtab.isOrdered ) obSat = obDone; break; + }else{ + pLoop->u.btree.nIdxCol = 0; } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; /* Mark off any ORDER BY term X that is a column in the table of ** the current loop for which there is term in the WHERE @@ -3675,10 +3677,11 @@ if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; } + pLoop->u.btree.nIdxCol = j+1; isMatch = 1; break; } if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ /* Make sure the sort order is compatible in an ORDER BY clause. @@ -4445,10 +4448,11 @@ goto whereBeginError; } pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; + pWInfo->pWhere = pWhere; pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; @@ -4755,10 +4759,11 @@ sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIx); if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 + && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ } VdbeComment((v, "%s", pIx->zName)); #ifdef SQLITE_ENABLE_COLUMN_USED_MASK @@ -4843,18 +4848,47 @@ sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; pLoop = pLevel->pWLoop; - sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + int addrSeek = 0; + Index *pIdx; + int n; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + && (pLoop->wsFlags & WHERE_INDEXED)!=0 + && (pIdx = pLoop->u.btree.pIndex)->hasStat1 + && (n = pLoop->u.btree.nIdxCol)>0 + && pIdx->aiRowLogEst[n]>=36 + ){ + int r1 = pParse->nMem+1; + int j, op; + for(j=0; jiIdxCur, j, r1+j); + } + pParse->nMem += n+1; + op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT; + addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n); + VdbeCoverageIf(v, op==OP_SeekLT); + VdbeCoverageIf(v, op==OP_SeekGT); + sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + } +#endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ + /* The common case: Advance to the next row */ + sqlite3VdbeResolveLabel(v, pLevel->addrCont); sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); VdbeCoverage(v); VdbeCoverageIf(v, pLevel->op==OP_Next); VdbeCoverageIf(v, pLevel->op==OP_Prev); VdbeCoverageIf(v, pLevel->op==OP_VNext); +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); +#endif + }else{ + sqlite3VdbeResolveLabel(v, pLevel->addrCont); } if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; sqlite3VdbeResolveLabel(v, pLevel->addrNxt); @@ -4973,10 +5007,12 @@ assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 || pWInfo->eOnePass ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; + }else if( pOp->opcode==OP_IfNullRow ){ + pOp->p1 = pLevel->iIdxCur; } } } } Index: src/whereInt.h ================================================================== --- src/whereInt.h +++ src/whereInt.h @@ -122,10 +122,11 @@ union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ u16 nBtm; /* Size of BTM vector */ u16 nTop; /* Size of TOP vector */ + u16 nIdxCol; /* Index column used for ORDER BY */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ int idxNum; /* Index number */ u8 needFree; /* True if sqlite3_free(idxStr) is needed */ @@ -415,10 +416,11 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pResultSet; /* Result set of the query */ + Expr *pWhere; /* The complete WHERE clause */ LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -1036,10 +1036,73 @@ }else{ assert( nReg==1 ); sqlite3ExprCode(pParse, p, iReg); } } + +/* An instance of the IdxExprTrans object carries information about a +** mapping from an expression on table columns into a column in an index +** down through the Walker. +*/ +typedef struct IdxExprTrans { + Expr *pIdxExpr; /* The index expression */ + int iTabCur; /* The cursor of the corresponding table */ + int iIdxCur; /* The cursor for the index */ + int iIdxCol; /* The column for the index */ +} IdxExprTrans; + +/* The walker node callback used to transform matching expressions into +** a reference to an index column for an index on an expression. +** +** If pExpr matches, then transform it into a reference to the index column +** that contains the value of pExpr. +*/ +static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ + IdxExprTrans *pX = p->u.pIdxTrans; + if( sqlite3ExprCompare(pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ + pExpr->op = TK_COLUMN; + pExpr->iTable = pX->iIdxCur; + pExpr->iColumn = pX->iIdxCol; + pExpr->pTab = 0; + return WRC_Prune; + }else{ + return WRC_Continue; + } +} + +/* +** For an indexes on expression X, locate every instance of expression X in pExpr +** and change that subexpression into a reference to the appropriate column of +** the index. +*/ +static void whereIndexExprTrans( + Index *pIdx, /* The Index */ + int iTabCur, /* Cursor of the table that is being indexed */ + int iIdxCur, /* Cursor of the index itself */ + WhereInfo *pWInfo /* Transform expressions in this WHERE clause */ +){ + int iIdxCol; /* Column number of the index */ + ExprList *aColExpr; /* Expressions that are indexed */ + Walker w; + IdxExprTrans x; + aColExpr = pIdx->aColExpr; + if( aColExpr==0 ) return; /* Not an index on expressions */ + memset(&w, 0, sizeof(w)); + w.xExprCallback = whereIndexExprTransNode; + w.u.pIdxTrans = &x; + x.iTabCur = iTabCur; + x.iIdxCur = iIdxCur; + for(iIdxCol=0; iIdxColnExpr; iIdxCol++){ + if( pIdx->aiColumn[iIdxCol]!=XN_EXPR ) continue; + assert( aColExpr->a[iIdxCol].pExpr!=0 ); + x.iIdxCol = iIdxCol; + x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; + sqlite3WalkExpr(&w, pWInfo->pWhere); + sqlite3WalkExprList(&w, pWInfo->pOrderBy); + sqlite3WalkExprList(&w, pWInfo->pResultSet); + } +} /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. */ @@ -1617,10 +1680,17 @@ sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); } sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } + + /* If pIdx is an index on one or more expressions, then look through + ** all the expressions in pWInfo and try to transform matching expressions + ** into reference to index columns. + */ + whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); + /* Record the instruction used to terminate the loop. */ if( pLoop->wsFlags & WHERE_ONEROW ){ pLevel->op = OP_Noop; }else if( bRev ){ Index: src/whereexpr.c ================================================================== --- src/whereexpr.c +++ src/whereexpr.c @@ -828,31 +828,50 @@ /* ** Expression pExpr is one operand of a comparison operator that might ** be useful for indexing. This routine checks to see if pExpr appears ** in any index. Return TRUE (1) if pExpr is an indexed term and return -** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor -** number of the table that is indexed and *piColumn to the column number +** FALSE (0) if not. If TRUE is returned, also set aiCurCol[0] to the cursor +** number of the table that is indexed and aiCurCol[1] to the column number ** of the column that is indexed, or XN_EXPR (-2) if an expression is being ** indexed. ** ** If pExpr is a TK_COLUMN column reference, then this routine always returns ** true even if that particular column is not indexed, because the column ** might be added to an automatic index later. */ -static int exprMightBeIndexed( +static SQLITE_NOINLINE int exprMightBeIndexed2( SrcList *pFrom, /* The FROM clause */ - int op, /* The specific comparison operator */ Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ - Expr *pExpr, /* An operand of a comparison operator */ - int *piCur, /* Write the referenced table cursor number here */ - int *piColumn /* Write the referenced table column number here */ + int *aiCurCol, /* Write the referenced table cursor and column here */ + Expr *pExpr /* An operand of a comparison operator */ ){ Index *pIdx; int i; int iCur; - + for(i=0; mPrereq>1; i++, mPrereq>>=1){} + iCur = pFrom->a[i].iCursor; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + aiCurCol[0] = iCur; + aiCurCol[1] = XN_EXPR; + return 1; + } + } + } + return 0; +} +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + int *aiCurCol, /* Write the referenced table cursor & column here */ + Expr *pExpr, /* An operand of a comparison operator */ + int op /* The specific comparison operator */ +){ /* If this expression is a vector to the left or right of a ** inequality constraint (>, <, >= or <=), perform the processing ** on the first element of the vector. */ assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE ); assert( TK_ISop==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){ pExpr = pExpr->x.pList->a[0].pExpr; } if( pExpr->op==TK_COLUMN ){ - *piCur = pExpr->iTable; - *piColumn = pExpr->iColumn; + aiCurCol[0] = pExpr->iTable; + aiCurCol[1] = pExpr->iColumn; return 1; } if( mPrereq==0 ) return 0; /* No table references */ if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ - for(i=0; mPrereq>1; i++, mPrereq>>=1){} - iCur = pFrom->a[i].iCursor; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->aColExpr==0 ) continue; - for(i=0; inKeyCol; i++){ - if( pIdx->aiColumn[i]!=XN_EXPR ) continue; - if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ - *piCur = iCur; - *piColumn = XN_EXPR; - return 1; - } - } - } - return 0; + return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr); } /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -959,11 +965,11 @@ pTerm->prereqAll = prereqAll; pTerm->leftCursor = -1; pTerm->iParent = -1; pTerm->eOperator = 0; if( allowedOp(op) ){ - int iCur, iColumn; + int aiCurCol[2]; Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; if( pTerm->iField>0 ){ @@ -970,18 +976,18 @@ assert( op==TK_IN ); assert( pLeft->op==TK_VECTOR ); pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr; } - if( exprMightBeIndexed(pSrc, op, prereqLeft, pLeft, &iCur, &iColumn) ){ - pTerm->leftCursor = iCur; - pTerm->u.leftColumn = iColumn; + if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){ + pTerm->leftCursor = aiCurCol[0]; + pTerm->u.leftColumn = aiCurCol[1]; pTerm->eOperator = operatorMask(op) & opMask; } if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; if( pRight - && exprMightBeIndexed(pSrc, op, pTerm->prereqRight, pRight, &iCur,&iColumn) + && exprMightBeIndexed(pSrc, pTerm->prereqRight, aiCurCol, pRight, op) ){ WhereTerm *pNew; Expr *pDup; u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ assert( pTerm->iField==0 ); @@ -1007,12 +1013,12 @@ }else{ pDup = pExpr; pNew = pTerm; } exprCommute(pParse, pDup); - pNew->leftCursor = iCur; - pNew->u.leftColumn = iColumn; + pNew->leftCursor = aiCurCol[0]; + pNew->u.leftColumn = aiCurCol[1]; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; } ADDED test/distinct2.test Index: test/distinct2.test ================================================================== --- test/distinct2.test +++ test/distinct2.test @@ -0,0 +1,183 @@ +# 2016-04-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is DISTINCT queries using the skip-ahead +# optimization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix distinct2 + +do_execsql_test 100 { + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(0),(1),(2); + CREATE TABLE t2 AS + SELECT DISTINCT a.x AS aa, b.x AS bb + FROM t1 a, t1 b; + SELECT *, '|' FROM t2 ORDER BY aa, bb; +} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |} +do_execsql_test 110 { + DROP TABLE t2; + CREATE TABLE t2 AS + SELECT DISTINCT a.x AS aa, b.x AS bb + FROM t1 a, t1 b + WHERE a.x IN t1 AND b.x IN t1; + SELECT *, '|' FROM t2 ORDER BY aa, bb; +} {0 0 | 0 1 | 0 2 | 1 0 | 1 1 | 1 2 | 2 0 | 2 1 | 2 2 |} +do_execsql_test 120 { + CREATE TABLE t102 (i0 TEXT UNIQUE NOT NULL); + INSERT INTO t102 VALUES ('0'),('1'),('2'); + DROP TABLE t2; + CREATE TABLE t2 AS + SELECT DISTINCT * + FROM t102 AS t0 + JOIN t102 AS t4 ON (t2.i0 IN t102) + NATURAL JOIN t102 AS t3 + JOIN t102 AS t1 ON (t0.i0 IN t102) + JOIN t102 AS t2 ON (t2.i0=+t0.i0 OR (t0.i0<>500 AND t2.i0=t1.i0)); + SELECT *, '|' FROM t2 ORDER BY 1, 2, 3, 4, 5; +} {0 0 0 0 | 0 0 1 0 | 0 0 1 1 | 0 0 2 0 | 0 0 2 2 | 0 1 0 0 | 0 1 1 0 | 0 1 1 1 | 0 1 2 0 | 0 1 2 2 | 0 2 0 0 | 0 2 1 0 | 0 2 1 1 | 0 2 2 0 | 0 2 2 2 | 1 0 0 0 | 1 0 0 1 | 1 0 1 1 | 1 0 2 1 | 1 0 2 2 | 1 1 0 0 | 1 1 0 1 | 1 1 1 1 | 1 1 2 1 | 1 1 2 2 | 1 2 0 0 | 1 2 0 1 | 1 2 1 1 | 1 2 2 1 | 1 2 2 2 | 2 0 0 0 | 2 0 0 2 | 2 0 1 1 | 2 0 1 2 | 2 0 2 2 | 2 1 0 0 | 2 1 0 2 | 2 1 1 1 | 2 1 1 2 | 2 1 2 2 | 2 2 0 0 | 2 2 0 2 | 2 2 1 1 | 2 2 1 2 | 2 2 2 2 |} + +do_execsql_test 400 { + CREATE TABLE t4(a,b,c,d,e,f,g,h,i,j); + INSERT INTO t4 VALUES(0,1,2,3,4,5,6,7,8,9); + INSERT INTO t4 SELECT * FROM t4; + INSERT INTO t4 SELECT * FROM t4; + CREATE INDEX t4x ON t4(c,d,e); + SELECT DISTINCT a,b,c FROM t4 WHERE a=0 AND b=1; +} {0 1 2} +do_execsql_test 410 { + SELECT DISTINCT a,b,c,d FROM t4 WHERE a=0 AND b=1; +} {0 1 2 3} +do_execsql_test 411 { + SELECT DISTINCT d,a,b,c FROM t4 WHERE a=0 AND b=1; +} {3 0 1 2} +do_execsql_test 420 { + SELECT DISTINCT a,b,c,d,e FROM t4 WHERE a=0 AND b=1; +} {0 1 2 3 4} +do_execsql_test 430 { + SELECT DISTINCT a,b,c,d,e,f FROM t4 WHERE a=0 AND b=1; +} {0 1 2 3 4 5} + +do_execsql_test 500 { + CREATE TABLE t5(a INT, b INT); + CREATE UNIQUE INDEX t5x ON t5(a+b); + INSERT INTO t5(a,b) VALUES(0,0),(1,0),(1,1),(0,3); + CREATE TEMP TABLE out AS SELECT DISTINCT a+b FROM t5; + SELECT * FROM out ORDER BY 1; +} {0 1 2 3} + +do_execsql_test 600 { + CREATE TABLE t6a(x INTEGER PRIMARY KEY); + INSERT INTO t6a VALUES(1); + CREATE TABLE t6b(y INTEGER PRIMARY KEY); + INSERT INTO t6b VALUES(2),(3); + SELECT DISTINCT x, x FROM t6a, t6b; +} {1 1} + +do_execsql_test 700 { + CREATE TABLE t7(a, b, c); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE (i+1)<200 + ) + INSERT INTO t7 SELECT i/100, i/50, i FROM s; +} +do_execsql_test 710 { + SELECT DISTINCT a, b FROM t7; +} { + 0 0 0 1 + 1 2 1 3 +} +do_execsql_test 720 { + SELECT DISTINCT a, b+1 FROM t7; +} { + 0 1 0 2 + 1 3 1 4 +} +do_execsql_test 730 { + CREATE INDEX i7 ON t7(a, b+1); + ANALYZE; + SELECT DISTINCT a, b+1 FROM t7; +} { + 0 1 0 2 + 1 3 1 4 +} + +do_execsql_test 800 { + CREATE TABLE t8(a, b, c); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE (i+1)<100 + ) + INSERT INTO t8 SELECT i/40, i/20, i/40 FROM s; +} + +do_execsql_test 820 { + SELECT DISTINCT a, b, c FROM t8; +} { + 0 0 0 0 1 0 + 1 2 1 1 3 1 + 2 4 2 +} + +do_execsql_test 820 { + SELECT DISTINCT a, b, c FROM t8 WHERE b=3; +} {1 3 1} + +do_execsql_test 830 { + CREATE INDEX i8 ON t8(a, c); + ANALYZE; + SELECT DISTINCT a, b, c FROM t8 WHERE b=3; +} {1 3 1} + +do_execsql_test 900 { + CREATE TABLE t9(v); + INSERT INTO t9 VALUES + ('abcd'), ('Abcd'), ('aBcd'), ('ABcd'), ('abCd'), ('AbCd'), ('aBCd'), + ('ABCd'), ('abcD'), ('AbcD'), ('aBcD'), ('ABcD'), ('abCD'), ('AbCD'), + ('aBCD'), ('ABCD'), + ('wxyz'), ('Wxyz'), ('wXyz'), ('WXyz'), ('wxYz'), ('WxYz'), ('wXYz'), + ('WXYz'), ('wxyZ'), ('WxyZ'), ('wXyZ'), ('WXyZ'), ('wxYZ'), ('WxYZ'), + ('wXYZ'), ('WXYZ'); +} + +do_execsql_test 910 { + SELECT DISTINCT v COLLATE NOCASE, v FROM t9 ORDER BY +v; +} { + ABCD ABCD ABCd ABCd ABcD ABcD ABcd ABcd AbCD + AbCD AbCd AbCd AbcD AbcD Abcd Abcd + WXYZ WXYZ WXYz WXYz WXyZ WXyZ WXyz WXyz WxYZ + WxYZ WxYz WxYz WxyZ WxyZ Wxyz Wxyz + aBCD aBCD aBCd aBCd aBcD aBcD aBcd aBcd abCD + abCD abCd abCd abcD abcD abcd abcd + wXYZ wXYZ wXYz wXYz wXyZ wXyZ wXyz wXyz wxYZ + wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz +} + +do_execsql_test 920 { + CREATE INDEX i9 ON t9(v COLLATE NOCASE, v); + ANALYZE; + + SELECT DISTINCT v COLLATE NOCASE, v FROM t9 ORDER BY +v; +} { + ABCD ABCD ABCd ABCd ABcD ABcD ABcd ABcd AbCD + AbCD AbCd AbCd AbcD AbcD Abcd Abcd + WXYZ WXYZ WXYz WXYz WXyZ WXyZ WXyz WXyz WxYZ + WxYZ WxYz WxYz WxyZ WxyZ Wxyz Wxyz + aBCD aBCD aBCd aBCd aBcD aBcD aBcd aBcd abCD + abCD abCd abCd abcD abcD abcd abcd + wXYZ wXYZ wXYz wXYz wXyZ wXyZ wXyz wXyz wxYZ + wxYZ wxYz wxYz wxyZ wxyZ wxyz wxyz +} + + +finish_test Index: test/fkey5.test ================================================================== --- test/fkey5.test +++ test/fkey5.test @@ -386,7 +386,42 @@ do_execsql_test 9.4 { INSERT INTO k2 VALUES('six', 'seven'); PRAGMA foreign_key_check(k2); } {k2 3 s1 0} +#------------------------------------------------------------------------- +# Test using a WITHOUT ROWID table as the child table with an INTEGER +# PRIMARY KEY as the parent key. +# +reset_db +do_execsql_test 10.1 { + CREATE TABLE p30 (id INTEGER PRIMARY KEY); + CREATE TABLE IF NOT EXISTS c30 ( + line INTEGER, + master REFERENCES p30(id), + PRIMARY KEY(master) + ) WITHOUT ROWID; + + INSERT INTO p30 (id) VALUES (1); + INSERT INTO c30 (master, line) VALUES (1, 999); +} +do_execsql_test 10.2 { + PRAGMA foreign_key_check; +} +do_execsql_test 10.3 { + INSERT INTO c30 VALUES(45, 45); + PRAGMA foreign_key_check; +} {c30 {} p30 0} + +#------------------------------------------------------------------------- +# Test "foreign key mismatch" errors. +# +reset_db +do_execsql_test 11.0 { + CREATE TABLE tt(y); + CREATE TABLE c11(x REFERENCES tt(y)); +} +do_catchsql_test 11.1 { + PRAGMA foreign_key_check; +} {1 {foreign key mismatch - "c11" referencing "tt"}} finish_test Index: test/fts3aa.test ================================================================== --- test/fts3aa.test +++ test/fts3aa.test @@ -241,7 +241,13 @@ } {6 5} do_execsql_test 8.5 { SELECT docid FROM t0 WHERE t0 MATCH '"abc abc"'; } {} +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t9 USING fts4(a, "", '---'); +} +do_execsql_test 9.2 { + CREATE VIRTUAL TABLE t10 USING fts3(<, b, c); +} finish_test ADDED test/fts3corrupt3.test Index: test/fts3corrupt3.test ================================================================== --- test/fts3corrupt3.test +++ test/fts3corrupt3.test @@ -0,0 +1,65 @@ +# 2010 October 27 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Test that the FTS3 extension does not crash when it encounters a +# corrupt data structure on disk. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS3 is not defined, omit this file. +ifcapable !fts3 { finish_test ; return } + +set ::testprefix fts3corrupt3 + +#------------------------------------------------------------------------- +# Test that fts3 does not choke on an oversized varint. +# +do_execsql_test 1.0 { + PRAGMA page_size = 512; + CREATE VIRTUAL TABLE t1 USING fts3; + BEGIN; + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('one'); + COMMIT; +} +do_execsql_test 1.1 { + SELECT quote(root) from t1_segdir; +} {X'00036F6E6509010200010200010200'} +do_execsql_test 1.2 { + UPDATE t1_segdir SET root = X'00036F6E650EFFFFFFFFFFFFFFFFFFFFFFFF0200'; +} +do_catchsql_test 1.3 { + SELECT rowid FROM t1 WHERE t1 MATCH 'one' +} {0 -1} + +#------------------------------------------------------------------------- +# Interior node with the prefix or suffix count of an entry set to a +# negative value. +# +set doc1 [string repeat "x " 600] +set doc2 [string repeat "y " 600] +set doc3 [string repeat "z " 600] + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts3; + BEGIN; + INSERT INTO t2 VALUES($doc1); + INSERT INTO t2 VALUES($doc2); + INSERT INTO t2 VALUES($doc3); + COMMIT; +} +do_execsql_test 2.1 { + SELECT quote(root) from t2_segdir; +} {X'0101017900017A'} + + + +finish_test Index: test/fts3fault.test ================================================================== --- test/fts3fault.test +++ test/fts3fault.test @@ -16,12 +16,10 @@ set ::testprefix fts3fault # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test ; return } -if 0 { - # Test error handling in the sqlite3Fts3Init() function. This is the # function that registers the FTS3 module and various support functions # with SQLite. # do_faultsim_test 1 -body { @@ -158,12 +156,10 @@ {1 {vtable constructor failed: t1}} \ {1 {SQL logic error or missing database}} } -} - proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* binary scan $blob $scan($::tcl_platform(byteOrder)) r return $r @@ -231,7 +227,15 @@ } -body { execsql { SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to*' } } -test { faultsim_test_result {0 {{0 0 20 39 0 0 64 2}}} } + +do_faultsim_test 10.1 -prep { + faultsim_delete_and_reopen +} -body { + execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, languageid=d) } +} -test { + faultsim_test_result {0 {}} +} finish_test Index: test/fts3fault2.test ================================================================== --- test/fts3fault2.test +++ test/fts3fault2.test @@ -171,7 +171,79 @@ } -body { execsql { SELECT docid FROM t6 WHERE t6 MATCH '"a* b"' } } -test { faultsim_test_result {0 -1} } + +#------------------------------------------------------------------------- +# Inject faults into a query for an N-byte prefix that uses a prefix=N+1 +# index. +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t7 USING fts4(x,prefix=2); + INSERT INTO t7 VALUES('the quick brown fox'); + INSERT INTO t7 VALUES('jumped over the'); + INSERT INTO t7 VALUES('lazy dog'); +} +do_faultsim_test 7.1 -faults oom* -body { + execsql { SELECT docid FROM t7 WHERE t7 MATCH 't*' } +} -test { + faultsim_test_result {0 {1 2}} +} + +#------------------------------------------------------------------------- +# Inject faults into a opening an existing fts3 table that has been +# upgraded to add an %_stat table. +# +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE t8 USING fts3; + INSERT INTO t8 VALUES('the quick brown fox'); + INSERT INTO t8 VALUES('jumped over the'); + INSERT INTO t8 VALUES('lazy dog'); + INSERT INTO t8(t8) VALUES('automerge=8'); + SELECT name FROM sqlite_master WHERE name LIKE 't8%'; +} { + t8 t8_content t8_segments t8_segdir t8_stat +} +faultsim_save_and_close + +do_faultsim_test 8.1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO t8 VALUES('one two three') } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 8.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { ALTER TABLE t8 RENAME TO t8ii } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +set chunkconfig [fts3_configure_incr_load 1 1] +do_execsql_test 9.0 { + PRAGMA page_size = 512; + CREATE VIRTUAL TABLE t9 USING fts3; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50 + ) + INSERT INTO t9 SELECT 'one two three' FROM s; +} + +do_faultsim_test 8.2 -faults io* -body { + execsql { SELECT count(*) FROM t9 WHERE t9 MATCH '"one two three"' } +} -test { + faultsim_test_result {0 50} +} + +eval fts3_configure_incr_load $chunkconfig + finish_test + + ADDED test/fts3misc.test Index: test/fts3misc.test ================================================================== --- test/fts3misc.test +++ test/fts3misc.test @@ -0,0 +1,228 @@ +# 2017 March 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3misc + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +#------------------------------------------------------------------------- +# A self-join. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts3(a, b); + INSERT INTO t1 VALUES('one', 'i'); + INSERT INTO t1 VALUES('one', 'ii'); + INSERT INTO t1 VALUES('two', 'i'); + INSERT INTO t1 VALUES('two', 'ii'); +} + +do_execsql_test 1.1 { + SELECT a.a, b.b FROM t1 a, t1 b WHERE a.t1 MATCH 'two' AND b.t1 MATCH 'i' +} {two i two i two i two i} + +#------------------------------------------------------------------------- +# FTS tables with 128 or more columns. +# +proc v1 {v} { + set vector [list a b c d e f g h] + set res [list] + for {set i 0} {$i<8} {incr i} { + if {$v & (1 << $i)} { lappend res [lindex $vector $i] } + } + set res +} +proc v2 {v} { + set vector [list d e f g h i j k] + set res [list] + for {set i 0} {$i<8} {incr i} { + if {$v & (1 << $i)} { lappend res [lindex $vector $i] } + } + set res +} +db func v1 v1 +db func v2 v2 + +do_test 2.0 { + set cols [list] + for {set i 0} {$i<200} {incr i} { + lappend cols "c$i" + } + execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])" + execsql { + WITH data(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM data WHERE i<200 + ) + INSERT INTO t2(c198, c199) SELECT v1(i), v2(i) FROM data; + } +} {} +do_execsql_test 2.1 { + SELECT rowid FROM t2 WHERE t2 MATCH '"a b c"' +} { + 7 15 23 31 39 47 55 63 71 79 87 95 103 111 + 119 127 135 143 151 159 167 175 183 191 199 +} +do_execsql_test 2.2 { + SELECT rowid FROM t2 WHERE t2 MATCH '"g h i"' +} { + 56 57 58 59 60 61 62 63 120 121 122 123 124 + 125 126 127 184 185 186 187 188 189 190 191 +} +do_execsql_test 2.3 { + SELECT rowid FROM t2 WHERE t2 MATCH '"i h"' +} { +} +do_execsql_test 2.4 { + SELECT rowid FROM t2 WHERE t2 MATCH '"f e"' +} { +} +do_execsql_test 2.5 { + SELECT rowid FROM t2 WHERE t2 MATCH '"e f"' +} { + 6 7 14 15 22 23 30 31 38 39 46 47 48 49 50 51 52 53 54 55 56 + 57 58 59 60 61 62 63 70 71 78 79 86 87 94 95 102 103 110 + 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 + 134 135 142 143 150 151 158 159 166 167 174 175 176 177 178 179 180 + 181 182 183 184 185 186 187 188 189 190 191 198 199 +} + +#------------------------------------------------------------------------- +# Range constraints on the docid using non-integer values. +# +do_execsql_test 2.6 { + SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN NULL AND 45; +} {} +do_execsql_test 2.7 { + SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN 11.5 AND 48.2; +} { + 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 + 29 30 31 34 35 38 39 42 43 46 47 48 +} +do_execsql_test 2.8 { + SELECT rowid FROM t2 WHERE t2 MATCH 'e' AND rowid BETWEEN '11.5' AND '48.2'; +} { + 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 + 29 30 31 34 35 38 39 42 43 46 47 48 +} + +#------------------------------------------------------------------------- +# Phrase query tests. +# +do_execsql_test 3.1.1 { + CREATE VIRTUAL TABLE t3 USING fts3; + INSERT INTO t3 VALUES('a b c'); + INSERT INTO t3 VALUES('d e f'); + INSERT INTO t3 VALUES('a b d'); + INSERT INTO t3 VALUES('1 2 3 4 5 6 7 8 9 10 11'); +} +do_execsql_test 3.1.2 { + SELECT * FROM t3 WHERE t3 MATCH '"a b x y"' ORDER BY docid DESC +} +do_execsql_test 3.1.3 { + SELECT * FROM t3 WHERE t3 MATCH '"a b c" OR "a b x y"' ORDER BY docid DESC +} {{a b c}} +do_execsql_test 3.1.4 { + SELECT * FROM t3 WHERE t3 MATCH '"a* b* x* a*"' +} +do_execsql_test 3.1.5 { + SELECT rowid FROM t3 WHERE t3 MATCH '"2 3 4 5 6 7 8 9"' +} {4} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + PRAGMA page_size = 512; + CREATE VIRTUAL TABLE t4 USING fts4; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<8000 ) + INSERT INTO t4 SELECT 'a b c a b c a b c' FROM s; +} +do_execsql_test 4.1 { + SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"' +} {8000} +do_execsql_test 4.2 { + SELECT quote(value) from t4_stat where id=0 +} {X'C03EC0B204C0A608'} +do_execsql_test 4.3 { + UPDATE t4_stat SET value = X'C03EC0B204C0A60800' WHERE id=0; +} +do_catchsql_test 4.4 { + SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"' +} {1 {database disk image is malformed}} +do_execsql_test 4.5 { + UPDATE t4_stat SET value = X'00C03EC0B204C0A608' WHERE id=0; +} +do_catchsql_test 4.6 { + SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"' +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t5 USING fts4; + INSERT INTO t5 VALUES('a x x x x b x x x x c'); + INSERT INTO t5 VALUES('a x x x x b x x x x c'); + INSERT INTO t5 VALUES('a x x x x b x x x x c'); +} +do_execsql_test 5.1 { + SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/4 b NEAR/4 c' +} {1 2 3} +do_execsql_test 5.2 { + SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/3 b NEAR/4 c' +} {} +do_execsql_test 5.3 { + SELECT rowid FROM t5 WHERE t5 MATCH 'a NEAR/4 b NEAR/3 c' +} {} +do_execsql_test 5.4 { + SELECT rowid FROM t5 WHERE t5 MATCH 'y NEAR/4 b NEAR/4 c' +} {} +do_execsql_test 5.5 { + SELECT rowid FROM t5 WHERE t5 MATCH 'x OR a NEAR/3 b NEAR/3 c' +} {1 2 3} +do_execsql_test 5.5 { + SELECT rowid FROM t5 WHERE t5 MATCH 'x OR y NEAR/3 b NEAR/3 c' +} {1 2 3} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t6 USING fts4; + + BEGIN; + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000) + INSERT INTO t6 SELECT 'x x x x x x x x x x x' FROM s; + + INSERT INTO t6 VALUES('x x x x x x x x x x x A'); + INSERT INTO t6 VALUES('x x x x x x x x x x x B'); + INSERT INTO t6 VALUES('x x x x x x x x x x x A'); + INSERT INTO t6 VALUES('x x x x x x x x x x x B'); + + WITH s(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000) + INSERT INTO t6 SELECT 'x x x x x x x x x x x' FROM s; + COMMIT; +} +breakpoint +do_execsql_test 6.1 { + SELECT rowid FROM t6 WHERE t6 MATCH 'b OR "x a"' +} {50001 50002 50003 50004} + + +finish_test Index: test/fts4langid.test ================================================================== --- test/fts4langid.test +++ test/fts4langid.test @@ -12,11 +12,10 @@ # focus of this script is testing the languageid=xxx FTS4 option. # set testdir [file dirname $argv0] source $testdir/tester.tcl -set ::testprefix fts4content # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { finish_test return @@ -338,10 +337,17 @@ and_merge_lists [rowid_list zero] [rowid_list one] [rowid_list two] } do_test_query1 3.3.4 {"zero one" OR "one two"} { or_merge_lists [rowid_list "zero one"] [rowid_list "one two"] } + +do_execsql_test 3.4 { + CREATE TABLE t8c(a, b); + CREATE VIRTUAL TABLE t8 USING fts4(content=t8c, languageid=langid); + INSERT INTO t8(docid, a, b) VALUES(-1, 'one two three', 'x y z'); + SELECT docid FROM t8 WHERE t8 MATCH 'one x' AND langid=0 +} {-1} #------------------------------------------------------------------------- # Test cases 4.* # proc build_multilingual_db_2 {db} { Index: test/in5.test ================================================================== --- test/in5.test +++ test/in5.test @@ -234,7 +234,19 @@ SELECT count(*) FROM n1 WHERE a IN (1, 2, 3) } 3 do_execsql_test 8.4 { SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1) } 3 + +#------------------------------------------------------------------------- +# Test that ticket 61fe97454c is fixed. +# +do_execsql_test 9.0 { + CREATE TABLE t9(a INTEGER PRIMARY KEY); + INSERT INTO t9 VALUES (44), (45); +} +do_execsql_test 9.1 { + SELECT * FROM t9 WHERE a IN (44, 45, 44, 45) +} {44 45} + finish_test ADDED test/indexexpr2.test Index: test/indexexpr2.test ================================================================== --- test/indexexpr2.test +++ test/indexexpr2.test @@ -0,0 +1,44 @@ +# 2017 April 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix indexexpr2 + +do_execsql_test 1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + + CREATE INDEX i1 ON t1(b || 'x'); +} + +do_execsql_test 1.1 { + SELECT 'TWOX' == (b || 'x') FROM t1 WHERE (b || 'x')>'onex' +} {0 0} + +do_execsql_test 1.2 { + SELECT 'TWOX' == (b || 'x') COLLATE nocase FROM t1 WHERE (b || 'x')>'onex' +} {0 1} + +do_execsql_test 2.0 { + CREATE INDEX i2 ON t1(a+1); +} + +do_execsql_test 2.1 { + SELECT a+1, quote(a+1) FROM t1 ORDER BY 1; +} {2 2 3 3 4 4} + +finish_test + Index: test/json101.test ================================================================== --- test/json101.test +++ test/json101.test @@ -353,10 +353,19 @@ SELECT b FROM t8; } {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}} do_execsql_test json-8.2 { SELECT a=json_extract(b,'$[0]') FROM t8; } {1} + +# 2017-04-12. Regression reported on the mailing list by Rolf Ade +# +do_execsql_test json-8.3 { + SELECT json_valid(char(0x22,0xe4,0x22)); +} {1} +do_execsql_test json-8.4 { + SELECT unicode(json_extract(char(0x22,228,0x22),'$')); +} {228} # The json_quote() function transforms an SQL value into a JSON value. # String values are quoted and interior quotes are escaped. NULL values # are rendered as the unquoted string "null". # @@ -686,7 +695,31 @@ } {0} do_execsql_test json-10.95 { SELECT json_valid('" \~ "'); } {0} +#-------------------------------------------------------------------------- +# 2017-04-11. https://www.sqlite.org/src/info/981329adeef51011 +# Stack overflow on deeply nested JSON. +# +# The following tests confirm that deeply nested JSON is considered invalid. +# +do_execsql_test json-11.0 { + /* Shallow enough to be parsed */ + SELECT json_valid(printf('%.2000c0%.2000c','[',']')); +} {1} +do_execsql_test json-11.1 { + /* Too deep by one */ + SELECT json_valid(printf('%.2001c0%.2001c','[',']')); +} {0} +do_execsql_test json-11.2 { + /* Shallow enough to be parsed { */ + SELECT json_valid(replace(printf('%.2000c0%.2000c','[','}'),'[','{"a":')); + /* } */ +} {1} +do_execsql_test json-11.3 { + /* Too deep by one { */ + SELECT json_valid(replace(printf('%.2001c0%.2001c','[','}'),'[','{"a":')); + /* } */ +} {0} finish_test Index: test/json102.test ================================================================== --- test/json102.test +++ test/json102.test @@ -316,7 +316,25 @@ do_execsql_test json102-1408 { SELECT json_valid('{"x":-0.0000}') } 1 do_execsql_test json102-1409 { SELECT json_valid('{"x":01.5}') } 0 do_execsql_test json102-1410 { SELECT json_valid('{"x":-01.5}') } 0 do_execsql_test json102-1411 { SELECT json_valid('{"x":00}') } 0 do_execsql_test json102-1412 { SELECT json_valid('{"x":-00}') } 0 + +#------------------------------------------------------------------------ +# 2017-04-10 ticket 6c9b5514077fed34551f98e64c09a10dc2fc8e16 +# JSON extension accepts strings containing control characters. +# +# The JSON spec requires that all control characters be escaped. +# +do_execsql_test json102-1500 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<0x20) + SELECT x FROM c WHERE json_valid(printf('{"a":"x%sz"}', char(x))) ORDER BY x; +} {32} + +# All control characters are escaped +# +do_execsql_test json102-1501 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<0x1f) + SELECT sum(json_valid(json_quote('a'||char(x)||'z'))) FROM c ORDER BY x; +} {31} finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -263,10 +263,12 @@ fts3tok_err.test fts3varint.test fts4aa.test fts4check.test fts4content.test fts4docid.test fts4growth2.test fts4growth.test fts4incr.test fts4langid.test fts4lastrowid.test fts4merge2.test fts4merge4.test fts4merge.test fts4noti.test fts4onepass.test fts4opt.test fts4unicode.test + fts3corrupt3.test + fts3misc.test } test_suite "fts5" -prefix "" -description { All FTS5 tests. } -files [glob -nocomplain $::testdir/../ext/fts5/test/*.test] Index: tool/addopcodes.tcl ================================================================== --- tool/addopcodes.tcl +++ tool/addopcodes.tcl @@ -37,10 +37,11 @@ UMINUS UPLUS REGISTER VECTOR SELECT_COLUMN + IF_NULL_ROW ASTERISK SPAN SPACE ILLEGAL } Index: tool/fuzzershell.c ================================================================== --- tool/fuzzershell.c +++ tool/fuzzershell.c @@ -248,21 +248,21 @@ if( p->n + n >= p->nAlloc ){ char *zNew; sqlite3_uint64 nNew; if( p->oomErr ) return; nNew = p->nAlloc*2 + 100 + n; - zNew = sqlite3_realloc(p->z, nNew); + zNew = sqlite3_realloc(p->z, (int)nNew); if( zNew==0 ){ sqlite3_free(p->z); memset(p, 0, sizeof(*p)); p->oomErr = 1; return; } p->z = zNew; p->nAlloc = nNew; } - memcpy(p->z + p->n, z, n); + memcpy(p->z + p->n, z, (size_t)n); p->n += n; p->z[p->n] = 0; } /* Return the current string content */ @@ -1114,19 +1114,19 @@ ** named "autoexec" with a column "sql", then replace the input SQL ** with the concatenated text of the autoexec table. In this way, ** if the database file is the input being fuzzed, the SQL text is ** fuzzed at the same time. */ if( sqlite3_table_column_metadata(db,0,"autoexec","sql",0,0,0,0,0)==0 ){ - sqlite3_stmt *pStmt; - rc = sqlite3_prepare_v2(db, "SELECT sql FROM autoexec", -1, &pStmt, 0); + sqlite3_stmt *pStmt2; + rc = sqlite3_prepare_v2(db,"SELECT sql FROM autoexec",-1,&pStmt2,0); if( rc==SQLITE_OK ){ - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - StrAppend(&sql, (const char*)sqlite3_column_text(pStmt, 0)); + while( sqlite3_step(pStmt2)==SQLITE_ROW ){ + StrAppend(&sql, (const char*)sqlite3_column_text(pStmt2, 0)); StrAppend(&sql, "\n"); } } - sqlite3_finalize(pStmt); + sqlite3_finalize(pStmt2); zSql = StrStr(&sql); } g.bOomEnable = 1; if( verboseFlag ){ Index: tool/lemon.c ================================================================== --- tool/lemon.c +++ tool/lemon.c @@ -166,16 +166,16 @@ static struct action *Action_new(void); static struct action *Action_sort(struct action *); /********** From the file "build.h" ************************************/ -void FindRulePrecedences(); -void FindFirstSets(); -void FindStates(); -void FindLinks(); -void FindFollowSets(); -void FindActions(); +void FindRulePrecedences(struct lemon*); +void FindFirstSets(struct lemon*); +void FindStates(struct lemon*); +void FindLinks(struct lemon*); +void FindFollowSets(struct lemon*); +void FindActions(struct lemon*); /********* From the file "configlist.h" *********************************/ void Configlist_init(void); struct config *Configlist_add(struct rule *, int); struct config *Configlist_addbasis(struct rule *, int); @@ -455,11 +455,11 @@ int Configcmp(const char *, const char *); struct state *State_new(void); void State_init(void); int State_insert(struct state *, struct config *); struct state *State_find(struct config *); -struct state **State_arrayof(/* */); +struct state **State_arrayof(void); /* Routines used for efficiency in Configlist_add */ void Configtable_init(void); int Configtable_insert(struct config *); @@ -559,12 +559,12 @@ ** of the lookahead input, then the value of the action_number output is ** aAction[X].action. If the lookaheads do not match then the ** default action for the state_number is returned. ** ** All actions associated with a single state_number are first entered -** into aLookahead[] using multiple calls to acttab_action(). Then the -** actions for that single state_number are placed into the aAction[] +** into aLookahead[] using multiple calls to acttab_action(). Then the +** actions for that single state_number are placed into the aAction[] ** array with a single call to acttab_insert(). The acttab_insert() call ** also resets the aLookahead[] array in preparation for the next ** state number. */ struct lookahead_action { @@ -610,11 +610,11 @@ } memset(p, 0, sizeof(*p)); return p; } -/* Add a new action to the current transaction set. +/* Add a new action to the current transaction set. ** ** This routine is called once for each lookahead for a particular ** state. */ void acttab_action(acttab *p, int lookahead, int action){ @@ -672,11 +672,11 @@ p->aAction[i].lookahead = -1; p->aAction[i].action = -1; } } - /* Scan the existing action table looking for an offset that is a + /* Scan the existing action table looking for an offset that is a ** duplicate of the current transaction set. Fall out of the loop ** if and when the duplicate is found. ** ** i is the index in p->aAction[] where p->mnLookahead is inserted. */ @@ -750,11 +750,11 @@ ** Routines to construction the finite state machine for the LEMON ** parser generator. */ /* Find a precedence symbol of every rule in the grammar. -** +** ** Those rules which have a precedence symbol coded in the input ** grammar using the "[symbol]" construct will already have the ** rp->precsym field filled. Other rules take as their precedence ** symbol the first RHS symbol with a defined precedence. If there ** are not RHS symbols with a defined precedence, the precedence @@ -1070,11 +1070,11 @@ for(i=0; instate; i++){ for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){ cfp->status = INCOMPLETE; } } - + do{ progress = 0; for(i=0; instate; i++){ for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){ if( cfp->status==COMPLETE ) continue; @@ -1101,11 +1101,11 @@ struct config *cfp; struct state *stp; struct symbol *sp; struct rule *rp; - /* Add all of the reduce actions + /* Add all of the reduce actions ** A reduce action is added for each element of the followset of ** a configuration which has its dot at the extreme right. */ for(i=0; instate; i++){ /* Loop over all states */ stp = lemp->sorted[i]; @@ -1218,11 +1218,11 @@ apy->type = RD_RESOLVED; }else if( spx->precprec ){ apx->type = RD_RESOLVED; } }else{ - assert( + assert( apx->type==SH_RESOLVED || apx->type==RD_RESOLVED || apx->type==SSCONFLICT || apx->type==SRCONFLICT || apx->type==RRCONFLICT || @@ -1249,11 +1249,11 @@ static struct config **currentend = 0; /* Last on list of configs */ static struct config *basis = 0; /* Top of list of basis configs */ static struct config **basisend = 0; /* End of list of basis configs */ /* Return a pointer to a new configuration */ -PRIVATE struct config *newconfig(){ +PRIVATE struct config *newconfig(void){ struct config *newcfg; if( freelist==0 ){ int i; int amt = 3; freelist = (struct config *)calloc( amt, sizeof(struct config) ); @@ -1275,21 +1275,21 @@ old->next = freelist; freelist = old; } /* Initialized the configuration list builder */ -void Configlist_init(){ +void Configlist_init(void){ current = 0; currentend = ¤t; basis = 0; basisend = &basis; Configtable_init(); return; } /* Initialized the configuration list builder */ -void Configlist_reset(){ +void Configlist_reset(void){ current = 0; currentend = ¤t; basis = 0; basisend = &basis; Configtable_clear(0); @@ -1395,38 +1395,38 @@ } return; } /* Sort the configuration list */ -void Configlist_sort(){ +void Configlist_sort(void){ current = (struct config*)msort((char*)current,(char**)&(current->next), Configcmp); currentend = 0; return; } /* Sort the basis configuration list */ -void Configlist_sortbasis(){ +void Configlist_sortbasis(void){ basis = (struct config*)msort((char*)current,(char**)&(current->bp), Configcmp); basisend = 0; return; } /* Return a pointer to the head of the configuration list and ** reset the list */ -struct config *Configlist_return(){ +struct config *Configlist_return(void){ struct config *old; old = current; current = 0; currentend = 0; return old; } /* Return a pointer to the head of the configuration list and ** reset the list */ -struct config *Configlist_basis(){ +struct config *Configlist_basis(void){ struct config *old; old = basis; basis = 0; basisend = 0; return old; @@ -1464,11 +1464,11 @@ */ /* Report an out-of-memory condition and abort. This function ** is used mostly by the "MemoryCheck" macro in struct.h */ -void memory_error(){ +void memory_error(void){ fprintf(stderr,"Out of memory. Aborting...\n"); exit(1); } static int nDefine = 0; /* Number of -D options on the command line */ @@ -1604,11 +1604,11 @@ struct rule *rp; OptInit(argv,options,stderr); if( version ){ printf("Lemon version 1.0\n"); - exit(0); + exit(0); } if( OptNArgs()!=1 ){ fprintf(stderr,"Exactly one filename argument is required.\n"); exit(1); } @@ -2049,11 +2049,11 @@ exit(1); } return 0; } -int OptNArgs(){ +int OptNArgs(void){ int cnt = 0; int dashdash = 0; int i; if( argv!=0 && argv[0]!=0 ){ for(i=1; argv[i]; i++){ @@ -2076,11 +2076,11 @@ int i; i = argindex(n); if( i>=0 ) errline(i,0,errstream); } -void OptPrint(){ +void OptPrint(void){ int i; int max, len; max = 0; for(i=0; op[i].label; i++){ len = lemonStrlen(op[i].label) + 1; @@ -2305,11 +2305,11 @@ } break; case IN_RHS: if( x[0]=='.' ){ struct rule *rp; - rp = (struct rule *)calloc( sizeof(struct rule) + + rp = (struct rule *)calloc( sizeof(struct rule) + sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1); if( rp==0 ){ ErrorMsg(psp->filename,psp->tokenlineno, "Can't allocate enough memory for this rule."); psp->errorcnt++; @@ -2894,11 +2894,11 @@ ** in the LEMON parser generator. */ static struct plink *plink_freelist = 0; /* Allocate a new plink */ -struct plink *Plink_new(){ +struct plink *Plink_new(void){ struct plink *newlink; if( plink_freelist==0 ){ int i; int amt = 100; @@ -2995,11 +2995,11 @@ return 0; } return fp; } -/* Duplicate the input file without comments and without actions +/* Duplicate the input file without comments and without actions ** on rules */ void Reprint(struct lemon *lemp) { struct rule *rp; struct symbol *sp; @@ -3146,11 +3146,11 @@ case RRCONFLICT: fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.rp->iRule); break; case SSCONFLICT: - fprintf(fp,"%*s shift %-7d ** Parsing conflict **", + fprintf(fp,"%*s shift %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.stp->statenum); break; case SH_RESOLVED: if( showPrecedenceConflict ){ fprintf(fp,"%*s shift %-7d -- dropped by precedence", @@ -3419,11 +3419,11 @@ if( str[-1]!='\n' ){ putc('\n',out); (*lineno)++; } if (!lemp->nolinenosflag) { - (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } return; } /* @@ -3464,12 +3464,12 @@ } if( *cp=='\n' ) (*lineno)++; fputc(*cp,out); } fprintf(out,"\n"); (*lineno)++; - if (!lemp->nolinenosflag) { - (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); + if (!lemp->nolinenosflag) { + (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } fprintf(out,"}\n"); (*lineno)++; return; } @@ -3588,11 +3588,11 @@ } }else if( rp->lhsalias==0 ){ /* There is no LHS value symbol. */ lhsdirect = 1; }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){ - /* The LHS symbol and the left-most RHS symbol are the same, so + /* The LHS symbol and the left-most RHS symbol are the same, so ** direct writing is allowed */ lhsdirect = 1; lhsused = 1; used[0] = 1; if( rp->lhs->dtnum!=rp->rhs[0]->dtnum ){ @@ -3599,11 +3599,11 @@ ErrorMsg(lemp->filename,rp->ruleline, "%s(%s) and %s(%s) share the same label but have " "different datatypes.", rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]); lemp->errorcnt++; - } + } }else{ lemon_sprintf(zOvwrt, "/*%s-overwrites-%s*/", rp->lhsalias, rp->rhsalias[0]); zSkip = strstr(rp->code, zOvwrt); if( zSkip!=0 ){ @@ -3738,11 +3738,11 @@ } return rc; } -/* +/* ** Generate code which executes when the rule "rp" is reduced. Write ** the code to "out". Make sure lineno stays up-to-date. */ PRIVATE void emit_code( FILE *out, @@ -4235,11 +4235,11 @@ while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++; fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; - fprintf(out, "static const %s yy_shift_ofst[] = {\n", + fprintf(out, "static const %s yy_shift_ofst[] = {\n", minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz)); lineno++; lemp->tablesize += n*sz; for(i=j=0; inxstate; while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--; fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; fprintf(out, "#define YY_REDUCE_MAX (%d)\n", mxNtOfst); lineno++; - fprintf(out, "static const %s yy_reduce_ofst[] = {\n", + fprintf(out, "static const %s yy_reduce_ofst[] = {\n", minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++; lemp->tablesize += n*sz; for(i=j=0; isorted[i]; @@ -4341,11 +4341,11 @@ fprintf(out,"\",\n"); lineno++; } tplt_xfer(lemp->name,in,out,&lineno); /* Generate code which executes every time a symbol is popped from - ** the stack while processing errors or while destroying the parser. + ** the stack while processing errors or while destroying the parser. ** (In other words, generate the %destructor actions) */ if( lemp->tokendest ){ int once = 1; for(i=0; insymbol; i++){ @@ -4407,11 +4407,11 @@ /* Generate code which executes whenever the parser stack overflows */ tplt_print(out,lemp,lemp->overflow,&lineno); tplt_xfer(lemp->name,in,out,&lineno); - /* Generate the table of rule information + /* Generate the table of rule information ** ** Note: This code depends on the fact that rules are number ** sequentually beginning with 0. */ for(rp=lemp->rule; rp; rp=rp->next){ @@ -4518,11 +4518,11 @@ out = file_open(lemp,".h","wb"); if( out ){ for(i=1; interminal; i++){ fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); } - fclose(out); + fclose(out); } return; } /* Reduce the size of the action tables, if possible, by making use @@ -4565,11 +4565,11 @@ if( n>nbest ){ nbest = n; rbest = rp; } } - + /* Do not make a default if the number of rules to default ** is not at least 1 or if the wildcard token is a possible ** lookahead. */ if( nbest<1 || usesWildcard ) continue; @@ -4722,11 +4722,11 @@ { size = n+1; } /* Allocate a new set */ -char *SetNew(){ +char *SetNew(void){ char *s; s = (char*)calloc( size, 1); if( s==0 ){ extern void memory_error(); memory_error(); @@ -4828,11 +4828,11 @@ /* There is only one instance of the array, which is the following */ static struct s_x1 *x1a; /* Allocate a new associative array */ -void Strsafe_init(){ +void Strsafe_init(void){ if( x1a ) return; x1a = (struct s_x1*)malloc( sizeof(struct s_x1) ); if( x1a ){ x1a->size = 1024; x1a->count = 0; @@ -4995,11 +4995,11 @@ /* There is only one instance of the array, which is the following */ static struct s_x2 *x2a; /* Allocate a new associative array */ -void Symbol_init(){ +void Symbol_init(void){ if( x2a ) return; x2a = (struct s_x2*)malloc( sizeof(struct s_x2) ); if( x2a ){ x2a->size = 128; x2a->count = 0; @@ -5192,11 +5192,11 @@ /* There is only one instance of the array, which is the following */ static struct s_x3 *x3a; /* Allocate a new associative array */ -void State_init(){ +void State_init(void){ if( x3a ) return; x3a = (struct s_x3*)malloc( sizeof(struct s_x3) ); if( x3a ){ x3a->size = 128; x3a->count = 0; @@ -5286,11 +5286,11 @@ } /* Return an array of pointers to all data in the table. ** The array is obtained from malloc. Return NULL if memory allocation ** problems, or if the array is empty. */ -struct state **State_arrayof() +struct state **State_arrayof(void) { struct state **array; int i,arrSize; if( x3a==0 ) return 0; arrSize = x3a->count; @@ -5332,11 +5332,11 @@ /* There is only one instance of the array, which is the following */ static struct s_x4 *x4a; /* Allocate a new associative array */ -void Configtable_init(){ +void Configtable_init(void){ if( x4a ) return; x4a = (struct s_x4*)malloc( sizeof(struct s_x4) ); if( x4a ){ x4a->size = 64; x4a->count = 0; Index: tool/showwal.c ================================================================== --- tool/showwal.c +++ tool/showwal.c @@ -10,10 +10,11 @@ #define ISDIGIT(X) isdigit((unsigned char)(X)) #define ISPRINT(X) isprint((unsigned char)(X)) #if !defined(_MSC_VER) #include +#include #else #include #endif #include @@ -577,10 +578,18 @@ ofst = 32 + hdrSize + (iStart-1)*(pagesize+24) + 24; a = getContent(ofst, nByte); decode_btree_page(a, iStart, hdrSize, zLeft+1); free(a); continue; +#if !defined(_MSC_VER) + }else if( zLeft && strcmp(zLeft,"truncate")==0 ){ + /* Frame number followed by "truncate" truncates the WAL file + ** after that frame */ + off_t newSize = 32 + iStart*(pagesize+24); + truncate(argv[1], newSize); + continue; +#endif }else{ iEnd = iStart; } if( iStart<1 || iEndmxFrame ){ fprintf(stderr,