Index: ext/fts5/fts5.c ================================================================== --- ext/fts5/fts5.c +++ ext/fts5/fts5.c @@ -527,50 +527,52 @@ /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); - Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; - Fts5Cursor **pp; - Fts5Auxdata *pData; - Fts5Auxdata *pNext; - - fts5CsrNewrow(pCsr); - if( pCsr->pStmt ){ - int eStmt = fts5StmtType(pCsr->idxNum); - sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); - } - if( pCsr->pSorter ){ - Fts5Sorter *pSorter = pCsr->pSorter; - sqlite3_finalize(pSorter->pStmt); - sqlite3_free(pSorter); - } - - if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){ - sqlite3Fts5ExprFree(pCsr->pExpr); - } - - for(pData=pCsr->pAuxdata; pData; pData=pNext){ - pNext = pData->pNext; - if( pData->xDelete ) pData->xDelete(pData->pPtr); - sqlite3_free(pData); - } - - /* Remove the cursor from the Fts5Global.pCsr list */ - for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); - *pp = pCsr->pNext; - - sqlite3_finalize(pCsr->pRankArgStmt); - sqlite3_free(pCsr->apRankArg); - - sqlite3_free(pCsr->zSpecial); - if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ - sqlite3_free(pCsr->zRank); - sqlite3_free(pCsr->zRankArgs); - } - sqlite3_free(pCsr); + if( pCursor ){ + Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; + Fts5Cursor **pp; + Fts5Auxdata *pData; + Fts5Auxdata *pNext; + + fts5CsrNewrow(pCsr); + if( pCsr->pStmt ){ + int eStmt = fts5StmtType(pCsr->idxNum); + sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); + } + if( pCsr->pSorter ){ + Fts5Sorter *pSorter = pCsr->pSorter; + sqlite3_finalize(pSorter->pStmt); + sqlite3_free(pSorter); + } + + if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){ + sqlite3Fts5ExprFree(pCsr->pExpr); + } + + for(pData=pCsr->pAuxdata; pData; pData=pNext){ + pNext = pData->pNext; + if( pData->xDelete ) pData->xDelete(pData->pPtr); + sqlite3_free(pData); + } + + /* Remove the cursor from the Fts5Global.pCsr list */ + for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); + *pp = pCsr->pNext; + + sqlite3_finalize(pCsr->pRankArgStmt); + sqlite3_free(pCsr->apRankArg); + + sqlite3_free(pCsr->zSpecial); + if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ + sqlite3_free(pCsr->zRank); + sqlite3_free(pCsr->zRankArgs); + } + sqlite3_free(pCsr); + } return SQLITE_OK; } static int fts5SorterNext(Fts5Cursor *pCsr){ Fts5Sorter *pSorter = pCsr->pSorter; @@ -891,11 +893,15 @@ if( pRank ){ const char *z = (const char*)sqlite3_value_text(pRank); char *zRank = 0; char *zRankArgs = 0; - rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); + if( z==0 ){ + if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR; + }else{ + rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs); + } if( rc==SQLITE_OK ){ pCsr->zRank = zRank; pCsr->zRankArgs = zRankArgs; CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK); }else if( rc==SQLITE_ERROR ){ @@ -1205,20 +1211,20 @@ ** 1. The "old" rowid, or NULL. ** 2. The "new" rowid. ** 3. Values for each of the nCol matchable columns. ** 4. Values for the two hidden columns ( and "rank"). */ - assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) ); eType0 = sqlite3_value_type(apVal[0]); eConflict = sqlite3_vtab_on_conflict(pConfig->db); assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); assert( pVtab->zErrMsg==0 ); + assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) ); fts5TripCursors(pTab); - if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){ + if( eType0==SQLITE_INTEGER ){ if( fts5IsContentless(pTab) ){ pTab->base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName ); @@ -1225,11 +1231,12 @@ rc = SQLITE_ERROR; }else{ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); } - }else if( nArg>1 ){ + }else{ + assert( nArg>1 ); sqlite3_value *pCmd = apVal[2 + pConfig->nCol]; if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){ const char *z = (const char*)sqlite3_value_text(pCmd); if( pConfig->eContent!=FTS5_CONTENT_NORMAL && 0==sqlite3_stricmp("delete", z) @@ -1469,10 +1476,11 @@ int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ i64 iRowid = fts5CursorRowid(pCsr); rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE); } if( iCol<0 ){ int i; *pnToken = 0; for(i=0; ipConfig->nCol; i++){ @@ -1876,10 +1884,27 @@ rc = SQLITE_NOMEM; } return rc; } + +static Fts5TokenizerModule *fts5LocateTokenizer( + Fts5Global *pGlobal, + const char *zName +){ + Fts5TokenizerModule *pMod = 0; + + if( zName==0 ){ + pMod = pGlobal->pDfltTok; + }else{ + for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ + if( sqlite3_stricmp(zName, pMod->zName)==0 ) break; + } + } + + return pMod; +} /* ** Find a tokenizer. This is the implementation of the ** fts5_api.xFindTokenizer() method. */ @@ -1887,25 +1912,17 @@ fts5_api *pApi, /* Global context (one per db handle) */ const char *zName, /* Name of new function */ void **ppUserData, fts5_tokenizer *pTokenizer /* Populate this object */ ){ - Fts5Global *pGlobal = (Fts5Global*)pApi; int rc = SQLITE_OK; - Fts5TokenizerModule *pTok; + Fts5TokenizerModule *pMod; - if( zName==0 ){ - pTok = pGlobal->pDfltTok; - }else{ - for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){ - if( sqlite3_stricmp(zName, pTok->zName)==0 ) break; - } - } - - if( pTok ){ - *pTokenizer = pTok->x; - *ppUserData = pTok->pUserData; + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + *pTokenizer = pMod->x; + *ppUserData = pMod->pUserData; }else{ memset(pTokenizer, 0, sizeof(fts5_tokenizer)); rc = SQLITE_ERROR; } @@ -1915,28 +1932,27 @@ int sqlite3Fts5GetTokenizer( Fts5Global *pGlobal, const char **azArg, int nArg, Fts5Tokenizer **ppTok, - fts5_tokenizer **ppTokApi + fts5_tokenizer **ppTokApi, + char **pzErr ){ - Fts5TokenizerModule *pMod = 0; + Fts5TokenizerModule *pMod; int rc = SQLITE_OK; - if( nArg==0 ){ - pMod = pGlobal->pDfltTok; - }else{ - for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){ - if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break; - } - } - + pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ + assert( nArg>0 ); rc = SQLITE_ERROR; + *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); }else{ rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok); *ppTokApi = &pMod->x; + if( rc!=SQLITE_OK && pzErr ){ + *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + } } if( rc!=SQLITE_OK ){ *ppTokApi = 0; *ppTok = 0; Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -59,11 +59,12 @@ int sqlite3Fts5GetTokenizer( Fts5Global*, const char **azArg, int nArg, Fts5Tokenizer**, - fts5_tokenizer** + fts5_tokenizer**, + char **pzErr ); /* ** End of interface to code in fts5.c. **************************************************************************/ Index: ext/fts5/fts5_aux.c ================================================================== --- ext/fts5/fts5_aux.c +++ ext/fts5/fts5_aux.c @@ -228,15 +228,16 @@ } fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); } sqlite3_free(ctx.zOut); } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } } /* ** End of highlight() implementation. **************************************************************************/ Index: ext/fts5/fts5_config.c ================================================================== --- ext/fts5/fts5_config.c +++ ext/fts5/fts5_config.c @@ -328,15 +328,13 @@ if( p==0 ){ *pzErr = sqlite3_mprintf("parse error in tokenize directive"); rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5GetTokenizer(pGlobal, - (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi + (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi, + pzErr ); - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("error in tokenizer constructor"); - } } } } sqlite3_free(azArg); @@ -385,11 +383,11 @@ ** code if an error occurs. */ static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); return sqlite3Fts5GetTokenizer( - pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi + pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0 ); } /* ** Gobble up the first bareword or quoted word from the input buffer zIn. @@ -561,11 +559,11 @@ if( z==0 ){ *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig); rc = SQLITE_ERROR; }else{ if( bOption ){ - rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo, pzErr); + rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr); }else{ rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr); zOne = 0; } } Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -278,19 +278,19 @@ Fts5ExprNode *pNode; Fts5ExprNearset *pNear; Fts5ExprPhrase *pCopy; pOrig = pExpr->apExprPhrase[iPhrase]; + pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm + ); pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr)); apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*)); pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode)); pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) ); - pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm - ); for(i=0; rc==SQLITE_OK && inTerm; i++){ pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm); pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; } Index: ext/fts5/fts5_tcl.c ================================================================== --- ext/fts5/fts5_tcl.c +++ ext/fts5/fts5_tcl.c @@ -34,10 +34,13 @@ ** This is a copy of the first part of the SqliteDb structure in ** tclsqlite.c. We need it here so that the get_sqlite_pointer routine ** can extract the sqlite3* pointer from an existing Tcl SQLite ** connection. */ + +extern const char *sqlite3ErrName(int); + struct SqliteDb { sqlite3 *db; }; /* @@ -388,11 +391,11 @@ break; } #undef CASE if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in api call", 0); + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_ERROR; } return TCL_OK; } @@ -725,12 +728,10 @@ pInst->pContext->pCtx = pOldCtx; pInst->pContext->xToken = xOldToken; return rc; } -extern const char *sqlite3ErrName(int); - /* ** sqlite3_fts5_token TEXT START END POS */ static int f5tTokenizerReturn( void * clientData, Index: ext/fts5/fts5_tokenize.c ================================================================== --- ext/fts5/fts5_tokenize.c +++ ext/fts5/fts5_tokenize.c @@ -527,15 +527,20 @@ ){ fts5_api *pApi = (fts5_api*)pCtx; int rc = SQLITE_OK; PorterTokenizer *pRet; void *pUserdata = 0; + const char *zBase = "unicode61"; + + if( nArg>0 ){ + zBase = azArg[0]; + } pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); - rc = pApi->xFindTokenizer(pApi, "unicode61", &pUserdata, &pRet->tokenizer); + rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); }else{ rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer); Index: ext/fts5/test/fts5aa.test ================================================================== --- ext/fts5/test/fts5aa.test +++ ext/fts5/test/fts5aa.test @@ -295,11 +295,11 @@ do_catchsql_test 12.2 { SELECT t2 FROM t2 WHERE t2 MATCH '*stuff' } {1 {unknown special query: stuff}} do_test 12.3 { - set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }] + set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }] string is integer $res } {1} #------------------------------------------------------------------------- # Index: ext/fts5/test/fts5al.test ================================================================== --- ext/fts5/test/fts5al.test +++ ext/fts5/test/fts5al.test @@ -265,9 +265,17 @@ WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' ORDER BY rank ASC } { {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 } + +do_catchsql_test 4.4.3 { + SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' +} {1 {no such function: xyz}} +do_catchsql_test 4.4.4 { + SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL +} {1 {parse error in rank function: }} + finish_test ADDED ext/fts5/test/fts5aux.test Index: ext/fts5/test/fts5aux.test ================================================================== --- ext/fts5/test/fts5aux.test +++ ext/fts5/test/fts5aux.test @@ -0,0 +1,57 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the auxiliary function APIs. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aux + +proc inst {cmd i} { + $cmd xInst $i +} +sqlite3_fts5_create_function db inst inst + +proc colsize {cmd i} { + $cmd xColumnSize $i +} +sqlite3_fts5_create_function db colsize colsize + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1 VALUES('one two', 'two one zero'); +} + +do_catchsql_test 1.1 { + SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} +do_catchsql_test 1.2 { + SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two'; +} {0 {{0 0 1}}} +do_catchsql_test 1.3 { + SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two'; +} {0 {{0 1 0}}} +do_catchsql_test 1.4 { + SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} + +do_catchsql_test 2.1 { + SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two'; +} {1 SQLITE_RANGE} + +do_execsql_test 2.2 { + SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero'; +} {2 3} + + + +finish_test + Index: ext/fts5/test/fts5content.test ================================================================== --- ext/fts5/test/fts5content.test +++ ext/fts5/test/fts5content.test @@ -212,10 +212,28 @@ INSERT INTO t2(t2) VALUES('integrity-check'); } do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {} do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40} + +#------------------------------------------------------------------------- +# Test that if the 'rowid' field of a 'delete' is not an integer, no +# changes are made to the FTS index. +# +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=); + INSERT INTO t5(rowid, a, b) VALUES(-1, 'one', 'two'); + INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four'); + INSERT INTO t5(rowid, a, b) VALUES( 1, 'five', 'six'); +} + +set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}] + +do_execsql_test 5.1 { + INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four'); + SELECT md5sum(id, block) FROM t5_data; +} $::checksum finish_test Index: ext/fts5/test/fts5corrupt.test ================================================================== --- ext/fts5/test/fts5corrupt.test +++ ext/fts5/test/fts5corrupt.test @@ -68,9 +68,26 @@ } execsql { INSERT INTO t2(t2) VALUES('integrity-check') } } {} #-------------------------------------------------------------------- +# A mundane test - missing row in the %_content table. # +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(x); + INSERT INTO t3 VALUES('one o'); + INSERT INTO t3 VALUES('two e'); + INSERT INTO t3 VALUES('three o'); + INSERT INTO t3 VALUES('four e'); + INSERT INTO t3 VALUES('five o'); +} +do_execsql_test 3.1 { + SELECT * FROM t3 WHERE t3 MATCH 'o' +} {{one o} {three o} {five o}} + +do_catchsql_test 3.1 { + DELETE FROM t3_content WHERE rowid = 3; + SELECT * FROM t3 WHERE t3 MATCH 'o'; +} {1 {database disk image is malformed}} finish_test ADDED ext/fts5/test/fts5doclist.test Index: ext/fts5/test/fts5doclist.test ================================================================== --- ext/fts5/test/fts5doclist.test +++ ext/fts5/test/fts5doclist.test @@ -0,0 +1,41 @@ +# 2015 April 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on edge cases in the doclist format. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5doclist + + +#------------------------------------------------------------------------- +# Create a table with 1000 columns. Then add some large documents to it. +# All text is in the right most column of the table. +# +do_test 1.0 { + set cols [list] + for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" } + execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])" +} {} + +db func rnddoc fts5_rnddoc +do_execsql_test 1.1 { + WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) + INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii; +} + +do_execsql_test 1.2 { + INSERT INTO ccc(ccc) VALUES('integrity-check'); +} + + +finish_test + Index: ext/fts5/test/fts5fault4.test ================================================================== --- ext/fts5/test/fts5fault4.test +++ ext/fts5/test/fts5fault4.test @@ -19,10 +19,12 @@ # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts5 { finish_test return } + +if 1 { #------------------------------------------------------------------------- # An OOM while dropping an fts5 table. # db func rnddoc fts5_rnddoc @@ -37,9 +39,157 @@ } -body { execsql { DROP TABLE xx } } -test { faultsim_test_result [list 0 {}] } + +#------------------------------------------------------------------------- +# An OOM within an "ORDER BY rank" query. +# +db func rnddoc fts5_rnddoc +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xx USING fts5(x); + INSERT INTO xx VALUES ('abc ' || rnddoc(10)); + INSERT INTO xx VALUES ('abc abc' || rnddoc(9)); + INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8)); +} {} +faultsim_save_and_close + +do_faultsim_test 2 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM xx } +} -body { + execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank } +} -test { + faultsim_test_result [list 0 {3 2 1}] +} + +#------------------------------------------------------------------------- +# An OOM while "reseeking" an FTS cursor. +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE jj USING fts5(j); + INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s'); + INSERT INTO jj(rowid, j) VALUES(202, 't w t f s'); + INSERT INTO jj(rowid, j) VALUES(303, 'w t f'); + INSERT INTO jj(rowid, j) VALUES(404, 't'); +} +faultsim_save_and_close + +do_faultsim_test 3 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM jj } +} -body { + set res [list] + db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } { + lappend res $rowid + if {$rowid==303} { + execsql { DELETE FROM jj WHERE rowid=404 } + } + } + set res +} -test { + faultsim_test_result [list 0 {101 202 303}] +} + +#------------------------------------------------------------------------- +# An OOM within a special "*reads" query. +# +reset_db +db func rnddoc fts5_rnddoc +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + + WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 ) + INSERT INTO x1 SELECT rnddoc(5) FROM ii; +} + +set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] + +do_faultsim_test 4 -faults oom-* -body { + db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} +} -test { + faultsim_test_result {0 {0 {} 3}} +} + +#------------------------------------------------------------------------- +# An OOM within a query that uses a custom rank function. +# +reset_db +do_execsql_test 5.0 { + PRAGMA encoding='utf16'; + CREATE VIRTUAL TABLE x2 USING fts5(x); + INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3 + INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6 + INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2 + INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5 + INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1 +} + +proc rowidmod {cmd mod} { + set row [$cmd xRowid] + expr {$row % $mod} +} +sqlite3_fts5_create_function db rowidmod rowidmod + +do_faultsim_test 5.1 -faults oom-* -body { + db eval { + SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND + rank MATCH "rowidmod('7')" ORDER BY rank + } +} -test { + faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}} +} + +proc rowidprefix {cmd prefix} { + set row [$cmd xRowid] + set {} "${row}-${prefix}" +} +sqlite3_fts5_create_function db rowidprefix rowidprefix + +set str [string repeat abcdefghijklmnopqrstuvwxyz 10] +do_faultsim_test 5.2 -faults oom-* -body { + db eval " + SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND + rank MATCH 'rowidprefix(''$::str'')' + LIMIT 1 + " +} -test { + faultsim_test_result "0 {10-$::str {a b c}}" +} + +} + + +#------------------------------------------------------------------------- +# OOM errors within auxiliary functions. +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x3 USING fts5(xxx); + INSERT INTO x3 VALUES('a b c d c b a'); +} + +do_faultsim_test 6.1 -faults oom-t* -body { + db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' } +} -test { + faultsim_test_result {0 {{a b *c* d *c* b a}}} +} + +proc firstinst {cmd} { + foreach {p c o} [$cmd xInst 0] {} + expr $c*100 + $o +} +sqlite3_fts5_create_function db firstinst firstinst + +do_faultsim_test 6.2 -faults oom-t* -body { + db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' } +} -test { + faultsim_test_result {0 2} {1 SQLITE_NOMEM} +} + + finish_test ADDED ext/fts5/test/fts5plan.test Index: ext/fts5/test/fts5plan.test ================================================================== --- ext/fts5/test/fts5plan.test +++ ext/fts5/test/fts5plan.test @@ -0,0 +1,61 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on testing the planner (xBestIndex function). +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5plan + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + CREATE VIRTUAL TABLE f1 USING fts5(ff); +} + +do_eqp_test 1.1 { + SELECT * FROM t1, f1 WHERE f1 MATCH t1.x +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} +} + +do_eqp_test 1.2 { + SELECT * FROM t1, f1 WHERE f1 > t1.x +} { + 0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 1 0 {SCAN TABLE t1} +} + +do_eqp_test 1.3 { + SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff +} { + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + +do_eqp_test 1.4 { + SELECT * FROM f1 ORDER BY rank +} { + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + +do_eqp_test 1.5 { + SELECT * FROM f1 WHERE rank MATCH ? +} { + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} +} + + + + +finish_test + ADDED ext/fts5/test/fts5rank.test Index: ext/fts5/test/fts5rank.test ================================================================== --- ext/fts5/test/fts5rank.test +++ ext/fts5/test/fts5rank.test @@ -0,0 +1,39 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file focuses on testing queries that use the "rank" column. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5rank + + +#------------------------------------------------------------------------- +# "ORDER BY rank" + highlight() + large poslists. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xyz USING fts5(z); +} +do_test 1.1 { + set doc [string trim [string repeat "x y " 500]] + execsql { INSERT INTO xyz VALUES($doc) } +} {} +do_execsql_test 1.2 { + SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank +} [list [string map {x [x]} $doc]] + +do_execsql_test 1.3 { + SELECT highlight(xyz, 0, '[', ']') FROM xyz + WHERE xyz MATCH 'x AND y' ORDER BY rank +} [list [string map {x [x] y [y]} $doc]] + +finish_test + Index: ext/fts5/test/fts5rebuild.test ================================================================== --- ext/fts5/test/fts5rebuild.test +++ ext/fts5/test/fts5rebuild.test @@ -44,7 +44,18 @@ do_execsql_test 1.7 { INSERT INTO f1(f1) VALUES('rebuild'); INSERT INTO f1(f1) VALUES('integrity-check'); } {} + +#------------------------------------------------------------------------- +# Check that 'rebuild' may not be used with a contentless table. +# +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE nc USING fts5(doc, content=); +} + +do_catchsql_test 2.2 { + INSERT INTO nc(nc) VALUES('rebuild'); +} {1 {'rebuild' may not be used with a contentless fts5 table}} finish_test Index: ext/fts5/test/fts5restart.test ================================================================== --- ext/fts5/test/fts5restart.test +++ ext/fts5/test/fts5restart.test @@ -17,10 +17,14 @@ do_execsql_test 1.0 { CREATE VIRTUAL TABLE f1 USING fts5(ff); } +#------------------------------------------------------------------------- +# Run the 'optimize' command. Check that it does not disturb ongoing +# full-text queries. +# do_test 1.1 { for {set i 1} {$i < 1000} {incr i} { execsql { INSERT INTO f1 VALUES('a b c d e') } lappend lRowid $i } @@ -28,11 +32,10 @@ do_execsql_test 1.2 { SELECT rowid FROM f1 WHERE f1 MATCH 'c'; } $lRowid -breakpoint do_test 1.3 { set res [list] db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } { if {$rowid == 100} { execsql { INSERT INTO f1(f1) VALUES('optimize') } @@ -39,10 +42,105 @@ } lappend res $rowid } set res } $lRowid + +do_test 1.4.1 { + sqlite3 db2 test.db + set res [list] + db2 eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } { + if {$rowid == 100} { + set cres [catchsql { INSERT INTO f1(f1) VALUES('optimize') }] + } + lappend res $rowid + } + set res +} $lRowid + +do_test 1.4.2 { + db2 close + set cres +} {1 {database is locked}} + +#------------------------------------------------------------------------- +# Open a couple of cursors. Then close them in the same order. +# +do_test 2.1 { + set ::s1 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'b'" -1 X] + set ::s2 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'c'" -1 X] + + sqlite3_step $::s1 +} {SQLITE_ROW} +do_test 2.2 { + sqlite3_step $::s2 +} {SQLITE_ROW} + +do_test 2.1 { + sqlite3_finalize $::s1 + sqlite3_finalize $::s2 +} {SQLITE_OK} + +#------------------------------------------------------------------------- +# Copy data between two FTS5 tables. +# +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE f2 USING fts5(gg); + INSERT INTO f2 SELECT ff FROM f1 WHERE f1 MATCH 'b+c+d'; +} +do_execsql_test 3.2 { + SELECT rowid FROM f2 WHERE f2 MATCH 'a+b+c+d+e' +} $lRowid + +#------------------------------------------------------------------------- +# Remove the row that an FTS5 cursor is currently pointing to. And +# various other similar things. Check that this does not disturb +# ongoing scans. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE n4 USING fts5(n); + INSERT INTO n4(rowid, n) VALUES(100, '1 2 3 4 5'); + INSERT INTO n4(rowid, n) VALUES(200, '1 2 3 4'); + INSERT INTO n4(rowid, n) VALUES(300, '2 3 4'); + INSERT INTO n4(rowid, n) VALUES(400, '2 3'); + INSERT INTO n4(rowid, n) VALUES(500, '3'); +} + +do_test 4.1 { + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' } { + if {$rowid==300} { + execsql { DELETE FROM n4 WHERE rowid=300 } + } + lappend res $rowid + } + set res +} {100 200 300 400 500} + +do_test 4.2 { + execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') } + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} { + if {$rowid==300} { + execsql { DELETE FROM n4 WHERE rowid=300 } + } + lappend res $rowid + } + set res +} {500 400 300 200 100} + +do_test 4.3 { + execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') } + set res [list] + db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} { + if {$rowid==300} { + execsql { DELETE FROM n4 } + } + lappend res $rowid + } + set res +} {500 400 300} finish_test Index: ext/fts5/test/fts5tokenizer.test ================================================================== --- ext/fts5/test/fts5tokenizer.test +++ ext/fts5/test/fts5tokenizer.test @@ -35,10 +35,18 @@ do_execsql_test 1.4 { CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter ascii'); DROP TABLE ft1; } +do_catchsql_test 1.5 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'nosuch'); +} {1 {no such tokenizer: nosuch}} + +do_catchsql_test 1.6 { + CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter nosuch'); +} {1 {error in tokenizer constructor}} + do_execsql_test 2.0 { CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter); INSERT INTO ft1 VALUES('embedded databases'); } do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1