Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -102,17 +102,17 @@ Fts5ExprPhrase **apPhrase; /* Array of all phrases */ Fts5ExprNode *pExpr; /* Result of a successful parse */ }; void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); if( pParse->rc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); pParse->zErr = sqlite3_vmprintf(zFmt, ap); - va_end(ap); pParse->rc = SQLITE_ERROR; } + va_end(ap); } static int fts5ExprIsspace(char t){ return t==' ' || t=='\t' || t=='\n' || t=='\r'; } @@ -267,56 +267,55 @@ Fts5Expr *pExpr, int iPhrase, Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ - int i; /* Used to iterate through phrase terms */ - - /* Components of the new expression object */ - Fts5Expr *pNew; - Fts5ExprPhrase **apPhrase; - Fts5ExprNode *pNode; - Fts5ExprNearset *pNear; - Fts5ExprPhrase *pCopy; + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5ExprPhrase *pCopy; /* Copy of pOrig */ + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ pOrig = pExpr->apExprPhrase[iPhrase]; pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm ); - pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr)); - apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*)); - pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode)); - pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) - ); - - for(i=0; rc==SQLITE_OK && inTerm; i++){ - pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm); - pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - } - - if( rc==SQLITE_OK ){ - /* All the allocations succeeded. Put the expression object together. */ - pNew->pIndex = pExpr->pIndex; - pNew->pRoot = pNode; - pNew->nPhrase = 1; - pNew->apExprPhrase = apPhrase; - pNew->apExprPhrase[0] = pCopy; - - pNode->eType = FTS5_STRING; - pNode->pNear = pNear; - - pNear->iCol = -1; - pNear->nPhrase = 1; - pNear->apPhrase[0] = pCopy; - - pCopy->nTerm = pOrig->nTerm; - pCopy->pNode = pNode; - }else{ - /* At least one allocation failed. Free them all. */ - if( pCopy ){ + if( pCopy ){ + int i; /* Used to iterate through phrase terms */ + Fts5ExprPhrase **apPhrase; + Fts5ExprNode *pNode; + Fts5ExprNearset *pNear; + + pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr)); + apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*)); + pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode)); + pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) + ); + + for(i=0; inTerm; i++){ + pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm); + pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } + + if( rc==SQLITE_OK ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->pRoot = pNode; + pNew->nPhrase = 1; + pNew->apExprPhrase = apPhrase; + pNew->apExprPhrase[0] = pCopy; + + pNode->eType = FTS5_STRING; + pNode->pNear = pNear; + + pNear->iCol = -1; + pNear->nPhrase = 1; + pNear->apPhrase[0] = pCopy; + + pCopy->nTerm = pOrig->nTerm; + pCopy->pNode = pNode; + }else{ + /* At least one allocation failed. Free them all. */ for(i=0; inTerm; i++){ sqlite3_free(pCopy->aTerm[i].zTerm); } sqlite3_free(pCopy); sqlite3_free(pNear); @@ -502,11 +501,11 @@ assert( pNear->nPhrase>1 ); /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pNear->nPhrase>(sizeof(aStatic) / sizeof(aStatic[0])) ){ - int nByte = sizeof(Fts5LookaheadReader) * pNear->nPhrase; + int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; a = (Fts5NearTrimmer*)sqlite3_malloc(nByte); if( !a ) return SQLITE_NOMEM; memset(a, 0, nByte); }else{ memset(aStatic, 0, sizeof(aStatic)); @@ -717,11 +716,11 @@ while( 1 ){ int i; /* Advance the iterators until they all point to the same rowid */ rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom); - if( pNode->bEof || rc!=SQLITE_OK ) break; + if( rc!=SQLITE_OK || pNode->bEof ) break; /* Check that each phrase in the nearset matches the current row. ** Populate the pPhrase->poslist buffers at the same time. If any ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && inPhrase; i++){ Index: ext/fts5/test/fts5ea.test ================================================================== --- ext/fts5/test/fts5ea.test +++ ext/fts5/test/fts5ea.test @@ -7,10 +7,13 @@ # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # +# Test the fts5 expression parser directly using the fts5_expr() SQL +# test function. +# source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ea # If SQLITE_ENABLE_FTS5 is defined, omit this file. @@ -38,10 +41,14 @@ 6 {one+two} {"one" + "two"} 7 {one AND two OR three} {("one" AND "two") OR "three"} 8 {one OR two AND three} {"one" OR ("two" AND "three")} 9 {NEAR(one two)} {NEAR("one" "two", 10)} 10 {NEAR("one three"* two, 5)} {NEAR("one" + "three" * "two", 5)} + 11 {a OR b NOT c} {"a" OR ("b" NOT "c")} + 12 "\x20one\x20two\x20three" {("one" AND "two") AND "three"} + 13 "\x09one\x0Atwo\x0Dthree" {("one" AND "two") AND "three"} + 14 {"abc""def"} {"abc" + "def"} } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } foreach {tn expr res} { @@ -51,33 +58,24 @@ {c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"} } { do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res] } -breakpoint foreach {tn expr err} { 1 {AND} {fts5: syntax error near "AND"} 2 {abc def AND} {fts5: syntax error near ""} 3 {abc OR AND} {fts5: syntax error near "AND"} 4 {(a OR b) abc} {fts5: syntax error near "abc"} 5 {NEaR (a b)} {fts5: syntax error near "NEaR"} 6 {(a OR b) NOT c)} {fts5: syntax error near ")"} 7 {nosuch: a nosuch2: b} {no such column: nosuch} 8 {addr: a nosuch2: b} {no such column: nosuch2} + 9 {NOT} {fts5: syntax error near "NOT"} + 10 {a AND "abc} {unterminated string} } { do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err] } -# do_syntax_error_test 1.0 {NOT} {syntax error near "NOT"} - - - -# do_catchsql_test 1.1 { - # SELECT fts5_expr('a OR b NOT c') -#} {0 {"a" OR "b" NOT "c"}} - - -#do_execsql_test 1.0 { SELECT fts5_expr('a') } {{"a"}} finish_test Index: ext/fts5/test/fts5fault4.test ================================================================== --- ext/fts5/test/fts5fault4.test +++ ext/fts5/test/fts5fault4.test @@ -156,12 +156,10 @@ " } -test { faultsim_test_result "0 {10-$::str {a b c}}" } -} - #------------------------------------------------------------------------- # OOM errors within auxiliary functions. # reset_db @@ -201,11 +199,69 @@ db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' } } -test { faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM} } +#------------------------------------------------------------------------- +# OOM error when querying for a phrase with many tokens. +# +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE tt USING fts5(x, y); + INSERT INTO tt VALUES('f b g b c b', 'f a d c c b'); -- 1 + INSERT INTO tt VALUES('d a e f e d', 'f b b d e e'); -- 2 + INSERT INTO tt VALUES('f b g a d c', 'e f c f a d'); -- 3 + INSERT INTO tt VALUES('f f c d g f', 'f a e b g b'); -- 4 + INSERT INTO tt VALUES('a g b d a g', 'e g a e a c'); -- 5 + INSERT INTO tt VALUES('c d b d e f', 'f g e g e e'); -- 6 + INSERT INTO tt VALUES('e g f f b c', 'f c e f g f'); -- 7 + INSERT INTO tt VALUES('e g c f c e', 'f e e a f g'); -- 8 + INSERT INTO tt VALUES('e a e b e e', 'd c c f f f'); -- 9 + INSERT INTO tt VALUES('f a g g c c', 'e g d g c e'); -- 10 + INSERT INTO tt VALUES('c d b a e f', 'f g e h e e'); -- 11 +} + +do_faultsim_test 7.2 -faults oom-* -body { + db eval { SELECT rowid FROM tt WHERE tt MATCH 'f+g+e+g+e+e' } +} -test { + faultsim_test_result {0 6} {1 SQLITE_NOMEM} +} + +do_faultsim_test 7.3 -faults oom-* -body { + db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d e f)' } +} -test { + faultsim_test_result {0 11} {1 SQLITE_NOMEM} +} + +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE tt USING fts5(x); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO tt(rowid, x) VALUES(1, 'a b c d x x'); + WITH ii(i) AS (SELECT 2 UNION ALL SELECT i+1 FROM ii WHERE i<99) + INSERT INTO tt(rowid, x) SELECT i, 'a b c x x d' FROM ii; + INSERT INTO tt(rowid, x) VALUES(100, 'a b c d x x'); + COMMIT; +} + +do_faultsim_test 8.1 -faults oom-t* -body { + db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d, 2)' } +} -test { + faultsim_test_result {0 {1 100}} {1 SQLITE_NOMEM} +} + +do_faultsim_test 8.2 -faults oom-t* -body { + db eval { SELECT count(*) FROM tt WHERE tt MATCH 'a OR d' } +} -test { + faultsim_test_result {0 100} {1 SQLITE_NOMEM} +} finish_test