Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fixes to fts5 snippet() function. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
d464a7b18d212720a4f4e2fa4e204f1a |
| User & Date: | dan 2016-08-27 18:34:06 |
Context
|
2016-08-27
| ||
| 18:35 | Fix some comments in sqlite3session.h. No changes to code. check-in: 78cd64e2 user: dan tags: trunk | |
| 18:34 | Fixes to fts5 snippet() function. check-in: d464a7b1 user: dan tags: trunk | |
| 14:05 | Fix the extra comments (added with -DSQLITE_ENABLE_EXPLAIN_COMMENTS) so that the sense of <, <=, >, and >= tests is correct and so that the SQLITE_STOREP2 version is shown correctly. These changes are already in the rowvalue branch but are added here since they are technically unrelated to rowvalue. check-in: 4d43c469 user: drh tags: trunk | |
|
2016-08-24
| ||
| 18:50 | Fix a bug in the fts5 snippet function causing it to return text containing zero phrase instances. Closed-Leaf check-in: b174ed2b user: dan tags: fts5-snippet-bias | |
Changes
Changes to ext/fts5/fts5_aux.c.
241 241 if( rc!=SQLITE_OK ){ 242 242 sqlite3_result_error_code(pCtx, rc); 243 243 } 244 244 } 245 245 /* 246 246 ** End of highlight() implementation. 247 247 **************************************************************************/ 248 + 249 +/* 250 +** Context object passed to the fts5SentenceFinderCb() function. 251 +*/ 252 +typedef struct Fts5SFinder Fts5SFinder; 253 +struct Fts5SFinder { 254 + int iPos; /* Current token position */ 255 + int nFirstAlloc; /* Allocated size of aFirst[] */ 256 + int nFirst; /* Number of entries in aFirst[] */ 257 + int *aFirst; /* Array of first token in each sentence */ 258 + const char *zDoc; /* Document being tokenized */ 259 +}; 260 + 261 +/* 262 +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if 263 +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an 264 +** error occurs. 265 +*/ 266 +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ 267 + if( p->nFirstAlloc==p->nFirst ){ 268 + int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; 269 + int *aNew; 270 + 271 + aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int)); 272 + if( aNew==0 ) return SQLITE_NOMEM; 273 + p->aFirst = aNew; 274 + p->nFirstAlloc = nNew; 275 + } 276 + p->aFirst[p->nFirst++] = iAdd; 277 + return SQLITE_OK; 278 +} 279 + 280 +/* 281 +** This function is an xTokenize() callback used by the auxiliary snippet() 282 +** function. Its job is to identify tokens that are the first in a sentence. 283 +** For each such token, an entry is added to the SFinder.aFirst[] array. 284 +*/ 285 +static int fts5SentenceFinderCb( 286 + void *pContext, /* Pointer to HighlightContext object */ 287 + int tflags, /* Mask of FTS5_TOKEN_* flags */ 288 + const char *pToken, /* Buffer containing token */ 289 + int nToken, /* Size of token in bytes */ 290 + int iStartOff, /* Start offset of token */ 291 + int iEndOff /* End offset of token */ 292 +){ 293 + int rc = SQLITE_OK; 294 + 295 + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ 296 + Fts5SFinder *p = (Fts5SFinder*)pContext; 297 + if( p->iPos>0 ){ 298 + int i; 299 + char c = 0; 300 + for(i=iStartOff-1; i>=0; i--){ 301 + c = p->zDoc[i]; 302 + if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; 303 + } 304 + if( i!=iStartOff-1 && (c=='.' || c==':') ){ 305 + rc = fts5SentenceFinderAdd(p, p->iPos); 306 + } 307 + }else{ 308 + rc = fts5SentenceFinderAdd(p, 0); 309 + } 310 + p->iPos++; 311 + } 312 + return rc; 313 +} 314 + 315 +static int fts5SnippetScore( 316 + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ 317 + Fts5Context *pFts, /* First arg to pass to pApi functions */ 318 + int nDocsize, /* Size of column in tokens */ 319 + unsigned char *aSeen, /* Array with one element per query phrase */ 320 + int iCol, /* Column to score */ 321 + int iPos, /* Starting offset to score */ 322 + int nToken, /* Max tokens per snippet */ 323 + int *pnScore, /* OUT: Score */ 324 + int *piPos /* OUT: Adjusted offset */ 325 +){ 326 + int rc; 327 + int i; 328 + int ip = 0; 329 + int ic = 0; 330 + int iOff = 0; 331 + int iFirst = -1; 332 + int nInst; 333 + int nScore = 0; 334 + int iLast = 0; 335 + 336 + rc = pApi->xInstCount(pFts, &nInst); 337 + for(i=0; i<nInst && rc==SQLITE_OK; i++){ 338 + rc = pApi->xInst(pFts, i, &ip, &ic, &iOff); 339 + if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){ 340 + nScore += (aSeen[ip] ? 1 : 1000); 341 + aSeen[ip] = 1; 342 + if( iFirst<0 ) iFirst = iOff; 343 + iLast = iOff + pApi->xPhraseSize(pFts, ip); 344 + } 345 + } 346 + 347 + *pnScore = nScore; 348 + if( piPos ){ 349 + int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; 350 + if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; 351 + if( iAdj<0 ) iAdj = 0; 352 + *piPos = iAdj; 353 + } 354 + 355 + return rc; 356 +} 248 357 249 358 /* 250 359 ** Implementation of snippet() function. 251 360 */ 252 361 static void fts5SnippetFunction( 253 362 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ 254 363 Fts5Context *pFts, /* First arg to pass to pApi functions */ ................................................................................ 263 372 int nToken; /* 5th argument to snippet() */ 264 373 int nInst = 0; /* Number of instance matches this row */ 265 374 int i; /* Used to iterate through instances */ 266 375 int nPhrase; /* Number of phrases in query */ 267 376 unsigned char *aSeen; /* Array of "seen instance" flags */ 268 377 int iBestCol; /* Column containing best snippet */ 269 378 int iBestStart = 0; /* First token of best snippet */ 270 - int iBestLast; /* Last token of best snippet */ 271 379 int nBestScore = 0; /* Score of best snippet */ 272 380 int nColSize = 0; /* Total size of iBestCol in tokens */ 381 + Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ 382 + int nCol; 273 383 274 384 if( nVal!=5 ){ 275 385 const char *zErr = "wrong number of arguments to function snippet()"; 276 386 sqlite3_result_error(pCtx, zErr, -1); 277 387 return; 278 388 } 279 389 390 + nCol = pApi->xColumnCount(pFts); 280 391 memset(&ctx, 0, sizeof(HighlightContext)); 281 392 iCol = sqlite3_value_int(apVal[0]); 282 393 ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); 283 394 ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); 284 395 zEllips = (const char*)sqlite3_value_text(apVal[3]); 285 396 nToken = sqlite3_value_int(apVal[4]); 286 - iBestLast = nToken-1; 287 397 288 398 iBestCol = (iCol>=0 ? iCol : 0); 289 399 nPhrase = pApi->xPhraseCount(pFts); 290 400 aSeen = sqlite3_malloc(nPhrase); 291 401 if( aSeen==0 ){ 292 402 rc = SQLITE_NOMEM; 293 403 } 294 - 295 404 if( rc==SQLITE_OK ){ 296 405 rc = pApi->xInstCount(pFts, &nInst); 297 406 } 298 - for(i=0; rc==SQLITE_OK && i<nInst; i++){ 299 - int ip, iSnippetCol, iStart; 300 - memset(aSeen, 0, nPhrase); 301 - rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart); 302 - if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){ 303 - int nScore = 1000; 304 - int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip); 305 - int j; 306 - aSeen[ip] = 1; 307 - 308 - for(j=i+1; rc==SQLITE_OK && j<nInst; j++){ 309 - int ic; int io; int iFinal; 310 - rc = pApi->xInst(pFts, j, &ip, &ic, &io); 311 - iFinal = io + pApi->xPhraseSize(pFts, ip) - 1; 312 - if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){ 313 - nScore += aSeen[ip] ? 1000 : 1; 314 - aSeen[ip] = 1; 315 - if( iFinal>iLast ) iLast = iFinal; 407 + 408 + memset(&sFinder, 0, sizeof(Fts5SFinder)); 409 + for(i=0; i<nCol; i++){ 410 + if( iCol<0 || iCol==i ){ 411 + int nDoc; 412 + int nDocsize; 413 + int ii; 414 + sFinder.iPos = 0; 415 + sFinder.nFirst = 0; 416 + rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc); 417 + if( rc!=SQLITE_OK ) break; 418 + rc = pApi->xTokenize(pFts, 419 + sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb 420 + ); 421 + if( rc!=SQLITE_OK ) break; 422 + rc = pApi->xColumnSize(pFts, i, &nDocsize); 423 + if( rc!=SQLITE_OK ) break; 424 + 425 + for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){ 426 + int ip, ic, io; 427 + int iAdj; 428 + int nScore; 429 + int jj; 430 + 431 + rc = pApi->xInst(pFts, ii, &ip, &ic, &io); 432 + if( ic!=i || rc!=SQLITE_OK ) continue; 433 + memset(aSeen, 0, nPhrase); 434 + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 435 + io, nToken, &nScore, &iAdj 436 + ); 437 + if( rc==SQLITE_OK && nScore>nBestScore ){ 438 + nBestScore = nScore; 439 + iBestCol = i; 440 + iBestStart = iAdj; 441 + nColSize = nDocsize; 316 442 } 317 - } 318 443 319 - if( rc==SQLITE_OK && nScore>nBestScore ){ 320 - iBestCol = iSnippetCol; 321 - iBestStart = iStart; 322 - iBestLast = iLast; 323 - nBestScore = nScore; 444 + if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ 445 + for(jj=0; jj<(sFinder.nFirst-1); jj++){ 446 + if( sFinder.aFirst[jj+1]>io ) break; 447 + } 448 + 449 + if( sFinder.aFirst[jj]<io ){ 450 + int nScore; 451 + memset(aSeen, 0, nPhrase); 452 + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 453 + sFinder.aFirst[jj], nToken, &nScore, 0 454 + ); 455 + 456 + nScore += (sFinder.aFirst[jj]==0 ? 120 : 100); 457 + if( rc==SQLITE_OK && nScore>nBestScore ){ 458 + nBestScore = nScore; 459 + iBestCol = i; 460 + iBestStart = sFinder.aFirst[jj]; 461 + nColSize = nDocsize; 462 + } 463 + } 464 + } 324 465 } 325 466 } 326 467 } 327 468 328 469 if( rc==SQLITE_OK ){ 329 - rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); 470 + rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); 330 471 } 331 - if( rc==SQLITE_OK ){ 332 - rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); 472 + if( rc==SQLITE_OK && nColSize==0 ){ 473 + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); 333 474 } 334 475 if( ctx.zIn ){ 335 476 if( rc==SQLITE_OK ){ 336 477 rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); 337 478 } 338 479 339 - if( (iBestStart+nToken-1)>iBestLast ){ 340 - iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; 341 - } 342 - if( iBestStart+nToken>nColSize ){ 343 - iBestStart = nColSize - nToken; 344 - } 345 - if( iBestStart<0 ) iBestStart = 0; 346 - 347 480 ctx.iRangeStart = iBestStart; 348 481 ctx.iRangeEnd = iBestStart + nToken - 1; 349 482 350 483 if( iBestStart>0 ){ 351 484 fts5HighlightAppend(&rc, &ctx, zEllips, -1); 352 485 } 353 486 ................................................................................ 361 494 rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); 362 495 } 363 496 if( ctx.iRangeEnd>=(nColSize-1) ){ 364 497 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); 365 498 }else{ 366 499 fts5HighlightAppend(&rc, &ctx, zEllips, -1); 367 500 } 368 - 369 - if( rc==SQLITE_OK ){ 370 - sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 371 - }else{ 372 - sqlite3_result_error_code(pCtx, rc); 373 - } 374 - sqlite3_free(ctx.zOut); 501 + } 502 + if( rc==SQLITE_OK ){ 503 + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); 504 + }else{ 505 + sqlite3_result_error_code(pCtx, rc); 375 506 } 507 + sqlite3_free(ctx.zOut); 376 508 sqlite3_free(aSeen); 509 + sqlite3_free(sFinder.aFirst); 377 510 } 378 511 379 512 /************************************************************************/ 380 513 381 514 /* 382 515 ** The first time the bm25() function is called for a query, an instance 383 516 ** of the following structure is allocated and populated.
Changes to ext/fts5/test/fts5af.test.
68 68 1.6 {o o o o o X o} {o o o o o [X] o} 69 69 1.7 {o o o o o o X} {o o o o o o [X]} 70 70 71 71 2.1 {X o o o o o o o} {[X] o o o o o o...} 72 72 2.2 {o X o o o o o o} {o [X] o o o o o...} 73 73 2.3 {o o X o o o o o} {o o [X] o o o o...} 74 74 2.4 {o o o X o o o o} {o o o [X] o o o...} 75 - 2.5 {o o o o X o o o} {...o o o [X] o o o} 76 - 2.6 {o o o o o X o o} {...o o o o [X] o o} 77 - 2.7 {o o o o o o X o} {...o o o o o [X] o} 75 + 2.5 {o o o o X o o o} {o o o o [X] o o...} 76 + 2.6 {o o o o o X o o} {o o o o o [X] o...} 77 + 2.7 {o o o o o o X o} {o o o o o o [X]...} 78 78 2.8 {o o o o o o o X} {...o o o o o o [X]} 79 + 80 + 2.9 {o o o o o o o X o} {...o o o o o [X] o} 81 + 2.10 {o o o o o o o X o o} {...o o o o [X] o o} 82 + 2.11 {o o o o o o o X o o o} {...o o o [X] o o o} 83 + 2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...} 84 + 79 85 80 86 3.1 {X o o o o o o o o} {[X] o o o o o o...} 81 87 3.2 {o X o o o o o o o} {o [X] o o o o o...} 82 88 3.3 {o o X o o o o o o} {o o [X] o o o o...} 83 89 3.4 {o o o X o o o o o} {o o o [X] o o o...} 84 - 3.5 {o o o o X o o o o} {...o o o [X] o o o...} 85 - 3.6 {o o o o o X o o o} {...o o o [X] o o o} 86 - 3.7 {o o o o o o X o o} {...o o o o [X] o o} 87 - 3.8 {o o o o o o o X o} {...o o o o o [X] o} 88 - 3.9 {o o o o o o o o X} {...o o o o o o [X]} 90 + 91 + 3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...} 92 + 3.6 {o o o o o o o o X o o o} {...o o o [X] o o o} 93 + 3.7 {o o o o o o o o o X o o} {...o o o o [X] o o} 94 + 3.8 {o o o o o o o o o o X o} {...o o o o o [X] o} 95 + 3.9 {o o o o o o o o o o o X} {...o o o o o o [X]} 89 96 90 97 4.1 {X o o o o o X o o} {[X] o o o o o [X]...} 91 - 4.2 {o X o o o o o X o} {...[X] o o o o o [X]...} 92 - 4.3 {o o X o o o o o X} {...[X] o o o o o [X]} 98 + 4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...} 99 + 4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]} 93 100 94 101 5.1 {X o o o o X o o o} {[X] o o o o [X] o...} 95 - 5.2 {o X o o o o X o o} {...[X] o o o o [X] o...} 96 - 5.3 {o o X o o o o X o} {...[X] o o o o [X] o} 97 - 5.4 {o o o X o o o o X} {...o [X] o o o o [X]} 102 + 5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...} 103 + 5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o} 104 + 5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]} 98 105 99 106 6.1 {X o o o X o o o} {[X] o o o [X] o o...} 100 107 6.2 {o X o o o X o o o} {o [X] o o o [X] o...} 101 - 6.3 {o o X o o o X o o} {...o [X] o o o [X] o...} 102 - 6.4 {o o o X o o o X o} {...o [X] o o o [X] o} 103 - 6.5 {o o o o X o o o X} {...o o [X] o o o [X]} 108 + 6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...} 109 + 6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o} 110 + 6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]} 104 111 105 112 7.1 {X o o X o o o o o} {[X] o o [X] o o o...} 106 113 7.2 {o X o o X o o o o} {o [X] o o [X] o o...} 107 - 7.3 {o o X o o X o o o} {...o [X] o o [X] o o...} 108 - 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} 109 - 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} 110 - 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} 114 + 7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...} 115 + 7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o} 116 + 7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o} 117 + 7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]} 111 118 112 - 8.1 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 119 + 8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o} 113 120 {...o o [X] [X] [X] o o...} 121 + 8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 122 + {...o o [X] o o o o...} 123 + 8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 124 + {o o o o [X] o o...} 114 125 } { 115 126 do_snippet_test 1.$tn $doc X $res 116 127 } 117 128 118 129 if {[detail_is_full]} { 119 130 foreach {tn doc res} { 120 131 1.1 {X Y o o o o o} {[X Y] o o o o o} ................................................................................ 123 134 1.4 {o o o X Y o o} {o o o [X Y] o o} 124 135 1.5 {o o o o X Y o} {o o o o [X Y] o} 125 136 1.6 {o o o o o X Y} {o o o o o [X Y]} 126 137 127 138 2.1 {X Y o o o o o o} {[X Y] o o o o o...} 128 139 2.2 {o X Y o o o o o} {o [X Y] o o o o...} 129 140 2.3 {o o X Y o o o o} {o o [X Y] o o o...} 130 - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} 131 - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} 132 - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} 133 - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} 141 + 2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o} 142 + 2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o} 143 + 2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o} 144 + 2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]} 134 145 135 146 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} 136 147 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} 137 148 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} 138 - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} 139 - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} 140 - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 141 - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 142 - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} 149 + 3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...} 150 + 3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o} 151 + 3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o} 152 + 3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o} 153 + 3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]} 143 154 } { 144 155 do_snippet_test 2.$tn $doc "X + Y" $res 145 156 } 146 157 } 147 158 159 +do_execsql_test 4.0 { 160 + CREATE VIRTUAL TABLE x1 USING fts5(a, b); 161 + INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13'); 162 + SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz'); 163 +} { 164 + {1 2 3 4 5...} 165 +} 166 + 167 +do_execsql_test 5.0 { 168 + CREATE VIRTUAL TABLE p1 USING fts5(a, b); 169 + INSERT INTO p1 VALUES( 170 + 'x a a a a a a a a a a', 171 + 'a a a a a a a a a a a a a a a a a a a x' 172 + ); 173 +} 174 +do_execsql_test 5.1 { 175 + SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); 176 +} {{[x] a a a a a...}} 177 + 148 178 } ;# foreach_detail_mode 149 179 150 180 finish_test 151 181
Changes to ext/fts5/test/fts5unicode2.test.
156 156 the maximum x value. 157 157 } 158 158 3 "ROW" { 159 159 ...returns the value of y on the same [row] that contains 160 160 the maximum x value. 161 161 } 162 162 4 "rollback" { 163 - ...[ROLLBACK]. Instead, the pending statement 164 - will return SQLITE_ABORT upon next access after the [ROLLBACK]. 163 + Pending statements no longer block [ROLLBACK]. Instead, the pending 164 + statement will return SQLITE_ABORT upon... 165 165 } 166 166 5 "rOllback" { 167 - ...[ROLLBACK]. Instead, the pending statement 168 - will return SQLITE_ABORT upon next access after the [ROLLBACK]. 167 + Pending statements no longer block [ROLLBACK]. Instead, the pending 168 + statement will return SQLITE_ABORT upon... 169 169 } 170 170 6 "lang*" { 171 171 Added support for the FTS4 [languageid] option. 172 172 } 173 173 } { 174 174 do_test 2.$tn { 175 175 set q [mapdoc $query]
Changes to src/main.c.
2951 2951 2952 2952 /* Register all built-in functions, but do not attempt to read the 2953 2953 ** database schema yet. This is delayed until the first time the database 2954 2954 ** is accessed. 2955 2955 */ 2956 2956 sqlite3Error(db, SQLITE_OK); 2957 2957 sqlite3RegisterPerConnectionBuiltinFunctions(db); 2958 + rc = sqlite3_errcode(db); 2959 + 2960 +#ifdef SQLITE_ENABLE_FTS5 2961 + /* Register any built-in FTS5 module before loading the automatic 2962 + ** extensions. This allows automatic extensions to register FTS5 2963 + ** tokenizers and auxiliary functions. */ 2964 + if( !db->mallocFailed && rc==SQLITE_OK ){ 2965 + rc = sqlite3Fts5Init(db); 2966 + } 2967 +#endif 2958 2968 2959 2969 /* Load automatic extensions - extensions that have been registered 2960 2970 ** using the sqlite3_automatic_extension() API. 2961 2971 */ 2962 - rc = sqlite3_errcode(db); 2963 2972 if( rc==SQLITE_OK ){ 2964 2973 sqlite3AutoLoadExtensions(db); 2965 2974 rc = sqlite3_errcode(db); 2966 2975 if( rc!=SQLITE_OK ){ 2967 2976 goto opendb_out; 2968 2977 } 2969 2978 } ................................................................................ 2982 2991 } 2983 2992 #endif 2984 2993 2985 2994 #ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ 2986 2995 if( !db->mallocFailed && rc==SQLITE_OK ){ 2987 2996 rc = sqlite3Fts3Init(db); 2988 2997 } 2989 -#endif 2990 - 2991 -#ifdef SQLITE_ENABLE_FTS5 2992 - if( !db->mallocFailed && rc==SQLITE_OK ){ 2993 - rc = sqlite3Fts5Init(db); 2994 - } 2995 2998 #endif 2996 2999 2997 3000 #ifdef SQLITE_ENABLE_ICU 2998 3001 if( !db->mallocFailed && rc==SQLITE_OK ){ 2999 3002 rc = sqlite3IcuInit(db); 3000 3003 } 3001 3004 #endif