Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Add hidden column "rank". Currently this always returns the same value as the bm25() function. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
4cc048c3651e830a6aeded924c7f3a60 |
| User & Date: | dan 2014-07-30 19:41:58 |
Context
|
2014-07-30
| ||
| 20:26 | Fix things so that the fts5 extension API works with "ORDER BY rank" queries. (check-in: f1b4e1a9 user: dan tags: fts5) | |
| 19:41 | Add hidden column "rank". Currently this always returns the same value as the bm25() function. (check-in: 4cc048c3 user: dan tags: fts5) | |
|
2014-07-28
| ||
| 20:14 | Add the "loadfts" program, for performance testing the loading of data into fts3/fts4/fts5 tables. (check-in: 770b9540 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
52 52 */ 53 53 struct Fts5Table { 54 54 sqlite3_vtab base; /* Base class used by SQLite core */ 55 55 Fts5Config *pConfig; /* Virtual table configuration */ 56 56 Fts5Index *pIndex; /* Full-text index */ 57 57 Fts5Storage *pStorage; /* Document store */ 58 58 Fts5Global *pGlobal; /* Global (connection wide) data */ 59 + Fts5Cursor *pSortCsr; /* Sort data from this cursor */ 59 60 }; 60 61 61 62 struct Fts5MatchPhrase { 62 63 Fts5Buffer *pPoslist; /* Pointer to current poslist */ 63 64 int nTerm; /* Size of phrase in terms */ 64 65 }; 66 + 67 +/* 68 +** Variable pStmt is set to a compiled SQL statement of the form: 69 +** 70 +** SELECT rowid, <fts> FROM <fts> ORDER BY +rank; 71 +** 72 +*/ 73 +struct Fts5Sorter { 74 + sqlite3_stmt *pStmt; 75 + i64 iRowid; /* Current rowid */ 76 + u8 *aPoslist; /* Position lists for current row */ 77 + int nIdx; /* Number of entries in aIdx[] */ 78 + int aIdx[0]; /* Offsets into aPoslist for current row */ 79 +}; 65 80 66 81 /* 67 82 ** Virtual-table cursor object. 68 83 */ 69 84 struct Fts5Cursor { 70 85 sqlite3_vtab_cursor base; /* Base class used by SQLite core */ 71 86 int idxNum; /* idxNum passed to xFilter() */ 72 87 sqlite3_stmt *pStmt; /* Statement used to read %_content */ 73 88 Fts5Expr *pExpr; /* Expression for MATCH queries */ 89 + Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ 74 90 int csrflags; /* Mask of cursor flags (see below) */ 75 91 Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ 92 + Fts5Auxiliary *pRank; /* Rank callback (or NULL) */ 76 93 77 94 /* Variables used by auxiliary functions */ 78 95 i64 iCsrId; /* Cursor id */ 79 96 Fts5Auxiliary *pAux; /* Currently executing extension function */ 80 - Fts5Auxdata *pAuxdata; /* First in linked list of aux-data */ 97 + Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ 81 98 int *aColumnSize; /* Values for xColumnSize() */ 82 99 }; 83 100 84 101 /* 85 102 ** Values for Fts5Cursor.csrflags 86 103 */ 87 104 #define FTS5CSR_REQUIRE_CONTENT 0x01 ................................................................................ 223 240 ){ 224 241 return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); 225 242 } 226 243 227 244 /* 228 245 ** The three query plans xBestIndex may choose between. 229 246 */ 230 -#define FTS5_PLAN_SCAN 1 /* No usable constraint */ 231 -#define FTS5_PLAN_MATCH 2 /* (<tbl> MATCH ?) */ 232 -#define FTS5_PLAN_ROWID 3 /* (rowid = ?) */ 247 +#define FTS5_PLAN_SCAN 1 /* No usable constraint */ 248 +#define FTS5_PLAN_MATCH 2 /* (<tbl> MATCH ?) */ 249 +#define FTS5_PLAN_SORTED_MATCH 3 /* (<tbl> MATCH ? ORDER BY rank) */ 250 +#define FTS5_PLAN_ROWID 4 /* (rowid = ?) */ 251 +#define FTS5_PLAN_SOURCE 5 /* A source cursor for SORTED_MATCH */ 233 252 234 253 #define FTS5_PLAN(idxNum) ((idxNum) & 0x7) 235 254 236 255 #define FTS5_ORDER_DESC 8 /* ORDER BY rowid DESC */ 237 256 #define FTS5_ORDER_ASC 16 /* ORDER BY rowid ASC */ 238 257 239 258 /* ................................................................................ 280 299 if( iCons>=0 ){ 281 300 pInfo->aConstraintUsage[iCons].argvIndex = 1; 282 301 pInfo->aConstraintUsage[iCons].omit = 1; 283 302 }else{ 284 303 pInfo->estimatedCost = 10000000.0; 285 304 } 286 305 287 - if( pInfo->nOrderBy==1 && pInfo->aOrderBy[0].iColumn<0 ){ 288 - pInfo->orderByConsumed = 1; 289 - ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC; 306 + if( pInfo->nOrderBy==1 ){ 307 + int iSort = pInfo->aOrderBy[0].iColumn; 308 + if( iSort<0 ){ 309 + /* ORDER BY rowid [ASC|DESC] */ 310 + pInfo->orderByConsumed = 1; 311 + }else if( iSort==(pConfig->nCol+1) && ePlan==FTS5_PLAN_MATCH ){ 312 + /* ORDER BY rank [ASC|DESC] */ 313 + pInfo->orderByConsumed = 1; 314 + ePlan = FTS5_PLAN_SORTED_MATCH; 315 + } 316 + 317 + if( pInfo->orderByConsumed ){ 318 + ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC; 319 + } 290 320 } 291 321 292 322 pInfo->idxNum = ePlan; 293 323 return SQLITE_OK; 294 324 } 295 325 296 326 /* ................................................................................ 337 367 Fts5Auxdata *pData; 338 368 Fts5Auxdata *pNext; 339 369 340 370 if( pCsr->pStmt ){ 341 371 int eStmt = fts5StmtType(pCsr->idxNum); 342 372 sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); 343 373 } 344 - sqlite3Fts5ExprFree(pCsr->pExpr); 374 + if( pCsr->pSorter ){ 375 + Fts5Sorter *pSorter = pCsr->pSorter; 376 + sqlite3_finalize(pSorter->pStmt); 377 + sqlite3_free(pSorter); 378 + } 379 + 380 + if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){ 381 + sqlite3Fts5ExprFree(pCsr->pExpr); 382 + } 345 383 346 384 for(pData=pCsr->pAuxdata; pData; pData=pNext){ 347 385 pNext = pData->pNext; 348 386 if( pData->xDelete ) pData->xDelete(pData->pPtr); 349 387 sqlite3_free(pData); 350 388 } 351 389 ................................................................................ 353 391 for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); 354 392 *pp = pCsr->pNext; 355 393 356 394 sqlite3_free(pCsr); 357 395 return SQLITE_OK; 358 396 } 359 397 398 +static int fts5SorterNext(Fts5Cursor *pCsr){ 399 + Fts5Sorter *pSorter = pCsr->pSorter; 400 + int rc; 401 + 402 + rc = sqlite3_step(pSorter->pStmt); 403 + if( rc==SQLITE_DONE ){ 404 + rc = SQLITE_OK; 405 + CsrFlagSet(pCsr, FTS5CSR_EOF); 406 + }else if( rc==SQLITE_ROW ){ 407 + rc = SQLITE_OK; 408 + pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); 409 + } 410 + 411 + return rc; 412 +} 360 413 361 414 /* 362 415 ** Advance the cursor to the next row in the table that matches the 363 416 ** search criteria. 364 417 ** 365 418 ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned 366 419 ** even if we reach end-of-file. The fts5EofMethod() will be called ................................................................................ 367 420 ** subsequently to determine whether or not an EOF was hit. 368 421 */ 369 422 static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ 370 423 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 371 424 int ePlan = FTS5_PLAN(pCsr->idxNum); 372 425 int rc = SQLITE_OK; 373 426 374 - if( ePlan!=FTS5_PLAN_MATCH ){ 375 - rc = sqlite3_step(pCsr->pStmt); 376 - if( rc!=SQLITE_ROW ){ 377 - CsrFlagSet(pCsr, FTS5CSR_EOF); 378 - rc = sqlite3_reset(pCsr->pStmt); 379 - }else{ 380 - rc = SQLITE_OK; 381 - } 382 - }else{ 383 - rc = sqlite3Fts5ExprNext(pCsr->pExpr); 384 - if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ 385 - CsrFlagSet(pCsr, FTS5CSR_EOF); 427 + switch( ePlan ){ 428 + case FTS5_PLAN_MATCH: 429 + case FTS5_PLAN_SOURCE: 430 + rc = sqlite3Fts5ExprNext(pCsr->pExpr); 431 + if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ 432 + CsrFlagSet(pCsr, FTS5CSR_EOF); 433 + } 434 + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE ); 435 + break; 436 + 437 + case FTS5_PLAN_SORTED_MATCH: { 438 + rc = fts5SorterNext(pCsr); 439 + break; 386 440 } 387 - CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE ); 441 + 442 + default: 443 + rc = sqlite3_step(pCsr->pStmt); 444 + if( rc!=SQLITE_ROW ){ 445 + CsrFlagSet(pCsr, FTS5CSR_EOF); 446 + rc = sqlite3_reset(pCsr->pStmt); 447 + }else{ 448 + rc = SQLITE_OK; 449 + } 450 + break; 388 451 } 389 452 453 + return rc; 454 +} 455 + 456 +static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){ 457 + Fts5Config *pConfig = pTab->pConfig; 458 + Fts5Sorter *pSorter; 459 + int nPhrase; 460 + int nByte; 461 + int rc = SQLITE_OK; 462 + char *zSql; 463 + ................................................................................ 464 + nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); 465 + nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase; 466 + pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); 467 + if( pSorter==0 ) return SQLITE_NOMEM; 468 + memset(pSorter, 0, nByte); 469 + 470 + zSql = sqlite3_mprintf("SELECT rowid, %Q FROM %Q.%Q ORDER BY +%s %s", 471 + pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME, 472 + bAsc ? "ASC" : "DESC" 473 + ); 474 + if( zSql==0 ){ 475 + rc = SQLITE_NOMEM; 476 + }else{ 477 + rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0); 478 + sqlite3_free(zSql); 479 + } 480 + 481 + pCsr->pSorter = pSorter; 482 + if( rc==SQLITE_OK ){ 483 + assert( pTab->pSortCsr==0 ); 484 + pTab->pSortCsr = pCsr; 485 + rc = fts5SorterNext(pCsr); 486 + pTab->pSortCsr = 0; 487 + } 488 + 489 + if( rc!=SQLITE_OK ){ 490 + sqlite3_finalize(pSorter->pStmt); 491 + sqlite3_free(pSorter); 492 + pCsr->pSorter = 0; 493 + } 494 + 390 495 return rc; 391 496 } 392 497 393 498 static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){ 394 499 int rc; 395 500 rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc); 396 501 if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ ................................................................................ 410 515 int idxNum, /* Strategy index */ 411 516 const char *idxStr, /* Unused */ 412 517 int nVal, /* Number of elements in apVal */ 413 518 sqlite3_value **apVal /* Arguments for the indexing scheme */ 414 519 ){ 415 520 Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); 416 521 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 417 - int rc = SQLITE_OK; 418 - int ePlan = FTS5_PLAN(idxNum); 419 - int eStmt = fts5StmtType(idxNum); 420 522 int bAsc = ((idxNum & FTS5_ORDER_ASC) ? 1 : 0); 523 + int rc = SQLITE_OK; 421 524 422 - pCsr->idxNum = idxNum; 423 525 assert( pCsr->pStmt==0 ); 424 526 assert( pCsr->pExpr==0 ); 425 527 assert( pCsr->csrflags==0 ); 426 - 427 - rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); 428 - if( rc==SQLITE_OK ){ 429 - if( ePlan==FTS5_PLAN_MATCH ){ 430 - char **pzErr = &pTab->base.zErrMsg; 431 - const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); 432 - rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); 433 - if( rc==SQLITE_OK ){ 434 - rc = fts5CursorFirst(pTab, pCsr, bAsc); 435 - } 436 - }else{ 437 - if( ePlan==FTS5_PLAN_ROWID ){ 438 - sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); 439 - } 440 - rc = fts5NextMethod(pCursor); 528 + assert( pCsr->pRank==0 ); 529 + 530 + if( pTab->pSortCsr ){ 531 + pCsr->idxNum = FTS5_PLAN_SOURCE; 532 + pCsr->pRank = pTab->pSortCsr->pRank; 533 + pCsr->pExpr = pTab->pSortCsr->pExpr; 534 + rc = fts5CursorFirst(pTab, pCsr, bAsc); 535 + }else{ 536 + int ePlan = FTS5_PLAN(idxNum); 537 + int eStmt = fts5StmtType(idxNum); 538 + pCsr->idxNum = idxNum; 539 + rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); 540 + if( rc==SQLITE_OK ){ 541 + if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){ 542 + char **pzErr = &pTab->base.zErrMsg; 543 + const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); 544 + pCsr->pRank = pTab->pGlobal->pAux; 545 + rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); 546 + if( rc==SQLITE_OK ){ 547 + if( ePlan==FTS5_PLAN_MATCH ){ 548 + rc = fts5CursorFirst(pTab, pCsr, bAsc); 549 + }else{ 550 + rc = fts5CursorFirstSorted(pTab, pCsr, bAsc); 551 + } 552 + } 553 + }else{ 554 + if( ePlan==FTS5_PLAN_ROWID ){ 555 + sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); 556 + } 557 + rc = fts5NextMethod(pCursor); 558 + } 441 559 } 442 560 } 443 561 444 562 return rc; 445 563 } 446 564 447 565 /* ................................................................................ 460 578 ** rowid should be written to *pRowid. 461 579 */ 462 580 static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ 463 581 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 464 582 int ePlan = FTS5_PLAN(pCsr->idxNum); 465 583 466 584 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); 467 - if( ePlan!=FTS5_PLAN_MATCH ){ 468 - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); 469 - }else{ 470 - *pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); 585 + switch( ePlan ){ 586 + case FTS5_PLAN_SOURCE: 587 + case FTS5_PLAN_MATCH: 588 + *pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); 589 + break; 590 + 591 + case FTS5_PLAN_SORTED_MATCH: 592 + *pRowid = pCsr->pSorter->iRowid; 593 + break; 594 + 595 + default: 596 + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); 597 + break; 471 598 } 472 599 473 600 return SQLITE_OK; 474 601 } 475 602 476 603 /* 477 604 ** If the cursor requires seeking (bSeekRequired flag is set), seek it. ................................................................................ 490 617 }else{ 491 618 rc = sqlite3_reset(pCsr->pStmt); 492 619 if( rc==SQLITE_OK ){ 493 620 rc = SQLITE_CORRUPT_VTAB; 494 621 } 495 622 } 496 623 } 497 - return rc; 498 -} 499 - 500 -/* 501 -** This is the xColumn method, called by SQLite to request a value from 502 -** the row that the supplied cursor currently points to. 503 -*/ 504 -static int fts5ColumnMethod( 505 - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ 506 - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ 507 - int iCol /* Index of column to read value from */ 508 -){ 509 - Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig; 510 - Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 511 - int rc = SQLITE_OK; 512 - 513 - assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); 514 - 515 - if( iCol==pConfig->nCol ){ 516 - /* User is requesting the value of the special column with the same name 517 - ** as the table. Return the cursor integer id number. This value is only 518 - ** useful in that it may be passed as the first argument to an FTS5 519 - ** auxiliary function. */ 520 - sqlite3_result_int64(pCtx, pCsr->iCsrId); 521 - }else{ 522 - rc = fts5SeekCursor(pCsr); 523 - if( rc==SQLITE_OK ){ 524 - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); 525 - } 526 - } 527 624 return rc; 528 625 } 529 626 530 627 /* 531 628 ** This function is called to handle an FTS INSERT command. In other words, 532 629 ** an INSERT statement of the form: 533 630 ** ................................................................................ 569 666 ){ 570 667 Fts5Table *pTab = (Fts5Table*)pVtab; 571 668 Fts5Config *pConfig = pTab->pConfig; 572 669 int eType0; /* value_type() of apVal[0] */ 573 670 int eConflict; /* ON CONFLICT for this DML */ 574 671 int rc = SQLITE_OK; /* Return code */ 575 672 576 - assert( nArg==1 || nArg==(2 + pConfig->nCol + 1) ); 673 + /* A delete specifies a single argument - the rowid of the row to remove. 674 + ** Update and insert operations pass: 675 + ** 676 + ** 1. The "old" rowid, or NULL. 677 + ** 2. The "new" rowid. 678 + ** 3. Values for each of the nCol matchable columns. 679 + ** 4. Values for the two hidden columns (<tablename> and "rank"). 680 + */ 681 + assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) ); 577 682 578 683 if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){ 579 684 return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]); 580 685 } 581 686 582 687 eType0 = sqlite3_value_type(apVal[0]); 583 688 eConflict = sqlite3_vtab_on_conflict(pConfig->db); ................................................................................ 835 940 } 836 941 } 837 942 } 838 943 839 944 fts5CloseMethod((sqlite3_vtab_cursor*)pNew); 840 945 return rc; 841 946 } 947 + 948 +static void fts5ApiInvoke( 949 + Fts5Auxiliary *pAux, 950 + Fts5Cursor *pCsr, 951 + sqlite3_context *context, 952 + int argc, 953 + sqlite3_value **argv 954 +){ 955 + assert( pCsr->pAux==0 ); 956 + pCsr->pAux = pAux; 957 + pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); 958 + pCsr->pAux = 0; 959 +} 842 960 843 961 static void fts5ApiCallback( 844 962 sqlite3_context *context, 845 963 int argc, 846 964 sqlite3_value **argv 847 965 ){ 848 966 ................................................................................ 857 975 for(pCsr=pAux->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ 858 976 if( pCsr->iCsrId==iCsrId ) break; 859 977 } 860 978 if( pCsr==0 ){ 861 979 char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); 862 980 sqlite3_result_error(context, zErr, -1); 863 981 }else{ 864 - assert( pCsr->pAux==0 ); 865 - pCsr->pAux = pAux; 866 - pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc-1, &argv[1]); 867 - pCsr->pAux = 0; 982 + fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); 983 + } 984 +} 985 + 986 +/* 987 +** This is the xColumn method, called by SQLite to request a value from 988 +** the row that the supplied cursor currently points to. 989 +*/ 990 +static int fts5ColumnMethod( 991 + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ 992 + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ 993 + int iCol /* Index of column to read value from */ 994 +){ 995 + Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig; 996 + Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 997 + int rc = SQLITE_OK; 998 + 999 + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); 1000 + 1001 + if( iCol==pConfig->nCol ){ 1002 + if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){ 1003 + /* todo */ 1004 + }else{ 1005 + /* User is requesting the value of the special column with the same name 1006 + ** as the table. Return the cursor integer id number. This value is only 1007 + ** useful in that it may be passed as the first argument to an FTS5 1008 + ** auxiliary function. */ 1009 + sqlite3_result_int64(pCtx, pCsr->iCsrId); 1010 + } 1011 + }else if( iCol==pConfig->nCol+1 ){ 1012 + /* The value of the "rank" column. */ 1013 + if( pCsr->pRank ){ 1014 + fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, 0, 0); 1015 + } 1016 + }else{ 1017 + rc = fts5SeekCursor(pCsr); 1018 + if( rc==SQLITE_OK ){ 1019 + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); 1020 + } 868 1021 } 1022 + return rc; 869 1023 } 870 1024 871 1025 872 1026 /* 873 1027 ** This routine implements the xFindFunction method for the FTS3 874 1028 ** virtual table. 875 1029 */
Changes to ext/fts5/fts5Int.h.
24 24 ** less than 32. If it is set to anything large than that, an #error 25 25 ** directive in fts5_index.c will cause the build to fail. 26 26 */ 27 27 #define FTS5_MAX_PREFIX_INDEXES 31 28 28 29 29 #define FTS5_DEFAULT_NEARDIST 10 30 30 31 +/* Name of rank column */ 32 +#define FTS5_RANK_NAME "rank" 33 + 31 34 /************************************************************************** 32 35 ** Interface to code in fts5_config.c. fts5_config.c contains contains code 33 36 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. 34 37 */ 35 38 36 39 typedef struct Fts5Config Fts5Config; 37 40 ................................................................................ 390 393 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); 391 394 392 395 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); 393 396 void sqlite3Fts5ParseSetColumn(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); 394 397 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); 395 398 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); 396 399 397 - 398 400 /* 399 401 ** End of interface to code in fts5_expr.c. 400 402 **************************************************************************/ 401 403 402 404 403 405 /************************************************************************** 404 406 ** Interface to code in fts5.c. ................................................................................ 419 421 420 422 /************************************************************************** 421 423 ** Interface to code in fts5_aux.c. 422 424 */ 423 425 424 426 int sqlite3Fts5AuxInit(Fts5Global*); 425 427 /* 426 -** End of interface to code in fts5_expr.c. 428 +** End of interface to code in fts5_aux.c. 429 +**************************************************************************/ 430 + 431 +/************************************************************************** 432 +** Interface to code in fts5_sorter.c. 433 +*/ 434 +typedef struct Fts5Sorter Fts5Sorter; 435 + 436 +int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp); 437 + 438 +/* 439 +** End of interface to code in fts5_sorter.c. 427 440 **************************************************************************/ 428 441 429 442 #endif
Changes to ext/fts5/fts5_aux.c.
952 952 int sqlite3Fts5AuxInit(Fts5Global *pGlobal){ 953 953 struct Builtin { 954 954 const char *zFunc; /* Function name (nul-terminated) */ 955 955 void *pUserData; /* User-data pointer */ 956 956 fts5_extension_function xFunc;/* Callback function */ 957 957 void (*xDestroy)(void*); /* Destructor function */ 958 958 } aBuiltin [] = { 959 - { "bm25", 0, fts5Bm25Function, 0 }, 960 959 { "bm25debug", (void*)1, fts5Bm25Function, 0 }, 961 960 { "snippet", 0, fts5SnippetFunction, 0 }, 962 961 { "fts5_test", 0, fts5TestFunction, 0 }, 962 + { "bm25", 0, fts5Bm25Function, 0 }, 963 963 }; 964 964 965 965 int rc = SQLITE_OK; /* Return code */ 966 966 int i; /* To iterate through builtin functions */ 967 967 968 968 for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){ 969 969 rc = sqlite3Fts5CreateAux(pGlobal,
Changes to ext/fts5/fts5_config.c.
161 161 if( pRet==0 ) return SQLITE_NOMEM; 162 162 memset(pRet, 0, sizeof(Fts5Config)); 163 163 pRet->db = db; 164 164 165 165 pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg); 166 166 pRet->zDb = fts5Strdup(azArg[1]); 167 167 pRet->zName = fts5Strdup(azArg[2]); 168 - if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){ 168 + if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ 169 + *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); 170 + rc = SQLITE_ERROR; 171 + }else if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){ 169 172 rc = SQLITE_NOMEM; 170 173 }else{ 171 174 int i; 172 175 for(i=3; rc==SQLITE_OK && i<nArg; i++){ 173 176 char *zDup = fts5Strdup(azArg[i]); 174 177 if( zDup==0 ){ 175 178 rc = SQLITE_NOMEM; ................................................................................ 185 188 sqlite3Fts5Dequote(zArg); 186 189 rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr); 187 190 sqlite3_free(zDup); 188 191 zDup = 0; 189 192 } 190 193 } 191 194 192 - /* If it is not a special directive, it must be a column name */ 195 + /* If it is not a special directive, it must be a column name. In 196 + ** this case, check that it is not the reserved column name "rank". */ 193 197 if( zDup ){ 194 198 sqlite3Fts5Dequote(zDup); 195 199 pRet->azCol[pRet->nCol++] = zDup; 200 + if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){ 201 + *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup); 202 + rc = SQLITE_ERROR; 203 + } 196 204 } 197 205 } 198 206 } 199 207 } 200 208 201 209 if( rc==SQLITE_OK && pRet->pTokenizer==0 ){ 202 210 rc = fts5ConfigDefaultTokenizer(pRet); ................................................................................ 245 253 zOld = zSql; 246 254 zSql = sqlite3_mprintf("%s%s%Q", zOld, (i==0?"":", "), pConfig->azCol[i]); 247 255 sqlite3_free(zOld); 248 256 } 249 257 250 258 if( zSql ){ 251 259 zOld = zSql; 252 - zSql = sqlite3_mprintf("%s, %Q HIDDEN)", zOld, pConfig->zName); 260 + zSql = sqlite3_mprintf("%s, %Q HIDDEN, %s HIDDEN)", 261 + zOld, pConfig->zName, FTS5_RANK_NAME 262 + ); 253 263 sqlite3_free(zOld); 254 264 } 255 265 256 266 if( zSql==0 ){ 257 267 rc = SQLITE_NOMEM; 258 268 }else{ 259 269 rc = sqlite3_declare_vtab(pConfig->db, zSql);
Changes to ext/fts5/fts5_expr.c.
30 30 void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); 31 31 32 32 struct Fts5Expr { 33 33 Fts5Index *pIndex; 34 34 Fts5ExprNode *pRoot; 35 35 int bAsc; 36 36 int nPhrase; /* Number of phrases in expression */ 37 - Fts5ExprPhrase **apPhrase; /* Pointers to phrase objects */ 37 + Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ 38 38 }; 39 39 40 40 /* 41 41 ** eType: 42 42 ** Expression node type. Always one of: 43 43 ** 44 44 ** FTS5_AND (pLeft, pRight valid) ................................................................................ 212 212 if( sParse.rc==SQLITE_OK ){ 213 213 *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); 214 214 if( pNew==0 ){ 215 215 sParse.rc = SQLITE_NOMEM; 216 216 }else{ 217 217 pNew->pRoot = sParse.pExpr; 218 218 pNew->pIndex = 0; 219 - pNew->apPhrase = sParse.apPhrase; 219 + pNew->apExprPhrase = sParse.apPhrase; 220 220 pNew->nPhrase = sParse.nPhrase; 221 221 sParse.apPhrase = 0; 222 222 } 223 223 } 224 224 225 225 sqlite3_free(sParse.apPhrase); 226 226 *pzErr = sParse.zErr; ................................................................................ 271 271 /* Components of the new expression object */ 272 272 Fts5Expr *pNew; 273 273 Fts5ExprPhrase **apPhrase; 274 274 Fts5ExprNode *pNode; 275 275 Fts5ExprNearset *pNear; 276 276 Fts5ExprPhrase *pCopy; 277 277 278 - pOrig = pExpr->apPhrase[iPhrase]; 278 + pOrig = pExpr->apExprPhrase[iPhrase]; 279 279 pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr)); 280 280 apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*)); 281 281 pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode)); 282 282 pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 283 283 sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) 284 284 ); 285 285 pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, ................................................................................ 292 292 } 293 293 294 294 if( rc==SQLITE_OK ){ 295 295 /* All the allocations succeeded. Put the expression object together. */ 296 296 pNew->pIndex = pExpr->pIndex; 297 297 pNew->pRoot = pNode; 298 298 pNew->nPhrase = 1; 299 - pNew->apPhrase = apPhrase; 300 - pNew->apPhrase[0] = pCopy; 299 + pNew->apExprPhrase = apPhrase; 300 + pNew->apExprPhrase[0] = pCopy; 301 301 302 302 pNode->eType = FTS5_STRING; 303 303 pNode->pNear = pNear; 304 304 305 305 pNear->iCol = -1; 306 306 pNear->nPhrase = 1; 307 307 pNear->apPhrase[0] = pCopy; ................................................................................ 341 341 342 342 /* 343 343 ** Free the expression object passed as the only argument. 344 344 */ 345 345 void sqlite3Fts5ExprFree(Fts5Expr *p){ 346 346 if( p ){ 347 347 sqlite3Fts5ParseNodeFree(p->pRoot); 348 - sqlite3_free(p->apPhrase); 348 + sqlite3_free(p->apExprPhrase); 349 349 sqlite3_free(p); 350 350 } 351 351 } 352 352 353 353 /* 354 354 ** All individual term iterators in pPhrase are guaranteed to be valid and 355 355 ** pointing to the same rowid when this function is called. This function ................................................................................ 1584 1584 } 1585 1585 1586 1586 /* 1587 1587 ** Return the number of terms in the iPhrase'th phrase in pExpr. 1588 1588 */ 1589 1589 int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){ 1590 1590 if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0; 1591 - return pExpr->apPhrase[iPhrase]->nTerm; 1591 + return pExpr->apExprPhrase[iPhrase]->nTerm; 1592 1592 } 1593 1593 1594 1594 /* 1595 1595 ** This function is used to access the current position list for phrase 1596 1596 ** iPhrase. 1597 1597 */ 1598 1598 int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ 1599 1599 if( iPhrase>=0 && iPhrase<pExpr->nPhrase ){ 1600 - Fts5ExprPhrase *pPhrase = pExpr->apPhrase[iPhrase]; 1600 + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; 1601 1601 Fts5ExprNode *pNode = pPhrase->pNode; 1602 1602 if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){ 1603 1603 *pa = pPhrase->poslist.p; 1604 1604 return pPhrase->poslist.n; 1605 1605 } 1606 1606 } 1607 1607 *pa = 0; 1608 1608 return 0; 1609 1609 } 1610 1610
Changes to test/fts5aa.test.
271 271 foreach {rowid x y} $d10 { 272 272 do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) } 273 273 do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') } 274 274 } 275 275 276 276 do_execsql_test 10.4.1 { DELETE FROM t1 } 277 277 do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } 278 + 279 +#------------------------------------------------------------------------- 280 +# 281 +do_catchsql_test 11.1 { 282 + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank); 283 +} {1 {reserved fts5 column name: rank}} 284 +do_catchsql_test 11.2 { 285 + CREATE VIRTUAL TABLE rank USING fts5(a, b, c); 286 +} {1 {reserved fts5 table name: rank}} 278 287 279 288 finish_test 280 289
Changes to test/fts5ae.test.
257 257 1 {a} {0 2} 258 258 2 {b} {3 1} 259 259 3 {c} {1 0} 260 260 4 {d} {2 3} 261 261 5 {g AND (e OR f)} {5 4} 262 262 6 {j AND (h OR i)} {5 6} 263 263 } { 264 - do_execsql_test 8.2.$tn { 264 + do_execsql_test 8.2.$tn.1 { 265 265 SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8) DESC; 266 266 } $res 267 + 268 + do_execsql_test 8.2.$tn.2 { 269 + SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank DESC; 270 + } $res 271 + 272 + do_execsql_test 8.3.$tn.3 { 273 + SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank DESC; 274 + } $res 267 275 } 268 276 269 277 270 278 finish_test 271 279