Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix things so that the fts5 extension API works with "ORDER BY rank" queries. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
f1b4e1a98d49ecaba962beba16f82241 |
User & Date: | dan 2014-07-30 20:26:24.443 |
Context
2014-07-31
| ||
11:57 | Add further tests for the extension APIs with "ORDER BY rank" queries. (check-in: 37a417d27e user: dan tags: fts5) | |
2014-07-30
| ||
20:26 | Fix things so that the fts5 extension API works with "ORDER BY rank" queries. (check-in: f1b4e1a98d user: dan tags: fts5) | |
19:41 | Add hidden column "rank". Currently this always returns the same value as the bm25() function. (check-in: 4cc048c365 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
︙ | ︙ | |||
69 70 71 72 73 74 75 | ** ** SELECT rowid, <fts> FROM <fts> ORDER BY +rank; ** */ struct Fts5Sorter { sqlite3_stmt *pStmt; i64 iRowid; /* Current rowid */ | | | 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | ** ** SELECT rowid, <fts> FROM <fts> ORDER BY +rank; ** */ struct Fts5Sorter { sqlite3_stmt *pStmt; i64 iRowid; /* Current rowid */ const u8 *aPoslist; /* Position lists for current row */ int nIdx; /* Number of entries in aIdx[] */ int aIdx[0]; /* Offsets into aPoslist for current row */ }; /* ** Virtual-table cursor object. */ |
︙ | ︙ | |||
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 | int rc; rc = sqlite3_step(pSorter->pStmt); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; CsrFlagSet(pCsr, FTS5CSR_EOF); }else if( rc==SQLITE_ROW ){ rc = SQLITE_OK; pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); } return rc; } /* ** Advance the cursor to the next row in the table that matches the | > > > > > > > > > > > > > > | 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 | int rc; rc = sqlite3_step(pSorter->pStmt); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; CsrFlagSet(pCsr, FTS5CSR_EOF); }else if( rc==SQLITE_ROW ){ const u8 *a; const u8 *aBlob; int nBlob; int i; rc = SQLITE_OK; pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); for(i=0; i<(pSorter->nIdx-1); i++){ a += getVarint32(a, pSorter->aIdx[i]); } pSorter->aIdx[i] = &aBlob[nBlob] - a; pSorter->aPoslist = a; CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE ); } return rc; } /* ** Advance the cursor to the next row in the table that matches the |
︙ | ︙ | |||
463 464 465 466 467 468 469 | nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase; pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); if( pSorter==0 ) return SQLITE_NOMEM; memset(pSorter, 0, nByte); | | | 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 | nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase; pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); if( pSorter==0 ) return SQLITE_NOMEM; memset(pSorter, 0, nByte); zSql = sqlite3_mprintf("SELECT rowid, %s FROM %Q.%Q ORDER BY +%s %s", pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME, bAsc ? "ASC" : "DESC" ); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0); |
︙ | ︙ | |||
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 | ** This is the xEof method of the virtual table. SQLite calls this ** routine to find out if it has reached the end of a result set. */ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0); } /* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. fts5 ** exposes %_content.docid as the rowid for the virtual table. The ** rowid should be written to *pRowid. */ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int ePlan = FTS5_PLAN(pCsr->idxNum); assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); switch( ePlan ){ case FTS5_PLAN_SOURCE: case FTS5_PLAN_MATCH: | > > > > > > > > > > > > > > > < < < | | | 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 | ** This is the xEof method of the virtual table. SQLite calls this ** routine to find out if it has reached the end of a result set. */ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0); } /* ** Return the rowid that the cursor currently points to. */ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ assert( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SORTED_MATCH || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ); if( pCsr->pSorter ){ return pCsr->pSorter->iRowid; }else{ return sqlite3Fts5ExprRowid(pCsr->pExpr); } } /* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. fts5 ** exposes %_content.docid as the rowid for the virtual table. The ** rowid should be written to *pRowid. */ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int ePlan = FTS5_PLAN(pCsr->idxNum); assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); switch( ePlan ){ case FTS5_PLAN_SOURCE: case FTS5_PLAN_MATCH: case FTS5_PLAN_SORTED_MATCH: *pRowid = fts5CursorRowid(pCsr); break; default: *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); break; } return SQLITE_OK; } /* ** If the cursor requires seeking (bSeekRequired flag is set), seek it. ** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. */ static int fts5SeekCursor(Fts5Cursor *pCsr){ int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ assert( pCsr->pExpr ); sqlite3_reset(pCsr->pStmt); sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ rc = SQLITE_OK; CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT); }else{ rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK ){ |
︙ | ︙ | |||
782 783 784 785 786 787 788 | static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; | | | 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 | static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return fts5CursorRowid((Fts5Cursor*)pCtx); } static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, const char **pz, int *pn |
︙ | ︙ | |||
806 807 808 809 810 811 812 | static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ | | > > > > > > | > | 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 | static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ i64 iRowid = fts5CursorRowid(pCsr); rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); } if( iCol>=0 && iCol<pTab->pConfig->nCol ){ *pnToken = pCsr->aColumnSize[iCol]; }else{ *pnToken = 0; } return rc; } static int fts5ApiPoslist( Fts5Context *pCtx, int iPhrase, int *pi, i64 *piPos ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; const u8 *a; int n; /* Poslist for phrase iPhrase */ if( pCsr->pSorter ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase ? 0 : pSorter->aIdx[iPhrase-1]); n = pSorter->aIdx[iPhrase] - i1; a = &pSorter->aPoslist[i1]; }else{ n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a); } return sqlite3Fts5PoslistNext64(a, n, pi, piPos); } static int fts5ApiSetAuxdata( Fts5Context *pCtx, /* Fts5 context */ void *pPtr, /* Pointer to save as auxdata */ void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */ |
︙ | ︙ | |||
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 | if( pCsr==0 ){ char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); sqlite3_result_error(context, zErr, -1); }else{ fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); } } /* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. */ static int fts5ColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig; Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); if( iCol==pConfig->nCol ){ if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){ | > > > > > > > > > > > > > > > > > > > > > > > > > | | 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 | if( pCsr==0 ){ char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); sqlite3_result_error(context, zErr, -1); }else{ fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); } } static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ int i; int rc = SQLITE_OK; int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); Fts5Buffer val; int iOff = 0; memset(&val, 0, sizeof(Fts5Buffer)); for(i=0; i<nPhrase; i++){ const u8 *dummy; if( i ) sqlite3Fts5BufferAppendVarint(&rc, &val, iOff); iOff += sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); } for(i=0; i<nPhrase; i++){ const u8 *pPoslist; int nPoslist; nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); } sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); return rc; } /* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. */ static int fts5ColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig; Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); if( iCol==pConfig->nCol ){ if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); }else{ /* User is requesting the value of the special column with the same name ** as the table. Return the cursor integer id number. This value is only ** useful in that it may be passed as the first argument to an FTS5 ** auxiliary function. */ sqlite3_result_int64(pCtx, pCsr->iCsrId); } |
︙ | ︙ |
Changes to test/fts5af.test.
︙ | ︙ | |||
42 43 44 45 46 47 48 49 50 51 52 53 54 55 | } [list $res] do_execsql_test $tn.2 { DELETE FROM t1; INSERT INTO t1 VALUES(NULL, $v1); SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2; } [list $res] } foreach {tn doc res} { 1.1 {X o o o o o o} {[X] o o o o o o} | > > > > > > > > | 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | } [list $res] do_execsql_test $tn.2 { DELETE FROM t1; INSERT INTO t1 VALUES(NULL, $v1); SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2; } [list $res] do_execsql_test $tn.3 { DELETE FROM t1; INSERT INTO t1 VALUES($v1, NULL); SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2 ORDER BY rank DESC; } [list $res] } foreach {tn doc res} { 1.1 {X o o o o o o} {[X] o o o o o o} |
︙ | ︙ |