Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix things so that the fts5 extension API works with "ORDER BY rank" queries. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
f1b4e1a98d49ecaba962beba16f82241 |
| User & Date: | dan 2014-07-30 20:26:24 |
Context
|
2014-07-31
| ||
| 11:57 | Add further tests for the extension APIs with "ORDER BY rank" queries. (check-in: 37a417d2 user: dan tags: fts5) | |
|
2014-07-30
| ||
| 20:26 | Fix things so that the fts5 extension API works with "ORDER BY rank" queries. (check-in: f1b4e1a9 user: dan tags: fts5) | |
| 19:41 | Add hidden column "rank". Currently this always returns the same value as the bm25() function. (check-in: 4cc048c3 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 ... 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 ... 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 ... 566 567 568 569 570 571 572 573 574 575 576 577 578 579 ... 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 ... 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 ... 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 ... 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 ... 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 ... 978 979 980 981 982 983 984 985 986 987 988 989 990 991 ... 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 |
**
** SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
**
*/
struct Fts5Sorter {
sqlite3_stmt *pStmt;
i64 iRowid; /* Current rowid */
u8 *aPoslist; /* Position lists for current row */
int nIdx; /* Number of entries in aIdx[] */
int aIdx[0]; /* Offsets into aPoslist for current row */
};
/*
** Virtual-table cursor object.
*/
................................................................................
int rc;
rc = sqlite3_step(pSorter->pStmt);
if( rc==SQLITE_DONE ){
rc = SQLITE_OK;
CsrFlagSet(pCsr, FTS5CSR_EOF);
}else if( rc==SQLITE_ROW ){
rc = SQLITE_OK;
pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
}
return rc;
}
/*
** Advance the cursor to the next row in the table that matches the
................................................................................
nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase;
pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
if( pSorter==0 ) return SQLITE_NOMEM;
memset(pSorter, 0, nByte);
zSql = sqlite3_mprintf("SELECT rowid, %Q FROM %Q.%Q ORDER BY +%s %s",
pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME,
bAsc ? "ASC" : "DESC"
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
................................................................................
** This is the xEof method of the virtual table. SQLite calls this
** routine to find out if it has reached the end of a result set.
*/
static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
}
/*
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. fts5
** exposes %_content.docid as the rowid for the virtual table. The
** rowid should be written to *pRowid.
*/
................................................................................
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int ePlan = FTS5_PLAN(pCsr->idxNum);
assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
switch( ePlan ){
case FTS5_PLAN_SOURCE:
case FTS5_PLAN_MATCH:
*pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
break;
case FTS5_PLAN_SORTED_MATCH:
*pRowid = pCsr->pSorter->iRowid;
break;
default:
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
break;
}
................................................................................
** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
int rc = SQLITE_OK;
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
assert( pCsr->pExpr );
sqlite3_reset(pCsr->pStmt);
sqlite3_bind_int64(pCsr->pStmt, 1, sqlite3Fts5ExprRowid(pCsr->pExpr));
rc = sqlite3_step(pCsr->pStmt);
if( rc==SQLITE_ROW ){
rc = SQLITE_OK;
CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
}else{
rc = sqlite3_reset(pCsr->pStmt);
if( rc==SQLITE_OK ){
................................................................................
static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}
static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return sqlite3Fts5ExprRowid(pCsr->pExpr);
}
static int fts5ApiColumnText(
Fts5Context *pCtx,
int iCol,
const char **pz,
int *pn
................................................................................
static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
int rc = SQLITE_OK;
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
}
if( iCol>=0 && iCol<pTab->pConfig->nCol ){
*pnToken = pCsr->aColumnSize[iCol];
}else{
*pnToken = 0;
}
................................................................................
Fts5Context *pCtx,
int iPhrase,
int *pi,
i64 *piPos
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
const u8 *a; int n; /* Poslist for phrase iPhrase */
n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);
return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}
static int fts5ApiSetAuxdata(
Fts5Context *pCtx, /* Fts5 context */
void *pPtr, /* Pointer to save as auxdata */
void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */
................................................................................
if( pCsr==0 ){
char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
sqlite3_result_error(context, zErr, -1);
}else{
fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
}
}
/*
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
*/
static int fts5ColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
................................................................................
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int rc = SQLITE_OK;
assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
if( iCol==pConfig->nCol ){
if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
/* todo */
}else{
/* User is requesting the value of the special column with the same name
** as the table. Return the cursor integer id number. This value is only
** useful in that it may be passed as the first argument to an FTS5
** auxiliary function. */
sqlite3_result_int64(pCtx, pCsr->iCsrId);
}
|
| > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > < < < < > | | | > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > | |
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 ... 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 ... 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 ... 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 ... 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 ... 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 ... 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 ... 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 ... 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 .... 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 .... 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 |
**
** SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
**
*/
struct Fts5Sorter {
sqlite3_stmt *pStmt;
i64 iRowid; /* Current rowid */
const u8 *aPoslist; /* Position lists for current row */
int nIdx; /* Number of entries in aIdx[] */
int aIdx[0]; /* Offsets into aPoslist for current row */
};
/*
** Virtual-table cursor object.
*/
................................................................................
int rc;
rc = sqlite3_step(pSorter->pStmt);
if( rc==SQLITE_DONE ){
rc = SQLITE_OK;
CsrFlagSet(pCsr, FTS5CSR_EOF);
}else if( rc==SQLITE_ROW ){
const u8 *a;
const u8 *aBlob;
int nBlob;
int i;
rc = SQLITE_OK;
pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
nBlob = sqlite3_column_bytes(pSorter->pStmt, 1);
aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1);
for(i=0; i<(pSorter->nIdx-1); i++){
a += getVarint32(a, pSorter->aIdx[i]);
}
pSorter->aIdx[i] = &aBlob[nBlob] - a;
pSorter->aPoslist = a;
CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
}
return rc;
}
/*
** Advance the cursor to the next row in the table that matches the
................................................................................
nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase;
pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
if( pSorter==0 ) return SQLITE_NOMEM;
memset(pSorter, 0, nByte);
zSql = sqlite3_mprintf("SELECT rowid, %s FROM %Q.%Q ORDER BY +%s %s",
pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME,
bAsc ? "ASC" : "DESC"
);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
................................................................................
** This is the xEof method of the virtual table. SQLite calls this
** routine to find out if it has reached the end of a result set.
*/
static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
}
/*
** Return the rowid that the cursor currently points to.
*/
static i64 fts5CursorRowid(Fts5Cursor *pCsr){
assert( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH
|| FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SORTED_MATCH
|| FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE
);
if( pCsr->pSorter ){
return pCsr->pSorter->iRowid;
}else{
return sqlite3Fts5ExprRowid(pCsr->pExpr);
}
}
/*
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. fts5
** exposes %_content.docid as the rowid for the virtual table. The
** rowid should be written to *pRowid.
*/
................................................................................
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int ePlan = FTS5_PLAN(pCsr->idxNum);
assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
switch( ePlan ){
case FTS5_PLAN_SOURCE:
case FTS5_PLAN_MATCH:
case FTS5_PLAN_SORTED_MATCH:
*pRowid = fts5CursorRowid(pCsr);
break;
default:
*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
break;
}
................................................................................
** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
*/
static int fts5SeekCursor(Fts5Cursor *pCsr){
int rc = SQLITE_OK;
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
assert( pCsr->pExpr );
sqlite3_reset(pCsr->pStmt);
sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
rc = sqlite3_step(pCsr->pStmt);
if( rc==SQLITE_ROW ){
rc = SQLITE_OK;
CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
}else{
rc = sqlite3_reset(pCsr->pStmt);
if( rc==SQLITE_OK ){
................................................................................
static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}
static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return fts5CursorRowid((Fts5Cursor*)pCtx);
}
static int fts5ApiColumnText(
Fts5Context *pCtx,
int iCol,
const char **pz,
int *pn
................................................................................
static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
int rc = SQLITE_OK;
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
i64 iRowid = fts5CursorRowid(pCsr);
rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
}
if( iCol>=0 && iCol<pTab->pConfig->nCol ){
*pnToken = pCsr->aColumnSize[iCol];
}else{
*pnToken = 0;
}
................................................................................
Fts5Context *pCtx,
int iPhrase,
int *pi,
i64 *piPos
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
const u8 *a; int n; /* Poslist for phrase iPhrase */
if( pCsr->pSorter ){
Fts5Sorter *pSorter = pCsr->pSorter;
int i1 = (iPhrase ? 0 : pSorter->aIdx[iPhrase-1]);
n = pSorter->aIdx[iPhrase] - i1;
a = &pSorter->aPoslist[i1];
}else{
n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);
}
return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}
static int fts5ApiSetAuxdata(
Fts5Context *pCtx, /* Fts5 context */
void *pPtr, /* Pointer to save as auxdata */
void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */
................................................................................
if( pCsr==0 ){
char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
sqlite3_result_error(context, zErr, -1);
}else{
fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
}
}
static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
int i;
int rc = SQLITE_OK;
int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
Fts5Buffer val;
int iOff = 0;
memset(&val, 0, sizeof(Fts5Buffer));
for(i=0; i<nPhrase; i++){
const u8 *dummy;
if( i ) sqlite3Fts5BufferAppendVarint(&rc, &val, iOff);
iOff += sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
}
for(i=0; i<nPhrase; i++){
const u8 *pPoslist;
int nPoslist;
nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
}
sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free);
return rc;
}
/*
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
*/
static int fts5ColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
................................................................................
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int rc = SQLITE_OK;
assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
if( iCol==pConfig->nCol ){
if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
fts5PoslistBlob(pCtx, pCsr);
}else{
/* User is requesting the value of the special column with the same name
** as the table. Return the cursor integer id number. This value is only
** useful in that it may be passed as the first argument to an FTS5
** auxiliary function. */
sqlite3_result_int64(pCtx, pCsr->iCsrId);
}
|
Changes to test/fts5af.test.
42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
} [list $res]
do_execsql_test $tn.2 {
DELETE FROM t1;
INSERT INTO t1 VALUES(NULL, $v1);
SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
} [list $res]
}
foreach {tn doc res} {
1.1 {X o o o o o o} {[X] o o o o o o}
|
> > > > > > > > |
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
} [list $res]
do_execsql_test $tn.2 {
DELETE FROM t1;
INSERT INTO t1 VALUES(NULL, $v1);
SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
} [list $res]
do_execsql_test $tn.3 {
DELETE FROM t1;
INSERT INTO t1 VALUES($v1, NULL);
SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2
ORDER BY rank DESC;
} [list $res]
}
foreach {tn doc res} {
1.1 {X o o o o o o} {[X] o o o o o o}
|