Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add extension apis xRowCount, xQueryPhrase, xSetAuxdata and xGetAuxdata. And a ranking function that uses all of the above. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
c4d50428ab97f77e6721c4f8d03eaaf3 |
User & Date: | dan 2014-07-25 20:30:47.445 |
Context
2014-07-26
| ||
18:38 | Add tests and fixes for bm25() function. (check-in: 71d32f53e8 user: dan tags: fts5) | |
2014-07-25
| ||
20:30 | Add extension apis xRowCount, xQueryPhrase, xSetAuxdata and xGetAuxdata. And a ranking function that uses all of the above. (check-in: c4d50428ab user: dan tags: fts5) | |
2014-07-23
| ||
19:31 | Add a snippet() function to fts5. (check-in: bdc58fd28a user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
︙ | ︙ | |||
15 16 17 18 19 20 21 22 23 24 25 26 27 28 | #include "fts5Int.h" typedef struct Fts5Table Fts5Table; typedef struct Fts5Cursor Fts5Cursor; typedef struct Fts5Global Fts5Global; typedef struct Fts5Auxiliary Fts5Auxiliary; /* ** A single object of this type is allocated when the FTS5 module is ** registered with a database handle. It is used to store pointers to ** all registered FTS5 extensions - tokenizers and auxiliary functions. */ struct Fts5Global { | > | 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | #include "fts5Int.h" typedef struct Fts5Table Fts5Table; typedef struct Fts5Cursor Fts5Cursor; typedef struct Fts5Global Fts5Global; typedef struct Fts5Auxiliary Fts5Auxiliary; typedef struct Fts5Auxdata Fts5Auxdata; /* ** A single object of this type is allocated when the FTS5 module is ** registered with a database handle. It is used to store pointers to ** all registered FTS5 extensions - tokenizers and auxiliary functions. */ struct Fts5Global { |
︙ | ︙ | |||
71 72 73 74 75 76 77 | sqlite3_stmt *pStmt; /* Statement used to read %_content */ Fts5Expr *pExpr; /* Expression for MATCH queries */ int csrflags; /* Mask of cursor flags (see below) */ Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ /* Variables used by auxiliary functions */ i64 iCsrId; /* Cursor id */ | | > > > > > > > > | 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | sqlite3_stmt *pStmt; /* Statement used to read %_content */ Fts5Expr *pExpr; /* Expression for MATCH queries */ int csrflags; /* Mask of cursor flags (see below) */ Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ /* Variables used by auxiliary functions */ i64 iCsrId; /* Cursor id */ Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of aux-data */ int *aColumnSize; /* Values for xColumnSize() */ }; /* ** Values for Fts5Cursor.csrflags */ #define FTS5CSR_REQUIRE_CONTENT 0x01 #define FTS5CSR_REQUIRE_DOCSIZE 0x02 #define FTS5CSR_EOF 0x04 /* ** Macros to Set(), Clear() and Test() cursor flags. */ #define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag)) #define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag)) #define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag)) struct Fts5Auxdata { Fts5Auxiliary *pAux; /* Extension to which this belongs */ void *pPtr; /* Pointer value */ void(*xDelete)(void*); /* Destructor */ Fts5Auxdata *pNext; /* Next object in linked list */ }; /* ** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy ** argument is non-zero, attempt delete the shadow tables from teh database */ static int fts5FreeVtab(Fts5Table *pTab, int bDestroy){ int rc = SQLITE_OK; |
︙ | ︙ | |||
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 | ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; Fts5Cursor **pp; if( pCsr->pStmt ){ int eStmt = fts5StmtType(pCsr->idxNum); sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); } sqlite3Fts5ExprFree(pCsr->pExpr); /* Remove the cursor from the Fts5Global.pCsr list */ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); *pp = pCsr->pNext; sqlite3_free(pCsr); return SQLITE_OK; | > > > > > > > > > | 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 | ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; Fts5Cursor **pp; Fts5Auxdata *pData; Fts5Auxdata *pNext; if( pCsr->pStmt ){ int eStmt = fts5StmtType(pCsr->idxNum); sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); } sqlite3Fts5ExprFree(pCsr->pExpr); for(pData=pCsr->pAuxdata; pData; pData=pNext){ pNext = pData->pNext; if( pData->xDelete ) pData->xDelete(pData->pPtr); sqlite3_free(pData); } /* Remove the cursor from the Fts5Global.pCsr list */ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); *pp = pCsr->pNext; sqlite3_free(pCsr); return SQLITE_OK; |
︙ | ︙ | |||
367 368 369 370 371 372 373 374 375 376 377 378 379 380 | CsrFlagSet(pCsr, FTS5CSR_EOF); } CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE ); } return rc; } /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. */ static int fts5FilterMethod( | > > > > > > > > > > | 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 | CsrFlagSet(pCsr, FTS5CSR_EOF); } CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE ); } return rc; } static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){ int rc; rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc); if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE ); return rc; } /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. */ static int fts5FilterMethod( |
︙ | ︙ | |||
399 400 401 402 403 404 405 | rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); if( rc==SQLITE_OK ){ if( ePlan==FTS5_PLAN_MATCH ){ char **pzErr = &pTab->base.zErrMsg; const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); if( rc==SQLITE_OK ){ | | < < < < | 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 | rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt); if( rc==SQLITE_OK ){ if( ePlan==FTS5_PLAN_MATCH ){ char **pzErr = &pTab->base.zErrMsg; const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); if( rc==SQLITE_OK ){ rc = fts5CursorFirst(pTab, pCsr, bAsc); } }else{ if( ePlan==FTS5_PLAN_ROWID ){ sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); } rc = fts5NextMethod(pCursor); } |
︙ | ︙ | |||
614 615 616 617 618 619 620 | } static int fts5ApiColumnCount(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; } | | > > > > | > > > > > > | 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 | } static int fts5ApiColumnCount(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; } static int fts5ApiColumnTotalSize( Fts5Context *pCtx, int iCol, sqlite3_int64 *pnToken ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken); } static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); } static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, int (*xToken)(void*, const char*, int, int, int, int) |
︙ | ︙ | |||
689 690 691 692 693 694 695 696 697 698 699 700 701 | i64 *piPos ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; const u8 *a; int n; /* Poslist for phrase iPhrase */ n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a); return sqlite3Fts5PoslistNext64(a, n, pi, piPos); } static void fts5ApiCallback( sqlite3_context *context, int argc, sqlite3_value **argv ){ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < < < < < < < < < < < | | 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 | i64 *piPos ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; const u8 *a; int n; /* Poslist for phrase iPhrase */ n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a); return sqlite3Fts5PoslistNext64(a, n, pi, piPos); } static int fts5ApiSetAuxdata( Fts5Context *pCtx, /* Fts5 context */ void *pPtr, /* Pointer to save as auxdata */ void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */ ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Auxdata *pData; for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ if( pData->pAux==pCsr->pAux ) break; } if( pData ){ if( pData->xDelete ){ pData->xDelete(pData->pPtr); } }else{ pData = (Fts5Auxdata*)sqlite3_malloc(sizeof(Fts5Auxdata)); if( pData==0 ) return SQLITE_NOMEM; memset(pData, 0, sizeof(Fts5Auxdata)); pData->pAux = pCsr->pAux; pData->pNext = pCsr->pAuxdata; pCsr->pAuxdata = pData; } pData->xDelete = xDelete; pData->pPtr = pPtr; return SQLITE_OK; } static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Auxdata *pData; void *pRet = 0; for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){ if( pData->pAux==pCsr->pAux ) break; } if( pData ){ pRet = pData->pPtr; if( bClear ){ pData->pPtr = 0; pData->xDelete = 0; } } return pRet; } static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { 1, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, fts5ApiColumnTotalSize, fts5ApiTokenize, fts5ApiPhraseCount, fts5ApiPhraseSize, fts5ApiRowid, fts5ApiColumnText, fts5ApiColumnSize, fts5ApiPoslist, fts5ApiQueryPhrase, fts5ApiSetAuxdata, fts5ApiGetAuxdata, }; /* ** Implementation of API function xQueryPhrase(). */ static int fts5ApiQueryPhrase( Fts5Context *pCtx, int iPhrase, void *pUserData, int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc; Fts5Cursor *pNew = 0; rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); if( rc==SQLITE_OK ){ Fts5Config *pConf = pTab->pConfig; pNew->idxNum = FTS5_PLAN_MATCH; pNew->base.pVtab = (sqlite3_vtab*)pTab; rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); } if( rc==SQLITE_OK ){ for(rc = fts5CursorFirst(pTab, pNew, 0); rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0; rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew) ){ rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData); if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ) rc = SQLITE_OK; break; } } } fts5CloseMethod((sqlite3_vtab_cursor*)pNew); return rc; } static void fts5ApiCallback( sqlite3_context *context, int argc, sqlite3_value **argv ){ Fts5Auxiliary *pAux; Fts5Cursor *pCsr; i64 iCsrId; assert( argc>=1 ); pAux = (Fts5Auxiliary*)sqlite3_user_data(context); iCsrId = sqlite3_value_int64(argv[0]); for(pCsr=pAux->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ if( pCsr->iCsrId==iCsrId ) break; } if( pCsr==0 ){ char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); sqlite3_result_error(context, zErr, -1); }else{ assert( pCsr->pAux==0 ); pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc-1, &argv[1]); pCsr->pAux = 0; } } /* ** This routine implements the xFindFunction method for the FTS3 |
︙ | ︙ |
Changes to ext/fts5/fts5.h.
︙ | ︙ | |||
39 40 41 42 43 44 45 | sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ); /* ** | | > > > > > > > > | 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ); /* ** ** xUserData(pFts): ** ** Return a copy of the context pointer the extension function was ** registered with. ** ** ** xColumnTotalSize(pFts, iCol, pnToken): ** ** Returns the total number of tokens in column iCol, considering all ** rows in the FTS5 table. ** ** ** xColumnCount: ** Returns the number of columns in the FTS5 table. ** ** xColumnSize: ** Reports the size in tokens of a column value from the current row. ** |
︙ | ︙ | |||
69 70 71 72 73 74 75 76 77 78 79 80 81 82 | ** xPoslist: ** Iterate through instances of phrase iPhrase in the current row. ** ** At EOF, a non-zero value is returned and output variable iPos set to -1. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > | 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | ** xPoslist: ** Iterate through instances of phrase iPhrase in the current row. ** ** At EOF, a non-zero value is returned and output variable iPos set to -1. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. ** ** ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): ** ** This API function is used to query the FTS table for phrase iPhrase ** of the current query. Specifically, a query equivalent to: ** ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC ** ** with $p set to a phrase equivalent to the phrase iPhrase of the ** current query is executed. For each row visited, the callback function ** passed as the fourth argument is invoked. The context and API objects ** passed to the callback function may be used to access the properties of ** each matched row. Invoking Api.xUserData() returns a copy of the pointer ** passed as the third argument to pUserData. ** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. ** Otherwise, the error code is propagated upwards. ** ** If the query runs to completion without incident, SQLITE_OK is returned. ** Or, if some error occurs before the query completes or is aborted by ** the callback, an SQLite error code is returned. ** ** ** xSetAuxdata(pFts5, pAux, xDelete) ** ** Save the pointer passed as the second argument as the extension functions ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of ** of the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot per ** query. If the extension function is invoked more than once by the SQL ** query, then all invocations share a single auxiliary data context. ** ** If there is already an auxiliary data pointer when this function is ** invoked, then it is replaced by the new pointer. If an xDelete callback ** was specified along with the original pointer, it is invoked at this ** point. ** ** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** ** ** xGetAuxdata(pFts5, bClear) ** ** Returns the current auxiliary data pointer for the fts5 extension ** function. See the xSetAuxdata() method for details. ** ** If the bClear argument is non-zero, then the auxiliary data is cleared ** (set to NULL) before this function returns. In this case the xDelete, ** if any, is not invoked. ** ** ** xRowCount(pFts5, pnRow) ** ** This function is used to retrieve the total number of rows in the table. ** In other words, the same value that would be returned by: ** ** SELECT count(*) FROM ftstable; */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ); int (*xPhraseCount)(Fts5Context*); int (*xPhraseSize)(Fts5Context*, int iPhrase); sqlite3_int64 (*xRowid)(Fts5Context*); int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, sqlite3_int64 *piPos); int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); }; #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF) /* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ #endif /* _FTS5_H */ |
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
296 297 298 299 300 301 302 | int sqlite3Fts5StorageIntegrity(Fts5Storage *p); int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); | | > | 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 | int sqlite3Fts5StorageIntegrity(Fts5Storage *p); int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); /* ** End of interface to code in fts5_storage.c. **************************************************************************/ |
︙ | ︙ | |||
349 350 351 352 353 354 355 356 357 358 359 360 361 362 | /* Called during startup to register a UDF with SQLite */ int sqlite3Fts5ExprInit(sqlite3*); int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by ** the parser code in fts5parse.y. */ void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); | > > | 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 | /* Called during startup to register a UDF with SQLite */ int sqlite3Fts5ExprInit(sqlite3*); int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by ** the parser code in fts5parse.y. */ void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); |
︙ | ︙ |
Changes to ext/fts5/fts5_aux.c.
︙ | ︙ | |||
8 9 10 11 12 13 14 15 16 17 18 19 20 21 | ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** */ #include "fts5Int.h" typedef struct SnippetPhrase SnippetPhrase; typedef struct SnippetIter SnippetIter; typedef struct SnippetCtx SnippetCtx; struct SnippetPhrase { u64 mask; /* Current mask */ | > | 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** */ #include "fts5Int.h" #include <math.h> typedef struct SnippetPhrase SnippetPhrase; typedef struct SnippetIter SnippetIter; typedef struct SnippetCtx SnippetCtx; struct SnippetPhrase { u64 mask; /* Current mask */ |
︙ | ︙ | |||
263 264 265 266 267 268 269 | rc = pApi->xTokenize(pFts, zCol, nCol, (void*)&ctx, fts5SnippetCallback); if( rc==SQLITE_OK ){ int i1; /* First token from input to include */ int i2; /* Last token from input to include */ int iPrint; int iMatchto; | < | 264 265 266 267 268 269 270 271 272 273 274 275 276 277 | rc = pApi->xTokenize(pFts, zCol, nCol, (void*)&ctx, fts5SnippetCallback); if( rc==SQLITE_OK ){ int i1; /* First token from input to include */ int i2; /* Last token from input to include */ int iPrint; int iMatchto; int iLast; int *aiStart = ctx.aiStart - ctx.iFirst; int *aiEnd = ctx.aiEnd - ctx.iFirst; /* Ideally we want to start the snippet with token (ctx.iFirst + nShift). ** However, this is only possible if there are sufficient tokens within |
︙ | ︙ | |||
363 364 365 366 367 368 369 | sqlite3_value **apVal /* Array of trailing arguments */ ){ const char *zStart = "<b>"; const char *zFinal = "</b>"; const char *zEllip = "<b>...</b>"; int nToken = -15; int nAbs; | < < < | 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 | sqlite3_value **apVal /* Array of trailing arguments */ ){ const char *zStart = "<b>"; const char *zFinal = "</b>"; const char *zEllip = "<b>...</b>"; int nToken = -15; int nAbs; int rc; SnippetIter *pIter = 0; if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]); if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]); if( nVal>=3 ) zEllip = (const char*)sqlite3_value_text(apVal[2]); if( nVal>=4 ){ nToken = sqlite3_value_int(apVal[3]); if( nToken==0 ) nToken = -15; } nAbs = nToken * (nToken<0 ? -1 : 1); rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter); if( rc==SQLITE_OK ){ Fts5Buffer buf; /* Result buffer */ int nBestScore = 0; /* Score of best snippet found */ for(fts5SnippetIterFirst(pIter); pIter->iLast>=0; fts5SnippetIterNext(pIter) ){ if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore; } |
︙ | ︙ | |||
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 | } fts5SnippetIterFree(pIter); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); } } static void fts5Bm25Function( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ){ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 | } fts5SnippetIterFree(pIter); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); } } typedef struct Fts5GatherCtx Fts5GatherCtx; struct Fts5GatherCtx { int nCol; int iPhrase; int *anVal; }; static int fts5GatherCallback( const Fts5ExtensionApi *pApi, Fts5Context *pFts, void *pUserData ){ Fts5GatherCtx *p = (Fts5GatherCtx*)pUserData; int i = 0; int iPrev = -1; i64 iPos = 0; while( 0==pApi->xPoslist(pFts, 0, &i, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); if( iCol!=iPrev ){ p->anVal[p->iPhrase * p->nCol + iCol]++; iPrev = iCol; } } return SQLITE_OK; } /* ** This function returns a pointer to an array of integers containing entries ** indicating the number of rows in the table for which each phrase features ** at least once in each column. ** ** If nCol is the number of matchable columns in the table, and nPhrase is ** the number of phrases in the query, the array contains a total of ** (nPhrase*nCol) entries. ** ** For phrase iPhrase and column iCol: ** ** anVal[iPhrase * nCol + iCol] ** ** is set to the number of rows in the table for which column iCol contains ** at least one instance of phrase iPhrase. */ static int fts5GatherTotals( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ Fts5Context *pFts, /* First arg to pass to pApi functions */ int **panVal ){ int rc = SQLITE_OK; int *anVal = 0; int i; /* For iterating through expression phrases */ int nPhrase = pApi->xPhraseCount(pFts); int nCol = pApi->xColumnCount(pFts); int nByte = nCol * nPhrase * sizeof(int); Fts5GatherCtx sCtx; sCtx.nCol = nCol; anVal = sCtx.anVal = (int*)sqlite3_malloc(nByte); if( anVal==0 ){ rc = SQLITE_NOMEM; }else{ memset(anVal, 0, nByte); } for(i=0; i<nPhrase && rc==SQLITE_OK; i++){ sCtx.iPhrase = i; rc = pApi->xQueryPhrase(pFts, i, (void*)&sCtx, fts5GatherCallback); } if( rc!=SQLITE_OK ){ sqlite3_free(anVal); anVal = 0; } *panVal = anVal; return rc; } typedef struct Fts5Bm25Context Fts5Bm25Context; struct Fts5Bm25Context { int nPhrase; int nCol; double *aIDF; /* Array of IDF values */ double *aAvg; /* Average size of each column in tokens */ }; static void fts5Bm25Function( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ){ const double k1 = 1.2; const double B = 0.75; int rc = SQLITE_OK; Fts5Bm25Context *p; p = pApi->xGetAuxdata(pFts, 0); if( p==0 ){ int *anVal = 0; int ic; /* For iterating through columns */ int ip; /* For iterating through phrases */ i64 nRow; /* Total number of rows in table */ int nPhrase = pApi->xPhraseCount(pFts); int nCol = pApi->xColumnCount(pFts); int nByte = sizeof(Fts5Bm25Context) + sizeof(double) * nPhrase * nCol /* aIDF[] */ + sizeof(double) * nCol; /* aAvg[] */ p = (Fts5Bm25Context*)sqlite3_malloc(nByte); if( p==0 ){ rc = SQLITE_NOMEM; }else{ memset(p, 0, nByte); p->aAvg = (double*)&p[1]; p->aIDF = (double*)&p->aAvg[nCol]; } if( rc==SQLITE_OK ){ rc = pApi->xRowCount(pFts, &nRow); assert( nRow>0 || rc!=SQLITE_OK ); } for(ic=0; rc==SQLITE_OK && ic<nCol; ic++){ i64 nToken = 0; rc = pApi->xColumnTotalSize(pFts, ic, &nToken); p->aAvg[ic] = (double)nToken / (double)nRow; } if( rc==SQLITE_OK ){ rc = fts5GatherTotals(pApi, pFts, &anVal); } for(ic=0; ic<nCol; ic++){ for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){ int idx = ip * nCol + ic; p->aIDF[idx] = log( (0.5 + nRow - anVal[idx]) / (0.5 + anVal[idx]) ); if( p->aIDF[idx]<0.0 ) p->aIDF[idx] = 0.0; } } sqlite3_free(anVal); if( rc==SQLITE_OK ){ rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); } if( rc!=SQLITE_OK ){ sqlite3_free(p); } } if( rc==SQLITE_OK ){ int ip; double score = 0.0; for(ip=0; rc==SQLITE_OK && ip<p->nPhrase; ip++){ int iPrev = 0; int nHit = 0; int i = 0; i64 iPos = 0; while( rc==SQLITE_OK && 0==pApi->xPoslist(pFts, ip, &i, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); if( iCol!=iPrev && nHit>0 ){ int sz = 0; int idx = ip * p->nCol + iPrev; rc = pApi->xColumnSize(pFts, iPrev, &sz); score += p->aIDF[idx] * nHit * (k1+1.0) / (nHit + k1 * (1.0 - B + B * sz / p->aAvg[iCol])); nHit = 0; } nHit++; iPrev = iCol; } } if( rc==SQLITE_OK ){ sqlite3_result_double(pCtx, score); } } if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); } } static int fts5TestCallback( void *pContext, /* Pointer to Fts5Buffer object */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ |
︙ | ︙ | |||
461 462 463 464 465 466 467 | zReq = (const char*)sqlite3_value_text(apVal[0]); } memset(&s, 0, sizeof(Fts5Buffer)); nCol = pApi->xColumnCount(pFts); if( zReq==0 ){ | | | | | | 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 | zReq = (const char*)sqlite3_value_text(apVal[0]); } memset(&s, 0, sizeof(Fts5Buffer)); nCol = pApi->xColumnCount(pFts); if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, "columntotalsize "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntotalsize") ){ if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{"); for(i=0; rc==SQLITE_OK && i<nCol; i++){ i64 colsz = 0; rc = pApi->xColumnTotalSize(pFts, i, &colsz); sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz); } if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}"); } if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount "); |
︙ | ︙ | |||
576 577 578 579 580 581 582 | }else{ sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p); } sqlite3_free(s3.p); } if( zReq==0 ){ | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > | 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 | }else{ sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p); } sqlite3_free(s3.p); } if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, " queryphrase "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "queryphrase") ){ int ic, ip; int *anVal = 0; Fts5Buffer buf1; memset(&buf1, 0, sizeof(Fts5Buffer)); if( rc==SQLITE_OK ){ anVal = (int*)pApi->xGetAuxdata(pFts, 0); if( anVal==0 ){ rc = fts5GatherTotals(pApi, pFts, &anVal); if( rc==SQLITE_OK ){ rc = pApi->xSetAuxdata(pFts, (void*)anVal, sqlite3_free); } } } for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){ if( ip>0 ) sqlite3Fts5BufferAppendString(&rc, &buf1, " "); if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "{"); for(ic=0; ic<nCol; ic++){ int iVal = anVal[ip * nCol + ic]; sqlite3Fts5BufferAppendPrintf(&rc, &buf1, "%s%d", ic==0?"":" ", iVal); } if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "}"); } if( zReq==0 ){ sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf1.p, buf1.n); }else{ sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf1.p); } sqlite3_free(buf1.p); } if( zReq==0 ){ sqlite3Fts5BufferAppendString(&rc, &s, " rowid "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){ iRowid = pApi->xRowid(pFts); sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid); } if( zReq==0 ){ sqlite3Fts5BufferAppendString(&rc, &s, " rowcount "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowcount") ){ i64 nRow; rc = pApi->xRowCount(pFts, &nRow); sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", nRow); } if( zReq==0 ){ sqlite3Fts5BufferAppendString(&rc, &s, " tokenize "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){ Fts5Buffer buf; memset(&buf, 0, sizeof(buf)); for(i=0; rc==SQLITE_OK && i<nCol; i++){ const char *z; int n; |
︙ | ︙ | |||
625 626 627 628 629 630 631 | int sqlite3Fts5AuxInit(Fts5Global *pGlobal){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ void *pUserData; /* User-data pointer */ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ } aBuiltin [] = { | > | | | 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 | int sqlite3Fts5AuxInit(Fts5Global *pGlobal){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ void *pUserData; /* User-data pointer */ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ } aBuiltin [] = { { "bm25", 0, fts5Bm25Function, 0 }, { "snippet", 0, fts5SnippetFunction, 0 }, { "fts5_test", 0, fts5TestFunction, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){ rc = sqlite3Fts5CreateAux(pGlobal, |
︙ | ︙ |
Changes to ext/fts5/fts5_expr.c.
︙ | ︙ | |||
222 223 224 225 226 227 228 229 230 231 232 233 234 235 | } } sqlite3_free(sParse.apPhrase); *pzErr = sParse.zErr; return sParse.rc; } /* ** Free the expression node object passed as the only argument. */ void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){ if( p ){ sqlite3Fts5ParseNodeFree(p->pLeft); | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 | } } sqlite3_free(sParse.apPhrase); *pzErr = sParse.zErr; return sParse.rc; } static char *fts5ExprStrdup(int *pRc, const char *zIn){ char *zRet = 0; if( *pRc==SQLITE_OK ){ int nByte = strlen(zIn) + 1; zRet = sqlite3_malloc(nByte); if( zRet ){ memcpy(zRet, zIn, nByte); }else{ *pRc = SQLITE_NOMEM; } } return zRet; } static void *fts5ExprMalloc(int *pRc, int nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ pRet = sqlite3_malloc(nByte); if( pRet ){ memset(pRet, 0, nByte); }else{ *pRc = SQLITE_NOMEM; } } return pRet; } /* ** Create a new FTS5 expression by cloning phrase iPhrase of the ** expression passed as the second argument. */ int sqlite3Fts5ExprPhraseExpr( Fts5Config *pConfig, Fts5Expr *pExpr, int iPhrase, Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ int i; /* Used to iterate through phrase terms */ /* Components of the new expression object */ Fts5Expr *pNew; Fts5ExprPhrase **apPhrase; Fts5ExprNode *pNode; Fts5ExprNearset *pNear; Fts5ExprPhrase *pCopy; pOrig = pExpr->apPhrase[iPhrase]; pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr)); apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*)); pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode)); pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) ); pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm ); for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){ pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm); pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; } if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; pNew->pRoot = pNode; pNew->nPhrase = 1; pNew->apPhrase = apPhrase; pNew->apPhrase[0] = pCopy; pNode->eType = FTS5_STRING; pNode->pNear = pNear; pNear->iCol = -1; pNear->nPhrase = 1; pNear->apPhrase[0] = pCopy; pCopy->nTerm = pOrig->nTerm; pCopy->pNode = pNode; }else{ /* At least one allocation failed. Free them all. */ if( pCopy ){ for(i=0; i<pOrig->nTerm; i++){ sqlite3_free(pCopy->aTerm[i].zTerm); } sqlite3_free(pCopy); sqlite3_free(pNear); sqlite3_free(pNode); sqlite3_free(apPhrase); sqlite3_free(pNew); pNew = 0; } } *ppNew = pNew; return rc; } /* ** Free the expression node object passed as the only argument. */ void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){ if( p ){ sqlite3Fts5ParseNodeFree(p->pLeft); |
︙ | ︙ |
Changes to ext/fts5/fts5_storage.c.
︙ | ︙ | |||
701 702 703 704 705 706 707 | if( bCorrupt && rc==SQLITE_OK ){ rc = SQLITE_CORRUPT_VTAB; } } return rc; } | | < < | < < | > > > > > > > | 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 | if( bCorrupt && rc==SQLITE_OK ){ rc = SQLITE_CORRUPT_VTAB; } } return rc; } int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){ int rc = fts5StorageLoadTotals(p); if( rc==SQLITE_OK ){ *pnToken = p->aTotalSize[iCol]; } return rc; } int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ int rc = fts5StorageLoadTotals(p); if( rc==SQLITE_OK ){ *pnRow = p->nTotalRow; } return rc; } |
Changes to test/fts5ae.test.
︙ | ︙ | |||
153 154 155 156 157 158 159 | } { 3 {a {}} 2 {{} a} 1 {{a b c d} {e f g h i j}} } do_execsql_test 5.3 { | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 | } { 3 {a {}} 2 {{} a} 1 {{a b c d} {e f g h i j}} } do_execsql_test 5.3 { SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a' ORDER BY rowid DESC; } { 3 {5 7} 2 {5 7} 1 {5 7} } do_execsql_test 5.4 { INSERT INTO t5 VALUES('x y z', 'v w x y z'); SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a' ORDER BY rowid DESC; } { 3 {8 12} 2 {8 12} 1 {8 12} } #------------------------------------------------------------------------- # Test the xTokenize() API # reset_db do_execsql_test 6.1 { CREATE VIRTUAL TABLE t6 USING fts5(x, y); INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); } do_execsql_test 6.2 { SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*' } { 2 {{horatio than are} {dreamt of in your philosophy}} 1 {{there are more} {things in heaven and earth}} } #------------------------------------------------------------------------- # Test the xQueryPhrase() API # reset_db do_execsql_test 7.1 { CREATE VIRTUAL TABLE t7 USING fts5(x, y); } do_test 7.2 { foreach {x y} { {q i b w s a a e l o} {i b z a l f p t e u} {b a z t a l o x d i} {b p a d b f h d w y} {z m h n p p u i e g} {v h d v b x j j c z} {a g i m v a u c b i} {p k s o t l r t b m} {v v c j o d a s c p} {f f v o k p o f o g} } { execsql {INSERT INTO t7 VALUES($x, $y)} } execsql { SELECT count(*) FROM t7 } } {5} foreach {tn q res} { 1 a {{4 2}} 2 b {{3 4}} 3 c {{2 1}} 4 d {{2 2}} 5 {a AND b} {{4 2} {3 4}} 6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}} } { do_execsql_test 7.3.$tn { SELECT fts5_test(t7, 'queryphrase') FROM t7 WHERE t7 MATCH $q LIMIT 1 } [list $res] } do_execsql_test 7.4 { SELECT fts5_test(t7, 'rowcount') FROM t7 WHERE t7 MATCH 'a'; } {5 5 5 5} finish_test |
Changes to test/fts5af.test.
︙ | ︙ | |||
26 27 28 29 30 31 32 33 34 35 36 37 38 39 | } do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, y); } foreach {tn doc res} { 1.1 {X o o o o o o} {[X] o o o o o o} 1.2 {o X o o o o o} {o [X] o o o o o} 1.3 {o o X o o o o} {o o [X] o o o o} 1.4 {o o o X o o o} {o o o [X] o o o} | > > > > > > > > > > > > > > > > > > > | 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | } do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, y); } proc do_snippet_test {tn doc match res} { uplevel #0 [list set v1 $doc] uplevel #0 [list set v2 $match] do_execsql_test $tn.1 { DELETE FROM t1; INSERT INTO t1 VALUES($v1, NULL); SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2; } [list $res] do_execsql_test $tn.2 { DELETE FROM t1; INSERT INTO t1 VALUES(NULL, $v1); SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2; } [list $res] } foreach {tn doc res} { 1.1 {X o o o o o o} {[X] o o o o o o} 1.2 {o X o o o o o} {o [X] o o o o o} 1.3 {o o X o o o o} {o o [X] o o o o} 1.4 {o o o X o o o} {o o o [X] o o o} |
︙ | ︙ | |||
78 79 80 81 82 83 84 | 7.1 {X o o X o o o o o} {[X] o o [X] o o o...} 7.2 {o X o o X o o o o} {o [X] o o [X] o o...} 7.3 {o o X o o X o o o} {...o [X] o o [X] o o...} 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} } { | < < < < < | < < < < < | 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | 7.1 {X o o X o o o o o} {[X] o o [X] o o o...} 7.2 {o X o o X o o o o} {o [X] o o [X] o o...} 7.3 {o o X o o X o o o} {...o [X] o o [X] o o...} 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} } { do_snippet_test 1.$tn $doc X $res } foreach {tn doc res} { 1.1 {X Y o o o o o} {[X Y] o o o o o} 1.2 {o X Y o o o o} {o [X Y] o o o o} 1.3 {o o X Y o o o} {o o [X Y] o o o} 1.4 {o o o X Y o o} {o o o [X Y] o o} |
︙ | ︙ | |||
117 118 119 120 121 122 123 | 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} } { | < < < < < | < < < < < | 126 127 128 129 130 131 132 133 134 135 136 137 | 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} } { do_snippet_test 1.$tn $doc "X + Y" $res } finish_test |