Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add the xPhraseFirst() and xPhraseNext() fts5 APIs, for faster iteration through a single phrases position list. Also optimize xInst() and xInstCount() a bit. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
f7682435278419829a46bb4cc9b5625d |
User & Date: | dan 2015-08-12 12:11:28.744 |
Context
2015-08-12
| ||
16:49 | Begin adding an extension that provides JSON SQL functions. (check-in: dde8afdd8d user: drh tags: json) | |
15:36 | Minor optimization for fts5 API xInst(). (check-in: efb7c9c5d0 user: dan tags: trunk) | |
12:11 | Add the xPhraseFirst() and xPhraseNext() fts5 APIs, for faster iteration through a single phrases position list. Also optimize xInst() and xInstCount() a bit. (check-in: f768243527 user: dan tags: trunk) | |
2015-08-11
| ||
14:25 | Merge fixes from the fts5NoWarn branch. (check-in: 61cb2fc6c1 user: dan tags: trunk) | |
Changes
Changes to ext/fts5/fts5.h.
︙ | ︙ | |||
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | ** ** Virtual table implementations may overload SQL functions by implementing ** the sqlite3_module.xFindFunction() method. */ typedef struct Fts5ExtensionApi Fts5ExtensionApi; typedef struct Fts5Context Fts5Context; typedef void (*fts5_extension_function)( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ); /* ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): ** Return a copy of the context pointer the extension function was ** registered with. | > > > > > > | 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | ** ** Virtual table implementations may overload SQL functions by implementing ** the sqlite3_module.xFindFunction() method. */ typedef struct Fts5ExtensionApi Fts5ExtensionApi; typedef struct Fts5Context Fts5Context; typedef struct Fts5PhraseIter Fts5PhraseIter; typedef void (*fts5_extension_function)( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ); struct Fts5PhraseIter { const unsigned char *a; const unsigned char *b; }; /* ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): ** Return a copy of the context pointer the extension function was ** registered with. |
︙ | ︙ | |||
170 171 172 173 174 175 176 177 178 179 180 181 182 183 | ** ** xRowCount(pFts5, pnRow) ** ** This function is used to retrieve the total number of rows in the table. ** In other words, the same value that would be returned by: ** ** SELECT count(*) FROM ftstable; */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); | > > > > > > > > > > > > > > > > > > > > > > > > | 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 | ** ** xRowCount(pFts5, pnRow) ** ** This function is used to retrieve the total number of rows in the table. ** In other words, the same value that would be returned by: ** ** SELECT count(*) FROM ftstable; ** ** xPhraseFirst() ** This function is used, along with type Fts5PhraseIter and the xPhraseNext ** method, to iterate through all instances of a single query phrase within ** the current row. This is the same information as is accessible via the ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient ** to use, this API may be faster under some circumstances. To iterate ** through instances of phrase iPhrase, use the following code: ** ** Fts5PhraseIter iter; ** int iCol, iOff; ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); ** iOff>=0; ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ** ){ ** // An instance of phrase iPhrase at offset iOff of column iCol ** } ** ** The Fts5PhraseIter structure is defined above. Applications should not ** modify this structure directly - it should only be used as shown above ** with the xPhraseFirst() and xPhraseNext() API methods. ** ** xPhraseNext() ** See xPhraseFirst above. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); |
︙ | ︙ | |||
201 202 203 204 205 206 207 208 209 210 211 212 213 214 | int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); }; /* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ /************************************************************************* | > > > | 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 | int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); }; /* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ /************************************************************************* |
︙ | ︙ |
Changes to ext/fts5/fts5_main.c.
︙ | ︙ | |||
193 194 195 196 197 198 199 200 201 202 203 204 205 206 | sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */ /* Auxiliary data storage */ Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ /* Cache used by auxiliary functions xInst() and xInstCount() */ int nInstCount; /* Number of phrase instances */ int *aInst; /* 3 integers per phrase instance */ }; /* ** Bits that make up the "idxNum" parameter passed indirectly by ** xBestIndex() to xFilter(). | > > | 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */ /* Auxiliary data storage */ Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ /* Cache used by auxiliary functions xInst() and xInstCount() */ Fts5PoslistReader *aInstIter; /* One for each phrase */ int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ int *aInst; /* 3 integers per phrase instance */ }; /* ** Bits that make up the "idxNum" parameter passed indirectly by ** xBestIndex() to xFilter(). |
︙ | ︙ | |||
613 614 615 616 617 618 619 620 621 622 623 624 625 626 | } static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); Fts5Auxdata *pData; Fts5Auxdata *pNext; sqlite3_free(pCsr->aInst); if( pCsr->pStmt ){ int eStmt = fts5StmtType(pCsr); sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); } if( pCsr->pSorter ){ Fts5Sorter *pSorter = pCsr->pSorter; | > | 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 | } static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); Fts5Auxdata *pData; Fts5Auxdata *pNext; sqlite3_free(pCsr->aInstIter); sqlite3_free(pCsr->aInst); if( pCsr->pStmt ){ int eStmt = fts5StmtType(pCsr); sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); } if( pCsr->pSorter ){ Fts5Sorter *pSorter = pCsr->pSorter; |
︙ | ︙ | |||
1531 1532 1533 1534 1535 1536 1537 | ** SQLite error code otherwise. */ static int fts5CacheInstArray(Fts5Cursor *pCsr){ int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST) ){ Fts5PoslistReader *aIter; /* One iterator for each phrase */ int nIter; /* Number of iterators/phrases */ | < > | | > > > < < > > > > | > > > > > > | > > | < < < | 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 | ** SQLite error code otherwise. */ static int fts5CacheInstArray(Fts5Cursor *pCsr){ int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST) ){ Fts5PoslistReader *aIter; /* One iterator for each phrase */ int nIter; /* Number of iterators/phrases */ nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); if( pCsr->aInstIter==0 ){ int nByte = sizeof(Fts5PoslistReader) * nIter; pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); } aIter = pCsr->aInstIter; if( aIter ){ int nInst = 0; /* Number instances seen so far */ int i; /* Initialize all iterators */ for(i=0; i<nIter; i++){ const u8 *a; int n = fts5CsrPoslist(pCsr, i, &a); sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[i]); } while( 1 ){ int *aInst; int iBest = -1; for(i=0; i<nIter; i++){ if( (aIter[i].bEof==0) && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) ){ iBest = i; } } if( iBest<0 ) break; nInst++; if( nInst>=pCsr->nInstAlloc ){ pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; aInst = (int*)sqlite3_realloc( pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 ); if( aInst ){ pCsr->aInst = aInst; }else{ rc = SQLITE_NOMEM; break; } } aInst = &pCsr->aInst[3 * (nInst-1)]; aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); sqlite3Fts5PoslistReaderNext(&aIter[iBest]); } pCsr->nInstCount = nInst; CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); } } return rc; } static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ |
︙ | ︙ | |||
1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 | pData->pPtr = 0; pData->xDelete = 0; } } return pRet; } static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > | 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 | pData->pPtr = 0; pData->xDelete = 0; } } return pRet; } static void fts5ApiPhraseNext( Fts5Context *pCtx, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; }else{ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); if( iVal==1 ){ pIter->a += fts5GetVarint32(pIter->a, iVal); *piCol = iVal; *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } *piOff += (iVal-2); } } static void fts5ApiPhraseFirst( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); pIter->b = &pIter->a[n]; *piCol = 0; *piOff = 0; fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); } static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { 2, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, fts5ApiColumnTotalSize, fts5ApiTokenize, fts5ApiPhraseCount, fts5ApiPhraseSize, fts5ApiInstCount, fts5ApiInst, fts5ApiRowid, fts5ApiColumnText, fts5ApiColumnSize, fts5ApiQueryPhrase, fts5ApiSetAuxdata, fts5ApiGetAuxdata, fts5ApiPhraseFirst, fts5ApiPhraseNext, }; /* ** Implementation of API function xQueryPhrase(). */ static int fts5ApiQueryPhrase( |
︙ | ︙ |
Changes to ext/fts5/fts5_test_mi.c.
︙ | ︙ | |||
124 125 126 127 128 129 130 131 | } static int fts5MatchinfoXCb( const Fts5ExtensionApi *pApi, Fts5Context *pFts, void *pUserData ){ u32 *aOut = (u32*)pUserData; | > > < < < < | < | | > | | | 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | } static int fts5MatchinfoXCb( const Fts5ExtensionApi *pApi, Fts5Context *pFts, void *pUserData ){ Fts5PhraseIter iter; int iCol, iOff; u32 *aOut = (u32*)pUserData; int iPrev = -1; for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); iOff>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ aOut[iCol*3+1]++; if( iCol!=iPrev ) aOut[iCol*3 + 2]++; iPrev = iCol; } return SQLITE_OK; } static int fts5MatchinfoGlobalCb( const Fts5ExtensionApi *pApi, Fts5Context *pFts, Fts5MatchinfoCtx *p, char f, |
︙ | ︙ | |||
212 213 214 215 216 217 218 | int i; int rc = SQLITE_OK; switch( f ){ case 'b': case 'x': case 'y': { | < > < | > | > > | > | | | | > | 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 | int i; int rc = SQLITE_OK; switch( f ){ case 'b': case 'x': case 'y': { int nMul = (f=='x' ? 3 : 1); int iPhrase; if( f=='b' ){ int nInt = ((p->nCol + 31) / 32) * p->nPhrase; for(i=0; i<nInt; i++) aOut[i] = 0; }else{ for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; } for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ Fts5PhraseIter iter; int iOff, iCol; for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); iOff>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ if( f=='b' ){ aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); }else{ aOut[nMul * (iCol + iPhrase * p->nCol)]++; } } } break; } case 'l': { |
︙ | ︙ | |||
383 384 385 386 387 388 389 | ** fts5_api_from_db() function above is copied verbatim from the ** FTS5 documentation. Refer there for details. */ pApi = fts5_api_from_db(db); /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered ** with this database handle, or an error (OOM perhaps?) has occurred. ** | | | < | 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 | ** fts5_api_from_db() function above is copied verbatim from the ** FTS5 documentation. Refer there for details. */ pApi = fts5_api_from_db(db); /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered ** with this database handle, or an error (OOM perhaps?) has occurred. ** ** Also check that the fts5_api object is version 2 or newer. */ if( pApi==0 || pApi->iVersion<1 ){ return SQLITE_ERROR; } /* Register the implementation of matchinfo() */ rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); return rc; } #endif /* SQLITE_ENABLE_FTS5 */ #endif /* SQLITE_TEST */ |