Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add the xTokenize extension API. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
8c6b0aff3443fae4b7f0b9adcbf15149 |
User & Date: | dan 2014-07-21 14:22:38.753 |
Context
2014-07-21
| ||
15:45 | Fix DELETE and UPDATE operations on fts5 tables. (check-in: d44d3a8518 user: dan tags: fts5) | |
14:22 | Add the xTokenize extension API. (check-in: 8c6b0aff34 user: dan tags: fts5) | |
11:44 | Fix the xColumnSize() extension API. (check-in: 19504c4108 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
︙ | ︙ | |||
626 627 628 629 630 631 632 | static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, int (*xToken)(void*, const char*, int, int, int, int) ){ | | > | | 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 | static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, int (*xToken)(void*, const char*, int, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return sqlite3Fts5ExprPhraseCount(pCsr->pExpr); } |
︙ | ︙ |
Changes to ext/fts5/fts5.h.
︙ | ︙ | |||
64 65 66 67 68 69 70 71 72 73 74 75 76 77 | ** are numbered starting from zero. ** ** xRowid: ** Returns the rowid of the current row. ** ** xPoslist: ** Iterate through instances of phrase iPhrase in the current row. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); | > > > | 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | ** are numbered starting from zero. ** ** xRowid: ** Returns the rowid of the current row. ** ** xPoslist: ** Iterate through instances of phrase iPhrase in the current row. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); |
︙ | ︙ |
Changes to ext/fts5/fts5_aux.c.
︙ | ︙ | |||
18 19 20 21 22 23 24 25 26 27 28 29 30 31 | Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ){ assert( 0 ); } static void fts5TestFunction( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ | > > > > > > > > > > > > > > > > > > | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ ){ assert( 0 ); } static int fts5TestCallback( void *pContext, /* Pointer to Fts5Buffer object */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd, /* End offset of token */ int iPos /* Position offset of token */ ){ int rc = SQLITE_OK; Fts5Buffer *pBuf = (Fts5Buffer*)pContext; if( pBuf->n!=0 ){ sqlite3Fts5BufferAppendString(&rc, pBuf, " "); } sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken); return rc; } static void fts5TestFunction( const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ Fts5Context *pFts, /* First arg to pass to pApi functions */ sqlite3_context *pCtx, /* Context for returning result/error */ int nVal, /* Number of values in apVal[] array */ sqlite3_value **apVal /* Array of trailing arguments */ |
︙ | ︙ | |||
162 163 164 165 166 167 168 169 170 171 172 173 174 175 | if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){ iRowid = pApi->xRowid(pFts); sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid); } if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT); }else{ sqlite3_result_error_code(pCtx, rc); } sqlite3Fts5BufferFree(&s); | > > > > > > > > > > > > > > > > > > > > > > > > > > > | 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 | if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){ iRowid = pApi->xRowid(pFts); sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid); } if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, " tokenize "); } if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){ Fts5Buffer buf; memset(&buf, 0, sizeof(buf)); for(i=0; rc==SQLITE_OK && i<nCol; i++){ const char *z; int n; rc = pApi->xColumnText(pFts, i, &z, &n); if( rc==SQLITE_OK ){ Fts5Buffer buf1; memset(&buf1, 0, sizeof(Fts5Buffer)); rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback); if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " "); sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n); sqlite3_free(buf1.p); } } if( zReq==0 ){ sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n); }else{ sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p); } sqlite3_free(buf.p); } if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT); }else{ sqlite3_result_error_code(pCtx, rc); } sqlite3Fts5BufferFree(&s); |
︙ | ︙ |
Changes to test/fts5ae.test.
︙ | ︙ | |||
171 172 173 174 175 176 177 | ORDER BY rowid DESC; } { 3 {2 3} 2 {2 3} 1 {2 3} } | > > | > > > > > > > > > > > > > | 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 | ORDER BY rowid DESC; } { 3 {2 3} 2 {2 3} 1 {2 3} } #------------------------------------------------------------------------- # Test the xTokenize() API # reset_db do_execsql_test 6.1 { CREATE VIRTUAL TABLE t6 USING fts5(x, y); INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); } do_execsql_test 6.2 { SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*' } { 2 {{horatio than are} {dreamt of in your philosophy}} 1 {{there are more} {things in heaven and earth}} } finish_test |