Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Add the xTokenize extension API. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
8c6b0aff3443fae4b7f0b9adcbf15149 |
| User & Date: | dan 2014-07-21 14:22:38 |
Context
|
2014-07-21
| ||
| 15:45 | Fix DELETE and UPDATE operations on fts5 tables. check-in: d44d3a85 user: dan tags: fts5 | |
| 14:22 | Add the xTokenize extension API. check-in: 8c6b0aff user: dan tags: fts5 | |
| 11:44 | Fix the xColumnSize() extension API. check-in: 19504c41 user: dan tags: fts5 | |
Changes
Changes to ext/fts5/fts5.c.
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 |
static int fts5ApiTokenize(
Fts5Context *pCtx,
const char *pText, int nText,
void *pUserData,
int (*xToken)(void*, const char*, int, int, int, int)
){
assert( 0 );
return SQLITE_OK;
}
static int fts5ApiPhraseCount(Fts5Context *pCtx){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
}
|
| | > |
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 |
static int fts5ApiTokenize(
Fts5Context *pCtx,
const char *pText, int nText,
void *pUserData,
int (*xToken)(void*, const char*, int, int, int, int)
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken);
}
static int fts5ApiPhraseCount(Fts5Context *pCtx){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
}
|
Changes to ext/fts5/fts5.h.
64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
** are numbered starting from zero.
**
** xRowid:
** Returns the rowid of the current row.
**
** xPoslist:
** Iterate through instances of phrase iPhrase in the current row.
*/
struct Fts5ExtensionApi {
int iVersion; /* Currently always set to 1 */
void *(*xUserData)(Fts5Context*);
int (*xColumnCount)(Fts5Context*);
|
> > > |
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 |
** are numbered starting from zero. ** ** xRowid: ** Returns the rowid of the current row. ** ** xPoslist: ** Iterate through instances of phrase iPhrase in the current row. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); |
Changes to ext/fts5/fts5_aux.c.
18
19
20
21
22
23
24
25
26
27
28
29
30
31
...
162
163
164
165
166
167
168
169
170
171
172
173
174
175
|
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
assert( 0 );
}
static void fts5TestFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
................................................................................
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid ");
}
if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
iRowid = pApi->xRowid(pFts);
sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(pCtx, rc);
}
sqlite3Fts5BufferFree(&s);
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
...
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
assert( 0 );
}
static int fts5TestCallback(
void *pContext, /* Pointer to Fts5Buffer object */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd, /* End offset of token */
int iPos /* Position offset of token */
){
int rc = SQLITE_OK;
Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
if( pBuf->n!=0 ){
sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
}
sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
return rc;
}
static void fts5TestFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
................................................................................
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid ");
}
if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
iRowid = pApi->xRowid(pFts);
sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
}
if( zReq==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &s, " tokenize ");
}
if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
Fts5Buffer buf;
memset(&buf, 0, sizeof(buf));
for(i=0; rc==SQLITE_OK && i<nCol; i++){
const char *z;
int n;
rc = pApi->xColumnText(pFts, i, &z, &n);
if( rc==SQLITE_OK ){
Fts5Buffer buf1;
memset(&buf1, 0, sizeof(Fts5Buffer));
rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
sqlite3_free(buf1.p);
}
}
if( zReq==0 ){
sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
}else{
sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
}
sqlite3_free(buf.p);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(pCtx, rc);
}
sqlite3Fts5BufferFree(&s);
|
Changes to test/fts5ae.test.
171 172 173 174 175 176 177 178 179 180 181 |
ORDER BY rowid DESC;
} {
3 {2 3}
2 {2 3}
1 {2 3}
}
finish_test
|
> > | > > > > > > > > > > > > > |
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 |
ORDER BY rowid DESC;
} {
3 {2 3}
2 {2 3}
1 {2 3}
}
#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE t6 USING fts5(x, y);
INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}
do_execsql_test 6.2 {
SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
} {
2 {{horatio than are} {dreamt of in your philosophy}}
1 {{there are more} {things in heaven and earth}}
}
finish_test
|