Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
| Comment: | Fix the xColumnSize() extension API. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
19504c4108472d2ad1281221642b8bd0 |
| User & Date: | dan 2014-07-21 11:44:47 |
|
2014-07-21
| ||
| 14:22 | Add the xTokenize extension API. (check-in: 8c6b0aff user: dan tags: fts5) | |
| 11:44 | Fix the xColumnSize() extension API. (check-in: 19504c41 user: dan tags: fts5) | |
|
2014-07-19
| ||
| 20:27 | Add simple tests for the xColumnText() extension api. (check-in: 1e9053ab user: dan tags: fts5) | |
Changes to ext/fts5/fts5.c.
615 615 616 616 static int fts5ApiColumnCount(Fts5Context *pCtx){ 617 617 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 618 618 return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; 619 619 } 620 620 621 621 static int fts5ApiColumnAvgSize(Fts5Context *pCtx, int iCol, int *pnToken){ 622 - assert( 0 ); 623 - return 0; 622 + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 623 + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); 624 + return sqlite3Fts5StorageAvgsize(pTab->pStorage, iCol, pnToken); 624 625 } 625 626 626 627 static int fts5ApiTokenize( 627 628 Fts5Context *pCtx, 628 629 const char *pText, int nText, 629 630 void *pUserData, 630 631 int (*xToken)(void*, const char*, int, int, int, int)
Changes to ext/fts5/fts5Int.h.
263 263 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); 264 264 265 265 /* Called during startup to register a UDF with SQLite */ 266 266 int sqlite3Fts5IndexInit(sqlite3*); 267 267 268 268 void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz); 269 269 270 +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf); 271 +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); 272 + 270 273 /* 271 274 ** End of interface to code in fts5_index.c. 272 275 **************************************************************************/ 273 276 274 277 /************************************************************************** 275 278 ** Interface to code in fts5_storage.c. fts5_storage.c contains contains 276 279 ** code to access the data stored in the %_content and %_docsize tables. ................................................................................ 293 296 294 297 int sqlite3Fts5StorageIntegrity(Fts5Storage *p); 295 298 296 299 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **); 297 300 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); 298 301 299 302 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); 300 -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol); 303 +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg); 301 304 302 305 303 306 /* 304 307 ** End of interface to code in fts5_storage.c. 305 308 **************************************************************************/ 306 309 307 310
Changes to ext/fts5/fts5_aux.c.
39 39 int i; 40 40 41 41 if( nVal>=1 ){ 42 42 zReq = (const char*)sqlite3_value_text(apVal[0]); 43 43 } 44 44 45 45 memset(&s, 0, sizeof(Fts5Buffer)); 46 + nCol = pApi->xColumnCount(pFts); 47 + 48 + if( zReq==0 ){ 49 + sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnavgsize "); 50 + } 51 + if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnavgsize") ){ 52 + if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{"); 53 + for(i=0; rc==SQLITE_OK && i<nCol; i++){ 54 + int colsz = 0; 55 + rc = pApi->xColumnAvgSize(pFts, i, &colsz); 56 + sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz); 57 + } 58 + if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}"); 59 + } 46 60 47 61 if( zReq==0 ){ 48 62 sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount "); 49 63 } 50 - nCol = pApi->xColumnCount(pFts); 51 64 if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){ 52 65 sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol); 53 66 } 54 67 55 68 if( zReq==0 ){ 56 69 sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnsize "); 57 70 }
Changes to ext/fts5/fts5_index.c.
730 730 static void fts5DataReference(Fts5Data *pData){ 731 731 pData->nRef++; 732 732 } 733 733 734 734 /* 735 735 ** INSERT OR REPLACE a record into the %_data table. 736 736 */ 737 -static void fts5DataWrite(Fts5Index *p, i64 iRowid, u8 *pData, int nData){ 737 +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ 738 738 if( p->rc!=SQLITE_OK ) return; 739 739 740 740 if( p->pWriter==0 ){ 741 741 int rc; 742 742 Fts5Config *pConfig = p->pConfig; 743 743 char *zSql = sqlite3_mprintf( 744 744 "REPLACE INTO '%q'.%Q(id, block) VALUES(?,?)", pConfig->zDb, p->zDataTbl ................................................................................ 2730 2730 if( rc==SQLITE_OK ){ 2731 2731 memset(&s, 0, sizeof(Fts5Structure)); 2732 2732 for(i=0; i<pConfig->nPrefix+1; i++){ 2733 2733 fts5StructureWrite(p, i, &s); 2734 2734 } 2735 2735 rc = p->rc; 2736 2736 } 2737 + sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); 2737 2738 } 2738 2739 2739 2740 if( rc ){ 2740 2741 sqlite3Fts5IndexClose(p, 0); 2741 2742 *pp = 0; 2742 2743 } 2743 2744 return rc; ................................................................................ 3614 3615 fts5StructureRelease(pIter->pStruct); 3615 3616 fts5BufferFree(&pIter->poslist); 3616 3617 } 3617 3618 fts5CloseReader(pIter->pIndex); 3618 3619 sqlite3_free(pIter); 3619 3620 } 3620 3621 } 3622 + 3623 +/* 3624 +** Read the "averages" record into the buffer supplied as the second 3625 +** argument. Return SQLITE_OK if successful, or an SQLite error code 3626 +** if an error occurs. 3627 +*/ 3628 +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){ 3629 + fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID); 3630 + return p->rc; 3631 +} 3632 + 3633 +/* 3634 +** Replace the current "averages" record with the contents of the buffer 3635 +** supplied as the second argument. 3636 +*/ 3637 +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ 3638 + fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); 3639 + return p->rc; 3640 +} 3621 3641
Changes to ext/fts5/fts5_storage.c.
13 13 */ 14 14 15 15 #include "fts5Int.h" 16 16 17 17 struct Fts5Storage { 18 18 Fts5Config *pConfig; 19 19 Fts5Index *pIndex; 20 + i64 nTotalRow; /* Total number of rows in FTS table */ 21 + i64 *aTotalSize; /* Total sizes of each column */ 20 22 sqlite3_stmt *aStmt[9]; 21 23 }; 22 24 23 25 24 26 #if FTS5_STMT_SCAN_ASC!=0 25 27 # error "FTS5_STMT_SCAN_ASC mismatch" 26 28 #endif ................................................................................ 164 166 Fts5Index *pIndex, 165 167 int bCreate, 166 168 Fts5Storage **pp, 167 169 char **pzErr /* OUT: Error message */ 168 170 ){ 169 171 int rc; 170 172 Fts5Storage *p; /* New object */ 173 + int nByte; /* Bytes of space to allocate */ 171 174 172 - *pp = p = (Fts5Storage*)sqlite3_malloc(sizeof(Fts5Storage)); 175 + nByte = sizeof(Fts5Storage) /* Fts5Storage object */ 176 + + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ 177 + *pp = p = (Fts5Storage*)sqlite3_malloc(nByte); 173 178 if( !p ) return SQLITE_NOMEM; 174 179 175 - memset(p, 0, sizeof(Fts5Storage)); 180 + memset(p, 0, nByte); 181 + p->aTotalSize = (i64*)&p[1]; 176 182 p->pConfig = pConfig; 177 183 p->pIndex = pIndex; 178 184 179 185 if( bCreate ){ 180 186 int i; 181 187 char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10); 182 188 if( zDefn==0 ){ ................................................................................ 281 287 for(iCol=1; iCol<=pConfig->nCol; iCol++){ 282 288 rc = sqlite3Fts5Tokenize(pConfig, 283 289 (const char*)sqlite3_column_text(pSeek, iCol), 284 290 sqlite3_column_bytes(pSeek, iCol), 285 291 (void*)&ctx, 286 292 fts5StorageInsertCallback 287 293 ); 294 + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; 288 295 } 296 + p->nTotalRow--; 289 297 } 290 298 rc2 = sqlite3_reset(pSeek); 291 299 if( rc==SQLITE_OK ) rc = rc2; 292 300 } 293 301 294 302 return rc; 295 303 } ................................................................................ 310 318 sqlite3_bind_int64(pReplace, 1, iRowid); 311 319 sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); 312 320 sqlite3_step(pReplace); 313 321 rc = sqlite3_reset(pReplace); 314 322 } 315 323 return rc; 316 324 } 325 + 326 +/* 327 +** Load the contents of the "averages" record from disk into the 328 +** p->nTotalRow and p->aTotalSize[] variables. 329 +** 330 +** Return SQLITE_OK if successful, or an SQLite error code if an error 331 +** occurs. 332 +*/ 333 +static int fts5StorageLoadTotals(Fts5Storage *p){ 334 + int nCol = p->pConfig->nCol; 335 + Fts5Buffer buf; 336 + int rc; 337 + memset(&buf, 0, sizeof(buf)); 338 + 339 + memset(p->aTotalSize, 0, sizeof(i64) * nCol); 340 + p->nTotalRow = 0; 341 + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf); 342 + if( rc==SQLITE_OK && buf.n ){ 343 + int i = 0; 344 + int iCol; 345 + i += getVarint(&buf.p[i], (u64*)&p->nTotalRow); 346 + for(iCol=0; i<buf.n && iCol<nCol; iCol++){ 347 + i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]); 348 + } 349 + } 350 + sqlite3_free(buf.p); 351 + 352 + return rc; 353 +} 354 + 355 +/* 356 +** Store the current contents of the p->nTotalRow and p->aTotalSize[] 357 +** variables in the "averages" record on disk. 358 +** 359 +** Return SQLITE_OK if successful, or an SQLite error code if an error 360 +** occurs. 361 +*/ 362 +static int fts5StorageSaveTotals(Fts5Storage *p){ 363 + int nCol = p->pConfig->nCol; 364 + int i; 365 + Fts5Buffer buf; 366 + int rc = SQLITE_OK; 367 + memset(&buf, 0, sizeof(buf)); 368 + 369 + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); 370 + for(i=0; i<nCol; i++){ 371 + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]); 372 + } 373 + if( rc==SQLITE_OK ){ 374 + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); 375 + } 376 + sqlite3_free(buf.p); 377 + 378 + return rc; 379 +} 380 + 317 381 318 382 /* 319 383 ** Insert a new row into the FTS table. 320 384 */ 321 385 int sqlite3Fts5StorageInsert( 322 386 Fts5Storage *p, /* Storage module to write to */ 323 387 sqlite3_value **apVal, /* Array of values passed to xUpdate() */ ................................................................................ 329 393 sqlite3_stmt *pInsert; /* Statement used to write %_content table */ 330 394 int eStmt; /* Type of statement used on %_content */ 331 395 int i; /* Counter variable */ 332 396 Fts5InsertCtx ctx; /* Tokenization callback context object */ 333 397 Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ 334 398 335 399 memset(&buf, 0, sizeof(Fts5Buffer)); 400 + rc = fts5StorageLoadTotals(p); 336 401 337 402 /* Insert the new row into the %_content table. */ 338 - if( eConflict==SQLITE_REPLACE ){ 339 - eStmt = FTS5_STMT_REPLACE_CONTENT; 340 - if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ 341 - rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); 403 + if( rc==SQLITE_OK ){ 404 + if( eConflict==SQLITE_REPLACE ){ 405 + eStmt = FTS5_STMT_REPLACE_CONTENT; 406 + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ 407 + rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); 408 + } 409 + }else{ 410 + eStmt = FTS5_STMT_INSERT_CONTENT; 342 411 } 343 - }else{ 344 - eStmt = FTS5_STMT_INSERT_CONTENT; 345 412 } 346 413 if( rc==SQLITE_OK ){ 347 414 rc = fts5StorageGetStmt(p, eStmt, &pInsert); 348 415 } 349 416 for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ 350 417 rc = sqlite3_bind_value(pInsert, i, apVal[i]); 351 418 } ................................................................................ 363 430 rc = sqlite3Fts5Tokenize(pConfig, 364 431 (const char*)sqlite3_value_text(apVal[ctx.iCol+2]), 365 432 sqlite3_value_bytes(apVal[ctx.iCol+2]), 366 433 (void*)&ctx, 367 434 fts5StorageInsertCallback 368 435 ); 369 436 sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); 437 + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; 370 438 } 439 + p->nTotalRow++; 371 440 372 441 /* Write the %_docsize record */ 373 442 if( rc==SQLITE_OK ){ 374 443 rc = fts5StorageInsertDocsize(p, *piRowid, &buf); 375 444 } 376 445 sqlite3_free(buf.p); 446 + 447 + /* Write the averages record */ 448 + if( rc==SQLITE_OK ){ 449 + rc = fts5StorageSaveTotals(p); 450 + } 377 451 378 452 return rc; 379 453 } 380 454 381 455 /* 382 456 ** Context object used by sqlite3Fts5StorageIntegrity(). 383 457 */ ................................................................................ 534 608 if( bCorrupt && rc==SQLITE_OK ){ 535 609 rc = SQLITE_CORRUPT_VTAB; 536 610 } 537 611 } 538 612 return rc; 539 613 } 540 614 541 -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol){ 542 - return 0; 615 +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg){ 616 + int rc = fts5StorageLoadTotals(p); 617 + if( rc==SQLITE_OK ){ 618 + int nAvg = 1; 619 + if( p->nTotalRow ){ 620 + nAvg = (int)((p->aTotalSize[iCol] + (p->nTotalRow/2)) / p->nTotalRow); 621 + if( nAvg<1 ) nAvg = 1; 622 + *pnAvg = nAvg; 623 + } 624 + } 625 + return rc; 543 626 } 544 627
Changes to test/fts5aa.test.
125 125 do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) } 126 126 do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') } 127 127 if {[set_test_counter errors]} break 128 128 } 129 129 130 130 #------------------------------------------------------------------------- 131 131 # 132 +breakpoint 132 133 reset_db 133 134 do_execsql_test 6.0 { 134 135 CREATE VIRTUAL TABLE t1 USING fts5(x,y); 135 136 INSERT INTO t1(t1) VALUES('pgsz=32'); 136 137 } 137 138 138 139 do_execsql_test 6.1 {
Changes to test/fts5ae.test.
124 124 do_execsql_test 4.1 { 125 125 SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c'; 126 126 } { 127 127 1 {0.5 {} {}} 128 128 } 129 129 130 130 #------------------------------------------------------------------------- 131 -# Test that the xColumnSize() API works. 131 +# Test that the xColumnSize() and xColumnAvgsize() APIs work. 132 132 # 133 133 134 134 reset_db 135 135 do_execsql_test 5.1 { 136 136 CREATE VIRTUAL TABLE t5 USING fts5(x, y); 137 137 INSERT INTO t5 VALUES('a b c d', 'e f g h i j'); 138 138 INSERT INTO t5 VALUES('', 'a'); ................................................................................ 151 151 SELECT rowid, fts5_test(t5, 'columntext') FROM t5 WHERE t5 MATCH 'a' 152 152 ORDER BY rowid DESC; 153 153 } { 154 154 3 {a {}} 155 155 2 {{} a} 156 156 1 {{a b c d} {e f g h i j}} 157 157 } 158 + 159 +do_execsql_test 5.3 { 160 + SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a' 161 + ORDER BY rowid DESC; 162 +} { 163 + 3 {2 2} 164 + 2 {2 2} 165 + 1 {2 2} 166 +} 167 + 168 +do_execsql_test 5.4 { 169 + INSERT INTO t5 VALUES('x y z', 'v w x y z'); 170 + SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a' 171 + ORDER BY rowid DESC; 172 +} { 173 + 3 {2 3} 174 + 2 {2 3} 175 + 1 {2 3} 176 +} 177 + 158 178 159 179 160 180 finish_test 161 181