Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Merge experimental fts3/fts4 changes with trunk. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
988164cf485300fb3d189fd1453c23c4 |
User & Date: | dan 2010-10-27 18:10:00.000 |
Context
2010-10-27
| ||
19:08 | Avoid trying to allocate a negative number of bytes of memory in the test wrapper for sqlite3_blob_read(). (check-in: 739b5d9aa4 user: dan tags: trunk) | |
18:10 | Merge experimental fts3/fts4 changes with trunk. (check-in: 988164cf48 user: dan tags: trunk) | |
16:52 | Fix a buffer overread in fts3 that can occur if the database is corrupt. (Closed-Leaf check-in: 84194c4195 user: dan tags: experimental) | |
15:36 | Fix a memory leak in the update_hook method of the TCL interface. (check-in: 1d17e3dc83 user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
437 438 439 440 441 442 443 444 445 446 447 448 | ** The xDisconnect() virtual table method. */ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table *)pVtab; int i; assert( p->nPendingData==0 ); /* Free any prepared statements held */ for(i=0; i<SizeofArray(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } | > < < < | < | | 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 | ** The xDisconnect() virtual table method. */ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table *)pVtab; int i; assert( p->nPendingData==0 ); assert( p->pSegments==0 ); /* Free any prepared statements held */ for(i=0; i<SizeofArray(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p->zSegmentsTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); return SQLITE_OK; } /* ** Construct one or more SQL statements from the format string given ** and then evaluate those statements. The success code is written ** into *pRc. ** ** If *pRc is initially non-zero then this routine is a no-op. */ static void fts3DbExec( int *pRc, /* Success code */ sqlite3 *db, /* Database in which to run SQL */ |
︙ | ︙ | |||
509 510 511 512 513 514 515 516 | } /* ** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table ** passed as the first argument. This is done as part of the xConnect() ** and xCreate() methods. */ | > > > > | > | | | | | | | | | | | | | < | | | | | | | | > < | | | | | | | | | | | | < | 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 | } /* ** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table ** passed as the first argument. This is done as part of the xConnect() ** and xCreate() methods. ** ** If *pRc is non-zero when this function is called, it is a no-op. ** Otherwise, if an error occurs, an SQLite error code is stored in *pRc ** before returning. */ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ if( *pRc==SQLITE_OK ){ int i; /* Iterator variable */ int rc; /* Return code */ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); for(i=1; zCols && i<p->nColumn; i++){ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); } /* Create the whole "CREATE TABLE" statement to pass to SQLite */ zSql = sqlite3_mprintf( "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName ); if( !zCols || !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_declare_vtab(p->db, zSql); } sqlite3_free(zSql); sqlite3_free(zCols); *pRc = rc; } } /* ** Create the backing store tables (%_content, %_segments and %_segdir) ** required by the FTS3 table passed as the only argument. This is done ** as part of the vtab xCreate() method. ** ** If the p->bHasDocsize boolean is true (indicating that this is an ** FTS4 table, not an FTS3 table) then also create the %_docsize and ** %_stat tables required by FTS4. */ static int fts3CreateTables(Fts3Table *p){ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ char *zContentCols; /* Columns of %_content table */ sqlite3 *db = p->db; /* The database connection */ /* Create a list of user columns for the content table */ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); for(i=0; zContentCols && i<p->nColumn; i++){ char *z = p->azColumn[i]; zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); } if( zContentCols==0 ) rc = SQLITE_NOMEM; /* Create the content table */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_content'(%s)", p->zDb, p->zName, zContentCols ); sqlite3_free(zContentCols); /* Create other tables */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", p->zDb, p->zName ); fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segdir'(" |
︙ | ︙ | |||
634 635 636 637 638 639 640 641 642 643 644 645 646 647 | zDb, zName, zSuffix ); rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0); sqlite3_free(zSql); *pResult = (u8)(res & 0xff); if( rc!=SQLITE_ABORT ) *pRc = rc; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 | zDb, zName, zSuffix ); rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0); sqlite3_free(zSql); *pResult = (u8)(res & 0xff); if( rc!=SQLITE_ABORT ) *pRc = rc; } /* ** Store the current database page-size in bytes in p->nPgsz. ** ** If *pRc is non-zero when this function is called, it is a no-op. ** Otherwise, if an error occurs, an SQLite error code is stored in *pRc ** before returning. */ static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ if( *pRc==SQLITE_OK ){ int rc; /* Return code */ char *zSql; /* SQL text "PRAGMA %Q.page_size" */ sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_step(pStmt); p->nPgsz = sqlite3_column_int(pStmt, 0); rc = sqlite3_finalize(pStmt); } } assert( p->nPgsz>0 || rc!=SQLITE_OK ); sqlite3_free(zSql); *pRc = rc; } } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** |
︙ | ︙ | |||
759 760 761 762 763 764 765 | p->bHasDocsize = argv[0][3]=='4'; rc = fts3CreateTables(p); }else{ rc = SQLITE_OK; fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent); fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize); } | < > > > > | < > | > > | 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 | p->bHasDocsize = argv[0][3]=='4'; rc = fts3CreateTables(p); }else{ rc = SQLITE_OK; fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent); fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize); } /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database (see ** function sqlite3Fts3SegReaderCost() for details). */ fts3DatabasePageSize(&rc, p); /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: assert( p || (pTokenizer && rc!=SQLITE_OK) ); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else{ pTokenizer->pModule->xDestroy(pTokenizer); } }else{ *ppVTab = &p->base; } return rc; } /* ** The xConnect() and xCreate() methods for the virtual table. All the ** work is done in function fts3InitVtab(). |
︙ | ︙ | |||
885 886 887 888 889 890 891 | return SQLITE_OK; } /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ | | > > | 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 | return SQLITE_OK; } /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_finalize(pCsr->pStmt); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); sqlite3_free(pCsr); return SQLITE_OK; } /* |
︙ | ︙ | |||
927 928 929 930 931 932 933 | } }else{ return SQLITE_OK; } } /* | | | | | > > | > > > > | < < | > > > > > > > > > > > > > > | > > > > > | > > > > > | > > > | > | > > > > > > | > | > > > > > | > > > > > > > > > > > > | > | > > > > | > > | > | < | > | | | | | | > > > > > | > | | | < | > | < < < | | < < < < | < > > | < < < > | | | < | | < < < < < < < < < | > | > > | | < < < | < < < < < < < < < < | < < > | < < < | | < > | < < < < | < | 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 | } }else{ return SQLITE_OK; } } /* ** This function is used to process a single interior node when searching ** a b-tree for a term or term prefix. The node data is passed to this ** function via the zNode/nNode parameters. The term to search for is ** passed in zTerm/nTerm. ** ** If piFirst is not NULL, then this function sets *piFirst to the blockid ** of the child node that heads the sub-tree that may contain the term. ** ** If piLast is not NULL, then *piLast is set to the right-most child node ** that heads a sub-tree that may contain a term for which zTerm/nTerm is ** a prefix. ** ** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ static int fts3ScanInteriorNode( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to select leaves for */ int nTerm, /* Size of term zTerm in bytes */ const char *zNode, /* Buffer containing segment interior node */ int nNode, /* Size of buffer at zNode */ sqlite3_int64 *piFirst, /* OUT: Selected child node */ sqlite3_int64 *piLast /* OUT: Selected child node */ ){ int rc = SQLITE_OK; /* Return code */ const char *zCsr = zNode; /* Cursor to iterate through node */ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ char *zBuffer = 0; /* Buffer to load terms into */ int nAlloc = 0; /* Size of allocated buffer */ int isFirstTerm = 1; /* True when processing first term on page */ sqlite3_int64 iChild; /* Block id of child node to descend to */ /* Skip over the 'height' varint that occurs at the start of every ** interior node. Then load the blockid of the left-child of the b-tree ** node into variable iChild. */ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); while( zCsr<zEnd && (piFirst || piLast) ){ int cmp; /* memcmp() result */ int nSuffix; /* Size of term suffix */ int nPrefix = 0; /* Size of term prefix */ int nBuffer; /* Total term size */ /* Load the next term on the node into zBuffer. Use realloc() to expand ** the size of zBuffer if required. */ if( !isFirstTerm ){ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix); } isFirstTerm = 0; zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix); if( nPrefix+nSuffix>nAlloc ){ char *zNew; nAlloc = (nPrefix+nSuffix) * 2; zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); if( !zNew ){ sqlite3_free(zBuffer); return SQLITE_NOMEM; } zBuffer = zNew; } memcpy(&zBuffer[nPrefix], zCsr, nSuffix); nBuffer = nPrefix + nSuffix; zCsr += nSuffix; /* Compare the term we are searching for with the term just loaded from ** the interior node. If the specified term is greater than or equal ** to the term from the interior node, then all terms on the sub-tree ** headed by node iChild are smaller than zTerm. No need to search ** iChild. ** ** If the interior node term is larger than the specified term, then ** the tree headed by iChild may contain the specified term. */ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ *piFirst = iChild; piFirst = 0; } if( piLast && cmp<0 ){ *piLast = iChild; piLast = 0; } iChild++; }; if( piFirst ) *piFirst = iChild; if( piLast ) *piLast = iChild; sqlite3_free(zBuffer); return rc; } /* ** The buffer pointed to by argument zNode (size nNode bytes) contains an ** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) ** contains a term. This function searches the sub-tree headed by the zNode ** node for the range of leaf nodes that may contain the specified term ** or terms for which the specified term is a prefix. ** ** If piLeaf is not NULL, then *piLeaf is set to the blockid of the ** left-most leaf node in the tree that may contain the specified term. ** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the ** right-most leaf node that may contain a term for which the specified ** term is a prefix. ** ** It is possible that the range of returned leaf nodes does not contain ** the specified term or any terms for which it is a prefix. However, if the ** segment does contain any such terms, they are stored within the identified ** range. Because this function only inspects interior segment nodes (and ** never loads leaf nodes into memory), it is not possible to be sure. ** ** If an error occurs, an error code other than SQLITE_OK is returned. */ static int fts3SelectLeaf( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to select leaves for */ int nTerm, /* Size of term zTerm in bytes */ const char *zNode, /* Buffer containing segment interior node */ int nNode, /* Size of buffer at zNode */ sqlite3_int64 *piLeaf, /* Selected leaf node */ sqlite3_int64 *piLeaf2 /* Selected leaf node */ ){ int rc; /* Return code */ int iHeight; /* Height of this node in tree */ assert( piLeaf || piLeaf2 ); sqlite3Fts3GetVarint32(zNode, &iHeight); rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); if( rc==SQLITE_OK && iHeight>1 ){ char *zBlob = 0; /* Blob read from %_segments table */ int nBlob; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } sqlite3_free(zBlob); piLeaf = 0; zBlob = 0; } if( rc==SQLITE_OK ){ rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); } if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); } sqlite3_free(zBlob); } return rc; } /* ** This function is used to create delta-encoded serialized lists of FTS3 ** varints. Each call to this function appends a single varint to a list. */ |
︙ | ︙ | |||
1282 1283 1284 1285 1286 1287 1288 1289 1290 | *pp = p; *pp1 = p1 + 1; *pp2 = p2 + 1; } /* ** nToken==1 searches for adjacent positions. */ static int fts3PoslistPhraseMerge( | > > > > > > > > > > > > > > > > > > > > | > | | < > > > > | 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 | *pp = p; *pp1 = p1 + 1; *pp2 = p2 + 1; } /* ** nToken==1 searches for adjacent positions. ** ** This function is used to merge two position lists into one. When it is ** called, *pp1 and *pp2 must both point to position lists. A position-list is ** the part of a doclist that follows each document id. For example, if a row ** contains: ** ** 'a b c'|'x y z'|'a b b a' ** ** Then the position list for this row for token 'b' would consist of: ** ** 0x02 0x01 0x02 0x03 0x03 0x00 ** ** When this function returns, both *pp1 and *pp2 are left pointing to the ** byte following the 0x00 terminator of their respective position lists. ** ** If isSaveLeft is 0, an entry is added to the output position list for ** each position in *pp2 for which there exists one or more positions in ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. ** when the *pp1 token appears before the *pp2 token, but not more than nToken ** slots before it. */ static int fts3PoslistPhraseMerge( char **pp, /* IN/OUT: Preallocated output buffer */ int nToken, /* Maximum difference in token positions */ int isSaveLeft, /* Save the left position */ int isExact, /* If *pp1 is exactly nTokens before *pp2 */ char **pp1, /* IN/OUT: Left input list */ char **pp2 /* IN/OUT: Right input list */ ){ char *p = (pp ? *pp : 0); char *p1 = *pp1; char *p2 = *pp2; int iCol1 = 0; int iCol2 = 0; /* Never set both isSaveLeft and isExact for the same invocation. */ assert( isSaveLeft==0 || isExact==0 ); assert( *p1!=0 && *p2!=0 ); if( *p1==POS_COLUMN ){ p1++; p1 += sqlite3Fts3GetVarint32(p1, &iCol1); } if( *p2==POS_COLUMN ){ p2++; |
︙ | ︙ | |||
1324 1325 1326 1327 1328 1329 1330 | assert( *p1!=POS_END && *p1!=POS_COLUMN ); assert( *p2!=POS_END && *p2!=POS_COLUMN ); fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; while( 1 ){ | > | > | 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 | assert( *p1!=POS_END && *p1!=POS_COLUMN ); assert( *p2!=POS_END && *p2!=POS_COLUMN ); fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; while( 1 ){ if( iPos2==iPos1+nToken || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) ){ sqlite3_int64 iSave; if( !pp ){ fts3PoslistCopy(0, &p2); fts3PoslistCopy(0, &p1); *pp1 = p1; *pp2 = p2; return 1; |
︙ | ︙ | |||
1407 1408 1409 1410 1411 1412 1413 | char **pp1, /* IN/OUT: Left input list */ char **pp2 /* IN/OUT: Right input list */ ){ char *p1 = *pp1; char *p2 = *pp2; if( !pp ){ | | | | | | 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 | char **pp1, /* IN/OUT: Left input list */ char **pp2 /* IN/OUT: Right input list */ ){ char *p1 = *pp1; char *p2 = *pp2; if( !pp ){ if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; *pp1 = p1; *pp2 = p2; return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); }else{ char *pTmp1 = aTmp; char *pTmp2; char *aTmp2; int res = 1; fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); aTmp2 = pTmp2 = pTmp1; *pp1 = p1; *pp2 = p2; fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ fts3PoslistMerge(pp, &aTmp, &aTmp2); }else if( pTmp1!=aTmp ){ fts3PoslistCopy(pp, &aTmp); }else if( pTmp2!=aTmp2 ){ fts3PoslistCopy(pp, &aTmp2); }else{ |
︙ | ︙ | |||
1467 1468 1469 1470 1471 1472 1473 | int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ char *aBuffer, /* Pre-allocated output buffer */ int *pnBuffer, /* OUT: Bytes written to aBuffer */ char *a1, /* Buffer containing first doclist */ int n1, /* Size of buffer a1 */ char *a2, /* Buffer containing second doclist */ | | > > | 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 | int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ char *aBuffer, /* Pre-allocated output buffer */ int *pnBuffer, /* OUT: Bytes written to aBuffer */ char *a1, /* Buffer containing first doclist */ int n1, /* Size of buffer a1 */ char *a2, /* Buffer containing second doclist */ int n2, /* Size of buffer a2 */ int *pnDoc /* OUT: Number of docids in output */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; char *p = aBuffer; char *p1 = a1; char *p2 = a2; char *pEnd1 = &a1[n1]; char *pEnd2 = &a2[n2]; int nDoc = 0; assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR || mergetype==MERGE_AND || mergetype==MERGE_NOT || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR ); |
︙ | ︙ | |||
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 | case MERGE_AND: while( p1 && p2 ){ if( i1==i2 ){ fts3PutDeltaVarint(&p, &iPrev, i1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); }else if( i1<i2 ){ fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; | > | 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 | case MERGE_AND: while( p1 && p2 ){ if( i1==i2 ){ fts3PutDeltaVarint(&p, &iPrev, i1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); nDoc++; }else if( i1<i2 ){ fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; |
︙ | ︙ | |||
1551 1552 1553 1554 1555 1556 1557 | case MERGE_PHRASE: { char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p); while( p1 && p2 ){ if( i1==i2 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; fts3PutDeltaVarint(&p, &iPrev, i1); | | > > | 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 | case MERGE_PHRASE: { char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p); while( p1 && p2 ){ if( i1==i2 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; fts3PutDeltaVarint(&p, &iPrev, i1); if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){ p = pSave; iPrev = iPrevSave; }else{ nDoc++; } fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); }else if( i1<i2 ){ fts3PoslistCopy(0, &p1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ |
︙ | ︙ | |||
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 | } } sqlite3_free(aTmp); break; } } *pnBuffer = (int)(p-aBuffer); return SQLITE_OK; } /* ** A pointer to an instance of this structure is used as the context ** argument to sqlite3Fts3SegReaderIterate() | > | 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 | } } sqlite3_free(aTmp); break; } } if( pnDoc ) *pnDoc = nDoc; *pnBuffer = (int)(p-aBuffer); return SQLITE_OK; } /* ** A pointer to an instance of this structure is used as the context ** argument to sqlite3Fts3SegReaderIterate() |
︙ | ︙ | |||
1644 1645 1646 1647 1648 1649 1650 | ** into a single doclist. */ for(i=0; i<SizeofArray(pTS->aaOutput); i++){ if( pTS->aaOutput[i] ){ if( !aOut ){ aOut = pTS->aaOutput[i]; nOut = pTS->anOutput[i]; | | | | 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 | ** into a single doclist. */ for(i=0; i<SizeofArray(pTS->aaOutput); i++){ if( pTS->aaOutput[i] ){ if( !aOut ){ aOut = pTS->aaOutput[i]; nOut = pTS->anOutput[i]; pTS->aaOutput[i] = 0; }else{ int nNew = nOut + pTS->anOutput[i]; char *aNew = sqlite3_malloc(nNew); if( !aNew ){ sqlite3_free(aOut); return SQLITE_NOMEM; } fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 ); sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); pTS->aaOutput[i] = 0; aOut = aNew; nOut = nNew; } |
︙ | ︙ | |||
1724 1725 1726 1727 1728 1729 1730 | aNew = sqlite3_malloc(nNew); if( !aNew ){ if( aMerge!=aDoclist ){ sqlite3_free(aMerge); } return SQLITE_NOMEM; } | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < | < < < < < < < < < < < < | < < < < < < < < < < < < < | < < < < < < < < < | | | | | | > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > | > | > > > > > > > | > > > > > > | > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > | > > > > > > > > > > | | > > > > > | > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > | > > > > > | > > > > | | < < < > | | | | > > > > > > > > > > > > > > > | > > | < > > > > > > > > > > > > | | | > > > > > > > > > > | > > > > > | | | 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 | aNew = sqlite3_malloc(nNew); if( !aNew ){ if( aMerge!=aDoclist ){ sqlite3_free(aMerge); } return SQLITE_NOMEM; } fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 ); if( iOut>0 ) sqlite3_free(aMerge); sqlite3_free(pTS->aaOutput[iOut]); pTS->aaOutput[iOut] = 0; aMerge = aNew; nMerge = nNew; if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ pTS->aaOutput[iOut] = aMerge; pTS->anOutput[iOut] = nMerge; } } } return SQLITE_OK; } static int fts3DeferredTermSelect( Fts3DeferredToken *pToken, /* Phrase token */ int isTermPos, /* True to include positions */ int *pnOut, /* OUT: Size of list */ char **ppOut /* OUT: Body of list */ ){ char *aSource; int nSource; aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); if( !aSource ){ *pnOut = 0; *ppOut = 0; }else if( isTermPos ){ *ppOut = sqlite3_malloc(nSource); if( !*ppOut ) return SQLITE_NOMEM; memcpy(*ppOut, aSource, nSource); *pnOut = nSource; }else{ sqlite3_int64 docid; *pnOut = sqlite3Fts3GetVarint(aSource, &docid); *ppOut = sqlite3_malloc(*pnOut); if( !*ppOut ) return SQLITE_NOMEM; sqlite3Fts3PutVarint(*ppOut, docid); } return SQLITE_OK; } /* ** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects. ** Elements are added to the array using fts3SegReaderArrayAdd(). */ struct Fts3SegReaderArray { int nSegment; /* Number of valid entries in apSegment[] */ int nAlloc; /* Allocated size of apSegment[] */ int nCost; /* The cost of executing SegReaderIterate() */ Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */ }; /* ** Free an Fts3SegReaderArray object. Also free all seg-readers in the ** array (using sqlite3Fts3SegReaderFree()). */ static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){ if( pArray ){ int i; for(i=0; i<pArray->nSegment; i++){ sqlite3Fts3SegReaderFree(0, pArray->apSegment[i]); } sqlite3_free(pArray); } } static int fts3SegReaderArrayAdd( Fts3SegReaderArray **ppArray, Fts3SegReader *pNew ){ Fts3SegReaderArray *pArray = *ppArray; if( !pArray || pArray->nAlloc==pArray->nSegment ){ int nNew = (pArray ? pArray->nAlloc+16 : 16); pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*) ); if( !pArray ){ sqlite3Fts3SegReaderFree(0, pNew); return SQLITE_NOMEM; } if( nNew==16 ){ pArray->nSegment = 0; pArray->nCost = 0; } pArray->nAlloc = nNew; *ppArray = pArray; } pArray->apSegment[pArray->nSegment++] = pNew; return SQLITE_OK; } static int fts3TermSegReaderArray( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */ ){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int rc; /* Return code */ Fts3SegReaderArray *pArray = 0; /* Array object to build */ Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */ sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ int iAge = 0; /* Used to assign ages to segments */ /* Allocate a seg-reader to scan the pending terms, if any. */ rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader); if( rc==SQLITE_OK && pReader ) { rc = fts3SegReaderArrayAdd(&pArray, pReader); } /* Loop through the entire %_segdir table. For each segment, create a ** Fts3SegReader to iterate through the subset of the segment leaves ** that may contain a term that matches zTerm/nTerm. For non-prefix ** searches, this is always a single leaf. For prefix searches, this ** may be a contiguous block of leaves. */ if( rc==SQLITE_OK ){ rc = sqlite3Fts3AllSegdirs(p, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ Fts3SegReader *pNew = 0; int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); if( sqlite3_column_int64(pStmt, 1)==0 ){ /* The entire segment is stored on the root node (which must be a ** leaf). Do not bother inspecting any data in this case, just ** create a Fts3SegReader to scan the single leaf. */ rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); }else{ sqlite3_int64 i1; /* First leaf that may contain zTerm */ sqlite3_int64 i2; /* Final leaf that may contain zTerm */ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); if( isPrefix==0 ) i2 = i1; if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); } } assert( (pNew==0)==(rc!=SQLITE_OK) ); /* If a new Fts3SegReader was allocated, add it to the array. */ if( rc==SQLITE_OK ){ rc = fts3SegReaderArrayAdd(&pArray, pNew); } if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost); } iAge++; } if( rc==SQLITE_DONE ){ rc = sqlite3_reset(pStmt); }else{ sqlite3_reset(pStmt); } if( rc!=SQLITE_OK ){ fts3SegReaderArrayFree(pArray); pArray = 0; } *ppArray = pArray; return rc; } /* ** This function retreives the doclist for the specified term (or term ** prefix) from the database. ** ** The returned doclist may be in one of two formats, depending on the ** value of parameter isReqPos. If isReqPos is zero, then the doclist is ** a sorted list of delta-compressed docids (a bare doclist). If isReqPos ** is non-zero, then the returned list is in the same format as is stored ** in the database without the found length specifier at the start of on-disk ** doclists. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ Fts3SegReaderArray *pArray; /* Seg-reader array for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ pArray = pTok->pArray; memset(&tsc, 0, sizeof(TermSelect)); tsc.isReqPos = isReqPos; filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; filter.nTerm = pTok->n; rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, &filter, fts3TermSelectCb, (void *)&tsc ); if( rc==SQLITE_OK ){ rc = fts3TermSelectMerge(&tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ int i; for(i=0; i<SizeofArray(tsc.aaOutput); i++){ sqlite3_free(tsc.aaOutput[i]); } } fts3SegReaderArrayFree(pArray); pTok->pArray = 0; return rc; } /* ** This function counts the total number of docids in the doclist stored ** in buffer aList[], size nList bytes. ** ** If the isPoslist argument is true, then it is assumed that the doclist ** contains a position-list following each docid. Otherwise, it is assumed ** that the doclist is simply a list of docids stored as delta encoded ** varints. */ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ int nDoc = 0; /* Return value */ if( aList ){ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ char *p = aList; /* Cursor */ if( !isPoslist ){ /* The number of docids in the list is the same as the number of ** varints. In FTS3 a varint consists of a single byte with the 0x80 ** bit cleared and zero or more bytes with the 0x80 bit set. So to ** count the varints in the buffer, just count the number of bytes ** with the 0x80 bit clear. */ while( p<aEnd ) nDoc += (((*p++)&0x80)==0); }else{ while( p<aEnd ){ nDoc++; while( (*p++)&0x80 ); /* Skip docid varint */ fts3PoslistCopy(0, &p); /* Skip over position list */ } } } return nDoc; } /* ** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. */ static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ int rc = SQLITE_OK; if( pExpr ){ rc = fts3DeferExpression(pCsr, pExpr->pLeft); if( rc==SQLITE_OK ){ rc = fts3DeferExpression(pCsr, pExpr->pRight); } if( pExpr->eType==FTSQUERY_PHRASE ){ int iCol = pExpr->pPhrase->iColumn; int i; for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; if( pToken->pDeferred==0 ){ rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); } } } } return rc; } /* ** This function removes the position information from a doclist. When ** called, buffer aList (size *pnList bytes) contains a doclist that includes ** position information. This function removes the position information so ** that aList contains only docids, and adjusts *pnList to reflect the new ** (possibly reduced) size of the doclist. */ static void fts3DoclistStripPositions( char *aList, /* IN/OUT: Buffer containing doclist */ int *pnList /* IN/OUT: Size of doclist in bytes */ ){ if( aList ){ char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ char *p = aList; /* Input cursor */ char *pOut = aList; /* Output cursor */ while( p<aEnd ){ sqlite3_int64 delta; p += sqlite3Fts3GetVarint(p, &delta); fts3PoslistCopy(0, &p); pOut += sqlite3Fts3PutVarint(pOut, delta); } *pnList = (pOut - aList); } } /* ** Return a DocList corresponding to the phrase *pPhrase. ** ** If this function returns SQLITE_OK, but *pnOut is set to a negative value, ** then no tokens in the phrase were looked up in the full-text index. This ** is only possible when this function is called from within xFilter(). The ** caller should assume that all documents match the phrase. The actual ** filtering will take place in xNext(). */ static int fts3PhraseSelect( Fts3Cursor *pCsr, /* Virtual table cursor handle */ Fts3Phrase *pPhrase, /* Phrase to return a doclist for */ int isReqPos, /* True if output should contain positions */ char **paOut, /* OUT: Pointer to malloc'd result buffer */ int *pnOut /* OUT: Size of buffer at *paOut */ ){ char *pOut = 0; int nOut = 0; int rc = SQLITE_OK; int ii; int iCol = pPhrase->iColumn; int isTermPos = (pPhrase->nToken>1 || isReqPos); Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int isFirst = 1; int iPrevTok = 0; int nDoc = 0; /* If this is an xFilter() evaluation, create a segment-reader for each ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo ** evaluation, only create segment-readers if there are no Fts3DeferredToken ** objects attached to the phrase-tokens. */ for(ii=0; ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pArray==0 ){ if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) ){ rc = fts3TermSegReaderArray( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray ); if( rc!=SQLITE_OK ) return rc; } } } for(ii=0; ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok; /* Token to find doclist for */ int iTok; /* The token being queried this iteration */ char *pList; /* Pointer to token doclist */ int nList; /* Size of buffer at pList */ /* Select a token to process. If this is an xFilter() call, then tokens ** are processed in order from least to most costly. Otherwise, tokens ** are processed in the order in which they occur in the phrase. */ if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ assert( isReqPos ); iTok = ii; pTok = &pPhrase->aToken[iTok]; if( pTok->bFulltext==0 ) continue; }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ iTok = ii; pTok = &pPhrase->aToken[iTok]; }else{ int nMinCost = 0x7FFFFFFF; int jj; /* Find the remaining token with the lowest cost. */ for(jj=0; jj<pPhrase->nToken; jj++){ Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray; if( pArray && pArray->nCost<nMinCost ){ iTok = jj; nMinCost = pArray->nCost; } } pTok = &pPhrase->aToken[iTok]; /* This branch is taken if it is determined that loading the doclist ** for the next token would require more IO than loading all documents ** currently identified by doclist pOut/nOut. No further doclists will ** be loaded from the full-text index for this phrase. */ if( nMinCost>nDoc && ii>0 ){ rc = fts3DeferExpression(pCsr, pCsr->pExpr); break; } } if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); }else{ assert( pTok->pArray ); rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); pTok->bFulltext = 1; } assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 ); if( rc!=SQLITE_OK ) break; if( isFirst ){ pOut = pList; nOut = nList; if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ nDoc = fts3DoclistCountDocids(1, pOut, nOut); } isFirst = 0; iPrevTok = iTok; }else{ /* Merge the new term list and the current output. */ char *aLeft, *aRight; int nLeft, nRight; int nDist; int mt; /* If this is the final token of the phrase, and positions were not ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. ** This drops the position information from the output list. */ mt = MERGE_POS_PHRASE; if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; assert( iPrevTok!=iTok ); if( iPrevTok<iTok ){ aLeft = pOut; nLeft = nOut; aRight = pList; nRight = nList; nDist = iTok-iPrevTok; iPrevTok = iTok; }else{ aRight = pOut; nRight = nOut; aLeft = pList; nLeft = nList; nDist = iPrevTok-iTok; } pOut = aRight; fts3DoclistMerge( mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc ); sqlite3_free(aLeft); } assert( nOut==0 || pOut!=0 ); } if( rc==SQLITE_OK ){ if( ii!=pPhrase->nToken ){ assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); fts3DoclistStripPositions(pOut, &nOut); } *paOut = pOut; *pnOut = nOut; }else{ sqlite3_free(pOut); } return rc; } /* ** This function merges two doclists according to the requirements of a ** NEAR operator. ** ** Both input doclists must include position information. The output doclist ** includes position information if the first argument to this function ** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. */ static int fts3NearMerge( int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ int nNear, /* Parameter to NEAR operator */ int nTokenLeft, /* Number of tokens in LHS phrase arg */ char *aLeft, /* Doclist for LHS (incl. positions) */ int nLeft, /* Size of LHS doclist in bytes */ int nTokenRight, /* As nTokenLeft */ char *aRight, /* As aLeft */ int nRight, /* As nRight */ char **paOut, /* OUT: Results of merge (malloced) */ int *pnOut /* OUT: Sized of output buffer */ ){ char *aOut; /* Buffer to write output doclist to */ int rc; /* Return code */ assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); aOut = sqlite3_malloc(nLeft+nRight+1); if( aOut==0 ){ rc = SQLITE_NOMEM; }else{ rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 ); if( rc!=SQLITE_OK ){ sqlite3_free(aOut); aOut = 0; } } *paOut = aOut; return rc; } /* ** This function is used as part of the processing for the snippet() and ** offsets() functions. ** ** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both ** have their respective doclists (including position information) loaded ** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from ** each doclist that are not within nNear tokens of a corresponding entry ** in the other doclist. */ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ int rc; /* Return code */ assert( pLeft->eType==FTSQUERY_PHRASE ); assert( pRight->eType==FTSQUERY_PHRASE ); assert( pLeft->isLoaded && pRight->isLoaded ); if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ sqlite3_free(pLeft->aDoclist); sqlite3_free(pRight->aDoclist); pRight->aDoclist = 0; pLeft->aDoclist = 0; rc = SQLITE_OK; }else{ char *aOut; /* Buffer in which to assemble new doclist */ int nOut; /* Size of buffer aOut in bytes */ rc = fts3NearMerge(MERGE_POS_NEAR, nNear, pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, &aOut, &nOut ); if( rc!=SQLITE_OK ) return rc; |
︙ | ︙ | |||
2014 2015 2016 2017 2018 2019 2020 2021 | sqlite3_free(pLeft->aDoclist); pLeft->aDoclist = aOut; pLeft->nDoclist = nOut; } return rc; } /* | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < | | > > | > > > > | < | 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 | sqlite3_free(pLeft->aDoclist); pLeft->aDoclist = aOut; pLeft->nDoclist = nOut; } return rc; } /* ** Allocate an Fts3SegReaderArray for each token in the expression pExpr. ** The allocated objects are stored in the Fts3PhraseToken.pArray member ** variables of each token structure. */ static int fts3ExprAllocateSegReaders( Fts3Cursor *pCsr, /* FTS3 table */ Fts3Expr *pExpr, /* Expression to create seg-readers for */ int *pnExpr /* OUT: Number of AND'd expressions */ ){ int rc = SQLITE_OK; /* Return code */ assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ (*pnExpr)++; pnExpr = 0; } if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pArray==0 ){ rc = fts3TermSegReaderArray( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray ); } } }else{ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); if( rc==SQLITE_OK ){ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); } } return rc; } /* ** Free the Fts3SegReaderArray objects associated with each token in the ** expression pExpr. In other words, this function frees the resources ** allocated by fts3ExprAllocateSegReaders(). */ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ int kk; for(kk=0; kk<pPhrase->nToken; kk++){ fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray); pPhrase->aToken[kk].pArray = 0; } } fts3ExprFreeSegReaders(pExpr->pLeft); fts3ExprFreeSegReaders(pExpr->pRight); } } /* ** Return the sum of the costs of all tokens in the expression pExpr. This ** function must be called after Fts3SegReaderArrays have been allocated ** for all tokens using fts3ExprAllocateSegReaders(). */ int fts3ExprCost(Fts3Expr *pExpr){ int nCost; /* Return value */ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; nCost = 0; for(ii=0; ii<pPhrase->nToken; ii++){ nCost += pPhrase->aToken[ii].pArray->nCost; } }else{ nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); } return nCost; } /* ** The following is a helper function (and type) for fts3EvalExpr(). It ** must be called after Fts3SegReaders have been allocated for every token ** in the expression. See the context it is called from in fts3EvalExpr() ** for further explanation. */ typedef struct ExprAndCost ExprAndCost; struct ExprAndCost { Fts3Expr *pExpr; int nCost; }; static void fts3ExprAssignCosts( Fts3Expr *pExpr, /* Expression to create seg-readers for */ ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ ){ if( pExpr->eType==FTSQUERY_AND ){ fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); fts3ExprAssignCosts(pExpr->pRight, ppExprCost); }else{ (*ppExprCost)->pExpr = pExpr; (*ppExprCost)->nCost = fts3ExprCost(pExpr);; (*ppExprCost)++; } } /* ** Evaluate the full-text expression pExpr against FTS3 table pTab. Store ** the resulting doclist in *paOut and *pnOut. This routine mallocs for ** the space needed to store the output. The caller is responsible for ** freeing the space when it has finished. ** ** This function is called in two distinct contexts: ** ** * From within the virtual table xFilter() method. In this case, the ** output doclist contains entries for all rows in the table, based on ** data read from the full-text index. ** ** In this case, if the query expression contains one or more tokens that ** are very common, then the returned doclist may contain a superset of ** the documents that actually match the expression. ** ** * From within the virtual table xNext() method. This call is only made ** if the call from within xFilter() found that there were very common ** tokens in the query expression and did return a superset of the ** matching documents. In this case the returned doclist contains only ** entries that correspond to the current row of the table. Instead of ** reading the data for each token from the full-text index, the data is ** already available in-memory in the Fts3PhraseToken.pDeferred structures. ** See fts3EvalDeferred() for how it gets there. ** ** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is ** required) Fts3Cursor.doDeferred==1. ** ** If the SQLite invokes the snippet(), offsets() or matchinfo() function ** as part of a SELECT on an FTS3 table, this function is called on each ** individual phrase expression in the query. If there were very common tokens ** found in the xFilter() call, then this function is called once for phrase ** for each row visited, and the returned doclist contains entries for the ** current row only. Otherwise, if there were no very common tokens, then this ** function is called once only for each phrase in the query and the returned ** doclist contains entries for all rows of the table. ** ** Fts3Cursor.doDeferred==1 when this function is called on phrases as a ** result of a snippet(), offsets() or matchinfo() invocation. */ static int fts3EvalExpr( Fts3Cursor *p, /* Virtual table cursor handle */ Fts3Expr *pExpr, /* Parsed fts3 expression */ char **paOut, /* OUT: Pointer to malloc'd result buffer */ int *pnOut, /* OUT: Size of buffer at *paOut */ int isReqPos /* Require positions in output buffer */ ){ int rc = SQLITE_OK; /* Return code */ /* Zero the output parameters. */ *paOut = 0; *pnOut = 0; if( pExpr ){ assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT || pExpr->eType==FTSQUERY_PHRASE ); assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); if( pExpr->eType==FTSQUERY_PHRASE ){ rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), paOut, pnOut ); fts3ExprFreeSegReaders(pExpr); }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ int nExpr = 0; /* Size of aExpr[] */ char *aRet = 0; /* Doclist to return to caller */ int nRet = 0; /* Length of aRet[] in bytes */ int nDoc = 0x7FFFFFFF; assert( !isReqPos ); rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); if( rc==SQLITE_OK ){ assert( nExpr>1 ); aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); if( !aExpr ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ int ii; /* Used to iterate through expressions */ fts3ExprAssignCosts(pExpr, &aExpr); aExpr -= nExpr; for(ii=0; ii<nExpr; ii++){ char *aNew; int nNew; int jj; ExprAndCost *pBest = 0; for(jj=0; jj<nExpr; jj++){ ExprAndCost *pCand = &aExpr[jj]; if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){ pBest = pCand; } } if( pBest->nCost>nDoc ){ rc = fts3DeferExpression(p, p->pExpr); break; }else{ rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); if( rc!=SQLITE_OK ) break; pBest->pExpr = 0; if( ii==0 ){ aRet = aNew; nRet = nNew; nDoc = fts3DoclistCountDocids(0, aRet, nRet); }else{ fts3DoclistMerge( MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc ); sqlite3_free(aNew); } } } } *paOut = aRet; *pnOut = nRet; sqlite3_free(aExpr); fts3ExprFreeSegReaders(pExpr); }else{ char *aLeft; char *aRight; int nLeft; int nRight; assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR || pExpr->eType==FTSQUERY_NOT || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) ); if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: { Fts3Expr *pLeft; Fts3Expr *pRight; int mergetype = MERGE_NEAR; if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ mergetype = MERGE_POS_NEAR; } pLeft = pExpr->pLeft; while( pLeft->eType==FTSQUERY_NEAR ){ pLeft=pLeft->pRight; } |
︙ | ︙ | |||
2089 2090 2091 2092 2093 2094 2095 | /* Allocate a buffer for the output. The maximum size is the ** sum of the sizes of the two input buffers. The +1 term is ** so that a buffer of zero bytes is never allocated - this can ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. */ char *aBuffer = sqlite3_malloc(nRight+nLeft+1); rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < < < | 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 | /* Allocate a buffer for the output. The maximum size is the ** sum of the sizes of the two input buffers. The +1 term is ** so that a buffer of zero bytes is never allocated - this can ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. */ char *aBuffer = sqlite3_malloc(nRight+nLeft+1); rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, aLeft, nLeft, aRight, nRight, 0 ); *paOut = aBuffer; sqlite3_free(aLeft); break; } default: { assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, aLeft, nLeft, aRight, nRight, 0 ); *paOut = aLeft; break; } } } sqlite3_free(aRight); } } return rc; } /* ** This function is called from within xNext() for each row visited by ** an FTS3 query. If evaluating the FTS3 query expression within xFilter() ** was able to determine the exact set of matching rows, this function sets ** *pbRes to true and returns SQLITE_IO immediately. ** ** Otherwise, if evaluating the query expression within xFilter() returned a ** superset of the matching documents instead of an exact set (this happens ** when the query includes very common tokens and it is deemed too expensive to ** load their doclists from disk), this function tests if the current row ** really does match the FTS3 query. ** ** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK ** is returned and *pbRes is set to true if the current row matches the ** FTS3 query (and should be included in the results returned to SQLite), or ** false otherwise. */ static int fts3EvalDeferred( Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ int *pbRes /* OUT: Set to true if row is a match */ ){ int rc = SQLITE_OK; if( pCsr->pDeferred==0 ){ *pbRes = 1; }else{ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ sqlite3Fts3FreeDeferredDoclists(pCsr); rc = sqlite3Fts3CacheDeferredDoclists(pCsr); } if( rc==SQLITE_OK ){ char *a = 0; int n = 0; rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); assert( n>=0 ); *pbRes = (n>0); sqlite3_free(a); } } return rc; } /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be ** the next row that matches. For a full-table scan, this will be ** simply the next row in the %_content table. For a docid lookup, ** this routine simply sets the EOF flag. ** ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned ** even if we reach end-of-file. The fts3EofMethod() will be called ** subsequently to determine whether or not an EOF was hit. */ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ int res; int rc = SQLITE_OK; /* Return code */ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; pCsr->eEvalmode = FTS3_EVAL_NEXT; do { if( pCsr->aDoclist==0 ){ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ pCsr->isEof = 1; rc = sqlite3_reset(pCsr->pStmt); break; } pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); }else{ if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ pCsr->isEof = 1; break; } sqlite3_reset(pCsr->pStmt); fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; } }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); return rc; } /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. ** ** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against ** the %_content table. ** ** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry ** in the %_content table. ** ** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The ** column on the left-hand side of the MATCH operator is column ** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand ** side of the MATCH operator. */ static int fts3FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ |
︙ | ︙ | |||
2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 | UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(nVal); assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( nVal==0 || nVal==1 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); | > < < < < < < < < < < < < < < < | < < | 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 | UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(nVal); assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( nVal==0 || nVal==1 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); assert( p->pSegments==0 ); /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ return SQLITE_NOMEM; } |
︙ | ︙ | |||
2202 2203 2204 2205 2206 2207 2208 | } return rc; } rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; | | > > > > > > > > > > > > > > > > > > > | 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 | } return rc; } rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; } /* Compile a SELECT statement for this cursor. For a full-table-scan, the ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); } if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); } pCsr->eSearch = (i16)idxNum; if( rc!=SQLITE_OK ) return rc; return fts3NextMethod(pCursor); } /* ** This is the xEof method of the virtual table. SQLite calls this |
︙ | ︙ | |||
2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 | ** rowid should be written to *pRowid. */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; if( pCsr->aDoclist ){ *pRowid = pCsr->iPrevId; }else{ *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); } return SQLITE_OK; } /* ** This is the xColumn method, called by SQLite to request a value from | > > > > > | 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 | ** rowid should be written to *pRowid. */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; if( pCsr->aDoclist ){ *pRowid = pCsr->iPrevId; }else{ /* This branch runs if the query is implemented using a full-table scan ** (not using the full-text index). In this case grab the rowid from the ** SELECT statement. */ assert( pCsr->isRequireSeek==0 ); *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); } return SQLITE_OK; } /* ** This is the xColumn method, called by SQLite to request a value from |
︙ | ︙ | |||
2292 2293 2294 2295 2296 2297 2298 | } /* ** Implementation of xSync() method. Flush the contents of the pending-terms ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ | | > > | 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 | } /* ** Implementation of xSync() method. Flush the contents of the pending-terms ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); return rc; } /* ** Implementation of xBegin() method. This is a no-op. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ UNUSED_PARAMETER(pVtab); |
︙ | ︙ | |||
2330 2331 2332 2333 2334 2335 2336 | /* ** Load the doclist associated with expression pExpr to pExpr->aDoclist. ** The loaded doclist contains positions as well as the document ids. ** This is used by the matchinfo(), snippet() and offsets() auxillary ** functions. */ | | > > > | > > > > > > > > > > > > > > > > | 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 | /* ** Load the doclist associated with expression pExpr to pExpr->aDoclist. ** The loaded doclist contains positions as well as the document ids. ** This is used by the matchinfo(), snippet() and offsets() auxillary ** functions. */ int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ int rc; assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); return rc; } int sqlite3Fts3ExprLoadFtDoclist( Fts3Cursor *pCsr, Fts3Expr *pExpr, char **paDoclist, int *pnDoclist ){ int rc; assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); pCsr->eEvalmode = FTS3_EVAL_NEXT; return rc; } /* ** After ExprLoadDoclist() (see above) has been called, this function is ** used to iterate/search through the position lists that make up the doclist ** stored in pExpr->aDoclist. */ |
︙ | ︙ | |||
2397 2398 2399 2400 2401 2402 2403 | ** message is written to context pContext and SQLITE_ERROR returned. The ** string passed via zFunc is used as part of the error message. */ static int fts3FunctionArg( sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ | | | 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 | ** message is written to context pContext and SQLITE_ERROR returned. The ** string passed via zFunc is used as part of the error message. */ static int fts3FunctionArg( sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ Fts3Cursor *pRet; if( sqlite3_value_type(pVal)!=SQLITE_BLOB || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) ){ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); sqlite3_result_error(pContext, zErr, -1); |
︙ | ︙ | |||
2523 2524 2525 2526 2527 2528 2529 | */ static void fts3MatchinfoFunc( sqlite3_context *pContext, /* SQLite function call context */ int nVal, /* Size of argument array */ sqlite3_value **apVal /* Array of arguments */ ){ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ | < | < < < < < | 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 | */ static void fts3MatchinfoFunc( sqlite3_context *pContext, /* SQLite function call context */ int nVal, /* Size of argument array */ sqlite3_value **apVal /* Array of arguments */ ){ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ assert( nVal==1 ); if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ sqlite3Fts3Matchinfo(pContext, pCsr); } } /* ** This routine implements the xFindFunction method for the FTS3 |
︙ | ︙ | |||
2592 2593 2594 2595 2596 2597 2598 | return rc; } fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", p->zDb, p->zName, zName ); | < | 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 | return rc; } fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", p->zDb, p->zName, zName ); if( p->bHasDocsize ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", p->zDb, p->zName, zName ); fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", |
︙ | ︙ | |||
2622 2623 2624 2625 2626 2627 2628 | /* iVersion */ 0, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, /* xDisconnect */ fts3DisconnectMethod, /* xDestroy */ fts3DestroyMethod, /* xOpen */ fts3OpenMethod, | | | 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 | /* iVersion */ 0, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, /* xDisconnect */ fts3DisconnectMethod, /* xDestroy */ fts3DestroyMethod, /* xOpen */ fts3OpenMethod, /* xClose */ fts3CloseMethod, /* xFilter */ fts3FilterMethod, /* xNext */ fts3NextMethod, /* xEof */ fts3EofMethod, /* xColumn */ fts3ColumnMethod, /* xRowid */ fts3RowidMethod, /* xUpdate */ fts3UpdateMethod, /* xBegin */ fts3BeginMethod, |
︙ | ︙ | |||
2649 2650 2651 2652 2653 2654 2655 | static void hashDestroy(void *p){ Fts3Hash *pHash = (Fts3Hash *)p; sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } /* | | | | | < | > > | 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 | static void hashDestroy(void *p){ Fts3Hash *pHash = (Fts3Hash *)p; sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } /* ** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are ** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c ** respectively. The following three forward declarations are for functions ** declared in these files used to retrieve the respective implementations. ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed ** to by the argument to point to the "simple" tokenizer implementation. ** And so on. */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); #ifdef SQLITE_ENABLE_ICU void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); #endif /* ** Initialise the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. */ |
︙ | ︙ | |||
2717 2718 2719 2720 2721 2722 2723 | ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) | | | 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 | ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ rc = sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); if( rc==SQLITE_OK ){ rc = sqlite3_create_module_v2( |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | ** amalgamation. */ #ifndef SQLITE_AMALGAMATION /* ** Macros indicating that conditional expressions are always true or ** false. */ # define ALWAYS(x) (x) # define NEVER(X) (x) /* ** Internal types used by SQLite. */ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ /* ** Macro used to suppress compiler warnings for unused parameters. */ #define UNUSED_PARAMETER(x) (void)(x) #endif typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3SegReader Fts3SegReader; | > > > > > > > > > > | | < < < < < < < < < < < > > > | 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | ** amalgamation. */ #ifndef SQLITE_AMALGAMATION /* ** Macros indicating that conditional expressions are always true or ** false. */ #ifdef SQLITE_COVERAGE_TEST # define ALWAYS(x) (1) # define NEVER(X) (0) #else # define ALWAYS(x) (x) # define NEVER(X) (x) #endif /* ** Internal types used by SQLite. */ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ /* ** Macro used to suppress compiler warnings for unused parameters. */ #define UNUSED_PARAMETER(x) (void)(x) #endif typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3SegReaderArray Fts3SegReaderArray; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. ** All other methods receive a pointer to the structure as one of their ** arguments. */ struct Fts3Table { sqlite3_vtab base; /* Base class used by SQLite core */ sqlite3 *db; /* The database connection */ const char *zDb; /* logical database name */ const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[24]; int nNodeSize; /* Soft limit for node size */ u8 bHasContent; /* True if %_content table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. |
︙ | ︙ | |||
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | struct Fts3Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ }; /* ** The Fts3Cursor.eSearch member is always set to one of the following. ** Actualy, Fts3Cursor.eSearch can be greater than or equal to ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index ** of the column to be searched. For example, in ** | > > > > > > > | 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 | struct Fts3Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ }; #define FTS3_EVAL_FILTER 0 #define FTS3_EVAL_NEXT 1 #define FTS3_EVAL_MATCHINFO 2 /* ** The Fts3Cursor.eSearch member is always set to one of the following. ** Actualy, Fts3Cursor.eSearch can be greater than or equal to ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index ** of the column to be searched. For example, in ** |
︙ | ︙ | |||
186 187 188 189 190 191 192 | #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. | | | > > > > > > > > > > > > > > > > < < < < | | 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in double-quotes (i.e. "one two three") ** nToken will be the number of tokens in the string. ** ** The nDocMatch and nMatch variables contain data that may be used by the ** matchinfo() function. They are populated when the full-text index is ** queried for hits on the phrase. If one or more tokens in the phrase ** are deferred, the nDocMatch and nMatch variables are populated based ** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ int bFulltext; /* True if full-text index was used */ Fts3SegReaderArray *pArray; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ }; struct Fts3Phrase { /* Variables populated by fts3_expr.c when parsing a MATCH expression */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ int isNot; /* Phrase prefixed by unary not (-) operator */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. ** ** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded ** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, |
︙ | ︙ | |||
247 248 249 250 251 252 253 | #define FTSQUERY_NEAR 1 #define FTSQUERY_NOT 2 #define FTSQUERY_AND 3 #define FTSQUERY_OR 4 #define FTSQUERY_PHRASE 5 | < < < < < | > > > > > > > > > | 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 | #define FTSQUERY_NEAR 1 #define FTSQUERY_NOT 2 #define FTSQUERY_AND 3 #define FTSQUERY_OR 4 #define FTSQUERY_PHRASE 5 /* fts3_write.c */ int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *); int sqlite3Fts3SegReaderIterate( Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, int (*)(Fts3Table *, void *, char *, int, char *, int), void * ); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 |
︙ | ︙ | |||
293 294 295 296 297 298 299 | int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); | | > | 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 | int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, sqlite3_tokenizer **, const char **, char ** |
︙ | ︙ |
Changes to ext/fts3/fts3_expr.c.
︙ | ︙ | |||
101 102 103 104 105 106 107 108 109 110 111 112 113 114 | ** is defined to accept an argument of type char, and always returns 0 for ** any values that fall outside of the range of the unsigned char type (i.e. ** negative values). */ static int fts3isspace(char c){ return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; } /* ** Extract the next token from buffer z (length n) using the tokenizer ** and other information (column names etc.) in pParse. Create an Fts3Expr ** structure of type FTSQUERY_PHRASE containing a phrase consisting of this ** single token and set *ppExpr to point to it. If the end of the buffer is ** reached before a token is found, set *ppExpr to zero. It is the | > > > > > > > > > > > > | 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 | ** is defined to accept an argument of type char, and always returns 0 for ** any values that fall outside of the range of the unsigned char type (i.e. ** negative values). */ static int fts3isspace(char c){ return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; } /* ** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, ** zero the memory before returning a pointer to it. If unsuccessful, ** return NULL. */ static void *fts3MallocZero(int nByte){ void *pRet = sqlite3_malloc(nByte); if( pRet ) memset(pRet, 0, nByte); return pRet; } /* ** Extract the next token from buffer z (length n) using the tokenizer ** and other information (column names etc.) in pParse. Create an Fts3Expr ** structure of type FTSQUERY_PHRASE containing a phrase consisting of this ** single token and set *ppExpr to point to it. If the end of the buffer is ** reached before a token is found, set *ppExpr to zero. It is the |
︙ | ︙ | |||
139 140 141 142 143 144 145 | int nByte; /* total space to allocate */ pCursor->pTokenizer = pTokenizer; rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; | | < | 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | int nByte; /* total space to allocate */ pCursor->pTokenizer = pTokenizer; rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; }else{ pRet->eType = FTSQUERY_PHRASE; pRet->pPhrase = (Fts3Phrase *)&pRet[1]; pRet->pPhrase->nToken = 1; pRet->pPhrase->iColumn = iCol; pRet->pPhrase->aToken[0].n = nToken; pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); |
︙ | ︙ | |||
219 220 221 222 223 224 225 | pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ const char *zToken; int nToken, iBegin, iEnd, iPos; rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); | | > | 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 | pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ const char *zToken; int nToken, iBegin, iEnd, iPos; rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); if( !p || !zTemp ){ goto no_mem; } if( ii==0 ){ memset(p, 0, nByte); p->pPhrase = (Fts3Phrase *)&p[1]; } p->pPhrase = (Fts3Phrase *)&p[1]; memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); p->pPhrase->nToken = ii+1; p->pPhrase->aToken[ii].n = nToken; memcpy(&zTemp[nTemp], zToken, nToken); nTemp += nToken; if( iEnd<nInput && zInput[iEnd]=='*' ){ p->pPhrase->aToken[ii].isPrefix = 1; }else{ |
︙ | ︙ | |||
250 251 252 253 254 255 256 | } if( rc==SQLITE_DONE ){ int jj; char *zNew = NULL; int nNew = 0; int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); | | | 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 | } if( rc==SQLITE_DONE ){ int jj; char *zNew = NULL; int nNew = 0; int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); p = fts3ReallocOrFree(p, nByte + nTemp); if( !p ){ goto no_mem; } if( zTemp ){ zNew = &(((char *)p)[nByte]); memcpy(zNew, zTemp, nTemp); |
︙ | ︙ | |||
368 369 370 371 372 373 374 | ** the next byte must contain either whitespace, an open or close ** parenthesis, a quote character, or EOF. */ cNext = zInput[nKey]; if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ | | < | 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 | ** the next byte must contain either whitespace, an open or close ** parenthesis, a quote character, or EOF. */ cNext = zInput[nKey]; if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); if( !pRet ){ return SQLITE_NOMEM; } pRet->eType = pKey->eType; pRet->nNear = nNear; *ppExpr = pRet; *pnConsumed = (int)((zInput - z) + nKey); return SQLITE_OK; } |
︙ | ︙ | |||
548 549 550 551 552 553 554 | if( rc==SQLITE_OK ){ int isPhrase; if( !sqlite3_fts3_enable_parentheses && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot ){ /* Create an implicit NOT operator. */ | | < | 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 | if( rc==SQLITE_OK ){ int isPhrase; if( !sqlite3_fts3_enable_parentheses && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot ){ /* Create an implicit NOT operator. */ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } pNot->eType = FTSQUERY_NOT; pNot->pRight = p; if( pNotBranch ){ pNot->pLeft = pNotBranch; } pNotBranch = pNot; p = pPrev; |
︙ | ︙ | |||
582 583 584 585 586 587 588 | goto exprparse_out; } if( isPhrase && !isRequirePhrase ){ /* Insert an implicit AND operator. */ Fts3Expr *pAnd; assert( pRet && pPrev ); | | < | 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 | goto exprparse_out; } if( isPhrase && !isRequirePhrase ){ /* Insert an implicit AND operator. */ Fts3Expr *pAnd; assert( pRet && pPrev ); pAnd = fts3MallocZero(sizeof(Fts3Expr)); if( !pAnd ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } pAnd->eType = FTSQUERY_AND; insertBinaryOperator(&pRet, pPrev, pAnd); pPrev = pAnd; } /* This test catches attempts to make either operand of a NEAR ** operator something other than a phrase. For example, either of |
︙ | ︙ |
Changes to ext/fts3/fts3_snippet.c.
︙ | ︙ | |||
20 21 22 23 24 25 26 | /* ** Used as an fts3ExprIterate() context when loading phrase doclists to ** Fts3Expr.aDoclist[]/nDoclist. */ typedef struct LoadDoclistCtx LoadDoclistCtx; struct LoadDoclistCtx { | | | 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | /* ** Used as an fts3ExprIterate() context when loading phrase doclists to ** Fts3Expr.aDoclist[]/nDoclist. */ typedef struct LoadDoclistCtx LoadDoclistCtx; struct LoadDoclistCtx { Fts3Cursor *pCsr; /* FTS3 Cursor */ int nPhrase; /* Number of phrases seen so far */ int nToken; /* Number of tokens seen so far */ }; /* ** The following types are used as part of the implementation of the ** fts3BestSnippet() routine. |
︙ | ︙ | |||
214 215 216 217 218 219 220 | UNUSED_PARAMETER(iPhrase); p->nPhrase++; p->nToken += pExpr->pPhrase->nToken; if( pExpr->isLoaded==0 ){ | | | 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 | UNUSED_PARAMETER(iPhrase); p->nPhrase++; p->nToken += pExpr->pPhrase->nToken; if( pExpr->isLoaded==0 ){ rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); pExpr->isLoaded = 1; if( rc==SQLITE_OK ){ rc = fts3ExprNearTrim(pExpr); } } return rc; |
︙ | ︙ | |||
257 258 259 260 261 262 263 | static int fts3ExprLoadDoclists( Fts3Cursor *pCsr, /* Fts3 cursor for current query */ int *pnPhrase, /* OUT: Number of phrases in query */ int *pnToken /* OUT: Number of tokens in query */ ){ int rc; /* Return Code */ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ | | | 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 | static int fts3ExprLoadDoclists( Fts3Cursor *pCsr, /* Fts3 cursor for current query */ int *pnPhrase, /* OUT: Number of phrases in query */ int *pnToken /* OUT: Number of tokens in query */ ){ int rc; /* Return Code */ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ sCtx.pCsr = pCsr; rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx); if( rc==SQLITE_OK ){ (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0); } if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; return rc; |
︙ | ︙ | |||
788 789 790 791 792 793 794 | */ static int fts3ExprGlobalMatchinfoCb( Fts3Expr *pExpr, /* Phrase expression node */ int iPhrase, /* Phrase number (numbered from zero) */ void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; | > | > > > > > > > > | > > > > > > > > > > > > > > | | > > > | | | > | 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 | */ static int fts3ExprGlobalMatchinfoCb( Fts3Expr *pExpr, /* Phrase expression node */ int iPhrase, /* Phrase number (numbered from zero) */ void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; Fts3Cursor *pCsr = p->pCursor; char *pIter; char *pEnd; char *pFree = 0; const int iStart = 2 + (iPhrase * p->nCol * 3) + 1; assert( pExpr->isLoaded ); assert( pExpr->eType==FTSQUERY_PHRASE ); if( pCsr->pDeferred ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; ii<pPhrase->nToken; ii++){ if( pPhrase->aToken[ii].bFulltext ) break; } if( ii<pPhrase->nToken ){ int nFree = 0; int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); if( rc!=SQLITE_OK ) return rc; pIter = pFree; pEnd = &pFree[nFree]; }else{ int nDoc = p->aMatchinfo[2 + 3*p->nCol*p->aMatchinfo[0]]; for(ii=0; ii<p->nCol; ii++){ p->aMatchinfo[iStart + ii*3] = nDoc; p->aMatchinfo[iStart + ii*3 + 1] = nDoc; } return SQLITE_OK; } }else{ pIter = pExpr->aDoclist; pEnd = &pExpr->aDoclist[pExpr->nDoclist]; } /* Fill in the global hit count matrix row for this phrase. */ while( pIter<pEnd ){ while( *pIter++ & 0x80 ); /* Skip past docid. */ fts3LoadColumnlistCounts(&pIter, &p->aMatchinfo[iStart], 1); } sqlite3_free(pFree); return SQLITE_OK; } /* ** fts3ExprIterate() callback used to collect the "local" matchinfo stats ** for a single query. The "local" stats are those elements of the matchinfo ** array that are different for each row returned by the query. |
︙ | ︙ | |||
870 871 872 873 874 875 876 | sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo); if( !sInfo.aMatchinfo ){ return SQLITE_NOMEM; } memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo); | < < > | 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 | sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo); if( !sInfo.aMatchinfo ){ return SQLITE_NOMEM; } memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo); /* First element of match-info is the number of phrases in the query */ sInfo.aMatchinfo[0] = nPhrase; sInfo.aMatchinfo[1] = sInfo.nCol; if( pTab->bHasDocsize ){ int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]); } (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo); pCsr->aMatchinfo = sInfo.aMatchinfo; pCsr->isMatchinfoNeeded = 1; } sInfo.aMatchinfo = pCsr->aMatchinfo; if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){ (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo); |
︙ | ︙ | |||
988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 | for(i=0; i<nSnippet && rc==SQLITE_OK; i++){ rc = fts3SnippetText(pCsr, &aSnippet[i], i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res ); } snippet_out: if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); }else{ sqlite3_result_text(pCtx, res.z, -1, sqlite3_free); } } | > | 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 | for(i=0; i<nSnippet && rc==SQLITE_OK; i++){ rc = fts3SnippetText(pCsr, &aSnippet[i], i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res ); } snippet_out: sqlite3Fts3SegmentsClose(pTab); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); }else{ sqlite3_result_text(pCtx, res.z, -1, sqlite3_free); } } |
︙ | ︙ | |||
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 | pMod->xClose(pC); if( rc!=SQLITE_OK ) goto offsets_out; } offsets_out: sqlite3_free(sCtx.aTerm); assert( rc!=SQLITE_DONE ); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); }else{ sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); } return; } /* ** Implementation of matchinfo() function. */ void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){ int rc; if( !pCsr->pExpr ){ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); return; } rc = fts3GetMatchinfo(pCsr); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pContext, rc); }else{ Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3); if( pTab->bHasDocsize ){ n += sizeof(u32)*(1 + 2*pTab->nColumn); } sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); } } #endif | > > | 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 | pMod->xClose(pC); if( rc!=SQLITE_OK ) goto offsets_out; } offsets_out: sqlite3_free(sCtx.aTerm); assert( rc!=SQLITE_DONE ); sqlite3Fts3SegmentsClose(pTab); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); }else{ sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); } return; } /* ** Implementation of matchinfo() function. */ void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){ int rc; if( !pCsr->pExpr ){ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); return; } rc = fts3GetMatchinfo(pCsr); sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab ); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pContext, rc); }else{ Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3); if( pTab->bHasDocsize ){ n += sizeof(u32)*(1 + 2*pTab->nColumn); } sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); } } #endif |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | char *aData; int nSpace; sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; }; /* ** An instance of this structure is used to iterate through the terms on ** a contiguous set of segment b-tree leaf nodes. Although the details of ** this structure are only manipulated by code in this file, opaque handles ** of type Fts3SegReader* are also used by code in fts3.c to iterate through ** terms when querying the full-text index. See functions: ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: ** ** fts3SegReaderNext() ** fts3SegReaderFirstDocid() ** fts3SegReaderNextDocid() */ struct Fts3SegReader { int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ | > > > > > > > > > > > > > | > | > | < > > | 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | char *aData; int nSpace; sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; }; /* ** Each cursor has a (possibly empty) linked list of the following objects. */ struct Fts3DeferredToken { Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ int iCol; /* Column token must occur in */ Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ PendingList *pList; /* Doclist is assembled here */ }; /* ** An instance of this structure is used to iterate through the terms on ** a contiguous set of segment b-tree leaf nodes. Although the details of ** this structure are only manipulated by code in this file, opaque handles ** of type Fts3SegReader* are also used by code in fts3.c to iterate through ** terms when querying the full-text index. See functions: ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() ** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: ** ** fts3SegReaderNext() ** fts3SegReaderFirstDocid() ** fts3SegReaderNextDocid() */ struct Fts3SegReader { int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly ** by the caller. They are valid from the time SegmentReaderNew() returns ** until SegmentReaderNext() returns something other than SQLITE_OK ** (i.e. SQLITE_DONE). */ int nTerm; /* Number of bytes in current term */ char *zTerm; /* Pointer to current term */ int nTermAlloc; /* Allocated size of zTerm buffer */ char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ /* The following variables are used to iterate through the current doclist */ char *pOffsetList; sqlite3_int64 iDocid; }; #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) /* ** An instance of this structure is used to create a segment b-tree in the ** database. The internal details of this type are only accessed by the ** following functions: ** ** fts3SegWriterAdd() |
︙ | ︙ | |||
149 150 151 152 153 154 155 | #define SQL_SELECT_LEVEL 12 #define SQL_SELECT_ALL_LEVEL 13 #define SQL_SELECT_LEVEL_COUNT 14 #define SQL_SELECT_SEGDIR_COUNT_MAX 15 #define SQL_DELETE_SEGDIR_BY_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 | < | | | | | | 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 | #define SQL_SELECT_LEVEL 12 #define SQL_SELECT_ALL_LEVEL 13 #define SQL_SELECT_LEVEL_COUNT 14 #define SQL_SELECT_SEGDIR_COUNT_MAX 15 #define SQL_DELETE_SEGDIR_BY_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** |
︙ | ︙ | |||
199 200 201 202 203 204 205 | /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", | < | | | | | | 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); |
︙ | ︙ | |||
279 280 281 282 283 284 285 | sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } *pRC = rc; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 293 294 295 296 297 298 299 300 301 302 303 304 305 306 | sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } *pRC = rc; } /* ** This function ensures that the caller has obtained a shared-cache ** table-lock on the %_content table. This is required before reading ** data from the fts3 table. If this lock is not acquired first, then ** the caller may end up holding read-locks on the %_segments and %_segdir ** tables, but no read-lock on the %_content table. If this happens ** a second connection will be able to write to the fts3 table, but |
︙ | ︙ | |||
486 487 488 489 490 491 492 | ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in ** p->iPrevDocid, and the column is specified by argument iCol. ** ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. */ static int fts3PendingTermsAdd( | | | | | | 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 | ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in ** p->iPrevDocid, and the column is specified by argument iCol. ** ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. */ static int fts3PendingTermsAdd( Fts3Table *p, /* Table into which text will be inserted */ const char *zText, /* Text of document to be inserted */ int iCol, /* Column into which text is being inserted */ u32 *pnWord /* OUT: Number of tokens inserted */ ){ int rc; int iStart; int iEnd; int iPos; int nWord = 0; |
︙ | ︙ | |||
574 575 576 577 578 579 580 581 582 583 584 585 586 587 | int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; return SQLITE_OK; } void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ sqlite3_free(fts3HashData(pElem)); } fts3HashClear(&p->pendingTerms); p->nPendingData = 0; | > > > | 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 | int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; return SQLITE_OK; } /* ** Discard the contents of the pending-terms hash table. */ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ sqlite3_free(fts3HashData(pElem)); } fts3HashClear(&p->pendingTerms); p->nPendingData = 0; |
︙ | ︙ | |||
601 602 603 604 605 606 607 608 609 610 611 612 613 614 | const char *zText = (const char *)sqlite3_value_text(apVal[i]); if( zText ){ int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); if( rc!=SQLITE_OK ){ return rc; } } } return SQLITE_OK; } /* ** This function is called by the xUpdate() method for an INSERT operation. ** The apVal parameter is passed a copy of the apVal argument passed by | > | 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 | const char *zText = (const char *)sqlite3_value_text(apVal[i]); if( zText ){ int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); if( rc!=SQLITE_OK ){ return rc; } } aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } return SQLITE_OK; } /* ** This function is called by the xUpdate() method for an INSERT operation. ** The apVal parameter is passed a copy of the apVal argument passed by |
︙ | ︙ | |||
698 699 700 701 702 703 704 | } /* ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ | | | 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 | } /* ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ u32 *aSz /* Sizes of deleted document written here */ ){ int rc; sqlite3_stmt *pSelect; |
︙ | ︙ | |||
720 721 722 723 724 725 726 727 728 729 730 731 732 733 | const char *zText = (const char *)sqlite3_column_text(pSelect, i); rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); if( rc!=SQLITE_OK ){ sqlite3_reset(pSelect); *pRC = rc; return; } } } rc = sqlite3_reset(pSelect); }else{ sqlite3_reset(pSelect); } *pRC = rc; | > | 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 | const char *zText = (const char *)sqlite3_column_text(pSelect, i); rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); if( rc!=SQLITE_OK ){ sqlite3_reset(pSelect); *pRC = rc; return; } aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } } rc = sqlite3_reset(pSelect); }else{ sqlite3_reset(pSelect); } *pRC = rc; |
︙ | ︙ | |||
781 782 783 784 785 786 787 788 789 790 791 792 793 | }else{ *piIdx = iNext; } } return rc; } /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > | > > | > | > > > > > < < < < | | | > > < > > > > > > > > > > > | 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 | }else{ *piIdx = iNext; } } return rc; } /* ** The %_segments table is declared as follows: ** ** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) ** ** This function reads data from a single row of the %_segments table. The ** specific row is identified by the iBlockid parameter. If paBlob is not ** NULL, then a buffer is allocated using sqlite3_malloc() and populated ** with the contents of the blob stored in the "block" column of the ** identified table row is. Whether or not paBlob is NULL, *pnBlob is set ** to the size of the blob in bytes before returning. ** ** If an error occurs, or the table does not contain the specified row, ** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If ** paBlob is non-NULL, then it is the responsibility of the caller to ** eventually free the returned buffer. ** ** This function may leave an open sqlite3_blob* handle in the ** Fts3Table.pSegments variable. This handle is reused by subsequent calls ** to this function. The handle may be closed by calling the ** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy ** performance improvement, but the blob handle should always be closed ** before control is returned to the user (to prevent a lock being held ** on the database file for longer than necessary). Thus, any virtual table ** method (xFilter etc.) that may directly or indirectly call this function ** must call sqlite3Fts3SegmentsClose() before returning. */ int sqlite3Fts3ReadBlock( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ char **paBlob, /* OUT: Blob data in malloc'd buffer */ int *pnBlob /* OUT: Size of blob data */ ){ int rc; /* Return code */ /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ assert( pnBlob); if( p->pSegments ){ rc = sqlite3_blob_reopen(p->pSegments, iBlockid); }else{ if( 0==p->zSegmentsTbl ){ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; } rc = sqlite3_blob_open( p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments ); } if( rc==SQLITE_OK ){ int nByte = sqlite3_blob_bytes(p->pSegments); if( paBlob ){ char *aByte = sqlite3_malloc(nByte); if( !aByte ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); if( rc!=SQLITE_OK ){ sqlite3_free(aByte); aByte = 0; } } *paBlob = aByte; } *pnBlob = nByte; } return rc; } /* ** Close the blob handle at p->pSegments, if it is open. See comments above ** the sqlite3Fts3ReadBlock() function for details. */ void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ if( !pReader->aDoclist ){ pNext = pReader->aNode; }else{ pNext = &pReader->aDoclist[pReader->nDoclist]; } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); if( pElem==0 ){ pReader->aNode = 0; }else{ PendingList *pList = (PendingList *)fts3HashData(pElem); pReader->zTerm = (char *)fts3HashKey(pElem); pReader->nTerm = fts3HashKeysize(pElem); pReader->nNode = pReader->nDoclist = pList->nData + 1; pReader->aNode = pReader->aDoclist = pList->aData; pReader->ppNextElem++; assert( pReader->aNode ); } return SQLITE_OK; } if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); } pReader->aNode = 0; /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf ** blocks have already been traversed. */ assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ return SQLITE_OK; } rc = sqlite3Fts3ReadBlock( p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode ); if( rc!=SQLITE_OK ) return rc; pNext = pReader->aNode; } pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix+nSuffix>pReader->nTermAlloc ){ int nNew = (nPrefix+nSuffix)*2; char *zNew = sqlite3_realloc(pReader->zTerm, nNew); if( !zNew ){ return SQLITE_NOMEM; } pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); pReader->aDoclist = pNext; pReader->pOffsetList = 0; /* Check that the doclist does not appear to extend past the end of the ** b-tree node. And that the final byte of the doclist is either an 0x00 ** or 0x01. If either of these statements is untrue, then the data structure ** is corrupt. */ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] || (pReader->aDoclist[pReader->nDoclist-1]&0xFE)!=0 ){ return SQLITE_CORRUPT; } return SQLITE_OK; } /* ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ |
︙ | ︙ | |||
909 910 911 912 913 914 915 916 917 918 919 920 921 | pReader->pOffsetList = 0; }else{ sqlite3_int64 iDelta; pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); pReader->iDocid += iDelta; } } /* ** Free all allocations associated with the iterator passed as the ** second argument. */ void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | < < < < < < < | | < > > > | < < < < < < < < < < < < < < | < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < | < | 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 | pReader->pOffsetList = 0; }else{ sqlite3_int64 iDelta; pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); pReader->iDocid += iDelta; } } /* ** This function is called to estimate the amount of data that will be ** loaded from the disk If SegReaderIterate() is called on this seg-reader, ** in units of average document size. ** ** This can be used as follows: If the caller has a small doclist that ** contains references to N documents, and is considering merging it with ** a large doclist (size X "average documents"), it may opt not to load ** the large doclist if X>N. */ int sqlite3Fts3SegReaderCost( Fts3Cursor *pCsr, /* FTS3 cursor handle */ Fts3SegReader *pReader, /* Segment-reader handle */ int *pnCost /* IN/OUT: Number of bytes read */ ){ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; int rc = SQLITE_OK; /* Return code */ int nCost = 0; /* Cost in bytes to return */ int pgsz = p->nPgsz; /* Database page size */ /* If this seg-reader is reading the pending-terms table, or if all data ** for the segment is stored on the root page of the b-tree, then the cost ** is zero. In this case all required data is already in main memory. */ if( p->bHasDocsize && !fts3SegReaderIsPending(pReader) && !fts3SegReaderIsRootOnly(pReader) ){ int nBlob = 0; sqlite3_int64 iBlock; if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost ** of each doclist, has not yet been determined. Read the required ** data from the %_stat table to calculate it. ** ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 ** varints, where nCol is the number of columns in the FTS3 table. ** The first varint is the number of documents currently stored in ** the table. The following nCol varints contain the total amount of ** data stored in all rows of each column of the table, from left ** to right. */ sqlite3_stmt *pStmt; rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); if( rc ) return rc; if( sqlite3_step(pStmt)==SQLITE_ROW ){ sqlite3_int64 nDoc = 0; sqlite3_int64 nByte = 0; const char *a = sqlite3_column_blob(pStmt, 0); if( a ){ const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; a += sqlite3Fts3GetVarint(a, &nDoc); while( a<pEnd ){ a += sqlite3Fts3GetVarint(a, &nByte); } } pCsr->nRowAvg = (((nByte / nDoc) + pgsz - 1) / pgsz); } rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc; } /* Assume that a blob flows over onto overflow pages if it is larger ** than (pgsz-35) bytes in size (the file-format documentation ** confirms this). */ for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); if( rc!=SQLITE_OK ) break; if( (nBlob+35)>pgsz ){ int nOvfl = (nBlob + 34)/pgsz; nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); } } } *pnCost += nCost; return rc; } /* ** Free all allocations associated with the iterator passed as the ** second argument. */ void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){ if( pReader && !fts3SegReaderIsPending(pReader) ){ sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); } } sqlite3_free(pReader); } /* ** Allocate a new SegReader object. */ int sqlite3Fts3SegReaderNew( Fts3Table *p, /* Virtual table handle */ int iAge, /* Segment "age". */ sqlite3_int64 iStartLeaf, /* First leaf to traverse */ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ sqlite3_int64 iEndBlock, /* Final block of segment */ const char *zRoot, /* Buffer containing root node */ int nRoot, /* Size of buffer containing root node */ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ ){ int rc = SQLITE_OK; /* Return code */ Fts3SegReader *pReader; /* Newly allocated SegReader object */ int nExtra = 0; /* Bytes to allocate segment root node */ assert( iStartLeaf<=iEndLeaf ); if( iStartLeaf==0 ){ nExtra = nRoot; } pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } memset(pReader, 0, sizeof(Fts3SegReader)); pReader->iIdx = iAge; pReader->iStartBlock = iStartLeaf; pReader->iLeafEndBlock = iEndLeaf; pReader->iEndBlock = iEndBlock; if( nExtra ){ /* The entire segment is stored in the root node. */ pReader->aNode = (char *)&pReader[1]; pReader->nNode = nRoot; memcpy(pReader->aNode, zRoot, nRoot); }else{ pReader->iCurrentBlock = iStartLeaf-1; } rc = fts3SegReaderNext(p, pReader); if( rc==SQLITE_OK ){ *ppReader = pReader; }else{ sqlite3Fts3SegReaderFree(p, pReader); } return rc; } |
︙ | ︙ | |||
1109 1110 1111 1112 1113 1114 1115 | if( !pReader ){ rc = SQLITE_NOMEM; }else{ memset(pReader, 0, nByte); pReader->iIdx = 0x7FFFFFFF; pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); | | | 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 | if( !pReader ){ rc = SQLITE_NOMEM; }else{ memset(pReader, 0, nByte); pReader->iIdx = 0x7FFFFFFF; pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); fts3SegReaderNext(p, pReader); } } if( isPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; |
︙ | ︙ | |||
1351 1352 1353 1354 1355 1356 1357 | } /* ** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger ** (according to memcmp) than the previous term. */ static int fts3NodeAddTerm( | | | 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 | } /* ** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger ** (according to memcmp) than the previous term. */ static int fts3NodeAddTerm( Fts3Table *p, /* Virtual table handle */ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ int isCopyTerm, /* True if zTerm/nTerm is transient */ const char *zTerm, /* Pointer to buffer containing term */ int nTerm /* Size of term in bytes */ ){ SegmentNode *pTree = *ppTree; int rc; |
︙ | ︙ | |||
1987 1988 1989 1990 1991 1992 1993 | */ if( pFilter->zTerm ){ int nTerm = pFilter->nTerm; const char *zTerm = pFilter->zTerm; for(i=0; i<nSegment; i++){ Fts3SegReader *pSeg = apSegment[i]; while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){ | | | 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 | */ if( pFilter->zTerm ){ int nTerm = pFilter->nTerm; const char *zTerm = pFilter->zTerm; for(i=0; i<nSegment; i++){ Fts3SegReader *pSeg = apSegment[i]; while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){ rc = fts3SegReaderNext(p, pSeg); if( rc!=SQLITE_OK ) goto finished; } } } fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp); while( apSegment[0]->aNode ){ int nTerm = apSegment[0]->nTerm; |
︙ | ︙ | |||
2098 2099 2100 2101 2102 2103 2104 | ** term (if such a term exists in the index) has already been made. */ if( pFilter->zTerm && !isPrefix ){ goto finished; } for(i=0; i<nMerge; i++){ | | | 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 | ** term (if such a term exists in the index) has already been made. */ if( pFilter->zTerm && !isPrefix ){ goto finished; } for(i=0; i<nMerge; i++){ rc = fts3SegReaderNext(p, apSegment[i]); if( rc!=SQLITE_OK ) goto finished; } fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp); } finished: sqlite3_free(aBuffer); |
︙ | ︙ | |||
2124 2125 2126 2127 2128 2129 2130 | ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ int i; /* Iterator variable */ int rc; /* Return code */ int iIdx; /* Index of new segment */ | | | 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 | ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ int i; /* Iterator variable */ int rc; /* Return code */ int iIdx; /* Index of new segment */ int iNewLevel = 0; /* Level to create new segment at */ sqlite3_stmt *pStmt = 0; SegmentWriter *pWriter = 0; int nSegment = 0; /* Number of segments being merged */ Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ Fts3SegReader *pPending = 0; /* Iterator for pending-terms */ Fts3SegFilter filter; /* Segment term filter condition */ |
︙ | ︙ | |||
2413 2414 2415 2416 2417 2418 2419 | sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); } /* | | > > | > > > > | > > > > | | | | | > > | | | | | | | 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 | sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); } /* ** Record 0 of the %_stat table contains a blob consisting of N varints, ** where N is the number of user defined columns in the fts3 table plus ** two. If nCol is the number of user defined columns, then values of the ** varints are set as follows: ** ** Varint 0: Total number of rows in the table. ** ** Varint 1..nCol: For each column, the total number of tokens stored in ** the column for all rows of the table. ** ** Varint 1+nCol: The total size, in bytes, of all text values in all ** columns of all rows of the table. ** */ static void fts3UpdateDocTotals( int *pRC, /* The result code */ Fts3Table *p, /* Table being updated */ u32 *aSzIns, /* Size increases */ u32 *aSzDel, /* Size decreases */ int nChng /* Change in the number of documents */ ){ char *pBlob; /* Storage for BLOB written into %_stat */ int nBlob; /* Size of BLOB written into %_stat */ u32 *a; /* Array of integers that becomes the BLOB */ sqlite3_stmt *pStmt; /* Statement for reading and writing */ int i; /* Loop counter */ int rc; /* Result code from subfunctions */ const int nStat = p->nColumn+2; if( *pRC ) return; a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; } pBlob = (char*)&a[nStat]; rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } if( sqlite3_step(pStmt)==SQLITE_ROW ){ fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), sqlite3_column_bytes(pStmt, 0)); }else{ memset(a, 0, sizeof(u32)*(nStat) ); } sqlite3_reset(pStmt); if( nChng<0 && a[0]<(u32)(-nChng) ){ a[0] = 0; }else{ a[0] += nChng; } for(i=0; i<p->nColumn+1; i++){ u32 x = a[i+1]; if( x+aSzIns[i] < aSzDel[i] ){ x = 0; }else{ x = x + aSzIns[i] - aSzDel[i]; } a[i+1] = x; } fts3EncodeIntArray(nStat, a, pBlob, &nBlob); rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); |
︙ | ︙ | |||
2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 | }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ p->nMaxPendingData = atoi(&zVal[11]); rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; } return rc; } /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. */ int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ int nArg, /* Size of argument array */ sqlite3_value **apVal, /* Array of arguments */ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ u32 *aSzIns; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ /* Allocate space to hold the change in document sizes */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 | }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ p->nMaxPendingData = atoi(&zVal[11]); rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; } sqlite3Fts3SegmentsClose(p); return rc; } /* ** Return the deferred doclist associated with deferred token pDeferred. ** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already ** been called to allocate and populate the doclist. */ char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ if( pDeferred->pList ){ *pnByte = pDeferred->pList->nData; return pDeferred->pList->aData; } *pnByte = 0; return 0; } /* ** Helper fucntion for FreeDeferredDoclists(). This function removes all ** references to deferred doclists from within the tree of Fts3Expr ** structures headed by */ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ if( pExpr ){ fts3DeferredDoclistClear(pExpr->pLeft); fts3DeferredDoclistClear(pExpr->pRight); if( pExpr->isLoaded ){ sqlite3_free(pExpr->aDoclist); pExpr->isLoaded = 0; pExpr->aDoclist = 0; pExpr->nDoclist = 0; pExpr->pCurrent = 0; pExpr->iCurrent = 0; } } } /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. */ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ sqlite3_free(pDef->pList); pDef->pList = 0; } if( pCsr->pDeferred ){ fts3DeferredDoclistClear(pCsr->pExpr); } } /* ** Free all entries in the pCsr->pDeffered list. Entries are added to ** this list using sqlite3Fts3DeferToken(). */ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; sqlite3_free(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; } /* ** Generate deferred-doclists for all tokens in the pCsr->pDeferred list ** based on the row that pCsr currently points to. ** ** A deferred-doclist is like any other doclist with position information ** included, except that it only contains entries for a single row of the ** table, not for all rows. */ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return code */ if( pCsr->pDeferred ){ int i; /* Used to iterate through table columns */ sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; sqlite3_tokenizer *pT = p->pTokenizer; sqlite3_tokenizer_module const *pModule = pT->pModule; assert( pCsr->isRequireSeek==0 ); iDocid = sqlite3_column_int64(pCsr->pStmt, 0); for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); sqlite3_tokenizer_cursor *pTC = 0; rc = pModule->xOpen(pT, zText, -1, &pTC); while( rc==SQLITE_OK ){ char const *zToken; /* Buffer containing token */ int nToken; /* Number of bytes in token */ int iDum1, iDum2; /* Dummy variables */ int iPos; /* Position of token in zText */ pTC->pTokenizer = pT; rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ Fts3PhraseToken *pPT = pDef->pToken; if( (pDef->iCol>=p->nColumn || pDef->iCol==i) && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken)) && (0==memcmp(zToken, pPT->z, pPT->n)) ){ fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); } } } if( pTC ) pModule->xClose(pTC); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ if( pDef->pList ){ rc = fts3PendingListAppendVarint(&pDef->pList, 0); } } } return rc; } /* ** Add an entry for token pToken to the pCsr->pDeferred list. */ int sqlite3Fts3DeferToken( Fts3Cursor *pCsr, /* Fts3 table cursor */ Fts3PhraseToken *pToken, /* Token to defer */ int iCol /* Column that token must appear in (or -1) */ ){ Fts3DeferredToken *pDeferred; pDeferred = sqlite3_malloc(sizeof(*pDeferred)); if( !pDeferred ){ return SQLITE_NOMEM; } memset(pDeferred, 0, sizeof(*pDeferred)); pDeferred->pToken = pToken; pDeferred->pNext = pCsr->pDeferred; pDeferred->iCol = iCol; pCsr->pDeferred = pDeferred; assert( pToken->pDeferred==0 ); pToken->pDeferred = pDeferred; return SQLITE_OK; } /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. */ int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ int nArg, /* Size of argument array */ sqlite3_value **apVal, /* Array of arguments */ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ u32 *aSzIns; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ assert( p->pSegments==0 ); /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); if( aSzIns==0 ) return SQLITE_NOMEM; aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ int isEmpty = 0; rc = fts3IsEmpty(p, apVal, &isEmpty); if( rc==SQLITE_OK ){ if( isEmpty ){ /* Deleting this row means the whole table is empty. In this case ** delete the contents of all three tables and throw away any ** data in the pendingTerms hash table. */ |
︙ | ︙ | |||
2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 | } if( p->bHasDocsize ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } sqlite3_free(aSzIns); return rc; } /* ** Flush any data in the pending-terms hash table to disk. If successful, ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. | > | 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 | } if( p->bHasDocsize ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; } /* ** Flush any data in the pending-terms hash table to disk. If successful, ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. |
︙ | ︙ | |||
2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 | sqlite3Fts3PendingTermsClear(p); } }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } return rc; } #endif | > | 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 | sqlite3Fts3PendingTermsClear(p); } }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } sqlite3Fts3SegmentsClose(p); return rc; } #endif |
Added ext/fts3/fts3speed.tcl.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | #-------------------------------------------------------------------------- # This script contains several sub-programs used to test FTS3/FTS4 # performance. It does not run the queries directly, but generates SQL # scripts that can be run using the shell tool. # # The following cases are tested: # # 1. Inserting documents into an FTS3 table. # 2. Optimizing an FTS3 table (i.e. "INSERT INTO t1 VALUES('optimize')"). # 3. Deleting documents from an FTS3 table. # 4. Querying FTS3 tables. # # Number of tokens in vocabulary. And number of tokens in each document. # set VOCAB_SIZE 2000 set DOC_SIZE 100 set NUM_INSERTS 100000 set NUM_SELECTS 1000 # Force everything in this script to be deterministic. # expr {srand(0)} proc usage {} { puts stderr "Usage: $::argv0 <rows> <selects>" exit -1 } proc sql {sql} { puts $::fd $sql } # Return a list of $nWord randomly generated tokens each between 2 and 10 # characters in length. # proc build_vocab {nWord} { set ret [list] set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z] for {set i 0} {$i<$nWord} {incr i} { set len [expr {int((rand()*9.0)+2)}] set term "" for {set j 0} {$j<$len} {incr j} { append term [lindex $chars [expr {int(rand()*[llength $chars])}]] } lappend ret $term } set ret } proc select_term {} { set n [llength $::vocab] set t [expr int(rand()*$n*3)] if {$t>=2*$n} { set t [expr {($t-2*$n)/100}] } if {$t>=$n} { set t [expr {($t-$n)/10}] } lindex $::vocab $t } proc select_doc {nTerm} { set ret [list] for {set i 0} {$i<$nTerm} {incr i} { lappend ret [select_term] } set ret } proc test_1 {nInsert} { sql "PRAGMA synchronous = OFF;" sql "DROP TABLE IF EXISTS t1;" sql "CREATE VIRTUAL TABLE t1 USING fts4;" for {set i 0} {$i < $nInsert} {incr i} { set doc [select_doc $::DOC_SIZE] sql "INSERT INTO t1 VALUES('$doc');" } } proc test_2 {} { sql "INSERT INTO t1(t1) VALUES('optimize');" } proc test_3 {nSelect} { for {set i 0} {$i < $nSelect} {incr i} { sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term]';" } } proc test_4 {nSelect} { for {set i 0} {$i < $nSelect} {incr i} { sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term] [select_term]';" } } if {[llength $argv]!=0} usage set ::vocab [build_vocab $::VOCAB_SIZE] set ::fd [open fts3speed_insert.sql w] test_1 $NUM_INSERTS close $::fd set ::fd [open fts3speed_select.sql w] test_3 $NUM_SELECTS close $::fd set ::fd [open fts3speed_select2.sql w] test_4 $NUM_SELECTS close $::fd set ::fd [open fts3speed_optimize.sql w] test_2 close $::fd puts "Success. Created files:" puts " fts3speed_insert.sql" puts " fts3speed_select.sql" puts " fts3speed_select2.sql" puts " fts3speed_optimize.sql" |
Changes to src/btree.c.
︙ | ︙ | |||
8092 8093 8094 8095 8096 8097 8098 | ** (stored in BtCursor.aOverflow[]) is allocated and used by function ** accessPayload() (the worker function for sqlite3BtreeData() and ** sqlite3BtreePutData()). */ void sqlite3BtreeCacheOverflow(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); | < | | 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 | ** (stored in BtCursor.aOverflow[]) is allocated and used by function ** accessPayload() (the worker function for sqlite3BtreeData() and ** sqlite3BtreePutData()). */ void sqlite3BtreeCacheOverflow(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); invalidateOverflowCache(pCur); pCur->isIncrblobHandle = 1; } #endif /* ** Set both the "read version" (single byte at byte offset 18) and ** "write version" (single byte at byte offset 19) fields in the database |
︙ | ︙ |
Changes to src/sqlite.h.in.
︙ | ︙ | |||
4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 | const char *zTable, const char *zColumn, sqlite3_int64 iRow, int flags, sqlite3_blob **ppBlob ); /* ** CAPI3REF: Close A BLOB Handle ** ** ^Closes an open [BLOB handle]. ** ** ^Closing a BLOB shall cause the current transaction to commit ** if there are no other BLOBs, no pending prepared statements, and the | > > > > > > > > > > > > > > > > > > > > > > > | 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 | const char *zTable, const char *zColumn, sqlite3_int64 iRow, int flags, sqlite3_blob **ppBlob ); /* ** CAPI3REF: Move a BLOB Handle to a New Row ** ** This function is used to move an existing blob handle so that it points ** to a different row of the same database table. The new row is identified ** by the rowid value passed as the second argument. Only the row can be ** changed, the database, table and column on which the blob handle is open ** remain the same. Moving an existing blob handle to a new row can be ** faster than closing the existing handle and opening a new one. ** ** The new row must meet the same criteria as for [sqlite3_blob_open()] - ** it must exist and there must be either a blob or text value stored in ** the nominated column. If the new row is not present in the table, or if ** it does not contain a blob or text value, or if another error occurs, an ** SQLite error code is returned and the blob handle is considered aborted. ** All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or ** [sqlite3_blob_reopen()] on an aborted blob handle immediately return ** SQLITE_ABORT. ** ** This function sets the database handle error code and message. */ SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle ** ** ^Closes an open [BLOB handle]. ** ** ^Closing a BLOB shall cause the current transaction to commit ** if there are no other BLOBs, no pending prepared statements, and the |
︙ | ︙ |
Changes to src/test1.c.
︙ | ︙ | |||
1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 | Tcl_SetObjResult(interp, pRet); return TCL_OK; } #endif #ifndef SQLITE_OMIT_INCRBLOB /* ** sqlite3_blob_read CHANNEL OFFSET N ** ** This command is used to test the sqlite3_blob_read() in ways that ** the Tcl channel interface does not. The first argument should ** be the name of a valid channel created by the [incrblob] method | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 | Tcl_SetObjResult(interp, pRet); return TCL_OK; } #endif #ifndef SQLITE_OMIT_INCRBLOB static int blobHandleFromObj( Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3_blob **ppBlob ){ char *z; int n; z = Tcl_GetStringFromObj(pObj, &n); if( n==0 ){ *ppBlob = 0; }else{ int notUsed; Tcl_Channel channel; ClientData instanceData; channel = Tcl_GetChannel(interp, z, ¬Used); if( !channel ) return TCL_ERROR; Tcl_Flush(channel); Tcl_Seek(channel, 0, SEEK_SET); instanceData = Tcl_GetChannelInstanceData(channel); *ppBlob = *((sqlite3_blob **)instanceData); } return TCL_OK; } /* ** sqlite3_blob_bytes CHANNEL */ static int test_blob_bytes( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3_blob *pBlob; int nByte; if( objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); return TCL_ERROR; } if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; nByte = sqlite3_blob_bytes(pBlob); Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); return TCL_OK; } /* ** sqlite3_blob_close CHANNEL */ static int test_blob_close( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3_blob *pBlob; int nByte; if( objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); return TCL_ERROR; } if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; sqlite3_blob_close(pBlob); return TCL_OK; } /* ** sqlite3_blob_read CHANNEL OFFSET N ** ** This command is used to test the sqlite3_blob_read() in ways that ** the Tcl channel interface does not. The first argument should ** be the name of a valid channel created by the [incrblob] method |
︙ | ︙ | |||
1607 1608 1609 1610 1611 1612 1613 | */ static int test_blob_read( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ | < < < < | | < < < < | 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 | */ static int test_blob_read( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3_blob *pBlob; int nByte; int iOffset; unsigned char *zBuf; int rc; if( objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); return TCL_ERROR; } if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) ){ return TCL_ERROR; } zBuf = (unsigned char *)Tcl_Alloc(nByte); rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); if( rc==SQLITE_OK ){ Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); }else{ Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); } |
︙ | ︙ | |||
1665 1666 1667 1668 1669 1670 1671 | */ static int test_blob_write( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ | < < < | | < < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 | */ static int test_blob_write( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3_blob *pBlob; int iOffset; int rc; unsigned char *zBuf; int nBuf; if( objc!=4 && objc!=5 ){ Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET DATA ?NDATA?"); return TCL_ERROR; } if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ return TCL_ERROR; } zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ return TCL_ERROR; } rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); if( rc!=SQLITE_OK ){ Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); } static int test_blob_reopen( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ Tcl_WideInt iRowid; sqlite3_blob *pBlob; int rc; unsigned char *zBuf; int nBuf; if( objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL ROWID"); return TCL_ERROR; } if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; if( Tcl_GetWideIntFromObj(interp, objv[2], &iRowid) ) return TCL_ERROR; rc = sqlite3_blob_reopen(pBlob, iRowid); if( rc!=SQLITE_OK ){ Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); } #endif /* ** Usage: sqlite3_create_collation_v2 DB-HANDLE NAME CMP-PROC DEL-PROC ** ** This Tcl proc is used for testing the experimental ** sqlite3_create_collation_v2() interface. |
︙ | ︙ | |||
5322 5323 5324 5325 5326 5327 5328 | { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0}, #endif { "sqlite3_libversion_number", test_libversion_number, 0 }, #ifdef SQLITE_ENABLE_COLUMN_METADATA { "sqlite3_table_column_metadata", test_table_column_metadata, 0 }, #endif #ifndef SQLITE_OMIT_INCRBLOB | | | > > > | 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 | { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0}, #endif { "sqlite3_libversion_number", test_libversion_number, 0 }, #ifdef SQLITE_ENABLE_COLUMN_METADATA { "sqlite3_table_column_metadata", test_table_column_metadata, 0 }, #endif #ifndef SQLITE_OMIT_INCRBLOB { "sqlite3_blob_read", test_blob_read, 0 }, { "sqlite3_blob_write", test_blob_write, 0 }, { "sqlite3_blob_reopen", test_blob_reopen, 0 }, { "sqlite3_blob_bytes", test_blob_bytes, 0 }, { "sqlite3_blob_close", test_blob_close, 0 }, #endif { "pcache_stats", test_pcache_stats, 0 }, #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY { "sqlite3_unlock_notify", test_unlock_notify, 0 }, #endif { "sqlite3_wal_checkpoint", test_wal_checkpoint, 0 }, { "test_sqlite3_log", test_sqlite3_log, 0 }, |
︙ | ︙ |
Changes to src/vdbeblob.c.
︙ | ︙ | |||
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | ** Valid sqlite3_blob* handles point to Incrblob structures. */ typedef struct Incrblob Incrblob; struct Incrblob { int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ }; /* ** Open a blob handle. */ int sqlite3_blob_open( sqlite3* db, /* The database connection */ const char *zDb, /* The attached database containing the blob */ const char *zTable, /* The table containing the blob */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | ** Valid sqlite3_blob* handles point to Incrblob structures. */ typedef struct Incrblob Incrblob; struct Incrblob { int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ int iCol; /* Table column this handle is open on */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ }; /* ** This function is used by both blob_open() and blob_reopen(). It seeks ** the b-tree cursor associated with blob handle p to point to row iRow. ** If successful, SQLITE_OK is returned and subsequent calls to ** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. ** ** If an error occurs, or if the specified row does not exist or does not ** contain a value of type TEXT or BLOB in the column nominated when the ** blob handle was opened, then an error code is returned and *pzErr may ** be set to point to a buffer containing an error message. It is the ** responsibility of the caller to free the error message buffer using ** sqlite3DbFree(). ** ** If an error does occur, then the b-tree cursor is closed. All subsequent ** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will ** immediately return SQLITE_ABORT. */ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ int rc; /* Error code */ char *zErr = 0; /* Error message */ Vdbe *v = (Vdbe *)p->pStmt; /* Set the value of the SQL statements only variable to integer iRow. ** This is done directly instead of using sqlite3_bind_int64() to avoid ** triggering asserts related to mutexes. */ assert( v->aVar[0].flags&MEM_Int ); v->aVar[0].u.i = iRow; rc = sqlite3_step(p->pStmt); if( rc==SQLITE_ROW ){ u32 type = v->apCsr[0]->aType[p->iCol]; if( type<12 ){ zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", type==0?"null": type==7?"real": "integer" ); rc = SQLITE_ERROR; sqlite3_finalize(p->pStmt); p->pStmt = 0; }else{ p->iOffset = v->apCsr[0]->aOffset[p->iCol]; p->nByte = sqlite3VdbeSerialTypeLen(type); p->pCsr = v->apCsr[0]->pCursor; sqlite3BtreeEnterCursor(p->pCsr); sqlite3BtreeCacheOverflow(p->pCsr); sqlite3BtreeLeaveCursor(p->pCsr); } } if( rc==SQLITE_ROW ){ rc = SQLITE_OK; }else if( p->pStmt ){ rc = sqlite3_finalize(p->pStmt); p->pStmt = 0; if( rc==SQLITE_OK ){ zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); rc = SQLITE_ERROR; }else{ zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); } } assert( rc!=SQLITE_OK || zErr==0 ); assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); *pzErr = zErr; return rc; } /* ** Open a blob handle. */ int sqlite3_blob_open( sqlite3* db, /* The database connection */ const char *zDb, /* The attached database containing the blob */ const char *zTable, /* The table containing the blob */ |
︙ | ︙ | |||
67 68 69 70 71 72 73 | {OP_TableLock, 0, 0, 0}, /* 2: Acquire a read or write lock */ /* One of the following two instructions is replaced by an OP_Noop. */ {OP_OpenRead, 0, 0, 0}, /* 3: Open cursor 0 for reading */ {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ | | > | | < | > > > > > > < < | | > > | 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | {OP_TableLock, 0, 0, 0}, /* 2: Acquire a read or write lock */ /* One of the following two instructions is replaced by an OP_Noop. */ {OP_OpenRead, 0, 0, 0}, /* 3: Open cursor 0 for reading */ {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ {OP_Column, 0, 0, 1}, /* 7 */ {OP_ResultRow, 1, 0, 0}, /* 8 */ {OP_Goto, 0, 5, 0}, /* 9 */ {OP_Close, 0, 0, 0}, /* 10 */ {OP_Halt, 0, 0, 0}, /* 11 */ }; int rc = SQLITE_OK; char *zErr = 0; Table *pTab; Parse *pParse = 0; Incrblob *pBlob = 0; flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; sqlite3_mutex_enter(db->mutex); pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); if( !pBlob ) goto blob_open_out; pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); if( !pParse ) goto blob_open_out; do { memset(pParse, 0, sizeof(Parse)); pParse->db = db; sqlite3DbFree(db, zErr); zErr = 0; sqlite3BtreeEnterAll(db); pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); if( pTab && IsVirtual(pTab) ){ pTab = 0; sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable); } |
︙ | ︙ | |||
115 116 117 118 119 120 121 | } rc = SQLITE_ERROR; sqlite3BtreeLeaveAll(db); goto blob_open_out; } /* Now search pTab for the exact column. */ | | | 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 | } rc = SQLITE_ERROR; sqlite3BtreeLeaveAll(db); goto blob_open_out; } /* Now search pTab for the exact column. */ for(iCol=0; iCol<pTab->nCol; iCol++) { if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ break; } } if( iCol==pTab->nCol ){ sqlite3DbFree(db, zErr); zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); |
︙ | ︙ | |||
169 170 171 172 173 174 175 | zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); rc = SQLITE_ERROR; sqlite3BtreeLeaveAll(db); goto blob_open_out; } } | | > | > > | | 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 | zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); rc = SQLITE_ERROR; sqlite3BtreeLeaveAll(db); goto blob_open_out; } } pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db); assert( pBlob->pStmt || db->mallocFailed ); if( pBlob->pStmt ){ Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); /* Configure the OP_Transaction */ sqlite3VdbeChangeP1(v, 0, iDb); sqlite3VdbeChangeP2(v, 0, flags); /* Configure the OP_VerifyCookie */ sqlite3VdbeChangeP1(v, 1, iDb); |
︙ | ︙ | |||
216 217 218 219 220 221 222 223 224 225 226 | sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); sqlite3VdbeChangeP2(v, 7, pTab->nCol); if( !db->mallocFailed ){ sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); } } sqlite3BtreeLeaveAll(db); if( db->mallocFailed ){ goto blob_open_out; } | > > > < | < < < < | < < < | < < < < < < < < < < < < < < | < < | < < < < < < < < < < < < < | > | < < < < < < < | 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 | sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); sqlite3VdbeChangeP2(v, 7, pTab->nCol); if( !db->mallocFailed ){ sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); } } pBlob->flags = flags; pBlob->iCol = iCol; pBlob->db = db; sqlite3BtreeLeaveAll(db); if( db->mallocFailed ){ goto blob_open_out; } sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); blob_open_out: if( rc==SQLITE_OK && db->mallocFailed==0 ){ *ppBlob = (sqlite3_blob *)pBlob; }else{ if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); sqlite3StackFree(db, pParse); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; |
︙ | ︙ | |||
327 328 329 330 331 332 333 | sqlite3_mutex_enter(db->mutex); v = (Vdbe*)p->pStmt; if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; sqlite3Error(db, SQLITE_ERROR, 0); | | | 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 | sqlite3_mutex_enter(db->mutex); v = (Vdbe*)p->pStmt; if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; sqlite3Error(db, SQLITE_ERROR, 0); }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. */ rc = SQLITE_ABORT; }else{ /* Call either BtreeData() or BtreePutData(). If SQLITE_ABORT is ** returned, clean-up the statement handle. |
︙ | ︙ | |||
377 378 379 380 381 382 383 384 385 | ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob ** so no mutex is required for access. */ int sqlite3_blob_bytes(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; return p ? p->nByte : 0; } #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 | ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob ** so no mutex is required for access. */ int sqlite3_blob_bytes(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; return p ? p->nByte : 0; } /* ** Move an existing blob handle to point to a different row of the same ** database table. ** ** If an error occurs, or if the specified row does not exist or does not ** contain a blob or text value, then an error code is returned and the ** database handle error code and message set. If this happens, then all ** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) ** immediately return SQLITE_ABORT. */ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ int rc; Incrblob *p = (Incrblob *)pBlob; sqlite3 *db; if( p==0 ) return SQLITE_MISUSE_BKPT; db = p->db; sqlite3_mutex_enter(db->mutex); if( p->pStmt==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. */ rc = SQLITE_ABORT; }else{ char *zErr; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); } rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ |
Changes to test/fts3ah.test.
|
| | | > > > > > | < < | < < < < < < < < | | | | | | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | # 2006 October 31 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The focus # here is testing correct handling of very long terms. # set testdir [file dirname $argv0] source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } # Generate a document of bigterms based on characters from the list # chars. proc bigtermdoc {chars len} { set doc "" foreach char $chars { append doc " " [string repeat $char $len] } return $doc } set len 5000 set doc1 [bigtermdoc {a b c d} $len] set doc2 [bigtermdoc {b d e f} $len] set doc3 [bigtermdoc {a c e} $len] set aterm [string repeat a $len] set bterm [string repeat b $len] set xterm [string repeat x $len] db eval { CREATE VIRTUAL TABLE t1 USING fts3(content); INSERT INTO t1 (rowid, content) VALUES(1, $doc1); INSERT INTO t1 (rowid, content) VALUES(2, $doc2); INSERT INTO t1 (rowid, content) VALUES(3, $doc3); } # No hits at all. Returns empty doclists from termSelect(). do_test fts3ah-1.1 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something'} } {} do_test fts3ah-1.2 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH $aterm} } {1 3} do_test fts3ah-1.3 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH $xterm} } {} do_test fts3ah-1.4 { execsql "SELECT rowid FROM t1 WHERE t1 MATCH '$aterm -$xterm'" } {1 3} do_test fts3ah-1.5 { execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"$aterm $bterm\"'" } {1} finish_test |
Changes to test/fts3ao.test.
︙ | ︙ | |||
18 19 20 21 22 23 24 25 26 27 28 29 30 31 | source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } #--------------------------------------------------------------------- # These tests, fts3ao-1.*, test that ticket #2429 is fixed. # db eval { CREATE VIRTUAL TABLE t1 USING fts3(a, b, c); INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one four two'); | > > | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } set ::testprefix fts3ao #--------------------------------------------------------------------- # These tests, fts3ao-1.*, test that ticket #2429 is fixed. # db eval { CREATE VIRTUAL TABLE t1 USING fts3(a, b, c); INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one four two'); |
︙ | ︙ | |||
194 195 196 197 198 199 200 201 202 203 | INSERT INTO t5 VALUES('Down came a jumbuck to drink at that billabong'); ALTER TABLE t5 RENAME TO t6; INSERT INTO t6 VALUES('Down came the troopers, one, two, three'); ROLLBACK; SELECT * FROM t5; } } {{the quick brown fox} {jumped over the} {lazy dog}} finish_test | > > > > > > > > > > > > > > > | 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 | INSERT INTO t5 VALUES('Down came a jumbuck to drink at that billabong'); ALTER TABLE t5 RENAME TO t6; INSERT INTO t6 VALUES('Down came the troopers, one, two, three'); ROLLBACK; SELECT * FROM t5; } } {{the quick brown fox} {jumped over the} {lazy dog}} # Test that it is possible to rename an FTS4 table. Renaming an FTS4 table # involves renaming the extra %_docsize and %_stat tables. # do_execsql_test 5.1 { CREATE VIRTUAL TABLE t7 USING FTS4; INSERT INTO t7 VALUES('coined by a German clinician'); SELECT count(*) FROM sqlite_master WHERE name LIKE 't7%'; SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%'; } {6 0} do_execsql_test 5.2 { ALTER TABLE t7 RENAME TO t8; SELECT count(*) FROM sqlite_master WHERE name LIKE 't7%'; SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%'; } {0 6} finish_test |
Added test/fts3corrupt.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | # 2010 October 27 # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # Test that the FTS3 extension does not crash when it encounters a # corrupt data structure on disk. # set testdir [file dirname $argv0] source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test ; return } set ::testprefix fts3corrupt do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts3; INSERT INTO t1 VALUES('hello'); } {} do_test fts3corrupt-1.1 { set blob [db one {SELECT root from t1_segdir}] set blob [binary format a7ca* $blob 24 [string range $blob 8 end]] execsql { UPDATE t1_segdir SET root = $blob } } {} do_test fts3corrupt-1.2 { foreach w {a b c d e f g h i j k l m n o} { execsql { INSERT INTO t1 VALUES($w) } } } {} do_catchsql_test 1.3 { INSERT INTO t1 VALUES('world'); } {1 {database disk image is malformed}} finish_test |
Changes to test/fts3cov.test.
1 2 3 4 5 6 7 8 | # 2009 December 03 # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # | | < < > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | # 2009 December 03 # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # The tests in this file are structural coverage tests for FTS3. # set testdir [file dirname $argv0] source $testdir/tester.tcl # If this build does not include FTS3, skip the tests in this file. # ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl source $testdir/malloc_common.tcl set DO_MALLOC_TEST 0 set testprefix fts3cov #-------------------------------------------------------------------------- # When it first needs to read a block from the %_segments table, the FTS3 # module compiles an SQL statement for that purpose. The statement is # stored and reused each subsequent time a block is read. This test case # tests the effects of an OOM error occuring while compiling the statement. # |
︙ | ︙ | |||
78 79 80 81 82 83 84 | INSERT INTO t1 VALUES('What makes her in the wood so late,'); INSERT INTO t1 VALUES('A furlong from the castle gate?'); INSERT INTO t1 VALUES('She had dreams all yesternight'); INSERT INTO t1 VALUES('Of her own betrothed knight;'); INSERT INTO t1 VALUES('And she in the midnight wood will pray'); INSERT INTO t1 VALUES('For the weal of her lover that''s far away.'); COMMIT; | | > | | | | | | | 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | INSERT INTO t1 VALUES('What makes her in the wood so late,'); INSERT INTO t1 VALUES('A furlong from the castle gate?'); INSERT INTO t1 VALUES('She had dreams all yesternight'); INSERT INTO t1 VALUES('Of her own betrothed knight;'); INSERT INTO t1 VALUES('And she in the midnight wood will pray'); INSERT INTO t1 VALUES('For the weal of her lover that''s far away.'); COMMIT; } execsql { INSERT INTO t1(t1) VALUES('optimize'); SELECT substr(hex(root), 1, 2) FROM t1_segdir; } } {03} # Test the "missing entry" case: do_test fts3cov-2.2 { set root [db one {SELECT root FROM t1_segdir}] read_fts3varint [string range $root 1 end] left_child execsql { DELETE FROM t1_segments WHERE blockid = $left_child } } {} do_error_test fts3cov-2.3 { SELECT * FROM t1 WHERE t1 MATCH 'c*' } {SQL logic error or missing database} # Test the "replaced with NULL" case: do_test fts3cov-2.4 { execsql { INSERT INTO t1_segments VALUES($left_child, NULL) } } {} do_error_test fts3cov-2.5 { SELECT * FROM t1 WHERE t1 MATCH 'cloud' } {SQL logic error or missing database} #-------------------------------------------------------------------------- # The following tests are to test the effects of OOM errors while storing # terms in the pending-hash table. Specifically, while creating doclist # blobs to store in the table. More specifically, to test OOM errors while # appending column numbers to doclists. For example, if a doclist consists # of: |
︙ | ︙ | |||
365 366 367 368 369 370 371 372 373 | INSERT INTO t13 VALUES('scalar two functions'); INSERT INTO t13 VALUES('functions scalar two'); } -sqlbody { SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two'; SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two'; SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two'; } finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 | INSERT INTO t13 VALUES('scalar two functions'); INSERT INTO t13 VALUES('functions scalar two'); } -sqlbody { SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two'; SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two'; SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two'; } do_execsql_test 14.0 { CREATE VIRTUAL TABLE t14 USING fts4(a, b); INSERT INTO t14 VALUES('one two three', 'one three four'); INSERT INTO t14 VALUES('a b c', 'd e a'); } do_execsql_test 14.1 { SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"' } {1} do_execsql_test 14.2 { SELECT rowid FROM t14 WHERE t14 MATCH '"one four"' } {} do_execsql_test 14.3 { SELECT rowid FROM t14 WHERE t14 MATCH '"e a"' } {2} do_execsql_test 14.5 { SELECT rowid FROM t14 WHERE t14 MATCH '"e b"' } {} do_catchsql_test 14.6 { SELECT rowid FROM t14 WHERE rowid MATCH 'one' } {1 {unable to use function MATCH in the requested context}} do_catchsql_test 14.7 { SELECT rowid FROM t14 WHERE docid MATCH 'one' } {1 {unable to use function MATCH in the requested context}} do_execsql_test 15.0 { CREATE VIRTUAL TABLE t15 USING fts4(a, b, c); INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3'); INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3'); } do_execsql_test 15.1 { SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"' } {1 2} # Test a corruption case. # do_execsql_test 16.1 { CREATE VIRTUAL TABLE t16 USING fts4; INSERT INTO t16 VALUES('theoretical work to examine the relationship'); INSERT INTO t16 VALUES('solution of our problems on the invisible'); DELETE FROM t16_content WHERE rowid = 2; } do_catchsql_test 16.2 { SELECT * FROM t16 WHERE t16 MATCH 'invisible' } {1 {database disk image is malformed}} # And another corruption test case. # do_execsql_test 17.1 { CREATE VIRTUAL TABLE t17 USING fts4; INSERT INTO t17(content) VALUES('one one one'); UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300' } {} do_catchsql_test 17.2 { SELECT * FROM t17 WHERE t17 MATCH 'one' } {1 {database disk image is malformed}} finish_test |
Added test/fts3defer.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 | # 2010 October 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl ifcapable !fts3 { finish_test return } set sqlite_fts3_enable_parentheses 1 set ::testprefix fts3defer #-------------------------------------------------------------------------- # Test cases fts3defer-1.* are the "warm body" cases. The database contains # one row with 15000 instances of the token "a". This makes the doclist for # "a" so large that FTS3 will avoid loading it in most cases. # # To show this, test cases fts3defer-1.2.* execute a bunch of FTS3 queries # involving token "a". Then, fts3defer-1.3.* replaces the doclist for token # "a" with all zeroes and fts3defer-1.4.* repeats the tests from 1.2. If # the tests still work, we can conclude that the doclist for "a" was not # used. # set aaa [string repeat "a " 15000] do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4; BEGIN; INSERT INTO t1 VALUES('this is a dog'); INSERT INTO t1 VALUES('an instance of a phrase'); INSERT INTO t1 VALUES('an instance of a longer phrase'); INSERT INTO t1 VALUES($aaa); COMMIT; } {} set tests { 1 {SELECT rowid FROM t1 WHERE t1 MATCH '"a dog"'} {1} 2 {SELECT rowid FROM t1 WHERE t1 MATCH '"is a dog"'} {1} 3 {SELECT rowid FROM t1 WHERE t1 MATCH '"a longer phrase"'} {3} 4 {SELECT snippet(t1) FROM t1 WHERE t1 MATCH '"a longer phrase"'} {"an instance of <b>a</b> <b>longer</b> <b>phrase</b>"} 5 {SELECT rowid FROM t1 WHERE t1 MATCH 'a dog'} {1} } do_select_tests 1.2 $tests do_execsql_test 1.3 { SELECT count(*) FROM t1_segments WHERE length(block)>10000; UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } {1} do_select_tests 1.4 $tests # Drop the table. It is corrupt now anyhow, so not useful for subsequent tests. # do_execsql_test 1.5 { DROP TABLE t1 } #-------------------------------------------------------------------------- # These tests - fts3defer-2.* - are more rigorous. They test that for a # variety of queries, FTS3 and FTS4 return the same results. And that # zeroing the very large doclists that FTS4 does not load does not change # the results. # # They use the following pseudo-randomly generated document data. The # tokens "zm" and "jk" are especially common in this dataset. Additionally, # two documents are added to the pseudo-random data before it is loaded # into FTS4 containing 100,000 instances of the "zm" and "jk" tokens. This # makes the doclists for those tokens so large that FTS4 avoids loading them # into memory if possible. # set data [list] lappend data [string repeat "zm " 100000] lappend data [string repeat "jk " 100000] lappend data {*}{ "zm zm agmckuiu uhzq nsab jk rrkx duszemmzl hyq jk" "jk uhzq zm zm rgpzmlnmd zm zk jk jk zm" "duszemmzl zm jk xldlpy zm jk sbptoa xh jk xldlpy" "zm xh zm xqf azavwm jk jk trqd rgpzmlnmd jk" "zm vwq urvysbnykk ubwrfqnbjf zk lsz jk doiwavhwwo jk jk" "jk xduvfhk orpfawpx zkhdvkw jk mjpavjuhw zm jk duszemmzl zm" "jk igju jk jk zm hmjf xh zm gwdfhwurx zk" "vgsld jk jk zm hrlipdm jn zm zsmhnf vgsld duszemmzl" "gtuiexzsu aayxpmve zm zm zm drir scpgna xh azavwm uhzq" "farlehdhq hkfoudzftq igju duszemmzl xnxhf ewle zm hrlipdm urvysbnykk kn" "xnxhf jk jk agmckuiu duszemmzl jk zm zm jk vgsld" "zm zm zm jk jk urvysbnykk ogttbykvt zm zm jk" "iasrqgqv zm azavwm zidhxhbtv jk jk mjpavjuhw zm zm ajmvcydy" "rgpzmlnmd tmt mjpavjuhw xh igju jk azavwm fibokdry vgsld ofm" "zm jk vgsld jk xh jk csjqxhgj drir jk pmrb" "xh jk jk zm rrkx duszemmzl mjpavjuhw xldlpy igju zm" "jk hkfoudzftq zf rrkx wdmy jupk jk zm urvysbnykk npywgdvgz" "zm jk zm zm zhbrzadb uenvbm aayxpmve urvysbnykk duszemmzl jk" "uenvbm jk zm fxw xh bdilwmjw mjpavjuhw uv jk zm" "nk jk bnhc pahlds jk igju dzadnqzprr jk jk jk" "uhzq uv zm duszemmzl tlqix jk jk xh jk zm" "jk zm agmckuiu urvysbnykk jk jk zm zm jk jk" "azavwm mjpavjuhw lsgshn trqd xldlpy ogyavjvv agmckuiu ryvwwhlbc jk jk" "tmt jk zk zm azavwm ofm acpgim bvgimjik iasrqgqv wuvajhwqz" "igju ogyavjvv xrbdak rrkx fibokdry zf ujfhmrllq jk zm hxgwvib" "zm pahlds jk uenvbm aayxpmve iaf hmjf xph vnlyvtkgx zm" "jk xnxhf igju jk xh jk nvfasfh zm js jk" "zm zm rwaj igju xr rrkx xnxhf nvfasfh skxbsqzvmt xatbxeqq" "vgsld zm ujfhmrllq uhzq ogyavjvv nsab azavwm zm vgsld jmfiqhwnjg" "ymjoym duszemmzl urvysbnykk azavwm jk jmfiqhwnjg bu qcdziqomqk vnlyvtkgx" "zm nbilqcnz dzadnqzprr xh bkfgzsxn urvysbnykk xrujfzxqf zm zf agmckuiu" "jk urvysbnykk nvfasfh zf xh zm zm qcdziqomqk qvxtclg wdmy" "fibokdry jk urvysbnykk jk xr osff zm cvnnsl zm vgsld" "jk mjpavjuhw hkfoudzftq jk zm xh xqf urvysbnykk jk iasrqgqv" "jk csjqxhgj duszemmzl iasrqgqv aayxpmve zm brsuoqww jk qpmhtvl wluvgsw" "jk mj azavwm jk zm jn dzadnqzprr zm jk uhzq" "zk xqf jupk fxw nbilqcnz zm jk jcpiwj tznlvbfcv nvfasfh" "jk jcpiwj zm xnxhf zm mjpavjuhw mj drir pa pvjrjlas" "duszemmzl dzadnqzprr jk swc duszemmzl tmt jk jk pahlds jk" "zk zm jk zm zm eczkjblu zm hi pmrb jk" "azavwm zm iz agmckuiu jk sntk jk duszemmzl duszemmzl zm" "jk zm jk eczkjblu urvysbnykk sk gnl jk ttvgf hmjf" "jk bnhc jjrxpjkb mjpavjuhw fibokdry igju jk zm zm xh" "wxe ogttbykvt uhzq xr iaf zf urvysbnykk aayxpmve oacaxgjoo mjpavjuhw" "gazrt jk ephknonq myjp uenvbm wuvajhwqz jk zm xnxhf nvfasfh" "zm aayxpmve csjqxhgj xnxhf xr jk aayxpmve xnxhf zm zm" "sokcyf zm ogyavjvv jk zm fibokdry zm jk igju igju" "vgsld bvgimjik xuprtlyle jk akmikrqyt jk aayxpmve hkfoudzftq ddjj ithtir" "zm uhzq ovkyevlgv zk uenvbm csjqxhgj jk vgsld pgybs jk" "zm agmckuiu zexh fibokdry jk uhzq bu tugflixoex xnxhf sk" "zm zf uenvbm jk azavwm zm zm agmckuiu zm jk" "rrkx jk zf jt zm oacaxgjoo fibokdry wdmy igju csjqxhgj" "hi igju zm jk zidhxhbtv dzadnqzprr jk jk trqd duszemmzl" "zm zm mjpavjuhw xrbdak qrvbjruc jk qzzqdxq guwq cvnnsl zm" "ithtir jk jk qcdziqomqk zm farlehdhq zm zm xrbdak jk" "ixfipk csjqxhgj azavwm sokcyf ttvgf vgsld jk sk xh zk" "nvfasfh azavwm zm zm zm fxw nvfasfh zk gnl trqd" "zm fibokdry csjqxhgj ofm dzadnqzprr jk akmikrqyt orpfawpx duszemmzl vwq" "csjqxhgj jk jk vgsld urvysbnykk jk nxum jk jk nxum" "zm hkfoudzftq jk ryvwwhlbc mjpavjuhw ephknonq jk zm ogyavjvv zm" "lwa hi xnxhf qdyerbws zk njtc jk uhzq zm jk" "trqd zm dzadnqzprr zm urvysbnykk jk lsz jk mjpavjuhw cmnnkna" "duszemmzl zk jk jk fibokdry jseuhjnzo zm aayxpmve zk jk" "fibokdry jk sviq qvxtclg wdmy jk doiwavhwwo zexh jk zm" "jupk zm xh jk mjpavjuhw zm jk nsab npywgdvgz duszemmzl" "zm igju zm zm nvfasfh eh hkfoudzftq fibokdry fxw xkblf" "jk zm jk jk zm xh zk abthnzcv zf csjqxhgj" "zm zm jk nkaotm urvysbnykk sbptoa bq jk ktxdty ubwrfqnbjf" "nvfasfh aayxpmve xdcuz zm tugflixoex jcpiwj zm mjpavjuhw fibokdry doiwavhwwo" "iaf jk mjpavjuhw zm duszemmzl jk jk uhzq pahlds fibokdry" "ddjj zk azavwm jk swc zm gjtexkv jk xh jk" "igju jk csjqxhgj zm jk dzadnqzprr duszemmzl ulvcbv jk jk" "jk fibokdry zm csjqxhgj jn zm zm zm zf uhzq" "duszemmzl jk xkblf zk hrlipdm aayxpmve uenvbm uhzq jk zf" "dzadnqzprr jk zm zdu nvfasfh zm jk urvysbnykk hmjf jk" "jk aayxpmve aserrdxm acpgim fibokdry jk drir wxe brsuoqww rrkx" "uhzq csjqxhgj nvfasfh jk rrkx qbamok trqd uenvbm sntk zm" "ps azavwm zkhdvkw jk zm jk jk zm csjqxhgj xedlrcfo" "jk jk ogyavjvv jk zm farlehdhq duszemmzl jk agitgxamxe jk" "qzzqdxq rwaj jk jk zm xqf jk uenvbm jk zk" "zm hxgwvib akmikrqyt zf agmckuiu uenvbm bq npywgdvgz azavwm jk" "zf jmfiqhwnjg js igju zm aayxpmve zm mbxnljomiv csjqxhgj nvfasfh" "zm jk jk gazrt jk jk lkc jk nvfasfh jk" "xldlpy orpfawpx zkhdvkw jk zm igju zm urvysbnykk dzadnqzprr mbxnljomiv" "urvysbnykk jk zk igju zm uenvbm jk zm ithtir jk" "zm zk zm zf ofm zm xdcuz dzadnqzprr zm vgsld" "sbptoa jk tugflixoex jk zm zm vgsld zm xh zm" "uhzq jk zk evvivo vgsld vniqnuynvf agmckuiu jk zm zm" "zm nvfasfh zm zm zm abthnzcv uenvbm jk zk dzadnqzprr" "zm azavwm igju qzzqdxq jk xnxhf abthnzcv jk nvfasfh zm" "qbamok fxw vgsld igju cmnnkna xnxhf vniqnuynvf zk xh zm" "nvfasfh zk zm mjpavjuhw dzadnqzprr jk jk duszemmzl xldlpy nvfasfh" "xnxhf sviq nsab npywgdvgz osff vgsld farlehdhq fibokdry wjbkhzsa hhac" "zm azavwm scpgna jk jk bq jk duszemmzl fibokdry ovkyevlgv" "csjqxhgj zm jk jk duszemmzl zk xh zm jk zf" "urvysbnykk dzadnqzprr csjqxhgj mjpavjuhw ubwrfqnbjf nkaotm jk jk zm drir" "nvfasfh xh igju zm wluvgsw jk zm srwwnezqk ewle ovnq" "jk nvfasfh eh ktxdty urvysbnykk vgsld zm jk eh uenvbm" "orpfawpx pahlds jk uhzq hi zm zm zf jk dzadnqzprr" "srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki" } proc add_empty_records {n} { execsql BEGIN for {set i 0} {$i < $n} {incr i} { execsql { INSERT INTO t1 VALUES('') } } execsql COMMIT } #set e [list] #foreach d $data {set e [concat $e $d]} #puts [lsort -unique $e] #exit set zero_long_doclists { UPDATE t1_segments SET block=zeroblob(length(block)) WHERE length(block)>10000 } foreach {tn setup} { 1 { set dmt_modes 0 execsql { CREATE VIRTUAL TABLE t1 USING FTS3 } foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } } 2 { set dmt_modes 0 execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } } 3 { set dmt_modes {0 1 2} execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } add_empty_records 1000 execsql $zero_long_doclists } 4 { set dmt_modes 0 execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } add_empty_records 1000 execsql "INSERT INTO t1(t1) VALUES('optimize')" execsql $zero_long_doclists } } { execsql { DROP TABLE IF EXISTS t1 } eval $setup set ::testprefix fts3defer-2.$tn set DO_MALLOC_TEST 0 do_execsql_test 0 { SELECT count(*) FROM t1_segments WHERE length(block)>10000 } {2} do_select_test 1.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk xnxhf' } {13 29 40 47 48 52 63 92} do_select_test 1.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh' } {100} if {$tn==3} breakpoint do_select_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf' } {7 70 98} do_select_test 1.4 { SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl jk' } {3 5 8 10 13 18 20 23 32 37 41 43 55 60 65 67 72 74 76 81 94 96 97} do_select_test 1.5 { SELECT rowid FROM t1 WHERE t1 MATCH 'ubwrfqnbjf jk' } {7 70 98} do_select_test 1.6 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf jk jk jk jk' } {7 70 98} do_select_test 1.7 { SELECT rowid FROM t1 WHERE t1 MATCH 'zm xnxhf' } {12 13 29 30 40 47 48 52 63 92 93} do_select_test 1.8 { SELECT rowid FROM t1 WHERE t1 MATCH 'zm eh' } {68 100} do_select_test 1.9 { SELECT rowid FROM t1 WHERE t1 MATCH 'zm ubwrfqnbjf' } {7 70 98} do_select_test 1.10 { SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' } {10 13 17 31 35 51 58 88 89 90 93 100} do_select_test 1.11 { SELECT rowid FROM t1 WHERE t1 MATCH '( zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR zk OR zkhdvkw OR zm OR zsmhnf ) vgsld' } {10 13 17 31 35 51 58 88 89 90 93 100} do_select_test 2.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"' } {3 24 52 53} do_select_test 2.2 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm zf"' } {33 53 75 88 101} do_select_test 2.3 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm aayxpmve"' } {48 65 84} do_select_test 2.4 { SELECT rowid FROM t1 WHERE t1 MATCH '"aayxpmve zm"' } {11 37 84} do_select_test 2.5 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk azavwm"' } {16 53} do_select_test 2.6 { SELECT rowid FROM t1 WHERE t1 MATCH '"xh jk jk"' } {18} do_select_test 2.7 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld"' } {13 17} do_select_test 2.8 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld lkjlkjlkj"' } {} do_select_test 3.1 { SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm agmckuiu"' } { {zm [zm] [agmckuiu] uhzq nsab jk rrkx duszemmzl hyq jk} {jk [zm] [agmckuiu] urvysbnykk jk jk zm zm jk jk} {[zm] [agmckuiu] zexh fibokdry jk uhzq bu tugflixoex xnxhf sk} {zm zf uenvbm jk azavwm zm [zm] [agmckuiu] zm jk} } do_select_test 3.2 { SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'xnxhf jk' } { {[xnxhf] [jk] [jk] agmckuiu duszemmzl [jk] zm zm [jk] vgsld} {[jk] [xnxhf] igju [jk] xh [jk] nvfasfh zm js [jk]} {[jk] jcpiwj zm [xnxhf] zm mjpavjuhw mj drir pa pvjrjlas} {gazrt [jk] ephknonq myjp uenvbm wuvajhwqz [jk] zm [xnxhf] nvfasfh} {zm aayxpmve csjqxhgj [xnxhf] xr [jk] aayxpmve [xnxhf] zm zm} {zm agmckuiu zexh fibokdry [jk] uhzq bu tugflixoex [xnxhf] sk} {lwa hi [xnxhf] qdyerbws zk njtc [jk] uhzq zm [jk]} {zm azavwm igju qzzqdxq [jk] [xnxhf] abthnzcv [jk] nvfasfh zm} } do_select_test 4.1 { SELECT offsets(t1) FROM t1 WHERE t1 MATCH '"jk uenvbm"' } { {0 0 10 2 0 1 13 6} {0 0 26 2 0 1 29 6} } do_select_test 4.2 { SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'duszemmzl jk fibokdry' } { {0 2 3 8 0 1 36 2 0 0 58 9} {0 0 0 9 0 1 13 2 0 1 16 2 0 2 19 8 0 1 53 2} {0 1 4 2 0 0 20 9 0 1 30 2 0 1 33 2 0 2 48 8} {0 1 17 2 0 1 20 2 0 1 26 2 0 0 29 9 0 2 39 8} } do_select_test 4.3 { SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'vgsld (hrlipdm OR (aapmve NEAR duszemmzl))' } {{0 0 0 5 0 1 15 7 0 0 36 5}} # The following block of tests runs normally with FTS3 or FTS4 without the # long doclists zeroed. And with OOM-injection for FTS4 with long doclists # zeroed. Change this by messing with the [set dmt_modes] commands above. # foreach DO_MALLOC_TEST $dmt_modes { # Phrase search. do_select_test 5.$DO_MALLOC_TEST.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"' } {8 15 36 64 67 72} # Multiple tokens search. do_select_test 5.$DO_MALLOC_TEST.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl zm' } {3 5 8 10 12 13 18 20 23 37 43 55 60 65 67 72 74 81 94 96 97} # snippet() function with phrase. do_select_test 5.$DO_MALLOC_TEST.3 { SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm aayxpmve"' } { {[zm] [aayxpmve] csjqxhgj xnxhf xr jk aayxpmve xnxhf zm zm} {duszemmzl zk jk jk fibokdry jseuhjnzo [zm] [aayxpmve] zk jk} {zf jmfiqhwnjg js igju [zm] [aayxpmve] zm mbxnljomiv csjqxhgj nvfasfh} } # snippet() function with multiple tokens. do_select_test 5.$DO_MALLOC_TEST.4 { SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'zm zhbrzadb' } { {[zm] jk [zm] [zm] [zhbrzadb] uenvbm aayxpmve urvysbnykk duszemmzl jk} } # snippet() function with phrase. do_select_test 5.$DO_MALLOC_TEST.5 { SELECT offsets(t1) FROM t1 WHERE t1 MATCH '"zm aayxpmve"' } { {0 0 0 2 0 1 3 8} {0 0 38 2 0 1 41 8} {0 0 22 2 0 1 25 8} } # snippet() function with multiple tokens. do_select_test 5.$DO_MALLOC_TEST.6 { SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'zm zhbrzadb' } { {0 0 0 2 0 0 6 2 0 0 9 2 0 1 12 8} } set DO_MALLOC_TEST 0 } do_select_test 6.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'vgsld (hrlipdm OR (aayxpmve duszemmzl))' } {10} do_select_test 6.2.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk"' } {8} do_select_test 6.2.2 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"' } {15 26 92 96} do_select_test 6.2.3 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' } {8 15 26 92 96} } set testprefix fts3defer do_execsql_test 3.1 { CREATE VIRTUAL TABLE x1 USING fts4(a, b); INSERT INTO x1 VALUES('a b c', 'd e f'); INSERT INTO x1 SELECT * FROM x1; INSERT INTO x1 SELECT * FROM x1; INSERT INTO x1 SELECT * FROM x1; INSERT INTO x1 SELECT * FROM x1; } do_execsql_test 3.2 " INSERT INTO x1 VALUES( '[string repeat {d } 3000]', '[string repeat {f } 30000]' ); INSERT INTO x1(x1) VALUES('optimize'); " do_execsql_test 3.3 { SELECT count(*) FROM x1 WHERE x1 MATCH '"d e f"' } {16} finish_test |
Added test/fts3defer2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | # 2010 October 23 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl ifcapable !fts3 { finish_test ; return } set testprefix fts3defer2 proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* binary scan $blob $scan($::tcl_platform(byteOrder)) r return $r } db func mit mit #----------------------------------------------------------------------------- # This block of tests - fts3defer2-1.* - test the interaction of deferred NEAR # expressions and the snippet, offsets and matchinfo functions. # do_execsql_test 1.1.1 { CREATE VIRTUAL TABLE t1 USING fts4; } do_execsql_test 1.1.2 "INSERT INTO t1 VALUES('[string repeat {a } 20000]')" do_execsql_test 1.1.3 "INSERT INTO t1 VALUES('[string repeat {z } 20000]')" do_execsql_test 1.1.4 { INSERT INTO t1 VALUES('a b c d e f a x y'); INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(''); INSERT INTO t1(t1) VALUES('optimize'); } do_execsql_test 1.1.4 { SELECT count(*) FROM t1_segments WHERE length(block)>10000; UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } {2} do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} do_execsql_test 1.2.2 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ [list 3 1 1 1 1 1 8 8 1 8 8 8 5001 9] ] do_execsql_test 1.2.3 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)'; } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ [list 3 1 1 1 1 1 8 8 2 8 8 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 } #----------------------------------------------------------------------------- # Test cases fts3defer2-2.* focus specifically on the matchinfo function. # do_execsql_test 2.1.1 "CREATE VIRTUAL TABLE t2 USING fts4" do_execsql_test 2.1.2 "INSERT INTO t2 VALUES('[string repeat {a } 10000]')" do_execsql_test 2.1.3 "INSERT INTO t2 VALUES('b [string repeat {z } 10000]')" do_execsql_test 2.1.4 [string repeat "INSERT INTO t2 VALUES('x');" 50] do_execsql_test 2.1.5 { INSERT INTO t2 VALUES('a b c d e f g'); INSERT INTO t2 VALUES('a b c d e f g'); } foreach {tn sql} { 1 {} 2 { INSERT INTO t2(t2) VALUES('optimize') } 3 { UPDATE t2_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } } { execsql $sql do_execsql_test 2.2.$tn { SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'a b'; } [list \ [list 2 1 1 54 54 1 3 3 54 372 7] \ [list 2 1 1 54 54 1 3 3 54 372 7] \ ] } do_execsql_test 2.3.1 { CREATE VIRTUAL TABLE t3 USING fts4; INSERT INTO t3 VALUES('a b c d e f'); INSERT INTO t3 VALUES('x b c d e f'); INSERT INTO t3 VALUES('d e f a b c'); INSERT INTO t3 VALUES('b c d e f'); INSERT INTO t3 VALUES(''); INSERT INTO t3 VALUES(''); INSERT INTO t3 VALUES(''); INSERT INTO t3 VALUES(''); INSERT INTO t3 VALUES(''); INSERT INTO t3 VALUES(''); } do_execsql_test 2.3.2 " INSERT INTO t3 VALUES('f e d c b [string repeat {a } 10000]') " foreach {tn sql} { 1 {} 2 { INSERT INTO t3(t3) VALUES('optimize') } 3 { UPDATE t3_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } } { execsql $sql do_execsql_test 2.4.$tn { SELECT docid, mit(matchinfo(t3)) FROM t3 WHERE t3 MATCH '"a b c"'; } {1 {1 1 1 4 4 11 912 6} 3 {1 1 1 4 4 11 912 6}} } finish_test |
Added test/fts3fault.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | # 2010 June 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # set testdir [file dirname $argv0] source $testdir/tester.tcl set ::testprefix fts3fault # Test error handling in the sqlite3Fts3Init() function. This is the # function that registers the FTS3 module and various support functions # with SQLite. # do_faultsim_test 1 -body { sqlite3 db test.db expr 0 } -test { catch { db close } } # Test error handling in an "ALTER TABLE ... RENAME TO" statement on an # FTS3 table. Specifically, test renaming the table within a transaction # after it has been written to. # faultsim_delete_and_reopen do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts3; INSERT INTO t1 VALUES('test renaming the table'); INSERT INTO t1 VALUES(' after it has been written'); } do_faultsim_test 2 -prep { sqlite3 db test.db execsql { BEGIN; INSERT INTO t1 VALUES('registers the FTS3 module'); INSERT INTO t1 VALUES('various support functions'); } } -body { execsql { ALTER TABLE t1 RENAME TO t2 } } -test { faultsim_test_result {0 {}} } # Test error handling in the special case where a single prefix query # matches terms that reside on a large range of leaf nodes. # do_test fts3fault-3.0 { sqlite3 db test.db execsql { CREATE VIRTUAL TABLE t3 USING fts4; } execsql { INSERT INTO t3(t3) VALUES('nodesize=50') } execsql { BEGIN } for {set i 0} {$i < 1000} {incr i} { execsql { INSERT INTO t3 VALUES('aaa' || $i) } } execsql { COMMIT } } {} do_faultsim_test 3 -faults oom-transient -prep { sqlite3 db test.db execsql { SELECT * FROM t3 WHERE t3 MATCH 'x' } } -body { execsql { SELECT count(rowid) FROM t3 WHERE t3 MATCH 'aa*' } } -test { faultsim_test_result {0 1000} } do_test fts3fault-4.0 { faultsim_delete_and_reopen execsql { CREATE VIRTUAL TABLE t4 USING fts4; INSERT INTO t4 VALUES('The British Government called on'); INSERT INTO t4 VALUES('as pesetas then became much'); } } {} faultsim_save_and_close do_faultsim_test 4 -prep { faultsim_restore_and_reopen execsql { SELECT content FROM t4 } } -body { execsql { SELECT optimize(t4) FROM t4 LIMIT 1 } } -test { faultsim_test_result {0 {{Index optimized}}} } do_test fts3fault-5.0 { faultsim_delete_and_reopen execsql { CREATE VIRTUAL TABLE t5 USING fts4; INSERT INTO t5 VALUES('The British Government called on'); INSERT INTO t5 VALUES('as pesetas then became much'); } } {} faultsim_save_and_close do_faultsim_test 5 -prep { faultsim_restore_and_reopen execsql { BEGIN; INSERT INTO t5 VALUES('influential in shaping his future outlook'); INSERT INTO t5 VALUES('might be acceptable to the British electorate'); } } -body { execsql { SELECT rowid FROM t5 WHERE t5 MATCH 'british' } } -test { faultsim_test_result {0 {1 4}} } do_test fts3fault-6.0 { faultsim_delete_and_reopen execsql { CREATE VIRTUAL TABLE t6 USING fts4 } } {} faultsim_save_and_close do_faultsim_test 6 -prep { faultsim_restore_and_reopen execsql { SELECT rowid FROM t6 } } -body { execsql { DROP TABLE t6 } } -test { faultsim_test_result {0 {}} } finish_test |
Changes to test/fts3malloc.test.
︙ | ︙ | |||
39 40 41 42 43 44 45 | proc normal_list {l} { set ret [list] foreach elem $l {lappend ret $elem} set ret } | < | 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | proc normal_list {l} { set ret [list] foreach elem $l {lappend ret $elem} set ret } do_write_test fts3_malloc-1.1 sqlite_master { CREATE VIRTUAL TABLE ft1 USING fts3(a, b) } do_write_test fts3_malloc-1.2 sqlite_master { CREATE VIRTUAL TABLE ft2 USING fts3([a], [b]); } |
︙ | ︙ |
Changes to test/fts3query.test.
︙ | ︙ | |||
20 21 22 23 24 25 26 27 28 29 30 31 32 33 | # If this build does not include FTS3, skip the tests in this file. # ifcapable !fts3 { finish_test ; return } source $testdir/malloc_common.tcl source $testdir/fts3_common.tcl set DO_MALLOC_TEST 0 do_test fts3query-1.1 { execsql { CREATE VIRTUAL TABLE t1 USING fts3(x); BEGIN; INSERT INTO t1 VALUES('The source code for SQLite is in the public'); } } {} | > > | 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | # If this build does not include FTS3, skip the tests in this file. # ifcapable !fts3 { finish_test ; return } source $testdir/malloc_common.tcl source $testdir/fts3_common.tcl set DO_MALLOC_TEST 0 set testprefix fts3query do_test fts3query-1.1 { execsql { CREATE VIRTUAL TABLE t1 USING fts3(x); BEGIN; INSERT INTO t1 VALUES('The source code for SQLite is in the public'); } } {} |
︙ | ︙ | |||
125 126 127 128 129 130 131 132 133 134 | do_test fts3query-4.4 { eqp "SELECT t1.number FROM t1, bt WHERE t1.number=bt.rowid ORDER BY t1.date" } {0 0 {TABLE t1 WITH INDEX i1 ORDER BY} 1 1 {TABLE bt USING PRIMARY KEY}} do_test fts3query-4.5 { eqp "SELECT t1.number FROM bt, t1 WHERE t1.number=bt.rowid ORDER BY t1.date" } {0 1 {TABLE t1 WITH INDEX i1 ORDER BY} 1 0 {TABLE bt USING PRIMARY KEY}} finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 | do_test fts3query-4.4 { eqp "SELECT t1.number FROM t1, bt WHERE t1.number=bt.rowid ORDER BY t1.date" } {0 0 {TABLE t1 WITH INDEX i1 ORDER BY} 1 1 {TABLE bt USING PRIMARY KEY}} do_test fts3query-4.5 { eqp "SELECT t1.number FROM bt, t1 WHERE t1.number=bt.rowid ORDER BY t1.date" } {0 1 {TABLE t1 WITH INDEX i1 ORDER BY} 1 0 {TABLE bt USING PRIMARY KEY}} # Test that calling matchinfo() with the wrong number of arguments, or with # an invalid argument returns an error. # do_execsql_test 5.1 { CREATE VIRTUAL TABLE t2 USING FTS4; INSERT INTO t2 VALUES('it was the first time in history'); } do_select_tests 5.2 -errorformat { wrong number of arguments to function %s() } { 1 "SELECT matchinfo() FROM t2 WHERE t2 MATCH 'history'" matchinfo 2 "SELECT matchinfo(t2, t2) FROM t2 WHERE t2 MATCH 'history'" matchinfo 3 "SELECT snippet(t2, 1, 2, 3, 4, 5, 6) FROM t2 WHERE t2 MATCH 'history'" snippet } do_select_tests 5.3 -errorformat { illegal first argument to %s } { 1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo 2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'" offsets 3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'" snippet 4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'" optimize } do_execsql_test 5.4.0 { UPDATE t2_content SET c0content = X'1234' } do_select_tests 5.4 -errorformat { illegal first argument to %s } { 1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo 2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'" offsets 3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'" snippet 4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'" optimize } do_execsql_test 5.5 { DROP TABLE t2 } # Test the snippet() function with 1 to 6 arguments. # do_execsql_test 6.1 { CREATE VIRTUAL TABLE t3 USING FTS4(a, b); INSERT INTO t3 VALUES('no gestures', 'another intriguing discovery by observing the hand gestures (called beats) people make while speaking. Research has shown that such gestures do more than add visual emphasis to our words (many people gesture while they''re on the telephone, for example); it seems they actually help our brains find words'); } do_select_tests 6.2 { 1 "SELECT snippet(t3) FROM t3 WHERE t3 MATCH 'gestures'" {{<b>...</b>hand <b>gestures</b> (called beats) people make while speaking. Research has shown that such <b>gestures</b> do<b>...</b>}} 2 "SELECT snippet(t3, 'XXX') FROM t3 WHERE t3 MATCH 'gestures'" {{<b>...</b>hand XXXgestures</b> (called beats) people make while speaking. Research has shown that such XXXgestures</b> do<b>...</b>}} 3 "SELECT snippet(t3, 'XXX', 'YYY') FROM t3 WHERE t3 MATCH 'gestures'" {{<b>...</b>hand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY do<b>...</b>}} 4 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ') FROM t3 WHERE t3 MATCH 'gestures'" {{ZZZhand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY doZZZ}} 5 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 1) FROM t3 WHERE t3 MATCH 'gestures'" {{ZZZhand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY doZZZ}} 6 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 0) FROM t3 WHERE t3 MATCH 'gestures'" {{no XXXgesturesYYY}} 7 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 1, 5) FROM t3 WHERE t3 MATCH 'gestures'" {{ZZZthe hand XXXgesturesYYY (called beatsZZZ}} } finish_test |
Changes to test/fts3snippet.test.
︙ | ︙ | |||
16 17 18 19 20 21 22 | set testdir [file dirname $argv0] source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl | < | 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | set testdir [file dirname $argv0] source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl set sqlite_fts3_enable_parentheses 1 set DO_MALLOC_TEST 0 # Transform the list $L to its "normal" form. So that it can be compared to # another list with the same set of elements using [string compare]. # |
︙ | ︙ | |||
455 456 457 458 459 460 461 | do_select_test $T.10.5 { SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft; } {0 blob 0 blob 0 blob} do_select_test $T.10.6 { SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r } {0 blob} } | < | 454 455 456 457 458 459 460 461 462 463 | do_select_test $T.10.5 { SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft; } {0 blob 0 blob 0 blob} do_select_test $T.10.6 { SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r } {0 blob} } set sqlite_fts3_enable_parentheses 0 finish_test |
Added test/incrblob3.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 | # 2010 October 20 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # set testdir [file dirname $argv0] source $testdir/tester.tcl sqlite3 db test.db sqlite3_db_config_lookaside db 0 0 0 do_execsql_test incrblob3-1.1 { CREATE TABLE blobs(k INTEGER PRIMARY KEY, v BLOB); INSERT INTO blobs VALUES(1, zeroblob(100)); INSERT INTO blobs VALUES(2, zeroblob(100)); } {} # Test the sqlite3_blob_reopen()/read()/write() functions. # do_test incrblob3-1.2 { set ::blob [db incrblob blobs v 1] puts $::blob "hello world" } {} do_test incrblob3-1.3 { sqlite3_blob_reopen $::blob 2 puts $::blob "world hello" } {} do_test incrblob3-1.4 { sqlite3_blob_reopen $::blob 1 gets $::blob } {hello world} do_test incrblob3-1.5 { sqlite3_blob_reopen $::blob 2 gets $::blob } {world hello} do_test incrblob3-1.6 { close $::blob } {} # Test some error conditions. # # incrblob3-2.1: Attempting to reopen a row that does not exist. # incrblob3-2.2: Attempting to reopen a row that does not contain a blob # or text value. # do_test incrblob3-2.1.1 { set ::blob [db incrblob blobs v 1] list [catch {sqlite3_blob_reopen $::blob 3} msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-2.1.2 { list [sqlite3_errcode db] [sqlite3_errmsg db] } {SQLITE_ERROR {no such rowid: 3}} do_test incrblob3-2.1.3 { list [catch {sqlite3_blob_reopen $::blob 1} msg] $msg } {1 SQLITE_ABORT} do_test incrblob3-2.1.4 { close $::blob } {} do_execsql_test incrblob3-2.2.1 { INSERT INTO blobs VALUES(3, 42); INSERT INTO blobs VALUES(4, 54.4); INSERT INTO blobs VALUES(5, NULL); } foreach {tn rowid type} { 1 3 integer 2 4 real 3 5 null } { do_test incrblob3-2.2.$tn.1 { set ::blob [db incrblob blobs v 1] list [catch {sqlite3_blob_reopen $::blob $rowid} msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-2.2.$tn.2 { list [sqlite3_errcode db] [sqlite3_errmsg db] } "SQLITE_ERROR {cannot open value of type $type}" do_test incrblob3-2.2.$tn.3 { list [catch {sqlite3_blob_reopen $::blob 1} msg] $msg } {1 SQLITE_ABORT} do_test incrblob3-2.2.$tn.4 { list [catch {sqlite3_blob_read $::blob 0 10} msg] $msg } {1 SQLITE_ABORT} do_test incrblob3-2.2.$tn.5 { list [catch {sqlite3_blob_write $::blob 0 "abcd"} msg] $msg } {1 SQLITE_ABORT} do_test incrblob3-2.2.$tn.4 { close $::blob } {} } # Test that passing NULL to sqlite3_blob_XXX() APIs returns SQLITE_MISUSE. # # incrblob3-3.1: sqlite3_blob_reopen() # incrblob3-3.2: sqlite3_blob_read() # incrblob3-3.3: sqlite3_blob_write() # incrblob3-3.4: sqlite3_blob_bytes() # do_test incrblob3-3.1 { list [catch {sqlite3_blob_reopen {} 3} msg] $msg } {1 SQLITE_MISUSE} do_test incrblob3-3.2 { list [catch {sqlite3_blob_read {} 0 10} msg] $msg } {1 SQLITE_MISUSE} do_test incrblob3-3.3 { list [catch {sqlite3_blob_write {} 0 "abcd"} msg] $msg } {1 SQLITE_MISUSE} do_test incrblob3-3.4 { sqlite3_blob_bytes {} } {0} do_test incrblob3-3.5 { sqlite3_blob_close {} } {} # Test out-of-range reading and writing # do_test incrblob3-4.1 { set ::blob [db incrblob blobs v 1] sqlite3_blob_bytes $::blob } {100} do_test incrblob3-4.2 { list [catch { sqlite3_blob_read $::blob -1 10 } msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-4.3 { list [catch { sqlite3_blob_read $::blob 0 -10 } msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-4.4 { list [catch { sqlite3_blob_read $::blob 95 10 } msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-4.5 { list [catch { sqlite3_blob_write $::blob -1 "abcdefghij" 10 } msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-4.6 { list [catch { sqlite3_blob_write $::blob 0 "abcdefghij" -10 } msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-4.7 { list [catch { sqlite3_blob_write $::blob 95 "abcdefghij" } msg] $msg } {1 SQLITE_ERROR} do_test incrblob3-4.8 { close $::blob } {} # Test that modifying the row a blob handle points to aborts the blob. # do_test incrblob3-5.1 { set ::blob [db incrblob blobs v 1] sqlite3_blob_bytes $::blob } {100} do_test incrblob3-5.2 { execsql { UPDATE blobs SET v = '123456789012345678901234567890' WHERE k = 1 } list [catch { sqlite3_blob_read $::blob 0 10 } msg] $msg } {1 SQLITE_ABORT} # Test various errors that can occur in sqlite3_blob_open(): # # 1. Trying to open a virtual table column. # 2. Trying to open a view column. # 3. Trying to open a column that does not exist. # 4. Trying to open a read/write handle on an indexed column. # 5. Trying to open a read/write handle on the child key of an FK constraint. # ifcapable fts3 { do_test incrblob3-6.1 { execsql { CREATE VIRTUAL TABLE ft USING fts3; INSERT INTO ft VALUES('rules to open a column to which'); } list [catch { db incrblob ft content 1 } msg] $msg } {1 {cannot open virtual table: ft}} } ifcapable view { do_test incrblob3-6.2 { execsql { CREATE VIEW v1 AS SELECT * FROM blobs } list [catch { db incrblob v1 content 1 } msg] $msg } {1 {cannot open view: v1}} } do_test incrblob3-6.3 { list [catch { db incrblob blobs content 1 } msg] $msg } {1 {no such column: "content"}} do_test incrblob3-6.4.1 { execsql { CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(b); INSERT INTO t1 VALUES(zeroblob(100), zeroblob(100)); } list [catch { db incrblob t1 b 1 } msg] $msg } {1 {cannot open indexed column for writing}} do_test incrblob3-6.4.2 { set ::blob [db incrblob t1 a 1] close $::blob } {} do_test incrblob3-6.4.3 { set ::blob [db incrblob -readonly t1 b 1] close $::blob } {} do_test incrblob3-6.5.1 { execsql { CREATE TABLE p1(a PRIMARY KEY); CREATE TABLE c1(a, b REFERENCES p1); PRAGMA foreign_keys = 1; INSERT INTO p1 VALUES(zeroblob(100)); INSERT INTO c1 VALUES(zeroblob(100), zeroblob(100)); } list [catch { db incrblob c1 b 1 } msg] $msg } {1 {cannot open foreign key column for writing}} do_test incrblob3-6.5.2 { set ::blob [db incrblob c1 a 1] close $::blob } {} do_test incrblob3-6.5.3 { set ::blob [db incrblob -readonly c1 b 1] close $::blob } {} do_test incrblob3-6.5.4 { execsql { PRAGMA foreign_keys = 0 } set ::blob [db incrblob c1 b 1] close $::blob } {} # Test that sqlite3_blob_open() handles transient and persistent schema # errors correctly. # do_test incrblob3-7.1 { sqlite3 db2 test.db sqlite3_db_config_lookaside db2 0 0 0 execsql { CREATE TABLE t2(x) } db2 set ::blob [db incrblob blobs v 1] close $::blob } {} db2 close testvfs tvfs -default 1 tvfs filter xAccess tvfs script access_method proc access_method {args} { set schemacookie [hexio_get_int [hexio_read test.db 40 4]] incr schemacookie hexio_write test.db 40 [hexio_render_int32 $schemacookie] set dbversion [hexio_get_int [hexio_read test.db 24 4]] incr dbversion hexio_write test.db 24 [hexio_render_int32 $dbversion] return "" } do_test incrblob3-7.2 { sqlite3 db test.db sqlite3_db_config_lookaside db 0 0 0 list [catch {db incrblob blobs v 1} msg] $msg } {1 {database schema has changed}} db close tvfs delete finish_test |
Added test/incrblobfault.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | # 2010 October 26 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix incrblobfault do_execsql_test 1.0 { CREATE TABLE blob(x INTEGER PRIMARY KEY, v BLOB); INSERT INTO blob VALUES(1, 'hello world'); INSERT INTO blob VALUES(2, 'world hello'); INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; INSERT INTO blob SELECT NULL, v FROM blob; } do_faultsim_test 1 -prep { sqlite3 db test.db set ::blob [db incrblob blob v 1] } -body { if {[catch {sqlite3_blob_reopen $::blob 1000}]} { error [sqlite3_errmsg db] } } -test { faultsim_test_result {0 {}} close $::blob } do_faultsim_test 2 -prep { sqlite3 db test.db set ::blob [db incrblob blob v 1] } -body { if {[catch {sqlite3_blob_reopen $::blob -1}]} { error [sqlite3_errmsg db] } } -test { faultsim_test_result {1 {no such rowid: -1}} close $::blob } do_faultsim_test 3 -prep { sqlite3 db test.db } -body { set ::blob [db incrblob blob v 1] gets $::blob } -test { faultsim_test_result {0 {hello world}} catch { close $::blob } } finish_test |
Changes to test/malloc_common.tcl.
︙ | ︙ | |||
109 110 111 112 113 114 115 116 117 118 119 120 121 122 | proc do_faultsim_test {name args} { global FAULTSIM set DEFAULT(-faults) [array names FAULTSIM] set DEFAULT(-prep) "" set DEFAULT(-body) "" set DEFAULT(-test) "" array set O [array get DEFAULT] array set O $args foreach o [array names O] { if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" } } | > > | 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | proc do_faultsim_test {name args} { global FAULTSIM set DEFAULT(-faults) [array names FAULTSIM] set DEFAULT(-prep) "" set DEFAULT(-body) "" set DEFAULT(-test) "" fix_testname name array set O [array get DEFAULT] array set O $args foreach o [array names O] { if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" } } |
︙ | ︙ | |||
522 523 524 525 526 527 528 | # by parameter $result, or (b) TCL throws an "out of memory" error. # # If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement # is executed just once. In this case the test case passes if the results # match the expected results passed via parameter $result. # proc do_select_test {name sql result} { | | > > > > > > | 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 | # by parameter $result, or (b) TCL throws an "out of memory" error. # # If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement # is executed just once. In this case the test case passes if the results # match the expected results passed via parameter $result. # proc do_select_test {name sql result} { uplevel [list doPassiveTest 0 $name $sql [list 0 [list {*}$result]]] } proc do_restart_select_test {name sql result} { uplevel [list doPassiveTest 1 $name $sql [list 0 $result]] } proc do_error_test {name sql error} { uplevel [list doPassiveTest 0 $name $sql [list 1 $error]] } proc doPassiveTest {isRestart name sql catchres} { if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } if {[info exists ::testprefix] && [string is integer [string range $name 0 0]] } { set name $::testprefix.$name } switch $::DO_MALLOC_TEST { 0 { # No malloc failures. do_test $name [list set {} [uplevel [list catchsql $sql]]] $catchres return } 1 { # Simulate transient failures. |
︙ | ︙ |
Changes to test/permutations.test.
︙ | ︙ | |||
155 156 157 158 159 160 161 | All multi-threaded tests. } -files { notify2.test thread001.test thread002.test thread003.test thread004.test thread005.test walthread.test } test_suite "fts3" -prefix "" -description { | | | | > > | 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 | All multi-threaded tests. } -files { notify2.test thread001.test thread002.test thread003.test thread004.test thread005.test walthread.test } test_suite "fts3" -prefix "" -description { All FTS3 tests except fts3rnd.test. } -files { fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3fault.test fts3malloc.test } lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the coverage related test suites: # |
︙ | ︙ |
Changes to test/tester.tcl.
︙ | ︙ | |||
1371 1372 1373 1374 1375 1376 1377 | } # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set # to non-zero, then set the global variable $AUTOVACUUM to 1. set AUTOVACUUM $sqlite_options(default_autovacuum) source $testdir/thread_common.tcl | > | 1371 1372 1373 1374 1375 1376 1377 1378 | } # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set # to non-zero, then set the global variable $AUTOVACUUM to 1. set AUTOVACUUM $sqlite_options(default_autovacuum) source $testdir/thread_common.tcl source $testdir/malloc_common.tcl |