Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Merge experimental fts3/fts4 changes with trunk. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
988164cf485300fb3d189fd1453c23c4 |
| User & Date: | dan 2010-10-27 18:10:00 |
Context
|
2010-10-27
| ||
| 19:08 | Avoid trying to allocate a negative number of bytes of memory in the test wrapper for sqlite3_blob_read(). (check-in: 739b5d9a user: dan tags: trunk) | |
| 18:10 | Merge experimental fts3/fts4 changes with trunk. (check-in: 988164cf user: dan tags: trunk) | |
| 16:52 | Fix a buffer overread in fts3 that can occur if the database is corrupt. (Closed-Leaf check-in: 84194c41 user: dan tags: experimental) | |
| 15:36 | Fix a memory leak in the update_hook method of the TCL interface. (check-in: 1d17e3dc user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
437 437 ** The xDisconnect() virtual table method. 438 438 */ 439 439 static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ 440 440 Fts3Table *p = (Fts3Table *)pVtab; 441 441 int i; 442 442 443 443 assert( p->nPendingData==0 ); 444 + assert( p->pSegments==0 ); 444 445 445 446 /* Free any prepared statements held */ 446 447 for(i=0; i<SizeofArray(p->aStmt); i++){ 447 448 sqlite3_finalize(p->aStmt[i]); 448 449 } 449 - for(i=0; i<p->nLeavesStmt; i++){ 450 - sqlite3_finalize(p->aLeavesStmt[i]); 451 - } 452 - sqlite3_free(p->zSelectLeaves); 453 - sqlite3_free(p->aLeavesStmt); 450 + sqlite3_free(p->zSegmentsTbl); 454 451 455 452 /* Invoke the tokenizer destructor to free the tokenizer. */ 456 453 p->pTokenizer->pModule->xDestroy(p->pTokenizer); 457 454 458 455 sqlite3_free(p); 459 456 return SQLITE_OK; 460 457 } 461 458 462 459 /* 463 460 ** Construct one or more SQL statements from the format string given 464 -** and then evaluate those statements. The success code is writting 461 +** and then evaluate those statements. The success code is written 465 462 ** into *pRc. 466 463 ** 467 464 ** If *pRc is initially non-zero then this routine is a no-op. 468 465 */ 469 466 static void fts3DbExec( 470 467 int *pRc, /* Success code */ 471 468 sqlite3 *db, /* Database in which to run SQL */ ................................................................................ 509 506 } 510 507 511 508 512 509 /* 513 510 ** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table 514 511 ** passed as the first argument. This is done as part of the xConnect() 515 512 ** and xCreate() methods. 516 -*/ 517 -static int fts3DeclareVtab(Fts3Table *p){ 518 - int i; /* Iterator variable */ 519 - int rc; /* Return code */ 520 - char *zSql; /* SQL statement passed to declare_vtab() */ 521 - char *zCols; /* List of user defined columns */ 522 - 523 - /* Create a list of user columns for the virtual table */ 524 - zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); 525 - for(i=1; zCols && i<p->nColumn; i++){ 526 - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); 527 - } 528 - 529 - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ 530 - zSql = sqlite3_mprintf( 531 - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName 532 - ); 533 - 534 - if( !zCols || !zSql ){ 535 - rc = SQLITE_NOMEM; 536 - }else{ 537 - rc = sqlite3_declare_vtab(p->db, zSql); 538 - } 539 - 540 - sqlite3_free(zSql); 541 - sqlite3_free(zCols); 542 - return rc; 513 +** 514 +** If *pRc is non-zero when this function is called, it is a no-op. 515 +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc 516 +** before returning. 517 +*/ 518 +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ 519 + if( *pRc==SQLITE_OK ){ 520 + int i; /* Iterator variable */ 521 + int rc; /* Return code */ 522 + char *zSql; /* SQL statement passed to declare_vtab() */ 523 + char *zCols; /* List of user defined columns */ 524 + 525 + /* Create a list of user columns for the virtual table */ 526 + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); 527 + for(i=1; zCols && i<p->nColumn; i++){ 528 + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); 529 + } 530 + 531 + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ 532 + zSql = sqlite3_mprintf( 533 + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName 534 + ); 535 + if( !zCols || !zSql ){ 536 + rc = SQLITE_NOMEM; 537 + }else{ 538 + rc = sqlite3_declare_vtab(p->db, zSql); 539 + } 540 + 541 + sqlite3_free(zSql); 542 + sqlite3_free(zCols); 543 + *pRc = rc; 544 + } 543 545 } 544 546 545 547 /* 546 548 ** Create the backing store tables (%_content, %_segments and %_segdir) 547 549 ** required by the FTS3 table passed as the only argument. This is done 548 550 ** as part of the vtab xCreate() method. 549 551 ** ................................................................................ 554 556 static int fts3CreateTables(Fts3Table *p){ 555 557 int rc = SQLITE_OK; /* Return code */ 556 558 int i; /* Iterator variable */ 557 559 char *zContentCols; /* Columns of %_content table */ 558 560 sqlite3 *db = p->db; /* The database connection */ 559 561 560 562 /* Create a list of user columns for the content table */ 561 - if( p->bHasContent ){ 562 - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); 563 - for(i=0; zContentCols && i<p->nColumn; i++){ 564 - char *z = p->azColumn[i]; 565 - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); 566 - } 567 - if( zContentCols==0 ) rc = SQLITE_NOMEM; 563 + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); 564 + for(i=0; zContentCols && i<p->nColumn; i++){ 565 + char *z = p->azColumn[i]; 566 + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); 567 + } 568 + if( zContentCols==0 ) rc = SQLITE_NOMEM; 568 569 569 - /* Create the content table */ 570 - fts3DbExec(&rc, db, 571 - "CREATE TABLE %Q.'%q_content'(%s)", 572 - p->zDb, p->zName, zContentCols 573 - ); 574 - sqlite3_free(zContentCols); 575 - } 570 + /* Create the content table */ 571 + fts3DbExec(&rc, db, 572 + "CREATE TABLE %Q.'%q_content'(%s)", 573 + p->zDb, p->zName, zContentCols 574 + ); 575 + sqlite3_free(zContentCols); 576 576 /* Create other tables */ 577 577 fts3DbExec(&rc, db, 578 578 "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", 579 579 p->zDb, p->zName 580 580 ); 581 581 fts3DbExec(&rc, db, 582 582 "CREATE TABLE %Q.'%q_segdir'(" ................................................................................ 634 634 zDb, zName, zSuffix 635 635 ); 636 636 rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0); 637 637 sqlite3_free(zSql); 638 638 *pResult = (u8)(res & 0xff); 639 639 if( rc!=SQLITE_ABORT ) *pRc = rc; 640 640 } 641 + 642 +/* 643 +** Store the current database page-size in bytes in p->nPgsz. 644 +** 645 +** If *pRc is non-zero when this function is called, it is a no-op. 646 +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc 647 +** before returning. 648 +*/ 649 +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ 650 + if( *pRc==SQLITE_OK ){ 651 + int rc; /* Return code */ 652 + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ 653 + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ 654 + 655 + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); 656 + if( !zSql ){ 657 + rc = SQLITE_NOMEM; 658 + }else{ 659 + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); 660 + if( rc==SQLITE_OK ){ 661 + sqlite3_step(pStmt); 662 + p->nPgsz = sqlite3_column_int(pStmt, 0); 663 + rc = sqlite3_finalize(pStmt); 664 + } 665 + } 666 + assert( p->nPgsz>0 || rc!=SQLITE_OK ); 667 + sqlite3_free(zSql); 668 + *pRc = rc; 669 + } 670 +} 641 671 642 672 /* 643 673 ** This function is the implementation of both the xConnect and xCreate 644 674 ** methods of the FTS3 virtual table. 645 675 ** 646 676 ** The argv[] array contains the following: 647 677 ** ................................................................................ 759 789 p->bHasDocsize = argv[0][3]=='4'; 760 790 rc = fts3CreateTables(p); 761 791 }else{ 762 792 rc = SQLITE_OK; 763 793 fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent); 764 794 fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize); 765 795 } 766 - if( rc!=SQLITE_OK ) goto fts3_init_out; 767 796 768 - rc = fts3DeclareVtab(p); 769 - if( rc!=SQLITE_OK ) goto fts3_init_out; 797 + /* Figure out the page-size for the database. This is required in order to 798 + ** estimate the cost of loading large doclists from the database (see 799 + ** function sqlite3Fts3SegReaderCost() for details). 800 + */ 801 + fts3DatabasePageSize(&rc, p); 770 802 771 - *ppVTab = &p->base; 803 + /* Declare the table schema to SQLite. */ 804 + fts3DeclareVtab(&rc, p); 772 805 773 806 fts3_init_out: 774 807 assert( p || (pTokenizer && rc!=SQLITE_OK) ); 775 808 if( rc!=SQLITE_OK ){ 776 809 if( p ){ 777 810 fts3DisconnectMethod((sqlite3_vtab *)p); 778 811 }else{ 779 812 pTokenizer->pModule->xDestroy(pTokenizer); 780 813 } 814 + }else{ 815 + *ppVTab = &p->base; 781 816 } 782 817 return rc; 783 818 } 784 819 785 820 /* 786 821 ** The xConnect() and xCreate() methods for the virtual table. All the 787 822 ** work is done in function fts3InitVtab(). ................................................................................ 885 920 return SQLITE_OK; 886 921 } 887 922 888 923 /* 889 924 ** Close the cursor. For additional information see the documentation 890 925 ** on the xClose method of the virtual table interface. 891 926 */ 892 -static int fulltextClose(sqlite3_vtab_cursor *pCursor){ 927 +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ 893 928 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; 929 + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); 894 930 sqlite3_finalize(pCsr->pStmt); 895 931 sqlite3Fts3ExprFree(pCsr->pExpr); 932 + sqlite3Fts3FreeDeferredTokens(pCsr); 896 933 sqlite3_free(pCsr->aDoclist); 897 934 sqlite3_free(pCsr->aMatchinfo); 898 935 sqlite3_free(pCsr); 899 936 return SQLITE_OK; 900 937 } 901 938 902 939 /* ................................................................................ 927 964 } 928 965 }else{ 929 966 return SQLITE_OK; 930 967 } 931 968 } 932 969 933 970 /* 934 -** Advance the cursor to the next row in the %_content table that 935 -** matches the search criteria. For a MATCH search, this will be 936 -** the next row that matches. For a full-table scan, this will be 937 -** simply the next row in the %_content table. For a docid lookup, 938 -** this routine simply sets the EOF flag. 971 +** This function is used to process a single interior node when searching 972 +** a b-tree for a term or term prefix. The node data is passed to this 973 +** function via the zNode/nNode parameters. The term to search for is 974 +** passed in zTerm/nTerm. 939 975 ** 940 -** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned 941 -** even if we reach end-of-file. The fts3EofMethod() will be called 942 -** subsequently to determine whether or not an EOF was hit. 976 +** If piFirst is not NULL, then this function sets *piFirst to the blockid 977 +** of the child node that heads the sub-tree that may contain the term. 978 +** 979 +** If piLast is not NULL, then *piLast is set to the right-most child node 980 +** that heads a sub-tree that may contain a term for which zTerm/nTerm is 981 +** a prefix. 982 +** 983 +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. 943 984 */ 944 -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ 985 +static int fts3ScanInteriorNode( 986 + Fts3Table *p, /* Virtual table handle */ 987 + const char *zTerm, /* Term to select leaves for */ 988 + int nTerm, /* Size of term zTerm in bytes */ 989 + const char *zNode, /* Buffer containing segment interior node */ 990 + int nNode, /* Size of buffer at zNode */ 991 + sqlite3_int64 *piFirst, /* OUT: Selected child node */ 992 + sqlite3_int64 *piLast /* OUT: Selected child node */ 993 +){ 945 994 int rc = SQLITE_OK; /* Return code */ 946 - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; 947 - 948 - if( pCsr->aDoclist==0 ){ 949 - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ 950 - pCsr->isEof = 1; 951 - rc = sqlite3_reset(pCsr->pStmt); 952 - } 953 - }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ 954 - pCsr->isEof = 1; 955 - }else{ 956 - sqlite3_reset(pCsr->pStmt); 957 - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); 958 - pCsr->isRequireSeek = 1; 959 - pCsr->isMatchinfoNeeded = 1; 960 - } 961 - return rc; 962 -} 963 - 964 - 965 -/* 966 -** The buffer pointed to by argument zNode (size nNode bytes) contains the 967 -** root node of a b-tree segment. The segment is guaranteed to be at least 968 -** one level high (i.e. the root node is not also a leaf). If successful, 969 -** this function locates the leaf node of the segment that may contain the 970 -** term specified by arguments zTerm and nTerm and writes its block number 971 -** to *piLeaf. 972 -** 973 -** It is possible that the returned leaf node does not contain the specified 974 -** term. However, if the segment does contain said term, it is stored on 975 -** the identified leaf node. Because this function only inspects interior 976 -** segment nodes (and never loads leaf nodes into memory), it is not possible 977 -** to be sure. 995 + const char *zCsr = zNode; /* Cursor to iterate through node */ 996 + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ 997 + char *zBuffer = 0; /* Buffer to load terms into */ 998 + int nAlloc = 0; /* Size of allocated buffer */ 999 + int isFirstTerm = 1; /* True when processing first term on page */ 1000 + sqlite3_int64 iChild; /* Block id of child node to descend to */ 1001 + 1002 + /* Skip over the 'height' varint that occurs at the start of every 1003 + ** interior node. Then load the blockid of the left-child of the b-tree 1004 + ** node into variable iChild. */ 1005 + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); 1006 + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); 1007 + 1008 + while( zCsr<zEnd && (piFirst || piLast) ){ 1009 + int cmp; /* memcmp() result */ 1010 + int nSuffix; /* Size of term suffix */ 1011 + int nPrefix = 0; /* Size of term prefix */ 1012 + int nBuffer; /* Total term size */ 1013 + 1014 + /* Load the next term on the node into zBuffer. Use realloc() to expand 1015 + ** the size of zBuffer if required. */ 1016 + if( !isFirstTerm ){ 1017 + zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix); 1018 + } 1019 + isFirstTerm = 0; 1020 + zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix); 1021 + if( nPrefix+nSuffix>nAlloc ){ 1022 + char *zNew; 1023 + nAlloc = (nPrefix+nSuffix) * 2; 1024 + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); 1025 + if( !zNew ){ 1026 + sqlite3_free(zBuffer); 1027 + return SQLITE_NOMEM; 1028 + } 1029 + zBuffer = zNew; 1030 + } 1031 + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); 1032 + nBuffer = nPrefix + nSuffix; 1033 + zCsr += nSuffix; 1034 + 1035 + /* Compare the term we are searching for with the term just loaded from 1036 + ** the interior node. If the specified term is greater than or equal 1037 + ** to the term from the interior node, then all terms on the sub-tree 1038 + ** headed by node iChild are smaller than zTerm. No need to search 1039 + ** iChild. 1040 + ** 1041 + ** If the interior node term is larger than the specified term, then 1042 + ** the tree headed by iChild may contain the specified term. 1043 + */ 1044 + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); 1045 + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ 1046 + *piFirst = iChild; 1047 + piFirst = 0; 1048 + } 1049 + 1050 + if( piLast && cmp<0 ){ 1051 + *piLast = iChild; 1052 + piLast = 0; 1053 + } 1054 + 1055 + iChild++; 1056 + }; 1057 + 1058 + if( piFirst ) *piFirst = iChild; 1059 + if( piLast ) *piLast = iChild; 1060 + 1061 + sqlite3_free(zBuffer); 1062 + return rc; 1063 +} 1064 + 1065 + 1066 +/* 1067 +** The buffer pointed to by argument zNode (size nNode bytes) contains an 1068 +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) 1069 +** contains a term. This function searches the sub-tree headed by the zNode 1070 +** node for the range of leaf nodes that may contain the specified term 1071 +** or terms for which the specified term is a prefix. 1072 +** 1073 +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the 1074 +** left-most leaf node in the tree that may contain the specified term. 1075 +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the 1076 +** right-most leaf node that may contain a term for which the specified 1077 +** term is a prefix. 1078 +** 1079 +** It is possible that the range of returned leaf nodes does not contain 1080 +** the specified term or any terms for which it is a prefix. However, if the 1081 +** segment does contain any such terms, they are stored within the identified 1082 +** range. Because this function only inspects interior segment nodes (and 1083 +** never loads leaf nodes into memory), it is not possible to be sure. 978 1084 ** 979 1085 ** If an error occurs, an error code other than SQLITE_OK is returned. 980 1086 */ 981 1087 static int fts3SelectLeaf( 982 1088 Fts3Table *p, /* Virtual table handle */ 983 1089 const char *zTerm, /* Term to select leaves for */ 984 1090 int nTerm, /* Size of term zTerm in bytes */ 985 1091 const char *zNode, /* Buffer containing segment interior node */ 986 1092 int nNode, /* Size of buffer at zNode */ 987 - sqlite3_int64 *piLeaf /* Selected leaf node */ 988 -){ 989 - int rc = SQLITE_OK; /* Return code */ 990 - const char *zCsr = zNode; /* Cursor to iterate through node */ 991 - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ 992 - char *zBuffer = 0; /* Buffer to load terms into */ 993 - int nAlloc = 0; /* Size of allocated buffer */ 994 - 995 - while( 1 ){ 996 - int isFirstTerm = 1; /* True when processing first term on page */ 997 - int iHeight; /* Height of this node in tree */ 998 - sqlite3_int64 iChild; /* Block id of child node to descend to */ 999 - int nBlock; /* Size of child node in bytes */ 1000 - 1001 - zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); 1002 - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); 1003 - 1004 - while( zCsr<zEnd ){ 1005 - int cmp; /* memcmp() result */ 1006 - int nSuffix; /* Size of term suffix */ 1007 - int nPrefix = 0; /* Size of term prefix */ 1008 - int nBuffer; /* Total term size */ 1009 - 1010 - /* Load the next term on the node into zBuffer */ 1011 - if( !isFirstTerm ){ 1012 - zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix); 1013 - } 1014 - isFirstTerm = 0; 1015 - zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix); 1016 - if( nPrefix+nSuffix>nAlloc ){ 1017 - char *zNew; 1018 - nAlloc = (nPrefix+nSuffix) * 2; 1019 - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); 1020 - if( !zNew ){ 1021 - sqlite3_free(zBuffer); 1022 - return SQLITE_NOMEM; 1023 - } 1024 - zBuffer = zNew; 1025 - } 1026 - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); 1027 - nBuffer = nPrefix + nSuffix; 1028 - zCsr += nSuffix; 1029 - 1030 - /* Compare the term we are searching for with the term just loaded from 1031 - ** the interior node. If the specified term is greater than or equal 1032 - ** to the term from the interior node, then all terms on the sub-tree 1033 - ** headed by node iChild are smaller than zTerm. No need to search 1034 - ** iChild. 1035 - ** 1036 - ** If the interior node term is larger than the specified term, then 1037 - ** the tree headed by iChild may contain the specified term. 1038 - */ 1039 - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); 1040 - if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break; 1041 - iChild++; 1042 - }; 1043 - 1044 - /* If (iHeight==1), the children of this interior node are leaves. The 1045 - ** specified term may be present on leaf node iChild. 1046 - */ 1047 - if( iHeight==1 ){ 1048 - *piLeaf = iChild; 1049 - break; 1050 - } 1051 - 1052 - /* Descend to interior node iChild. */ 1053 - rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock); 1054 - if( rc!=SQLITE_OK ) break; 1055 - zEnd = &zCsr[nBlock]; 1056 - } 1057 - sqlite3_free(zBuffer); 1093 + sqlite3_int64 *piLeaf, /* Selected leaf node */ 1094 + sqlite3_int64 *piLeaf2 /* Selected leaf node */ 1095 +){ 1096 + int rc; /* Return code */ 1097 + int iHeight; /* Height of this node in tree */ 1098 + 1099 + assert( piLeaf || piLeaf2 ); 1100 + 1101 + sqlite3Fts3GetVarint32(zNode, &iHeight); 1102 + rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); 1103 + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); 1104 + 1105 + if( rc==SQLITE_OK && iHeight>1 ){ 1106 + char *zBlob = 0; /* Blob read from %_segments table */ 1107 + int nBlob; /* Size of zBlob in bytes */ 1108 + 1109 + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ 1110 + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); 1111 + if( rc==SQLITE_OK ){ 1112 + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); 1113 + } 1114 + sqlite3_free(zBlob); 1115 + piLeaf = 0; 1116 + zBlob = 0; 1117 + } 1118 + 1119 + if( rc==SQLITE_OK ){ 1120 + rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); 1121 + } 1122 + if( rc==SQLITE_OK ){ 1123 + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); 1124 + } 1125 + sqlite3_free(zBlob); 1126 + } 1127 + 1058 1128 return rc; 1059 1129 } 1060 1130 1061 1131 /* 1062 1132 ** This function is used to create delta-encoded serialized lists of FTS3 1063 1133 ** varints. Each call to this function appends a single varint to a list. 1064 1134 */ ................................................................................ 1282 1352 *pp = p; 1283 1353 *pp1 = p1 + 1; 1284 1354 *pp2 = p2 + 1; 1285 1355 } 1286 1356 1287 1357 /* 1288 1358 ** nToken==1 searches for adjacent positions. 1359 +** 1360 +** This function is used to merge two position lists into one. When it is 1361 +** called, *pp1 and *pp2 must both point to position lists. A position-list is 1362 +** the part of a doclist that follows each document id. For example, if a row 1363 +** contains: 1364 +** 1365 +** 'a b c'|'x y z'|'a b b a' 1366 +** 1367 +** Then the position list for this row for token 'b' would consist of: 1368 +** 1369 +** 0x02 0x01 0x02 0x03 0x03 0x00 1370 +** 1371 +** When this function returns, both *pp1 and *pp2 are left pointing to the 1372 +** byte following the 0x00 terminator of their respective position lists. 1373 +** 1374 +** If isSaveLeft is 0, an entry is added to the output position list for 1375 +** each position in *pp2 for which there exists one or more positions in 1376 +** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. 1377 +** when the *pp1 token appears before the *pp2 token, but not more than nToken 1378 +** slots before it. 1289 1379 */ 1290 1380 static int fts3PoslistPhraseMerge( 1291 - char **pp, /* Output buffer */ 1381 + char **pp, /* IN/OUT: Preallocated output buffer */ 1292 1382 int nToken, /* Maximum difference in token positions */ 1293 1383 int isSaveLeft, /* Save the left position */ 1294 - char **pp1, /* Left input list */ 1295 - char **pp2 /* Right input list */ 1384 + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ 1385 + char **pp1, /* IN/OUT: Left input list */ 1386 + char **pp2 /* IN/OUT: Right input list */ 1296 1387 ){ 1297 1388 char *p = (pp ? *pp : 0); 1298 1389 char *p1 = *pp1; 1299 1390 char *p2 = *pp2; 1300 - 1301 1391 int iCol1 = 0; 1302 1392 int iCol2 = 0; 1393 + 1394 + /* Never set both isSaveLeft and isExact for the same invocation. */ 1395 + assert( isSaveLeft==0 || isExact==0 ); 1396 + 1303 1397 assert( *p1!=0 && *p2!=0 ); 1304 1398 if( *p1==POS_COLUMN ){ 1305 1399 p1++; 1306 1400 p1 += sqlite3Fts3GetVarint32(p1, &iCol1); 1307 1401 } 1308 1402 if( *p2==POS_COLUMN ){ 1309 1403 p2++; ................................................................................ 1324 1418 1325 1419 assert( *p1!=POS_END && *p1!=POS_COLUMN ); 1326 1420 assert( *p2!=POS_END && *p2!=POS_COLUMN ); 1327 1421 fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; 1328 1422 fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; 1329 1423 1330 1424 while( 1 ){ 1331 - if( iPos2>iPos1 && iPos2<=iPos1+nToken ){ 1425 + if( iPos2==iPos1+nToken 1426 + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) 1427 + ){ 1332 1428 sqlite3_int64 iSave; 1333 1429 if( !pp ){ 1334 1430 fts3PoslistCopy(0, &p2); 1335 1431 fts3PoslistCopy(0, &p1); 1336 1432 *pp1 = p1; 1337 1433 *pp2 = p2; 1338 1434 return 1; ................................................................................ 1407 1503 char **pp1, /* IN/OUT: Left input list */ 1408 1504 char **pp2 /* IN/OUT: Right input list */ 1409 1505 ){ 1410 1506 char *p1 = *pp1; 1411 1507 char *p2 = *pp2; 1412 1508 1413 1509 if( !pp ){ 1414 - if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1; 1510 + if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; 1415 1511 *pp1 = p1; 1416 1512 *pp2 = p2; 1417 - return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1); 1513 + return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); 1418 1514 }else{ 1419 1515 char *pTmp1 = aTmp; 1420 1516 char *pTmp2; 1421 1517 char *aTmp2; 1422 1518 int res = 1; 1423 1519 1424 - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2); 1520 + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); 1425 1521 aTmp2 = pTmp2 = pTmp1; 1426 1522 *pp1 = p1; 1427 1523 *pp2 = p2; 1428 - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1); 1524 + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); 1429 1525 if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ 1430 1526 fts3PoslistMerge(pp, &aTmp, &aTmp2); 1431 1527 }else if( pTmp1!=aTmp ){ 1432 1528 fts3PoslistCopy(pp, &aTmp); 1433 1529 }else if( pTmp2!=aTmp2 ){ 1434 1530 fts3PoslistCopy(pp, &aTmp2); 1435 1531 }else{ ................................................................................ 1467 1563 int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ 1468 1564 int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ 1469 1565 char *aBuffer, /* Pre-allocated output buffer */ 1470 1566 int *pnBuffer, /* OUT: Bytes written to aBuffer */ 1471 1567 char *a1, /* Buffer containing first doclist */ 1472 1568 int n1, /* Size of buffer a1 */ 1473 1569 char *a2, /* Buffer containing second doclist */ 1474 - int n2 /* Size of buffer a2 */ 1570 + int n2, /* Size of buffer a2 */ 1571 + int *pnDoc /* OUT: Number of docids in output */ 1475 1572 ){ 1476 1573 sqlite3_int64 i1 = 0; 1477 1574 sqlite3_int64 i2 = 0; 1478 1575 sqlite3_int64 iPrev = 0; 1479 1576 1480 1577 char *p = aBuffer; 1481 1578 char *p1 = a1; 1482 1579 char *p2 = a2; 1483 1580 char *pEnd1 = &a1[n1]; 1484 1581 char *pEnd2 = &a2[n2]; 1582 + int nDoc = 0; 1485 1583 1486 1584 assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR 1487 1585 || mergetype==MERGE_AND || mergetype==MERGE_NOT 1488 1586 || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE 1489 1587 || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR 1490 1588 ); 1491 1589 ................................................................................ 1521 1619 1522 1620 case MERGE_AND: 1523 1621 while( p1 && p2 ){ 1524 1622 if( i1==i2 ){ 1525 1623 fts3PutDeltaVarint(&p, &iPrev, i1); 1526 1624 fts3GetDeltaVarint2(&p1, pEnd1, &i1); 1527 1625 fts3GetDeltaVarint2(&p2, pEnd2, &i2); 1626 + nDoc++; 1528 1627 }else if( i1<i2 ){ 1529 1628 fts3GetDeltaVarint2(&p1, pEnd1, &i1); 1530 1629 }else{ 1531 1630 fts3GetDeltaVarint2(&p2, pEnd2, &i2); 1532 1631 } 1533 1632 } 1534 1633 break; ................................................................................ 1551 1650 case MERGE_PHRASE: { 1552 1651 char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p); 1553 1652 while( p1 && p2 ){ 1554 1653 if( i1==i2 ){ 1555 1654 char *pSave = p; 1556 1655 sqlite3_int64 iPrevSave = iPrev; 1557 1656 fts3PutDeltaVarint(&p, &iPrev, i1); 1558 - if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){ 1657 + if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){ 1559 1658 p = pSave; 1560 1659 iPrev = iPrevSave; 1660 + }else{ 1661 + nDoc++; 1561 1662 } 1562 1663 fts3GetDeltaVarint2(&p1, pEnd1, &i1); 1563 1664 fts3GetDeltaVarint2(&p2, pEnd2, &i2); 1564 1665 }else if( i1<i2 ){ 1565 1666 fts3PoslistCopy(0, &p1); 1566 1667 fts3GetDeltaVarint2(&p1, pEnd1, &i1); 1567 1668 }else{ ................................................................................ 1606 1707 } 1607 1708 } 1608 1709 sqlite3_free(aTmp); 1609 1710 break; 1610 1711 } 1611 1712 } 1612 1713 1714 + if( pnDoc ) *pnDoc = nDoc; 1613 1715 *pnBuffer = (int)(p-aBuffer); 1614 1716 return SQLITE_OK; 1615 1717 } 1616 1718 1617 1719 /* 1618 1720 ** A pointer to an instance of this structure is used as the context 1619 1721 ** argument to sqlite3Fts3SegReaderIterate() ................................................................................ 1644 1746 ** into a single doclist. 1645 1747 */ 1646 1748 for(i=0; i<SizeofArray(pTS->aaOutput); i++){ 1647 1749 if( pTS->aaOutput[i] ){ 1648 1750 if( !aOut ){ 1649 1751 aOut = pTS->aaOutput[i]; 1650 1752 nOut = pTS->anOutput[i]; 1651 - pTS->aaOutput[0] = 0; 1753 + pTS->aaOutput[i] = 0; 1652 1754 }else{ 1653 1755 int nNew = nOut + pTS->anOutput[i]; 1654 1756 char *aNew = sqlite3_malloc(nNew); 1655 1757 if( !aNew ){ 1656 1758 sqlite3_free(aOut); 1657 1759 return SQLITE_NOMEM; 1658 1760 } 1659 1761 fts3DoclistMerge(mergetype, 0, 0, 1660 - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut 1762 + aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 1661 1763 ); 1662 1764 sqlite3_free(pTS->aaOutput[i]); 1663 1765 sqlite3_free(aOut); 1664 1766 pTS->aaOutput[i] = 0; 1665 1767 aOut = aNew; 1666 1768 nOut = nNew; 1667 1769 } ................................................................................ 1724 1826 aNew = sqlite3_malloc(nNew); 1725 1827 if( !aNew ){ 1726 1828 if( aMerge!=aDoclist ){ 1727 1829 sqlite3_free(aMerge); 1728 1830 } 1729 1831 return SQLITE_NOMEM; 1730 1832 } 1731 - fts3DoclistMerge(mergetype, 0, 0, 1732 - aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge 1833 + fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, 1834 + pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 1733 1835 ); 1734 1836 1735 1837 if( iOut>0 ) sqlite3_free(aMerge); 1736 1838 sqlite3_free(pTS->aaOutput[iOut]); 1737 1839 pTS->aaOutput[iOut] = 0; 1738 1840 1739 1841 aMerge = aNew; ................................................................................ 1742 1844 pTS->aaOutput[iOut] = aMerge; 1743 1845 pTS->anOutput[iOut] = nMerge; 1744 1846 } 1745 1847 } 1746 1848 } 1747 1849 return SQLITE_OK; 1748 1850 } 1851 + 1852 +static int fts3DeferredTermSelect( 1853 + Fts3DeferredToken *pToken, /* Phrase token */ 1854 + int isTermPos, /* True to include positions */ 1855 + int *pnOut, /* OUT: Size of list */ 1856 + char **ppOut /* OUT: Body of list */ 1857 +){ 1858 + char *aSource; 1859 + int nSource; 1860 + 1861 + aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); 1862 + if( !aSource ){ 1863 + *pnOut = 0; 1864 + *ppOut = 0; 1865 + }else if( isTermPos ){ 1866 + *ppOut = sqlite3_malloc(nSource); 1867 + if( !*ppOut ) return SQLITE_NOMEM; 1868 + memcpy(*ppOut, aSource, nSource); 1869 + *pnOut = nSource; 1870 + }else{ 1871 + sqlite3_int64 docid; 1872 + *pnOut = sqlite3Fts3GetVarint(aSource, &docid); 1873 + *ppOut = sqlite3_malloc(*pnOut); 1874 + if( !*ppOut ) return SQLITE_NOMEM; 1875 + sqlite3Fts3PutVarint(*ppOut, docid); 1876 + } 1877 + 1878 + return SQLITE_OK; 1879 +} 1880 + 1881 +/* 1882 +** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects. 1883 +** Elements are added to the array using fts3SegReaderArrayAdd(). 1884 +*/ 1885 +struct Fts3SegReaderArray { 1886 + int nSegment; /* Number of valid entries in apSegment[] */ 1887 + int nAlloc; /* Allocated size of apSegment[] */ 1888 + int nCost; /* The cost of executing SegReaderIterate() */ 1889 + Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */ 1890 +}; 1891 + 1892 + 1893 +/* 1894 +** Free an Fts3SegReaderArray object. Also free all seg-readers in the 1895 +** array (using sqlite3Fts3SegReaderFree()). 1896 +*/ 1897 +static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){ 1898 + if( pArray ){ 1899 + int i; 1900 + for(i=0; i<pArray->nSegment; i++){ 1901 + sqlite3Fts3SegReaderFree(0, pArray->apSegment[i]); 1902 + } 1903 + sqlite3_free(pArray); 1904 + } 1905 +} 1906 + 1907 +static int fts3SegReaderArrayAdd( 1908 + Fts3SegReaderArray **ppArray, 1909 + Fts3SegReader *pNew 1910 +){ 1911 + Fts3SegReaderArray *pArray = *ppArray; 1912 + 1913 + if( !pArray || pArray->nAlloc==pArray->nSegment ){ 1914 + int nNew = (pArray ? pArray->nAlloc+16 : 16); 1915 + pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, 1916 + sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*) 1917 + ); 1918 + if( !pArray ){ 1919 + sqlite3Fts3SegReaderFree(0, pNew); 1920 + return SQLITE_NOMEM; 1921 + } 1922 + if( nNew==16 ){ 1923 + pArray->nSegment = 0; 1924 + pArray->nCost = 0; 1925 + } 1926 + pArray->nAlloc = nNew; 1927 + *ppArray = pArray; 1928 + } 1929 + 1930 + pArray->apSegment[pArray->nSegment++] = pNew; 1931 + return SQLITE_OK; 1932 +} 1933 + 1934 +static int fts3TermSegReaderArray( 1935 + Fts3Cursor *pCsr, /* Virtual table cursor handle */ 1936 + const char *zTerm, /* Term to query for */ 1937 + int nTerm, /* Size of zTerm in bytes */ 1938 + int isPrefix, /* True for a prefix search */ 1939 + Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */ 1940 +){ 1941 + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; 1942 + int rc; /* Return code */ 1943 + Fts3SegReaderArray *pArray = 0; /* Array object to build */ 1944 + Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */ 1945 + sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ 1946 + int iAge = 0; /* Used to assign ages to segments */ 1947 + 1948 + /* Allocate a seg-reader to scan the pending terms, if any. */ 1949 + rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader); 1950 + if( rc==SQLITE_OK && pReader ) { 1951 + rc = fts3SegReaderArrayAdd(&pArray, pReader); 1952 + } 1953 + 1954 + /* Loop through the entire %_segdir table. For each segment, create a 1955 + ** Fts3SegReader to iterate through the subset of the segment leaves 1956 + ** that may contain a term that matches zTerm/nTerm. For non-prefix 1957 + ** searches, this is always a single leaf. For prefix searches, this 1958 + ** may be a contiguous block of leaves. 1959 + */ 1960 + if( rc==SQLITE_OK ){ 1961 + rc = sqlite3Fts3AllSegdirs(p, &pStmt); 1962 + } 1963 + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ 1964 + Fts3SegReader *pNew = 0; 1965 + int nRoot = sqlite3_column_bytes(pStmt, 4); 1966 + char const *zRoot = sqlite3_column_blob(pStmt, 4); 1967 + if( sqlite3_column_int64(pStmt, 1)==0 ){ 1968 + /* The entire segment is stored on the root node (which must be a 1969 + ** leaf). Do not bother inspecting any data in this case, just 1970 + ** create a Fts3SegReader to scan the single leaf. 1971 + */ 1972 + rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); 1973 + }else{ 1974 + sqlite3_int64 i1; /* First leaf that may contain zTerm */ 1975 + sqlite3_int64 i2; /* Final leaf that may contain zTerm */ 1976 + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); 1977 + if( isPrefix==0 ) i2 = i1; 1978 + if( rc==SQLITE_OK ){ 1979 + rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); 1980 + } 1981 + } 1982 + assert( (pNew==0)==(rc!=SQLITE_OK) ); 1983 + 1984 + /* If a new Fts3SegReader was allocated, add it to the array. */ 1985 + if( rc==SQLITE_OK ){ 1986 + rc = fts3SegReaderArrayAdd(&pArray, pNew); 1987 + } 1988 + if( rc==SQLITE_OK ){ 1989 + rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost); 1990 + } 1991 + iAge++; 1992 + } 1993 + 1994 + if( rc==SQLITE_DONE ){ 1995 + rc = sqlite3_reset(pStmt); 1996 + }else{ 1997 + sqlite3_reset(pStmt); 1998 + } 1999 + if( rc!=SQLITE_OK ){ 2000 + fts3SegReaderArrayFree(pArray); 2001 + pArray = 0; 2002 + } 2003 + *ppArray = pArray; 2004 + return rc; 2005 +} 1749 2006 1750 2007 /* 1751 2008 ** This function retreives the doclist for the specified term (or term 1752 2009 ** prefix) from the database. 1753 2010 ** 1754 2011 ** The returned doclist may be in one of two formats, depending on the 1755 2012 ** value of parameter isReqPos. If isReqPos is zero, then the doclist is ................................................................................ 1756 2013 ** a sorted list of delta-compressed docids (a bare doclist). If isReqPos 1757 2014 ** is non-zero, then the returned list is in the same format as is stored 1758 2015 ** in the database without the found length specifier at the start of on-disk 1759 2016 ** doclists. 1760 2017 */ 1761 2018 static int fts3TermSelect( 1762 2019 Fts3Table *p, /* Virtual table handle */ 2020 + Fts3PhraseToken *pTok, /* Token to query for */ 1763 2021 int iColumn, /* Column to query (or -ve for all columns) */ 1764 - const char *zTerm, /* Term to query for */ 1765 - int nTerm, /* Size of zTerm in bytes */ 1766 - int isPrefix, /* True for a prefix search */ 1767 2022 int isReqPos, /* True to include position lists in output */ 1768 2023 int *pnOut, /* OUT: Size of buffer at *ppOut */ 1769 2024 char **ppOut /* OUT: Malloced result buffer */ 1770 2025 ){ 1771 - int i; 1772 - TermSelect tsc; 1773 - Fts3SegFilter filter; /* Segment term filter configuration */ 1774 - Fts3SegReader **apSegment; /* Array of segments to read data from */ 1775 - int nSegment = 0; /* Size of apSegment array */ 1776 - int nAlloc = 16; /* Allocated size of segment array */ 1777 2026 int rc; /* Return code */ 1778 - sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ 1779 - int iAge = 0; /* Used to assign ages to segments */ 2027 + Fts3SegReaderArray *pArray; /* Seg-reader array for this term */ 2028 + TermSelect tsc; /* Context object for fts3TermSelectCb() */ 2029 + Fts3SegFilter filter; /* Segment term filter configuration */ 1780 2030 1781 - apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc); 1782 - if( !apSegment ) return SQLITE_NOMEM; 1783 - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]); 1784 - if( rc!=SQLITE_OK ) goto finished; 1785 - if( apSegment[0] ){ 1786 - nSegment = 1; 1787 - } 1788 - 1789 - /* Loop through the entire %_segdir table. For each segment, create a 1790 - ** Fts3SegReader to iterate through the subset of the segment leaves 1791 - ** that may contain a term that matches zTerm/nTerm. For non-prefix 1792 - ** searches, this is always a single leaf. For prefix searches, this 1793 - ** may be a contiguous block of leaves. 1794 - ** 1795 - ** The code in this loop does not actually load any leaves into memory 1796 - ** (unless the root node happens to be a leaf). It simply examines the 1797 - ** b-tree structure to determine which leaves need to be inspected. 1798 - */ 1799 - rc = sqlite3Fts3AllSegdirs(p, &pStmt); 1800 - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ 1801 - Fts3SegReader *pNew = 0; 1802 - int nRoot = sqlite3_column_bytes(pStmt, 4); 1803 - char const *zRoot = sqlite3_column_blob(pStmt, 4); 1804 - if( sqlite3_column_int64(pStmt, 1)==0 ){ 1805 - /* The entire segment is stored on the root node (which must be a 1806 - ** leaf). Do not bother inspecting any data in this case, just 1807 - ** create a Fts3SegReader to scan the single leaf. 1808 - */ 1809 - rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); 1810 - }else{ 1811 - int rc2; /* Return value of sqlite3Fts3ReadBlock() */ 1812 - sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */ 1813 - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1); 1814 - if( rc==SQLITE_OK ){ 1815 - sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2); 1816 - rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); 1817 - } 1818 - 1819 - /* The following call to ReadBlock() serves to reset the SQL statement 1820 - ** used to retrieve blocks of data from the %_segments table. If it is 1821 - ** not reset here, then it may remain classified as an active statement 1822 - ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands 1823 - ** failing. 1824 - */ 1825 - rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0); 1826 - if( rc==SQLITE_OK ){ 1827 - rc = rc2; 1828 - } 1829 - } 1830 - iAge++; 1831 - 1832 - /* If a new Fts3SegReader was allocated, add it to the apSegment array. */ 1833 - assert( pNew!=0 || rc!=SQLITE_OK ); 1834 - if( pNew ){ 1835 - if( nSegment==nAlloc ){ 1836 - Fts3SegReader **pArray; 1837 - nAlloc += 16; 1838 - pArray = (Fts3SegReader **)sqlite3_realloc( 1839 - apSegment, nAlloc*sizeof(Fts3SegReader *) 1840 - ); 1841 - if( !pArray ){ 1842 - sqlite3Fts3SegReaderFree(p, pNew); 1843 - rc = SQLITE_NOMEM; 1844 - goto finished; 1845 - } 1846 - apSegment = pArray; 1847 - } 1848 - apSegment[nSegment++] = pNew; 1849 - } 1850 - } 1851 - if( rc!=SQLITE_DONE ){ 1852 - assert( rc!=SQLITE_OK ); 1853 - goto finished; 1854 - } 1855 - 2031 + pArray = pTok->pArray; 1856 2032 memset(&tsc, 0, sizeof(TermSelect)); 1857 2033 tsc.isReqPos = isReqPos; 1858 2034 1859 2035 filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 1860 - | (isPrefix ? FTS3_SEGMENT_PREFIX : 0) 2036 + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) 1861 2037 | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) 1862 2038 | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); 1863 2039 filter.iCol = iColumn; 1864 - filter.zTerm = zTerm; 1865 - filter.nTerm = nTerm; 2040 + filter.zTerm = pTok->z; 2041 + filter.nTerm = pTok->n; 1866 2042 1867 - rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter, 1868 - fts3TermSelectCb, (void *)&tsc 2043 + rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, 2044 + &filter, fts3TermSelectCb, (void *)&tsc 1869 2045 ); 1870 2046 if( rc==SQLITE_OK ){ 1871 2047 rc = fts3TermSelectMerge(&tsc); 1872 2048 } 1873 2049 1874 2050 if( rc==SQLITE_OK ){ 1875 2051 *ppOut = tsc.aaOutput[0]; 1876 2052 *pnOut = tsc.anOutput[0]; 1877 2053 }else{ 2054 + int i; 1878 2055 for(i=0; i<SizeofArray(tsc.aaOutput); i++){ 1879 2056 sqlite3_free(tsc.aaOutput[i]); 1880 2057 } 1881 2058 } 1882 2059 1883 -finished: 1884 - sqlite3_reset(pStmt); 1885 - for(i=0; i<nSegment; i++){ 1886 - sqlite3Fts3SegReaderFree(p, apSegment[i]); 1887 - } 1888 - sqlite3_free(apSegment); 1889 - return rc; 1890 -} 1891 - 2060 + fts3SegReaderArrayFree(pArray); 2061 + pTok->pArray = 0; 2062 + return rc; 2063 +} 2064 + 2065 +/* 2066 +** This function counts the total number of docids in the doclist stored 2067 +** in buffer aList[], size nList bytes. 2068 +** 2069 +** If the isPoslist argument is true, then it is assumed that the doclist 2070 +** contains a position-list following each docid. Otherwise, it is assumed 2071 +** that the doclist is simply a list of docids stored as delta encoded 2072 +** varints. 2073 +*/ 2074 +static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ 2075 + int nDoc = 0; /* Return value */ 2076 + if( aList ){ 2077 + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ 2078 + char *p = aList; /* Cursor */ 2079 + if( !isPoslist ){ 2080 + /* The number of docids in the list is the same as the number of 2081 + ** varints. In FTS3 a varint consists of a single byte with the 0x80 2082 + ** bit cleared and zero or more bytes with the 0x80 bit set. So to 2083 + ** count the varints in the buffer, just count the number of bytes 2084 + ** with the 0x80 bit clear. */ 2085 + while( p<aEnd ) nDoc += (((*p++)&0x80)==0); 2086 + }else{ 2087 + while( p<aEnd ){ 2088 + nDoc++; 2089 + while( (*p++)&0x80 ); /* Skip docid varint */ 2090 + fts3PoslistCopy(0, &p); /* Skip over position list */ 2091 + } 2092 + } 2093 + } 2094 + 2095 + return nDoc; 2096 +} 2097 + 2098 +/* 2099 +** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. 2100 +*/ 2101 +static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ 2102 + int rc = SQLITE_OK; 2103 + if( pExpr ){ 2104 + rc = fts3DeferExpression(pCsr, pExpr->pLeft); 2105 + if( rc==SQLITE_OK ){ 2106 + rc = fts3DeferExpression(pCsr, pExpr->pRight); 2107 + } 2108 + if( pExpr->eType==FTSQUERY_PHRASE ){ 2109 + int iCol = pExpr->pPhrase->iColumn; 2110 + int i; 2111 + for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){ 2112 + Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; 2113 + if( pToken->pDeferred==0 ){ 2114 + rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); 2115 + } 2116 + } 2117 + } 2118 + } 2119 + return rc; 2120 +} 2121 + 2122 +/* 2123 +** This function removes the position information from a doclist. When 2124 +** called, buffer aList (size *pnList bytes) contains a doclist that includes 2125 +** position information. This function removes the position information so 2126 +** that aList contains only docids, and adjusts *pnList to reflect the new 2127 +** (possibly reduced) size of the doclist. 2128 +*/ 2129 +static void fts3DoclistStripPositions( 2130 + char *aList, /* IN/OUT: Buffer containing doclist */ 2131 + int *pnList /* IN/OUT: Size of doclist in bytes */ 2132 +){ 2133 + if( aList ){ 2134 + char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ 2135 + char *p = aList; /* Input cursor */ 2136 + char *pOut = aList; /* Output cursor */ 2137 + 2138 + while( p<aEnd ){ 2139 + sqlite3_int64 delta; 2140 + p += sqlite3Fts3GetVarint(p, &delta); 2141 + fts3PoslistCopy(0, &p); 2142 + pOut += sqlite3Fts3PutVarint(pOut, delta); 2143 + } 2144 + 2145 + *pnList = (pOut - aList); 2146 + } 2147 +} 1892 2148 1893 2149 /* 1894 2150 ** Return a DocList corresponding to the phrase *pPhrase. 2151 +** 2152 +** If this function returns SQLITE_OK, but *pnOut is set to a negative value, 2153 +** then no tokens in the phrase were looked up in the full-text index. This 2154 +** is only possible when this function is called from within xFilter(). The 2155 +** caller should assume that all documents match the phrase. The actual 2156 +** filtering will take place in xNext(). 1895 2157 */ 1896 2158 static int fts3PhraseSelect( 1897 - Fts3Table *p, /* Virtual table handle */ 2159 + Fts3Cursor *pCsr, /* Virtual table cursor handle */ 1898 2160 Fts3Phrase *pPhrase, /* Phrase to return a doclist for */ 1899 2161 int isReqPos, /* True if output should contain positions */ 1900 2162 char **paOut, /* OUT: Pointer to malloc'd result buffer */ 1901 2163 int *pnOut /* OUT: Size of buffer at *paOut */ 1902 2164 ){ 1903 2165 char *pOut = 0; 1904 2166 int nOut = 0; 1905 2167 int rc = SQLITE_OK; 1906 2168 int ii; 1907 2169 int iCol = pPhrase->iColumn; 1908 2170 int isTermPos = (pPhrase->nToken>1 || isReqPos); 2171 + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; 2172 + int isFirst = 1; 2173 + 2174 + int iPrevTok = 0; 2175 + int nDoc = 0; 2176 + 2177 + /* If this is an xFilter() evaluation, create a segment-reader for each 2178 + ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo 2179 + ** evaluation, only create segment-readers if there are no Fts3DeferredToken 2180 + ** objects attached to the phrase-tokens. 2181 + */ 2182 + for(ii=0; ii<pPhrase->nToken; ii++){ 2183 + Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; 2184 + if( pTok->pArray==0 ){ 2185 + if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) 2186 + || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) 2187 + || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) 2188 + ){ 2189 + rc = fts3TermSegReaderArray( 2190 + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray 2191 + ); 2192 + if( rc!=SQLITE_OK ) return rc; 2193 + } 2194 + } 2195 + } 1909 2196 1910 2197 for(ii=0; ii<pPhrase->nToken; ii++){ 1911 - struct PhraseToken *pTok = &pPhrase->aToken[ii]; 1912 - char *z = pTok->z; /* Next token of the phrase */ 1913 - int n = pTok->n; /* Size of z in bytes */ 1914 - int isPrefix = pTok->isPrefix;/* True if token is a prefix */ 2198 + Fts3PhraseToken *pTok; /* Token to find doclist for */ 2199 + int iTok; /* The token being queried this iteration */ 1915 2200 char *pList; /* Pointer to token doclist */ 1916 2201 int nList; /* Size of buffer at pList */ 1917 2202 1918 - rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList); 2203 + /* Select a token to process. If this is an xFilter() call, then tokens 2204 + ** are processed in order from least to most costly. Otherwise, tokens 2205 + ** are processed in the order in which they occur in the phrase. 2206 + */ 2207 + if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ 2208 + assert( isReqPos ); 2209 + iTok = ii; 2210 + pTok = &pPhrase->aToken[iTok]; 2211 + if( pTok->bFulltext==0 ) continue; 2212 + }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ 2213 + iTok = ii; 2214 + pTok = &pPhrase->aToken[iTok]; 2215 + }else{ 2216 + int nMinCost = 0x7FFFFFFF; 2217 + int jj; 2218 + 2219 + /* Find the remaining token with the lowest cost. */ 2220 + for(jj=0; jj<pPhrase->nToken; jj++){ 2221 + Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray; 2222 + if( pArray && pArray->nCost<nMinCost ){ 2223 + iTok = jj; 2224 + nMinCost = pArray->nCost; 2225 + } 2226 + } 2227 + pTok = &pPhrase->aToken[iTok]; 2228 + 2229 + /* This branch is taken if it is determined that loading the doclist 2230 + ** for the next token would require more IO than loading all documents 2231 + ** currently identified by doclist pOut/nOut. No further doclists will 2232 + ** be loaded from the full-text index for this phrase. 2233 + */ 2234 + if( nMinCost>nDoc && ii>0 ){ 2235 + rc = fts3DeferExpression(pCsr, pCsr->pExpr); 2236 + break; 2237 + } 2238 + } 2239 + 2240 + if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ 2241 + rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); 2242 + }else{ 2243 + assert( pTok->pArray ); 2244 + rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); 2245 + pTok->bFulltext = 1; 2246 + } 2247 + assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 ); 1919 2248 if( rc!=SQLITE_OK ) break; 1920 2249 1921 - if( ii==0 ){ 2250 + if( isFirst ){ 1922 2251 pOut = pList; 1923 2252 nOut = nList; 2253 + if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ 2254 + nDoc = fts3DoclistCountDocids(1, pOut, nOut); 2255 + } 2256 + isFirst = 0; 2257 + iPrevTok = iTok; 1924 2258 }else{ 1925 - /* Merge the new term list and the current output. If this is the 1926 - ** last term in the phrase, and positions are not required in the 1927 - ** output of this function, the positions can be dropped as part 1928 - ** of this merge. Either way, the result of this merge will be 1929 - ** smaller than nList bytes. The code in fts3DoclistMerge() is written 1930 - ** so that it is safe to use pList as the output as well as an input 1931 - ** in this case. 2259 + /* Merge the new term list and the current output. */ 2260 + char *aLeft, *aRight; 2261 + int nLeft, nRight; 2262 + int nDist; 2263 + int mt; 2264 + 2265 + /* If this is the final token of the phrase, and positions were not 2266 + ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. 2267 + ** This drops the position information from the output list. 1932 2268 */ 1933 - int mergetype = MERGE_POS_PHRASE; 1934 - if( ii==pPhrase->nToken-1 && !isReqPos ){ 1935 - mergetype = MERGE_PHRASE; 2269 + mt = MERGE_POS_PHRASE; 2270 + if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; 2271 + 2272 + assert( iPrevTok!=iTok ); 2273 + if( iPrevTok<iTok ){ 2274 + aLeft = pOut; 2275 + nLeft = nOut; 2276 + aRight = pList; 2277 + nRight = nList; 2278 + nDist = iTok-iPrevTok; 2279 + iPrevTok = iTok; 2280 + }else{ 2281 + aRight = pOut; 2282 + nRight = nOut; 2283 + aLeft = pList; 2284 + nLeft = nList; 2285 + nDist = iPrevTok-iTok; 1936 2286 } 1937 - fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList); 1938 - sqlite3_free(pOut); 1939 - pOut = pList; 2287 + pOut = aRight; 2288 + fts3DoclistMerge( 2289 + mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc 2290 + ); 2291 + sqlite3_free(aLeft); 1940 2292 } 1941 2293 assert( nOut==0 || pOut!=0 ); 1942 2294 } 1943 2295 1944 2296 if( rc==SQLITE_OK ){ 2297 + if( ii!=pPhrase->nToken ){ 2298 + assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); 2299 + fts3DoclistStripPositions(pOut, &nOut); 2300 + } 1945 2301 *paOut = pOut; 1946 2302 *pnOut = nOut; 1947 2303 }else{ 1948 2304 sqlite3_free(pOut); 1949 2305 } 1950 2306 return rc; 1951 2307 } 1952 2308 2309 +/* 2310 +** This function merges two doclists according to the requirements of a 2311 +** NEAR operator. 2312 +** 2313 +** Both input doclists must include position information. The output doclist 2314 +** includes position information if the first argument to this function 2315 +** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. 2316 +*/ 1953 2317 static int fts3NearMerge( 1954 2318 int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ 1955 2319 int nNear, /* Parameter to NEAR operator */ 1956 2320 int nTokenLeft, /* Number of tokens in LHS phrase arg */ 1957 2321 char *aLeft, /* Doclist for LHS (incl. positions) */ 1958 2322 int nLeft, /* Size of LHS doclist in bytes */ 1959 2323 int nTokenRight, /* As nTokenLeft */ 1960 2324 char *aRight, /* As aLeft */ 1961 2325 int nRight, /* As nRight */ 1962 2326 char **paOut, /* OUT: Results of merge (malloced) */ 1963 2327 int *pnOut /* OUT: Sized of output buffer */ 1964 2328 ){ 1965 - char *aOut; 1966 - int rc; 2329 + char *aOut; /* Buffer to write output doclist to */ 2330 + int rc; /* Return code */ 1967 2331 1968 2332 assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); 1969 2333 1970 2334 aOut = sqlite3_malloc(nLeft+nRight+1); 1971 2335 if( aOut==0 ){ 1972 2336 rc = SQLITE_NOMEM; 1973 2337 }else{ 1974 2338 rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, 1975 - aOut, pnOut, aLeft, nLeft, aRight, nRight 2339 + aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 1976 2340 ); 1977 2341 if( rc!=SQLITE_OK ){ 1978 2342 sqlite3_free(aOut); 1979 2343 aOut = 0; 1980 2344 } 1981 2345 } 1982 2346 1983 2347 *paOut = aOut; 1984 2348 return rc; 1985 2349 } 1986 2350 2351 +/* 2352 +** This function is used as part of the processing for the snippet() and 2353 +** offsets() functions. 2354 +** 2355 +** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both 2356 +** have their respective doclists (including position information) loaded 2357 +** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from 2358 +** each doclist that are not within nNear tokens of a corresponding entry 2359 +** in the other doclist. 2360 +*/ 1987 2361 int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ 1988 - int rc; 2362 + int rc; /* Return code */ 2363 + 2364 + assert( pLeft->eType==FTSQUERY_PHRASE ); 2365 + assert( pRight->eType==FTSQUERY_PHRASE ); 2366 + assert( pLeft->isLoaded && pRight->isLoaded ); 2367 + 1989 2368 if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ 1990 2369 sqlite3_free(pLeft->aDoclist); 1991 2370 sqlite3_free(pRight->aDoclist); 1992 2371 pRight->aDoclist = 0; 1993 2372 pLeft->aDoclist = 0; 1994 2373 rc = SQLITE_OK; 1995 2374 }else{ 1996 - char *aOut; 1997 - int nOut; 2375 + char *aOut; /* Buffer in which to assemble new doclist */ 2376 + int nOut; /* Size of buffer aOut in bytes */ 1998 2377 1999 2378 rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 2000 2379 pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, 2001 2380 pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, 2002 2381 &aOut, &nOut 2003 2382 ); 2004 2383 if( rc!=SQLITE_OK ) return rc; ................................................................................ 2014 2393 sqlite3_free(pLeft->aDoclist); 2015 2394 pLeft->aDoclist = aOut; 2016 2395 pLeft->nDoclist = nOut; 2017 2396 } 2018 2397 return rc; 2019 2398 } 2020 2399 2021 -/* 2022 -** Evaluate the full-text expression pExpr against fts3 table pTab. Store 2023 -** the resulting doclist in *paOut and *pnOut. This routine mallocs for 2024 -** the space needed to store the output. The caller is responsible for 2400 + 2401 +/* 2402 +** Allocate an Fts3SegReaderArray for each token in the expression pExpr. 2403 +** The allocated objects are stored in the Fts3PhraseToken.pArray member 2404 +** variables of each token structure. 2405 +*/ 2406 +static int fts3ExprAllocateSegReaders( 2407 + Fts3Cursor *pCsr, /* FTS3 table */ 2408 + Fts3Expr *pExpr, /* Expression to create seg-readers for */ 2409 + int *pnExpr /* OUT: Number of AND'd expressions */ 2410 +){ 2411 + int rc = SQLITE_OK; /* Return code */ 2412 + 2413 + assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); 2414 + if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ 2415 + (*pnExpr)++; 2416 + pnExpr = 0; 2417 + } 2418 + 2419 + if( pExpr->eType==FTSQUERY_PHRASE ){ 2420 + Fts3Phrase *pPhrase = pExpr->pPhrase; 2421 + int ii; 2422 + 2423 + for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){ 2424 + Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; 2425 + if( pTok->pArray==0 ){ 2426 + rc = fts3TermSegReaderArray( 2427 + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray 2428 + ); 2429 + } 2430 + } 2431 + }else{ 2432 + rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); 2433 + if( rc==SQLITE_OK ){ 2434 + rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); 2435 + } 2436 + } 2437 + return rc; 2438 +} 2439 + 2440 +/* 2441 +** Free the Fts3SegReaderArray objects associated with each token in the 2442 +** expression pExpr. In other words, this function frees the resources 2443 +** allocated by fts3ExprAllocateSegReaders(). 2444 +*/ 2445 +static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ 2446 + if( pExpr ){ 2447 + Fts3Phrase *pPhrase = pExpr->pPhrase; 2448 + if( pPhrase ){ 2449 + int kk; 2450 + for(kk=0; kk<pPhrase->nToken; kk++){ 2451 + fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray); 2452 + pPhrase->aToken[kk].pArray = 0; 2453 + } 2454 + } 2455 + fts3ExprFreeSegReaders(pExpr->pLeft); 2456 + fts3ExprFreeSegReaders(pExpr->pRight); 2457 + } 2458 +} 2459 + 2460 +/* 2461 +** Return the sum of the costs of all tokens in the expression pExpr. This 2462 +** function must be called after Fts3SegReaderArrays have been allocated 2463 +** for all tokens using fts3ExprAllocateSegReaders(). 2464 +*/ 2465 +int fts3ExprCost(Fts3Expr *pExpr){ 2466 + int nCost; /* Return value */ 2467 + if( pExpr->eType==FTSQUERY_PHRASE ){ 2468 + Fts3Phrase *pPhrase = pExpr->pPhrase; 2469 + int ii; 2470 + nCost = 0; 2471 + for(ii=0; ii<pPhrase->nToken; ii++){ 2472 + nCost += pPhrase->aToken[ii].pArray->nCost; 2473 + } 2474 + }else{ 2475 + nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); 2476 + } 2477 + return nCost; 2478 +} 2479 + 2480 +/* 2481 +** The following is a helper function (and type) for fts3EvalExpr(). It 2482 +** must be called after Fts3SegReaders have been allocated for every token 2483 +** in the expression. See the context it is called from in fts3EvalExpr() 2484 +** for further explanation. 2485 +*/ 2486 +typedef struct ExprAndCost ExprAndCost; 2487 +struct ExprAndCost { 2488 + Fts3Expr *pExpr; 2489 + int nCost; 2490 +}; 2491 +static void fts3ExprAssignCosts( 2492 + Fts3Expr *pExpr, /* Expression to create seg-readers for */ 2493 + ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ 2494 +){ 2495 + if( pExpr->eType==FTSQUERY_AND ){ 2496 + fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); 2497 + fts3ExprAssignCosts(pExpr->pRight, ppExprCost); 2498 + }else{ 2499 + (*ppExprCost)->pExpr = pExpr; 2500 + (*ppExprCost)->nCost = fts3ExprCost(pExpr);; 2501 + (*ppExprCost)++; 2502 + } 2503 +} 2504 + 2505 +/* 2506 +** Evaluate the full-text expression pExpr against FTS3 table pTab. Store 2507 +** the resulting doclist in *paOut and *pnOut. This routine mallocs for 2508 +** the space needed to store the output. The caller is responsible for 2025 2509 ** freeing the space when it has finished. 2510 +** 2511 +** This function is called in two distinct contexts: 2512 +** 2513 +** * From within the virtual table xFilter() method. In this case, the 2514 +** output doclist contains entries for all rows in the table, based on 2515 +** data read from the full-text index. 2516 +** 2517 +** In this case, if the query expression contains one or more tokens that 2518 +** are very common, then the returned doclist may contain a superset of 2519 +** the documents that actually match the expression. 2520 +** 2521 +** * From within the virtual table xNext() method. This call is only made 2522 +** if the call from within xFilter() found that there were very common 2523 +** tokens in the query expression and did return a superset of the 2524 +** matching documents. In this case the returned doclist contains only 2525 +** entries that correspond to the current row of the table. Instead of 2526 +** reading the data for each token from the full-text index, the data is 2527 +** already available in-memory in the Fts3PhraseToken.pDeferred structures. 2528 +** See fts3EvalDeferred() for how it gets there. 2529 +** 2530 +** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is 2531 +** required) Fts3Cursor.doDeferred==1. 2532 +** 2533 +** If the SQLite invokes the snippet(), offsets() or matchinfo() function 2534 +** as part of a SELECT on an FTS3 table, this function is called on each 2535 +** individual phrase expression in the query. If there were very common tokens 2536 +** found in the xFilter() call, then this function is called once for phrase 2537 +** for each row visited, and the returned doclist contains entries for the 2538 +** current row only. Otherwise, if there were no very common tokens, then this 2539 +** function is called once only for each phrase in the query and the returned 2540 +** doclist contains entries for all rows of the table. 2541 +** 2542 +** Fts3Cursor.doDeferred==1 when this function is called on phrases as a 2543 +** result of a snippet(), offsets() or matchinfo() invocation. 2026 2544 */ 2027 -static int evalFts3Expr( 2028 - Fts3Table *p, /* Virtual table handle */ 2545 +static int fts3EvalExpr( 2546 + Fts3Cursor *p, /* Virtual table cursor handle */ 2029 2547 Fts3Expr *pExpr, /* Parsed fts3 expression */ 2030 2548 char **paOut, /* OUT: Pointer to malloc'd result buffer */ 2031 2549 int *pnOut, /* OUT: Size of buffer at *paOut */ 2032 2550 int isReqPos /* Require positions in output buffer */ 2033 2551 ){ 2034 2552 int rc = SQLITE_OK; /* Return code */ 2035 2553 2036 2554 /* Zero the output parameters. */ 2037 2555 *paOut = 0; 2038 2556 *pnOut = 0; 2039 2557 2040 2558 if( pExpr ){ 2041 - assert( pExpr->eType==FTSQUERY_PHRASE 2042 - || pExpr->eType==FTSQUERY_NEAR 2043 - || isReqPos==0 2559 + assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR 2560 + || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT 2561 + || pExpr->eType==FTSQUERY_PHRASE 2044 2562 ); 2563 + assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); 2564 + 2045 2565 if( pExpr->eType==FTSQUERY_PHRASE ){ 2046 - rc = fts3PhraseSelect(p, pExpr->pPhrase, 2566 + rc = fts3PhraseSelect(p, pExpr->pPhrase, 2047 2567 isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), 2048 2568 paOut, pnOut 2049 2569 ); 2570 + fts3ExprFreeSegReaders(pExpr); 2571 + }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ 2572 + ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ 2573 + int nExpr = 0; /* Size of aExpr[] */ 2574 + char *aRet = 0; /* Doclist to return to caller */ 2575 + int nRet = 0; /* Length of aRet[] in bytes */ 2576 + int nDoc = 0x7FFFFFFF; 2577 + 2578 + assert( !isReqPos ); 2579 + 2580 + rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); 2581 + if( rc==SQLITE_OK ){ 2582 + assert( nExpr>1 ); 2583 + aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); 2584 + if( !aExpr ) rc = SQLITE_NOMEM; 2585 + } 2586 + if( rc==SQLITE_OK ){ 2587 + int ii; /* Used to iterate through expressions */ 2588 + 2589 + fts3ExprAssignCosts(pExpr, &aExpr); 2590 + aExpr -= nExpr; 2591 + for(ii=0; ii<nExpr; ii++){ 2592 + char *aNew; 2593 + int nNew; 2594 + int jj; 2595 + ExprAndCost *pBest = 0; 2596 + 2597 + for(jj=0; jj<nExpr; jj++){ 2598 + ExprAndCost *pCand = &aExpr[jj]; 2599 + if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){ 2600 + pBest = pCand; 2601 + } 2602 + } 2603 + 2604 + if( pBest->nCost>nDoc ){ 2605 + rc = fts3DeferExpression(p, p->pExpr); 2606 + break; 2607 + }else{ 2608 + rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); 2609 + if( rc!=SQLITE_OK ) break; 2610 + pBest->pExpr = 0; 2611 + if( ii==0 ){ 2612 + aRet = aNew; 2613 + nRet = nNew; 2614 + nDoc = fts3DoclistCountDocids(0, aRet, nRet); 2615 + }else{ 2616 + fts3DoclistMerge( 2617 + MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc 2618 + ); 2619 + sqlite3_free(aNew); 2620 + } 2621 + } 2622 + } 2623 + } 2624 + 2625 + *paOut = aRet; 2626 + *pnOut = nRet; 2627 + sqlite3_free(aExpr); 2628 + fts3ExprFreeSegReaders(pExpr); 2629 + 2050 2630 }else{ 2051 2631 char *aLeft; 2052 2632 char *aRight; 2053 2633 int nLeft; 2054 2634 int nRight; 2055 2635 2056 - if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) 2057 - && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) 2636 + assert( pExpr->eType==FTSQUERY_NEAR 2637 + || pExpr->eType==FTSQUERY_OR 2638 + || pExpr->eType==FTSQUERY_NOT 2639 + || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) 2640 + ); 2641 + 2642 + if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) 2643 + && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) 2058 2644 ){ 2059 - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR 2060 - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT 2061 - ); 2062 2645 switch( pExpr->eType ){ 2063 2646 case FTSQUERY_NEAR: { 2064 2647 Fts3Expr *pLeft; 2065 2648 Fts3Expr *pRight; 2066 - int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR; 2067 - 2649 + int mergetype = MERGE_NEAR; 2068 2650 if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ 2069 2651 mergetype = MERGE_POS_NEAR; 2070 2652 } 2071 2653 pLeft = pExpr->pLeft; 2072 2654 while( pLeft->eType==FTSQUERY_NEAR ){ 2073 2655 pLeft=pLeft->pRight; 2074 2656 } ................................................................................ 2089 2671 /* Allocate a buffer for the output. The maximum size is the 2090 2672 ** sum of the sizes of the two input buffers. The +1 term is 2091 2673 ** so that a buffer of zero bytes is never allocated - this can 2092 2674 ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. 2093 2675 */ 2094 2676 char *aBuffer = sqlite3_malloc(nRight+nLeft+1); 2095 2677 rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, 2096 - aLeft, nLeft, aRight, nRight 2678 + aLeft, nLeft, aRight, nRight, 0 2097 2679 ); 2098 2680 *paOut = aBuffer; 2099 2681 sqlite3_free(aLeft); 2100 2682 break; 2101 2683 } 2102 2684 2103 2685 default: { 2104 2686 assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); 2105 2687 fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, 2106 - aLeft, nLeft, aRight, nRight 2688 + aLeft, nLeft, aRight, nRight, 0 2107 2689 ); 2108 2690 *paOut = aLeft; 2109 2691 break; 2110 2692 } 2111 2693 } 2112 2694 } 2113 2695 sqlite3_free(aRight); 2114 2696 } 2115 2697 } 2116 2698 2699 + return rc; 2700 +} 2701 + 2702 +/* 2703 +** This function is called from within xNext() for each row visited by 2704 +** an FTS3 query. If evaluating the FTS3 query expression within xFilter() 2705 +** was able to determine the exact set of matching rows, this function sets 2706 +** *pbRes to true and returns SQLITE_IO immediately. 2707 +** 2708 +** Otherwise, if evaluating the query expression within xFilter() returned a 2709 +** superset of the matching documents instead of an exact set (this happens 2710 +** when the query includes very common tokens and it is deemed too expensive to 2711 +** load their doclists from disk), this function tests if the current row 2712 +** really does match the FTS3 query. 2713 +** 2714 +** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK 2715 +** is returned and *pbRes is set to true if the current row matches the 2716 +** FTS3 query (and should be included in the results returned to SQLite), or 2717 +** false otherwise. 2718 +*/ 2719 +static int fts3EvalDeferred( 2720 + Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ 2721 + int *pbRes /* OUT: Set to true if row is a match */ 2722 +){ 2723 + int rc = SQLITE_OK; 2724 + if( pCsr->pDeferred==0 ){ 2725 + *pbRes = 1; 2726 + }else{ 2727 + rc = fts3CursorSeek(0, pCsr); 2728 + if( rc==SQLITE_OK ){ 2729 + sqlite3Fts3FreeDeferredDoclists(pCsr); 2730 + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); 2731 + } 2732 + if( rc==SQLITE_OK ){ 2733 + char *a = 0; 2734 + int n = 0; 2735 + rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); 2736 + assert( n>=0 ); 2737 + *pbRes = (n>0); 2738 + sqlite3_free(a); 2739 + } 2740 + } 2741 + return rc; 2742 +} 2743 + 2744 +/* 2745 +** Advance the cursor to the next row in the %_content table that 2746 +** matches the search criteria. For a MATCH search, this will be 2747 +** the next row that matches. For a full-table scan, this will be 2748 +** simply the next row in the %_content table. For a docid lookup, 2749 +** this routine simply sets the EOF flag. 2750 +** 2751 +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned 2752 +** even if we reach end-of-file. The fts3EofMethod() will be called 2753 +** subsequently to determine whether or not an EOF was hit. 2754 +*/ 2755 +static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ 2756 + int res; 2757 + int rc = SQLITE_OK; /* Return code */ 2758 + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; 2759 + 2760 + pCsr->eEvalmode = FTS3_EVAL_NEXT; 2761 + do { 2762 + if( pCsr->aDoclist==0 ){ 2763 + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ 2764 + pCsr->isEof = 1; 2765 + rc = sqlite3_reset(pCsr->pStmt); 2766 + break; 2767 + } 2768 + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); 2769 + }else{ 2770 + if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ 2771 + pCsr->isEof = 1; 2772 + break; 2773 + } 2774 + sqlite3_reset(pCsr->pStmt); 2775 + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); 2776 + pCsr->isRequireSeek = 1; 2777 + pCsr->isMatchinfoNeeded = 1; 2778 + } 2779 + }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); 2780 + 2117 2781 return rc; 2118 2782 } 2119 2783 2120 2784 /* 2121 2785 ** This is the xFilter interface for the virtual table. See 2122 2786 ** the virtual table xFilter method documentation for additional 2123 2787 ** information. ................................................................................ 2129 2793 ** in the %_content table. 2130 2794 ** 2131 2795 ** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The 2132 2796 ** column on the left-hand side of the MATCH operator is column 2133 2797 ** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand 2134 2798 ** side of the MATCH operator. 2135 2799 */ 2136 -/* TODO(shess) Upgrade the cursor initialization and destruction to 2137 -** account for fts3FilterMethod() being called multiple times on the 2138 -** same cursor. The current solution is very fragile. Apply fix to 2139 -** fts3 as appropriate. 2140 -*/ 2141 2800 static int fts3FilterMethod( 2142 2801 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ 2143 2802 int idxNum, /* Strategy index */ 2144 2803 const char *idxStr, /* Unused */ 2145 2804 int nVal, /* Number of elements in apVal */ 2146 2805 sqlite3_value **apVal /* Arguments for the indexing scheme */ 2147 2806 ){ ................................................................................ 2156 2815 2157 2816 UNUSED_PARAMETER(idxStr); 2158 2817 UNUSED_PARAMETER(nVal); 2159 2818 2160 2819 assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); 2161 2820 assert( nVal==0 || nVal==1 ); 2162 2821 assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); 2822 + assert( p->pSegments==0 ); 2163 2823 2164 2824 /* In case the cursor has been used before, clear it now. */ 2165 2825 sqlite3_finalize(pCsr->pStmt); 2166 2826 sqlite3_free(pCsr->aDoclist); 2167 2827 sqlite3Fts3ExprFree(pCsr->pExpr); 2168 2828 memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); 2169 2829 2170 - /* Compile a SELECT statement for this cursor. For a full-table-scan, the 2171 - ** statement loops through all rows of the %_content table. For a 2172 - ** full-text query or docid lookup, the statement retrieves a single 2173 - ** row by docid. 2174 - */ 2175 - zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); 2176 - if( !zSql ){ 2177 - rc = SQLITE_NOMEM; 2178 - }else{ 2179 - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); 2180 - sqlite3_free(zSql); 2181 - } 2182 - if( rc!=SQLITE_OK ) return rc; 2183 - pCsr->eSearch = (i16)idxNum; 2184 - 2185 - if( idxNum==FTS3_DOCID_SEARCH ){ 2186 - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); 2187 - }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){ 2830 + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ 2188 2831 int iCol = idxNum-FTS3_FULLTEXT_SEARCH; 2189 2832 const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); 2190 2833 2191 2834 if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ 2192 2835 return SQLITE_NOMEM; 2193 2836 } 2194 2837 ................................................................................ 2202 2845 } 2203 2846 return rc; 2204 2847 } 2205 2848 2206 2849 rc = sqlite3Fts3ReadLock(p); 2207 2850 if( rc!=SQLITE_OK ) return rc; 2208 2851 2209 - rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); 2852 + rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); 2853 + sqlite3Fts3SegmentsClose(p); 2854 + if( rc!=SQLITE_OK ) return rc; 2210 2855 pCsr->pNextId = pCsr->aDoclist; 2211 2856 pCsr->iPrevId = 0; 2212 2857 } 2858 + 2859 + /* Compile a SELECT statement for this cursor. For a full-table-scan, the 2860 + ** statement loops through all rows of the %_content table. For a 2861 + ** full-text query or docid lookup, the statement retrieves a single 2862 + ** row by docid. 2863 + */ 2864 + zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); 2865 + if( !zSql ){ 2866 + rc = SQLITE_NOMEM; 2867 + }else{ 2868 + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); 2869 + sqlite3_free(zSql); 2870 + } 2871 + if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ 2872 + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); 2873 + } 2874 + pCsr->eSearch = (i16)idxNum; 2213 2875 2214 2876 if( rc!=SQLITE_OK ) return rc; 2215 2877 return fts3NextMethod(pCursor); 2216 2878 } 2217 2879 2218 2880 /* 2219 2881 ** This is the xEof method of the virtual table. SQLite calls this ................................................................................ 2230 2892 ** rowid should be written to *pRowid. 2231 2893 */ 2232 2894 static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ 2233 2895 Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; 2234 2896 if( pCsr->aDoclist ){ 2235 2897 *pRowid = pCsr->iPrevId; 2236 2898 }else{ 2899 + /* This branch runs if the query is implemented using a full-table scan 2900 + ** (not using the full-text index). In this case grab the rowid from the 2901 + ** SELECT statement. 2902 + */ 2903 + assert( pCsr->isRequireSeek==0 ); 2237 2904 *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); 2238 2905 } 2239 2906 return SQLITE_OK; 2240 2907 } 2241 2908 2242 2909 /* 2243 2910 ** This is the xColumn method, called by SQLite to request a value from ................................................................................ 2292 2959 } 2293 2960 2294 2961 /* 2295 2962 ** Implementation of xSync() method. Flush the contents of the pending-terms 2296 2963 ** hash-table to the database. 2297 2964 */ 2298 2965 static int fts3SyncMethod(sqlite3_vtab *pVtab){ 2299 - return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); 2966 + int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); 2967 + sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); 2968 + return rc; 2300 2969 } 2301 2970 2302 2971 /* 2303 2972 ** Implementation of xBegin() method. This is a no-op. 2304 2973 */ 2305 2974 static int fts3BeginMethod(sqlite3_vtab *pVtab){ 2306 2975 UNUSED_PARAMETER(pVtab); ................................................................................ 2330 2999 2331 3000 /* 2332 3001 ** Load the doclist associated with expression pExpr to pExpr->aDoclist. 2333 3002 ** The loaded doclist contains positions as well as the document ids. 2334 3003 ** This is used by the matchinfo(), snippet() and offsets() auxillary 2335 3004 ** functions. 2336 3005 */ 2337 -int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){ 2338 - return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); 3006 +int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ 3007 + int rc; 3008 + assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); 3009 + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); 3010 + rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); 3011 + return rc; 3012 +} 3013 + 3014 +int sqlite3Fts3ExprLoadFtDoclist( 3015 + Fts3Cursor *pCsr, 3016 + Fts3Expr *pExpr, 3017 + char **paDoclist, 3018 + int *pnDoclist 3019 +){ 3020 + int rc; 3021 + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); 3022 + assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); 3023 + pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; 3024 + rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); 3025 + pCsr->eEvalmode = FTS3_EVAL_NEXT; 3026 + return rc; 2339 3027 } 2340 3028 2341 3029 /* 2342 3030 ** After ExprLoadDoclist() (see above) has been called, this function is 2343 3031 ** used to iterate/search through the position lists that make up the doclist 2344 3032 ** stored in pExpr->aDoclist. 2345 3033 */ ................................................................................ 2397 3085 ** message is written to context pContext and SQLITE_ERROR returned. The 2398 3086 ** string passed via zFunc is used as part of the error message. 2399 3087 */ 2400 3088 static int fts3FunctionArg( 2401 3089 sqlite3_context *pContext, /* SQL function call context */ 2402 3090 const char *zFunc, /* Function name */ 2403 3091 sqlite3_value *pVal, /* argv[0] passed to function */ 2404 - Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ 3092 + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ 2405 3093 ){ 2406 3094 Fts3Cursor *pRet; 2407 3095 if( sqlite3_value_type(pVal)!=SQLITE_BLOB 2408 3096 || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) 2409 3097 ){ 2410 3098 char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); 2411 3099 sqlite3_result_error(pContext, zErr, -1); ................................................................................ 2523 3211 */ 2524 3212 static void fts3MatchinfoFunc( 2525 3213 sqlite3_context *pContext, /* SQLite function call context */ 2526 3214 int nVal, /* Size of argument array */ 2527 3215 sqlite3_value **apVal /* Array of arguments */ 2528 3216 ){ 2529 3217 Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ 2530 - 2531 - if( nVal!=1 ){ 2532 - sqlite3_result_error(pContext, 2533 - "wrong number of arguments to function matchinfo()", -1); 2534 - return; 2535 - } 2536 - 3218 + assert( nVal==1 ); 2537 3219 if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ 2538 3220 sqlite3Fts3Matchinfo(pContext, pCsr); 2539 3221 } 2540 3222 } 2541 3223 2542 3224 /* 2543 3225 ** This routine implements the xFindFunction method for the FTS3 ................................................................................ 2592 3274 return rc; 2593 3275 } 2594 3276 2595 3277 fts3DbExec(&rc, db, 2596 3278 "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", 2597 3279 p->zDb, p->zName, zName 2598 3280 ); 2599 - if( rc==SQLITE_ERROR ) rc = SQLITE_OK; 2600 3281 if( p->bHasDocsize ){ 2601 3282 fts3DbExec(&rc, db, 2602 3283 "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", 2603 3284 p->zDb, p->zName, zName 2604 3285 ); 2605 3286 fts3DbExec(&rc, db, 2606 3287 "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", ................................................................................ 2622 3303 /* iVersion */ 0, 2623 3304 /* xCreate */ fts3CreateMethod, 2624 3305 /* xConnect */ fts3ConnectMethod, 2625 3306 /* xBestIndex */ fts3BestIndexMethod, 2626 3307 /* xDisconnect */ fts3DisconnectMethod, 2627 3308 /* xDestroy */ fts3DestroyMethod, 2628 3309 /* xOpen */ fts3OpenMethod, 2629 - /* xClose */ fulltextClose, 3310 + /* xClose */ fts3CloseMethod, 2630 3311 /* xFilter */ fts3FilterMethod, 2631 3312 /* xNext */ fts3NextMethod, 2632 3313 /* xEof */ fts3EofMethod, 2633 3314 /* xColumn */ fts3ColumnMethod, 2634 3315 /* xRowid */ fts3RowidMethod, 2635 3316 /* xUpdate */ fts3UpdateMethod, 2636 3317 /* xBegin */ fts3BeginMethod, ................................................................................ 2649 3330 static void hashDestroy(void *p){ 2650 3331 Fts3Hash *pHash = (Fts3Hash *)p; 2651 3332 sqlite3Fts3HashClear(pHash); 2652 3333 sqlite3_free(pHash); 2653 3334 } 2654 3335 2655 3336 /* 2656 -** The fts3 built-in tokenizers - "simple" and "porter" - are implemented 2657 -** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following 2658 -** two forward declarations are for functions declared in these files 2659 -** used to retrieve the respective implementations. 3337 +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are 3338 +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c 3339 +** respectively. The following three forward declarations are for functions 3340 +** declared in these files used to retrieve the respective implementations. 2660 3341 ** 2661 3342 ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed 2662 3343 ** to by the argument to point to the "simple" tokenizer implementation. 2663 -** Function ...PorterTokenizerModule() sets *pModule to point to the 2664 -** porter tokenizer/stemmer implementation. 3344 +** And so on. 2665 3345 */ 2666 3346 void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); 2667 3347 void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); 3348 +#ifdef SQLITE_ENABLE_ICU 2668 3349 void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); 3350 +#endif 2669 3351 2670 3352 /* 2671 3353 ** Initialise the fts3 extension. If this extension is built as part 2672 3354 ** of the sqlite library, then this function is called directly by 2673 3355 ** SQLite. If fts3 is built as a dynamically loadable extension, this 2674 3356 ** function is called by the sqlite3_extension_init() entry point. 2675 3357 */ ................................................................................ 2717 3399 ** the two scalar functions. If this is successful, register the 2718 3400 ** module with sqlite. 2719 3401 */ 2720 3402 if( SQLITE_OK==rc 2721 3403 && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) 2722 3404 && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) 2723 3405 && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) 2724 - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1)) 3406 + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) 2725 3407 && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) 2726 3408 ){ 2727 3409 rc = sqlite3_create_module_v2( 2728 3410 db, "fts3", &fts3Module, (void *)pHash, hashDestroy 2729 3411 ); 2730 3412 if( rc==SQLITE_OK ){ 2731 3413 rc = sqlite3_create_module_v2(
Changes to ext/fts3/fts3Int.h.
73 73 ** amalgamation. 74 74 */ 75 75 #ifndef SQLITE_AMALGAMATION 76 76 /* 77 77 ** Macros indicating that conditional expressions are always true or 78 78 ** false. 79 79 */ 80 +#ifdef SQLITE_COVERAGE_TEST 81 +# define ALWAYS(x) (1) 82 +# define NEVER(X) (0) 83 +#else 80 84 # define ALWAYS(x) (x) 81 85 # define NEVER(X) (x) 86 +#endif 87 + 82 88 /* 83 89 ** Internal types used by SQLite. 84 90 */ 85 91 typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ 86 92 typedef short int i16; /* 2-byte (or larger) signed integer */ 87 93 typedef unsigned int u32; /* 4-byte unsigned integer */ 88 94 typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ ................................................................................ 92 98 #define UNUSED_PARAMETER(x) (void)(x) 93 99 #endif 94 100 95 101 typedef struct Fts3Table Fts3Table; 96 102 typedef struct Fts3Cursor Fts3Cursor; 97 103 typedef struct Fts3Expr Fts3Expr; 98 104 typedef struct Fts3Phrase Fts3Phrase; 99 -typedef struct Fts3SegReader Fts3SegReader; 105 +typedef struct Fts3PhraseToken Fts3PhraseToken; 106 + 100 107 typedef struct Fts3SegFilter Fts3SegFilter; 108 +typedef struct Fts3DeferredToken Fts3DeferredToken; 109 +typedef struct Fts3SegReader Fts3SegReader; 110 +typedef struct Fts3SegReaderArray Fts3SegReaderArray; 101 111 102 112 /* 103 113 ** A connection to a fulltext index is an instance of the following 104 114 ** structure. The xCreate and xConnect methods create an instance 105 115 ** of this structure and xDestroy and xDisconnect free that instance. 106 116 ** All other methods receive a pointer to the structure as one of their 107 117 ** arguments. ................................................................................ 114 124 int nColumn; /* number of named columns in virtual table */ 115 125 char **azColumn; /* column names. malloced */ 116 126 sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ 117 127 118 128 /* Precompiled statements used by the implementation. Each of these 119 129 ** statements is run and reset within a single virtual table API call. 120 130 */ 121 - sqlite3_stmt *aStmt[25]; 122 - 123 - /* Pointer to string containing the SQL: 124 - ** 125 - ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ? 126 - ** ORDER BY blockid" 127 - */ 128 - char *zSelectLeaves; 129 - int nLeavesStmt; /* Valid statements in aLeavesStmt */ 130 - int nLeavesTotal; /* Total number of prepared leaves stmts */ 131 - int nLeavesAlloc; /* Allocated size of aLeavesStmt */ 132 - sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */ 131 + sqlite3_stmt *aStmt[24]; 133 132 134 133 int nNodeSize; /* Soft limit for node size */ 135 134 u8 bHasContent; /* True if %_content table exists */ 136 135 u8 bHasDocsize; /* True if %_docsize table exists */ 136 + int nPgsz; /* Page size for host database */ 137 + char *zSegmentsTbl; /* Name of %_segments table */ 138 + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ 137 139 138 140 /* The following hash table is used to buffer pending index updates during 139 141 ** transactions. Variable nPendingData estimates the memory size of the 140 142 ** pending data, including hash table overhead, but not malloc overhead. 141 143 ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 142 144 ** automatically. Variable iPrevDocid is the docid of the most recently 143 145 ** inserted record. ................................................................................ 156 158 struct Fts3Cursor { 157 159 sqlite3_vtab_cursor base; /* Base class used by SQLite core */ 158 160 i16 eSearch; /* Search strategy (see below) */ 159 161 u8 isEof; /* True if at End Of Results */ 160 162 u8 isRequireSeek; /* True if must seek pStmt to %_content row */ 161 163 sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ 162 164 Fts3Expr *pExpr; /* Parsed MATCH query string */ 165 + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ 163 166 sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ 164 167 char *pNextId; /* Pointer into the body of aDoclist */ 165 168 char *aDoclist; /* List of docids for full-text queries */ 166 169 int nDoclist; /* Size of buffer at aDoclist */ 167 170 int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ 168 171 u32 *aMatchinfo; /* Information about most recent match */ 172 + int eEvalmode; /* An FTS3_EVAL_XX constant */ 173 + int nRowAvg; /* Average size of database rows, in pages */ 169 174 }; 175 + 176 +#define FTS3_EVAL_FILTER 0 177 +#define FTS3_EVAL_NEXT 1 178 +#define FTS3_EVAL_MATCHINFO 2 170 179 171 180 /* 172 181 ** The Fts3Cursor.eSearch member is always set to one of the following. 173 182 ** Actualy, Fts3Cursor.eSearch can be greater than or equal to 174 183 ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index 175 184 ** of the column to be searched. For example, in 176 185 ** ................................................................................ 186 195 #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ 187 196 #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ 188 197 #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ 189 198 190 199 /* 191 200 ** A "phrase" is a sequence of one or more tokens that must match in 192 201 ** sequence. A single token is the base case and the most common case. 193 -** For a sequence of tokens contained in "...", nToken will be the number 194 -** of tokens in the string. 202 +** For a sequence of tokens contained in double-quotes (i.e. "one two three") 203 +** nToken will be the number of tokens in the string. 204 +** 205 +** The nDocMatch and nMatch variables contain data that may be used by the 206 +** matchinfo() function. They are populated when the full-text index is 207 +** queried for hits on the phrase. If one or more tokens in the phrase 208 +** are deferred, the nDocMatch and nMatch variables are populated based 209 +** on the assumption that the 195 210 */ 211 +struct Fts3PhraseToken { 212 + char *z; /* Text of the token */ 213 + int n; /* Number of bytes in buffer z */ 214 + int isPrefix; /* True if token ends with a "*" character */ 215 + int bFulltext; /* True if full-text index was used */ 216 + Fts3SegReaderArray *pArray; /* Segment-reader for this token */ 217 + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ 218 +}; 219 + 196 220 struct Fts3Phrase { 221 + /* Variables populated by fts3_expr.c when parsing a MATCH expression */ 197 222 int nToken; /* Number of tokens in the phrase */ 198 223 int iColumn; /* Index of column this phrase must match */ 199 224 int isNot; /* Phrase prefixed by unary not (-) operator */ 200 - struct PhraseToken { 201 - char *z; /* Text of the token */ 202 - int n; /* Number of bytes in buffer pointed to by z */ 203 - int isPrefix; /* True if token ends in with a "*" character */ 204 - } aToken[1]; /* One entry for each token in the phrase */ 225 + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ 205 226 }; 206 227 207 228 /* 208 229 ** A tree of these objects forms the RHS of a MATCH operator. 209 230 ** 210 231 ** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded 211 232 ** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, ................................................................................ 247 268 #define FTSQUERY_NEAR 1 248 269 #define FTSQUERY_NOT 2 249 270 #define FTSQUERY_AND 3 250 271 #define FTSQUERY_OR 4 251 272 #define FTSQUERY_PHRASE 5 252 273 253 274 254 -/* fts3_init.c */ 255 -int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *); 256 -int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, 257 - sqlite3_vtab **, char **); 258 - 259 275 /* fts3_write.c */ 260 276 int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); 261 277 int sqlite3Fts3PendingTermsFlush(Fts3Table *); 262 278 void sqlite3Fts3PendingTermsClear(Fts3Table *); 263 279 int sqlite3Fts3Optimize(Fts3Table *); 264 280 int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, 265 281 sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); 266 282 int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); 267 283 void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *); 268 284 int sqlite3Fts3SegReaderIterate( 269 285 Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, 270 286 int (*)(Fts3Table *, void *, char *, int, char *, int), void * 271 287 ); 272 -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); 288 +int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); 273 289 int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); 274 290 int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); 275 291 int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); 276 292 int sqlite3Fts3ReadLock(Fts3Table *); 293 +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); 294 + 295 +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); 296 +int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); 297 +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); 298 +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); 299 +char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); 300 + 301 +void sqlite3Fts3SegmentsClose(Fts3Table *); 277 302 278 303 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ 279 304 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 280 305 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 281 306 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 282 307 #define FTS3_SEGMENT_PREFIX 0x00000008 283 308 ................................................................................ 293 318 int sqlite3Fts3PutVarint(char *, sqlite3_int64); 294 319 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); 295 320 int sqlite3Fts3GetVarint32(const char *, int *); 296 321 int sqlite3Fts3VarintLen(sqlite3_uint64); 297 322 void sqlite3Fts3Dequote(char *); 298 323 299 324 char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); 300 -int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *); 325 +int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); 326 +int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); 301 327 int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); 302 328 303 329 /* fts3_tokenizer.c */ 304 330 const char *sqlite3Fts3NextToken(const char *, int *); 305 331 int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); 306 332 int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, 307 333 const char *, sqlite3_tokenizer **, const char **, char **
Changes to ext/fts3/fts3_expr.c.
101 101 ** is defined to accept an argument of type char, and always returns 0 for 102 102 ** any values that fall outside of the range of the unsigned char type (i.e. 103 103 ** negative values). 104 104 */ 105 105 static int fts3isspace(char c){ 106 106 return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; 107 107 } 108 + 109 +/* 110 +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, 111 +** zero the memory before returning a pointer to it. If unsuccessful, 112 +** return NULL. 113 +*/ 114 +static void *fts3MallocZero(int nByte){ 115 + void *pRet = sqlite3_malloc(nByte); 116 + if( pRet ) memset(pRet, 0, nByte); 117 + return pRet; 118 +} 119 + 108 120 109 121 /* 110 122 ** Extract the next token from buffer z (length n) using the tokenizer 111 123 ** and other information (column names etc.) in pParse. Create an Fts3Expr 112 124 ** structure of type FTSQUERY_PHRASE containing a phrase consisting of this 113 125 ** single token and set *ppExpr to point to it. If the end of the buffer is 114 126 ** reached before a token is found, set *ppExpr to zero. It is the ................................................................................ 139 151 int nByte; /* total space to allocate */ 140 152 141 153 pCursor->pTokenizer = pTokenizer; 142 154 rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); 143 155 144 156 if( rc==SQLITE_OK ){ 145 157 nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; 146 - pRet = (Fts3Expr *)sqlite3_malloc(nByte); 158 + pRet = (Fts3Expr *)fts3MallocZero(nByte); 147 159 if( !pRet ){ 148 160 rc = SQLITE_NOMEM; 149 161 }else{ 150 - memset(pRet, 0, nByte); 151 162 pRet->eType = FTSQUERY_PHRASE; 152 163 pRet->pPhrase = (Fts3Phrase *)&pRet[1]; 153 164 pRet->pPhrase->nToken = 1; 154 165 pRet->pPhrase->iColumn = iCol; 155 166 pRet->pPhrase->aToken[0].n = nToken; 156 167 pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; 157 168 memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); ................................................................................ 219 230 pCursor->pTokenizer = pTokenizer; 220 231 for(ii=0; rc==SQLITE_OK; ii++){ 221 232 const char *zToken; 222 233 int nToken, iBegin, iEnd, iPos; 223 234 rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); 224 235 if( rc==SQLITE_OK ){ 225 236 int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); 226 - p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken)); 237 + p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); 227 238 zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); 228 239 if( !p || !zTemp ){ 229 240 goto no_mem; 230 241 } 231 242 if( ii==0 ){ 232 243 memset(p, 0, nByte); 233 244 p->pPhrase = (Fts3Phrase *)&p[1]; 234 245 } 235 246 p->pPhrase = (Fts3Phrase *)&p[1]; 247 + memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); 236 248 p->pPhrase->nToken = ii+1; 237 249 p->pPhrase->aToken[ii].n = nToken; 238 250 memcpy(&zTemp[nTemp], zToken, nToken); 239 251 nTemp += nToken; 240 252 if( iEnd<nInput && zInput[iEnd]=='*' ){ 241 253 p->pPhrase->aToken[ii].isPrefix = 1; 242 254 }else{ ................................................................................ 250 262 } 251 263 252 264 if( rc==SQLITE_DONE ){ 253 265 int jj; 254 266 char *zNew = NULL; 255 267 int nNew = 0; 256 268 int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); 257 - nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken); 269 + nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); 258 270 p = fts3ReallocOrFree(p, nByte + nTemp); 259 271 if( !p ){ 260 272 goto no_mem; 261 273 } 262 274 if( zTemp ){ 263 275 zNew = &(((char *)p)[nByte]); 264 276 memcpy(zNew, zTemp, nTemp); ................................................................................ 368 380 ** the next byte must contain either whitespace, an open or close 369 381 ** parenthesis, a quote character, or EOF. 370 382 */ 371 383 cNext = zInput[nKey]; 372 384 if( fts3isspace(cNext) 373 385 || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 374 386 ){ 375 - pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); 387 + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); 376 388 if( !pRet ){ 377 389 return SQLITE_NOMEM; 378 390 } 379 - memset(pRet, 0, sizeof(Fts3Expr)); 380 391 pRet->eType = pKey->eType; 381 392 pRet->nNear = nNear; 382 393 *ppExpr = pRet; 383 394 *pnConsumed = (int)((zInput - z) + nKey); 384 395 return SQLITE_OK; 385 396 } 386 397 ................................................................................ 548 559 if( rc==SQLITE_OK ){ 549 560 int isPhrase; 550 561 551 562 if( !sqlite3_fts3_enable_parentheses 552 563 && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot 553 564 ){ 554 565 /* Create an implicit NOT operator. */ 555 - Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr)); 566 + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); 556 567 if( !pNot ){ 557 568 sqlite3Fts3ExprFree(p); 558 569 rc = SQLITE_NOMEM; 559 570 goto exprparse_out; 560 571 } 561 - memset(pNot, 0, sizeof(Fts3Expr)); 562 572 pNot->eType = FTSQUERY_NOT; 563 573 pNot->pRight = p; 564 574 if( pNotBranch ){ 565 575 pNot->pLeft = pNotBranch; 566 576 } 567 577 pNotBranch = pNot; 568 578 p = pPrev; ................................................................................ 582 592 goto exprparse_out; 583 593 } 584 594 585 595 if( isPhrase && !isRequirePhrase ){ 586 596 /* Insert an implicit AND operator. */ 587 597 Fts3Expr *pAnd; 588 598 assert( pRet && pPrev ); 589 - pAnd = sqlite3_malloc(sizeof(Fts3Expr)); 599 + pAnd = fts3MallocZero(sizeof(Fts3Expr)); 590 600 if( !pAnd ){ 591 601 sqlite3Fts3ExprFree(p); 592 602 rc = SQLITE_NOMEM; 593 603 goto exprparse_out; 594 604 } 595 - memset(pAnd, 0, sizeof(Fts3Expr)); 596 605 pAnd->eType = FTSQUERY_AND; 597 606 insertBinaryOperator(&pRet, pPrev, pAnd); 598 607 pPrev = pAnd; 599 608 } 600 609 601 610 /* This test catches attempts to make either operand of a NEAR 602 611 ** operator something other than a phrase. For example, either of
Changes to ext/fts3/fts3_snippet.c.
20 20 21 21 /* 22 22 ** Used as an fts3ExprIterate() context when loading phrase doclists to 23 23 ** Fts3Expr.aDoclist[]/nDoclist. 24 24 */ 25 25 typedef struct LoadDoclistCtx LoadDoclistCtx; 26 26 struct LoadDoclistCtx { 27 - Fts3Table *pTab; /* FTS3 Table */ 27 + Fts3Cursor *pCsr; /* FTS3 Cursor */ 28 28 int nPhrase; /* Number of phrases seen so far */ 29 29 int nToken; /* Number of tokens seen so far */ 30 30 }; 31 31 32 32 /* 33 33 ** The following types are used as part of the implementation of the 34 34 ** fts3BestSnippet() routine. ................................................................................ 214 214 215 215 UNUSED_PARAMETER(iPhrase); 216 216 217 217 p->nPhrase++; 218 218 p->nToken += pExpr->pPhrase->nToken; 219 219 220 220 if( pExpr->isLoaded==0 ){ 221 - rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr); 221 + rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); 222 222 pExpr->isLoaded = 1; 223 223 if( rc==SQLITE_OK ){ 224 224 rc = fts3ExprNearTrim(pExpr); 225 225 } 226 226 } 227 227 228 228 return rc; ................................................................................ 257 257 static int fts3ExprLoadDoclists( 258 258 Fts3Cursor *pCsr, /* Fts3 cursor for current query */ 259 259 int *pnPhrase, /* OUT: Number of phrases in query */ 260 260 int *pnToken /* OUT: Number of tokens in query */ 261 261 ){ 262 262 int rc; /* Return Code */ 263 263 LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ 264 - sCtx.pTab = (Fts3Table *)pCsr->base.pVtab; 264 + sCtx.pCsr = pCsr; 265 265 rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx); 266 266 if( rc==SQLITE_OK ){ 267 267 (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0); 268 268 } 269 269 if( pnPhrase ) *pnPhrase = sCtx.nPhrase; 270 270 if( pnToken ) *pnToken = sCtx.nToken; 271 271 return rc; ................................................................................ 788 788 */ 789 789 static int fts3ExprGlobalMatchinfoCb( 790 790 Fts3Expr *pExpr, /* Phrase expression node */ 791 791 int iPhrase, /* Phrase number (numbered from zero) */ 792 792 void *pCtx /* Pointer to MatchInfo structure */ 793 793 ){ 794 794 MatchInfo *p = (MatchInfo *)pCtx; 795 - char *pCsr; 795 + Fts3Cursor *pCsr = p->pCursor; 796 + char *pIter; 796 797 char *pEnd; 798 + char *pFree = 0; 797 799 const int iStart = 2 + (iPhrase * p->nCol * 3) + 1; 798 800 799 801 assert( pExpr->isLoaded ); 802 + assert( pExpr->eType==FTSQUERY_PHRASE ); 803 + 804 + if( pCsr->pDeferred ){ 805 + Fts3Phrase *pPhrase = pExpr->pPhrase; 806 + int ii; 807 + for(ii=0; ii<pPhrase->nToken; ii++){ 808 + if( pPhrase->aToken[ii].bFulltext ) break; 809 + } 810 + if( ii<pPhrase->nToken ){ 811 + int nFree = 0; 812 + int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); 813 + if( rc!=SQLITE_OK ) return rc; 814 + pIter = pFree; 815 + pEnd = &pFree[nFree]; 816 + }else{ 817 + int nDoc = p->aMatchinfo[2 + 3*p->nCol*p->aMatchinfo[0]]; 818 + for(ii=0; ii<p->nCol; ii++){ 819 + p->aMatchinfo[iStart + ii*3] = nDoc; 820 + p->aMatchinfo[iStart + ii*3 + 1] = nDoc; 821 + } 822 + return SQLITE_OK; 823 + } 824 + }else{ 825 + pIter = pExpr->aDoclist; 826 + pEnd = &pExpr->aDoclist[pExpr->nDoclist]; 827 + } 800 828 801 829 /* Fill in the global hit count matrix row for this phrase. */ 802 - pCsr = pExpr->aDoclist; 803 - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; 804 - while( pCsr<pEnd ){ 805 - while( *pCsr++ & 0x80 ); /* Skip past docid. */ 806 - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 1); 830 + while( pIter<pEnd ){ 831 + while( *pIter++ & 0x80 ); /* Skip past docid. */ 832 + fts3LoadColumnlistCounts(&pIter, &p->aMatchinfo[iStart], 1); 807 833 } 808 834 835 + sqlite3_free(pFree); 809 836 return SQLITE_OK; 810 837 } 811 838 812 839 /* 813 840 ** fts3ExprIterate() callback used to collect the "local" matchinfo stats 814 841 ** for a single query. The "local" stats are those elements of the matchinfo 815 842 ** array that are different for each row returned by the query. ................................................................................ 870 897 871 898 sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo); 872 899 if( !sInfo.aMatchinfo ){ 873 900 return SQLITE_NOMEM; 874 901 } 875 902 memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo); 876 903 877 - 878 904 /* First element of match-info is the number of phrases in the query */ 879 905 sInfo.aMatchinfo[0] = nPhrase; 880 906 sInfo.aMatchinfo[1] = sInfo.nCol; 881 - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo); 882 907 if( pTab->bHasDocsize ){ 883 908 int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; 884 909 rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]); 885 910 } 911 + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo); 886 912 pCsr->aMatchinfo = sInfo.aMatchinfo; 887 913 pCsr->isMatchinfoNeeded = 1; 888 914 } 889 915 890 916 sInfo.aMatchinfo = pCsr->aMatchinfo; 891 917 if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){ 892 918 (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo); ................................................................................ 988 1014 for(i=0; i<nSnippet && rc==SQLITE_OK; i++){ 989 1015 rc = fts3SnippetText(pCsr, &aSnippet[i], 990 1016 i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res 991 1017 ); 992 1018 } 993 1019 994 1020 snippet_out: 1021 + sqlite3Fts3SegmentsClose(pTab); 995 1022 if( rc!=SQLITE_OK ){ 996 1023 sqlite3_result_error_code(pCtx, rc); 997 1024 sqlite3_free(res.z); 998 1025 }else{ 999 1026 sqlite3_result_text(pCtx, res.z, -1, sqlite3_free); 1000 1027 } 1001 1028 } ................................................................................ 1167 1194 pMod->xClose(pC); 1168 1195 if( rc!=SQLITE_OK ) goto offsets_out; 1169 1196 } 1170 1197 1171 1198 offsets_out: 1172 1199 sqlite3_free(sCtx.aTerm); 1173 1200 assert( rc!=SQLITE_DONE ); 1201 + sqlite3Fts3SegmentsClose(pTab); 1174 1202 if( rc!=SQLITE_OK ){ 1175 1203 sqlite3_result_error_code(pCtx, rc); 1176 1204 sqlite3_free(res.z); 1177 1205 }else{ 1178 1206 sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); 1179 1207 } 1180 1208 return; ................................................................................ 1186 1214 void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){ 1187 1215 int rc; 1188 1216 if( !pCsr->pExpr ){ 1189 1217 sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); 1190 1218 return; 1191 1219 } 1192 1220 rc = fts3GetMatchinfo(pCsr); 1221 + sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab ); 1193 1222 if( rc!=SQLITE_OK ){ 1194 1223 sqlite3_result_error_code(pContext, rc); 1195 1224 }else{ 1196 1225 Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; 1197 1226 int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3); 1198 1227 if( pTab->bHasDocsize ){ 1199 1228 n += sizeof(u32)*(1 + 2*pTab->nColumn); 1200 1229 } 1201 1230 sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); 1202 1231 } 1203 1232 } 1204 1233 1205 1234 #endif
Changes to ext/fts3/fts3_write.c.
38 38 char *aData; 39 39 int nSpace; 40 40 sqlite3_int64 iLastDocid; 41 41 sqlite3_int64 iLastCol; 42 42 sqlite3_int64 iLastPos; 43 43 }; 44 44 45 + 46 +/* 47 +** Each cursor has a (possibly empty) linked list of the following objects. 48 +*/ 49 +struct Fts3DeferredToken { 50 + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ 51 + int iCol; /* Column token must occur in */ 52 + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ 53 + PendingList *pList; /* Doclist is assembled here */ 54 +}; 55 + 45 56 /* 46 57 ** An instance of this structure is used to iterate through the terms on 47 58 ** a contiguous set of segment b-tree leaf nodes. Although the details of 48 59 ** this structure are only manipulated by code in this file, opaque handles 49 60 ** of type Fts3SegReader* are also used by code in fts3.c to iterate through 50 61 ** terms when querying the full-text index. See functions: 51 62 ** 52 63 ** sqlite3Fts3SegReaderNew() 53 64 ** sqlite3Fts3SegReaderFree() 65 +** sqlite3Fts3SegReaderCost() 54 66 ** sqlite3Fts3SegReaderIterate() 55 67 ** 56 68 ** Methods used to manipulate Fts3SegReader structures: 57 69 ** 58 70 ** fts3SegReaderNext() 59 71 ** fts3SegReaderFirstDocid() 60 72 ** fts3SegReaderNextDocid() 61 73 */ 62 74 struct Fts3SegReader { 63 75 int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ 64 - sqlite3_int64 iStartBlock; 65 - sqlite3_int64 iEndBlock; 66 - sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */ 76 + 77 + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ 78 + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ 79 + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ 80 + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ 81 + 67 82 char *aNode; /* Pointer to node data (or NULL) */ 68 83 int nNode; /* Size of buffer at aNode (or 0) */ 69 - int nTermAlloc; /* Allocated size of zTerm buffer */ 70 84 Fts3HashElem **ppNextElem; 71 85 72 86 /* Variables set by fts3SegReaderNext(). These may be read directly 73 87 ** by the caller. They are valid from the time SegmentReaderNew() returns 74 88 ** until SegmentReaderNext() returns something other than SQLITE_OK 75 89 ** (i.e. SQLITE_DONE). 76 90 */ 77 91 int nTerm; /* Number of bytes in current term */ 78 92 char *zTerm; /* Pointer to current term */ 93 + int nTermAlloc; /* Allocated size of zTerm buffer */ 79 94 char *aDoclist; /* Pointer to doclist of current entry */ 80 95 int nDoclist; /* Size of doclist in current entry */ 81 96 82 97 /* The following variables are used to iterate through the current doclist */ 83 98 char *pOffsetList; 84 99 sqlite3_int64 iDocid; 85 100 }; 86 101 87 102 #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) 103 +#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) 88 104 89 105 /* 90 106 ** An instance of this structure is used to create a segment b-tree in the 91 107 ** database. The internal details of this type are only accessed by the 92 108 ** following functions: 93 109 ** 94 110 ** fts3SegWriterAdd() ................................................................................ 149 165 #define SQL_SELECT_LEVEL 12 150 166 #define SQL_SELECT_ALL_LEVEL 13 151 167 #define SQL_SELECT_LEVEL_COUNT 14 152 168 #define SQL_SELECT_SEGDIR_COUNT_MAX 15 153 169 #define SQL_DELETE_SEGDIR_BY_LEVEL 16 154 170 #define SQL_DELETE_SEGMENTS_RANGE 17 155 171 #define SQL_CONTENT_INSERT 18 156 -#define SQL_GET_BLOCK 19 157 -#define SQL_DELETE_DOCSIZE 20 158 -#define SQL_REPLACE_DOCSIZE 21 159 -#define SQL_SELECT_DOCSIZE 22 160 -#define SQL_SELECT_DOCTOTAL 23 161 -#define SQL_REPLACE_DOCTOTAL 24 172 +#define SQL_DELETE_DOCSIZE 19 173 +#define SQL_REPLACE_DOCSIZE 20 174 +#define SQL_SELECT_DOCSIZE 21 175 +#define SQL_SELECT_DOCTOTAL 22 176 +#define SQL_REPLACE_DOCTOTAL 23 162 177 163 178 /* 164 179 ** This function is used to obtain an SQLite prepared statement handle 165 180 ** for the statement identified by the second argument. If successful, 166 181 ** *pp is set to the requested statement handle and SQLITE_OK returned. 167 182 ** Otherwise, an SQLite error code is returned and *pp is set to 0. 168 183 ** ................................................................................ 199 214 200 215 /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", 201 216 /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", 202 217 203 218 /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", 204 219 /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", 205 220 /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", 206 -/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", 207 -/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", 208 -/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", 209 -/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", 210 -/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", 211 -/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", 221 +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", 222 +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", 223 +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", 224 +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", 225 +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", 212 226 }; 213 227 int rc = SQLITE_OK; 214 228 sqlite3_stmt *pStmt; 215 229 216 230 assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); 217 231 assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); 218 232 ................................................................................ 279 293 sqlite3_step(pStmt); 280 294 rc = sqlite3_reset(pStmt); 281 295 } 282 296 *pRC = rc; 283 297 } 284 298 285 299 286 -/* 287 -** Read a single block from the %_segments table. If the specified block 288 -** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO 289 -** etc.) occurs, return the appropriate SQLite error code. 290 -** 291 -** Otherwise, if successful, set *pzBlock to point to a buffer containing 292 -** the block read from the database, and *pnBlock to the size of the read 293 -** block in bytes. 294 -** 295 -** WARNING: The returned buffer is only valid until the next call to 296 -** sqlite3Fts3ReadBlock(). 297 -*/ 298 -int sqlite3Fts3ReadBlock( 299 - Fts3Table *p, 300 - sqlite3_int64 iBlock, 301 - char const **pzBlock, 302 - int *pnBlock 303 -){ 304 - sqlite3_stmt *pStmt; 305 - int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0); 306 - if( rc!=SQLITE_OK ) return rc; 307 - sqlite3_reset(pStmt); 308 - 309 - if( pzBlock ){ 310 - sqlite3_bind_int64(pStmt, 1, iBlock); 311 - rc = sqlite3_step(pStmt); 312 - if( rc!=SQLITE_ROW ){ 313 - return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc); 314 - } 315 - 316 - *pnBlock = sqlite3_column_bytes(pStmt, 0); 317 - *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); 318 - if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ 319 - return SQLITE_CORRUPT; 320 - } 321 - } 322 - return SQLITE_OK; 323 -} 324 - 325 300 /* 326 301 ** This function ensures that the caller has obtained a shared-cache 327 302 ** table-lock on the %_content table. This is required before reading 328 303 ** data from the fts3 table. If this lock is not acquired first, then 329 304 ** the caller may end up holding read-locks on the %_segments and %_segdir 330 305 ** tables, but no read-lock on the %_content table. If this happens 331 306 ** a second connection will be able to write to the fts3 table, but ................................................................................ 486 461 ** Tokenize the nul-terminated string zText and add all tokens to the 487 462 ** pending-terms hash-table. The docid used is that currently stored in 488 463 ** p->iPrevDocid, and the column is specified by argument iCol. 489 464 ** 490 465 ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. 491 466 */ 492 467 static int fts3PendingTermsAdd( 493 - Fts3Table *p, /* FTS table into which text will be inserted */ 494 - const char *zText, /* Text of document to be inseted */ 495 - int iCol, /* Column number into which text is inserted */ 496 - u32 *pnWord /* OUT: Number of tokens inserted */ 468 + Fts3Table *p, /* Table into which text will be inserted */ 469 + const char *zText, /* Text of document to be inserted */ 470 + int iCol, /* Column into which text is being inserted */ 471 + u32 *pnWord /* OUT: Number of tokens inserted */ 497 472 ){ 498 473 int rc; 499 474 int iStart; 500 475 int iEnd; 501 476 int iPos; 502 477 int nWord = 0; 503 478 ................................................................................ 574 549 int rc = sqlite3Fts3PendingTermsFlush(p); 575 550 if( rc!=SQLITE_OK ) return rc; 576 551 } 577 552 p->iPrevDocid = iDocid; 578 553 return SQLITE_OK; 579 554 } 580 555 556 +/* 557 +** Discard the contents of the pending-terms hash table. 558 +*/ 581 559 void sqlite3Fts3PendingTermsClear(Fts3Table *p){ 582 560 Fts3HashElem *pElem; 583 561 for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ 584 562 sqlite3_free(fts3HashData(pElem)); 585 563 } 586 564 fts3HashClear(&p->pendingTerms); 587 565 p->nPendingData = 0; ................................................................................ 601 579 const char *zText = (const char *)sqlite3_value_text(apVal[i]); 602 580 if( zText ){ 603 581 int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); 604 582 if( rc!=SQLITE_OK ){ 605 583 return rc; 606 584 } 607 585 } 586 + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); 608 587 } 609 588 return SQLITE_OK; 610 589 } 611 590 612 591 /* 613 592 ** This function is called by the xUpdate() method for an INSERT operation. 614 593 ** The apVal parameter is passed a copy of the apVal argument passed by ................................................................................ 698 677 } 699 678 700 679 /* 701 680 ** The first element in the apVal[] array is assumed to contain the docid 702 681 ** (an integer) of a row about to be deleted. Remove all terms from the 703 682 ** full-text index. 704 683 */ 705 -static void fts3DeleteTerms( 684 +static void fts3DeleteTerms( 706 685 int *pRC, /* Result code */ 707 686 Fts3Table *p, /* The FTS table to delete from */ 708 687 sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ 709 688 u32 *aSz /* Sizes of deleted document written here */ 710 689 ){ 711 690 int rc; 712 691 sqlite3_stmt *pSelect; ................................................................................ 720 699 const char *zText = (const char *)sqlite3_column_text(pSelect, i); 721 700 rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); 722 701 if( rc!=SQLITE_OK ){ 723 702 sqlite3_reset(pSelect); 724 703 *pRC = rc; 725 704 return; 726 705 } 706 + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); 727 707 } 728 708 } 729 709 rc = sqlite3_reset(pSelect); 730 710 }else{ 731 711 sqlite3_reset(pSelect); 732 712 } 733 713 *pRC = rc; ................................................................................ 781 761 }else{ 782 762 *piIdx = iNext; 783 763 } 784 764 } 785 765 786 766 return rc; 787 767 } 768 + 769 +/* 770 +** The %_segments table is declared as follows: 771 +** 772 +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) 773 +** 774 +** This function reads data from a single row of the %_segments table. The 775 +** specific row is identified by the iBlockid parameter. If paBlob is not 776 +** NULL, then a buffer is allocated using sqlite3_malloc() and populated 777 +** with the contents of the blob stored in the "block" column of the 778 +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set 779 +** to the size of the blob in bytes before returning. 780 +** 781 +** If an error occurs, or the table does not contain the specified row, 782 +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If 783 +** paBlob is non-NULL, then it is the responsibility of the caller to 784 +** eventually free the returned buffer. 785 +** 786 +** This function may leave an open sqlite3_blob* handle in the 787 +** Fts3Table.pSegments variable. This handle is reused by subsequent calls 788 +** to this function. The handle may be closed by calling the 789 +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy 790 +** performance improvement, but the blob handle should always be closed 791 +** before control is returned to the user (to prevent a lock being held 792 +** on the database file for longer than necessary). Thus, any virtual table 793 +** method (xFilter etc.) that may directly or indirectly call this function 794 +** must call sqlite3Fts3SegmentsClose() before returning. 795 +*/ 796 +int sqlite3Fts3ReadBlock( 797 + Fts3Table *p, /* FTS3 table handle */ 798 + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ 799 + char **paBlob, /* OUT: Blob data in malloc'd buffer */ 800 + int *pnBlob /* OUT: Size of blob data */ 801 +){ 802 + int rc; /* Return code */ 803 + 804 + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ 805 + assert( pnBlob); 806 + 807 + if( p->pSegments ){ 808 + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); 809 + }else{ 810 + if( 0==p->zSegmentsTbl ){ 811 + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); 812 + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; 813 + } 814 + rc = sqlite3_blob_open( 815 + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments 816 + ); 817 + } 818 + 819 + if( rc==SQLITE_OK ){ 820 + int nByte = sqlite3_blob_bytes(p->pSegments); 821 + if( paBlob ){ 822 + char *aByte = sqlite3_malloc(nByte); 823 + if( !aByte ){ 824 + rc = SQLITE_NOMEM; 825 + }else{ 826 + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); 827 + if( rc!=SQLITE_OK ){ 828 + sqlite3_free(aByte); 829 + aByte = 0; 830 + } 831 + } 832 + *paBlob = aByte; 833 + } 834 + *pnBlob = nByte; 835 + } 836 + 837 + return rc; 838 +} 839 + 840 +/* 841 +** Close the blob handle at p->pSegments, if it is open. See comments above 842 +** the sqlite3Fts3ReadBlock() function for details. 843 +*/ 844 +void sqlite3Fts3SegmentsClose(Fts3Table *p){ 845 + sqlite3_blob_close(p->pSegments); 846 + p->pSegments = 0; 847 +} 788 848 789 849 /* 790 850 ** Move the iterator passed as the first argument to the next term in the 791 851 ** segment. If successful, SQLITE_OK is returned. If there is no next term, 792 852 ** SQLITE_DONE. Otherwise, an SQLite error code. 793 853 */ 794 -static int fts3SegReaderNext(Fts3SegReader *pReader){ 854 +static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ 795 855 char *pNext; /* Cursor variable */ 796 856 int nPrefix; /* Number of bytes in term prefix */ 797 857 int nSuffix; /* Number of bytes in term suffix */ 798 858 799 859 if( !pReader->aDoclist ){ 800 860 pNext = pReader->aNode; 801 861 }else{ 802 862 pNext = &pReader->aDoclist[pReader->nDoclist]; 803 863 } 804 864 805 865 if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ 806 - int rc; 866 + int rc; /* Return code from Fts3ReadBlock() */ 867 + 807 868 if( fts3SegReaderIsPending(pReader) ){ 808 869 Fts3HashElem *pElem = *(pReader->ppNextElem); 809 870 if( pElem==0 ){ 810 871 pReader->aNode = 0; 811 872 }else{ 812 873 PendingList *pList = (PendingList *)fts3HashData(pElem); 813 874 pReader->zTerm = (char *)fts3HashKey(pElem); ................................................................................ 815 876 pReader->nNode = pReader->nDoclist = pList->nData + 1; 816 877 pReader->aNode = pReader->aDoclist = pList->aData; 817 878 pReader->ppNextElem++; 818 879 assert( pReader->aNode ); 819 880 } 820 881 return SQLITE_OK; 821 882 } 822 - if( !pReader->pStmt ){ 823 - pReader->aNode = 0; 883 + 884 + if( !fts3SegReaderIsRootOnly(pReader) ){ 885 + sqlite3_free(pReader->aNode); 886 + } 887 + pReader->aNode = 0; 888 + 889 + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 890 + ** blocks have already been traversed. */ 891 + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); 892 + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ 824 893 return SQLITE_OK; 825 894 } 826 - rc = sqlite3_step(pReader->pStmt); 827 - if( rc!=SQLITE_ROW ){ 828 - pReader->aNode = 0; 829 - return (rc==SQLITE_DONE ? SQLITE_OK : rc); 830 - } 831 - pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0); 832 - pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0); 895 + 896 + rc = sqlite3Fts3ReadBlock( 897 + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode 898 + ); 899 + if( rc!=SQLITE_OK ) return rc; 833 900 pNext = pReader->aNode; 834 901 } 835 902 836 903 pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); 837 904 pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); 838 905 839 906 if( nPrefix+nSuffix>pReader->nTermAlloc ){ ................................................................................ 845 912 pReader->zTerm = zNew; 846 913 pReader->nTermAlloc = nNew; 847 914 } 848 915 memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); 849 916 pReader->nTerm = nPrefix+nSuffix; 850 917 pNext += nSuffix; 851 918 pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); 852 - assert( pNext<&pReader->aNode[pReader->nNode] ); 853 919 pReader->aDoclist = pNext; 854 920 pReader->pOffsetList = 0; 921 + 922 + /* Check that the doclist does not appear to extend past the end of the 923 + ** b-tree node. And that the final byte of the doclist is either an 0x00 924 + ** or 0x01. If either of these statements is untrue, then the data structure 925 + ** is corrupt. 926 + */ 927 + if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 928 + || (pReader->aDoclist[pReader->nDoclist-1]&0xFE)!=0 929 + ){ 930 + return SQLITE_CORRUPT; 931 + } 855 932 return SQLITE_OK; 856 933 } 857 934 858 935 /* 859 936 ** Set the SegReader to point to the first docid in the doclist associated 860 937 ** with the current term. 861 938 */ ................................................................................ 909 986 pReader->pOffsetList = 0; 910 987 }else{ 911 988 sqlite3_int64 iDelta; 912 989 pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); 913 990 pReader->iDocid += iDelta; 914 991 } 915 992 } 993 + 994 +/* 995 +** This function is called to estimate the amount of data that will be 996 +** loaded from the disk If SegReaderIterate() is called on this seg-reader, 997 +** in units of average document size. 998 +** 999 +** This can be used as follows: If the caller has a small doclist that 1000 +** contains references to N documents, and is considering merging it with 1001 +** a large doclist (size X "average documents"), it may opt not to load 1002 +** the large doclist if X>N. 1003 +*/ 1004 +int sqlite3Fts3SegReaderCost( 1005 + Fts3Cursor *pCsr, /* FTS3 cursor handle */ 1006 + Fts3SegReader *pReader, /* Segment-reader handle */ 1007 + int *pnCost /* IN/OUT: Number of bytes read */ 1008 +){ 1009 + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; 1010 + int rc = SQLITE_OK; /* Return code */ 1011 + int nCost = 0; /* Cost in bytes to return */ 1012 + int pgsz = p->nPgsz; /* Database page size */ 1013 + 1014 + /* If this seg-reader is reading the pending-terms table, or if all data 1015 + ** for the segment is stored on the root page of the b-tree, then the cost 1016 + ** is zero. In this case all required data is already in main memory. 1017 + */ 1018 + if( p->bHasDocsize 1019 + && !fts3SegReaderIsPending(pReader) 1020 + && !fts3SegReaderIsRootOnly(pReader) 1021 + ){ 1022 + int nBlob = 0; 1023 + sqlite3_int64 iBlock; 1024 + 1025 + if( pCsr->nRowAvg==0 ){ 1026 + /* The average document size, which is required to calculate the cost 1027 + ** of each doclist, has not yet been determined. Read the required 1028 + ** data from the %_stat table to calculate it. 1029 + ** 1030 + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 1031 + ** varints, where nCol is the number of columns in the FTS3 table. 1032 + ** The first varint is the number of documents currently stored in 1033 + ** the table. The following nCol varints contain the total amount of 1034 + ** data stored in all rows of each column of the table, from left 1035 + ** to right. 1036 + */ 1037 + sqlite3_stmt *pStmt; 1038 + rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); 1039 + if( rc ) return rc; 1040 + if( sqlite3_step(pStmt)==SQLITE_ROW ){ 1041 + sqlite3_int64 nDoc = 0; 1042 + sqlite3_int64 nByte = 0; 1043 + const char *a = sqlite3_column_blob(pStmt, 0); 1044 + if( a ){ 1045 + const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; 1046 + a += sqlite3Fts3GetVarint(a, &nDoc); 1047 + while( a<pEnd ){ 1048 + a += sqlite3Fts3GetVarint(a, &nByte); 1049 + } 1050 + } 1051 + 1052 + pCsr->nRowAvg = (((nByte / nDoc) + pgsz - 1) / pgsz); 1053 + } 1054 + rc = sqlite3_reset(pStmt); 1055 + if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc; 1056 + } 1057 + 1058 + /* Assume that a blob flows over onto overflow pages if it is larger 1059 + ** than (pgsz-35) bytes in size (the file-format documentation 1060 + ** confirms this). 1061 + */ 1062 + for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ 1063 + rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); 1064 + if( rc!=SQLITE_OK ) break; 1065 + if( (nBlob+35)>pgsz ){ 1066 + int nOvfl = (nBlob + 34)/pgsz; 1067 + nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); 1068 + } 1069 + } 1070 + } 1071 + 1072 + *pnCost += nCost; 1073 + return rc; 1074 +} 916 1075 917 1076 /* 918 1077 ** Free all allocations associated with the iterator passed as the 919 1078 ** second argument. 920 1079 */ 921 1080 void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){ 922 - if( pReader ){ 923 - if( pReader->pStmt ){ 924 - /* Move the leaf-range SELECT statement to the aLeavesStmt[] array, 925 - ** so that it can be reused when required by another query. 926 - */ 927 - assert( p->nLeavesStmt<p->nLeavesTotal ); 928 - sqlite3_reset(pReader->pStmt); 929 - p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt; 1081 + if( pReader && !fts3SegReaderIsPending(pReader) ){ 1082 + sqlite3_free(pReader->zTerm); 1083 + if( !fts3SegReaderIsRootOnly(pReader) ){ 1084 + sqlite3_free(pReader->aNode); 930 1085 } 931 - if( !fts3SegReaderIsPending(pReader) ){ 932 - sqlite3_free(pReader->zTerm); 933 - } 934 - sqlite3_free(pReader); 935 1086 } 1087 + sqlite3_free(pReader); 936 1088 } 937 1089 938 1090 /* 939 1091 ** Allocate a new SegReader object. 940 1092 */ 941 1093 int sqlite3Fts3SegReaderNew( 942 1094 Fts3Table *p, /* Virtual table handle */ ................................................................................ 948 1100 int nRoot, /* Size of buffer containing root node */ 949 1101 Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ 950 1102 ){ 951 1103 int rc = SQLITE_OK; /* Return code */ 952 1104 Fts3SegReader *pReader; /* Newly allocated SegReader object */ 953 1105 int nExtra = 0; /* Bytes to allocate segment root node */ 954 1106 1107 + assert( iStartLeaf<=iEndLeaf ); 955 1108 if( iStartLeaf==0 ){ 956 1109 nExtra = nRoot; 957 1110 } 958 1111 959 1112 pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); 960 1113 if( !pReader ){ 961 1114 return SQLITE_NOMEM; 962 1115 } 963 1116 memset(pReader, 0, sizeof(Fts3SegReader)); 964 - pReader->iStartBlock = iStartLeaf; 965 1117 pReader->iIdx = iAge; 1118 + pReader->iStartBlock = iStartLeaf; 1119 + pReader->iLeafEndBlock = iEndLeaf; 966 1120 pReader->iEndBlock = iEndBlock; 967 1121 968 1122 if( nExtra ){ 969 1123 /* The entire segment is stored in the root node. */ 970 1124 pReader->aNode = (char *)&pReader[1]; 971 1125 pReader->nNode = nRoot; 972 1126 memcpy(pReader->aNode, zRoot, nRoot); 973 1127 }else{ 974 - /* If the text of the SQL statement to iterate through a contiguous 975 - ** set of entries in the %_segments table has not yet been composed, 976 - ** compose it now. 977 - */ 978 - if( !p->zSelectLeaves ){ 979 - p->zSelectLeaves = sqlite3_mprintf( 980 - "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? " 981 - "ORDER BY blockid", p->zDb, p->zName 982 - ); 983 - if( !p->zSelectLeaves ){ 984 - rc = SQLITE_NOMEM; 985 - goto finished; 986 - } 987 - } 1128 + pReader->iCurrentBlock = iStartLeaf-1; 1129 + } 1130 + rc = fts3SegReaderNext(p, pReader); 988 1131 989 - /* If there are no free statements in the aLeavesStmt[] array, prepare 990 - ** a new statement now. Otherwise, reuse a prepared statement from 991 - ** aLeavesStmt[]. 992 - */ 993 - if( p->nLeavesStmt==0 ){ 994 - if( p->nLeavesTotal==p->nLeavesAlloc ){ 995 - int nNew = p->nLeavesAlloc + 16; 996 - sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc( 997 - p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *) 998 - ); 999 - if( !aNew ){ 1000 - rc = SQLITE_NOMEM; 1001 - goto finished; 1002 - } 1003 - p->nLeavesAlloc = nNew; 1004 - p->aLeavesStmt = aNew; 1005 - } 1006 - rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0); 1007 - if( rc!=SQLITE_OK ){ 1008 - goto finished; 1009 - } 1010 - p->nLeavesTotal++; 1011 - }else{ 1012 - pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt]; 1013 - } 1014 - 1015 - /* Bind the start and end leaf blockids to the prepared SQL statement. */ 1016 - sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf); 1017 - sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf); 1018 - } 1019 - rc = fts3SegReaderNext(pReader); 1020 - 1021 - finished: 1022 1132 if( rc==SQLITE_OK ){ 1023 1133 *ppReader = pReader; 1024 1134 }else{ 1025 1135 sqlite3Fts3SegReaderFree(p, pReader); 1026 1136 } 1027 1137 return rc; 1028 1138 } ................................................................................ 1109 1219 if( !pReader ){ 1110 1220 rc = SQLITE_NOMEM; 1111 1221 }else{ 1112 1222 memset(pReader, 0, nByte); 1113 1223 pReader->iIdx = 0x7FFFFFFF; 1114 1224 pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; 1115 1225 memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); 1116 - fts3SegReaderNext(pReader); 1226 + fts3SegReaderNext(p, pReader); 1117 1227 } 1118 1228 } 1119 1229 1120 1230 if( isPrefix ){ 1121 1231 sqlite3_free(aElem); 1122 1232 } 1123 1233 *ppReader = pReader; ................................................................................ 1351 1461 } 1352 1462 1353 1463 /* 1354 1464 ** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger 1355 1465 ** (according to memcmp) than the previous term. 1356 1466 */ 1357 1467 static int fts3NodeAddTerm( 1358 - Fts3Table *p, /* Virtual table handle */ 1468 + Fts3Table *p, /* Virtual table handle */ 1359 1469 SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ 1360 1470 int isCopyTerm, /* True if zTerm/nTerm is transient */ 1361 1471 const char *zTerm, /* Pointer to buffer containing term */ 1362 1472 int nTerm /* Size of term in bytes */ 1363 1473 ){ 1364 1474 SegmentNode *pTree = *ppTree; 1365 1475 int rc; ................................................................................ 1987 2097 */ 1988 2098 if( pFilter->zTerm ){ 1989 2099 int nTerm = pFilter->nTerm; 1990 2100 const char *zTerm = pFilter->zTerm; 1991 2101 for(i=0; i<nSegment; i++){ 1992 2102 Fts3SegReader *pSeg = apSegment[i]; 1993 2103 while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){ 1994 - rc = fts3SegReaderNext(pSeg); 2104 + rc = fts3SegReaderNext(p, pSeg); 1995 2105 if( rc!=SQLITE_OK ) goto finished; } 1996 2106 } 1997 2107 } 1998 2108 1999 2109 fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp); 2000 2110 while( apSegment[0]->aNode ){ 2001 2111 int nTerm = apSegment[0]->nTerm; ................................................................................ 2098 2208 ** term (if such a term exists in the index) has already been made. 2099 2209 */ 2100 2210 if( pFilter->zTerm && !isPrefix ){ 2101 2211 goto finished; 2102 2212 } 2103 2213 2104 2214 for(i=0; i<nMerge; i++){ 2105 - rc = fts3SegReaderNext(apSegment[i]); 2215 + rc = fts3SegReaderNext(p, apSegment[i]); 2106 2216 if( rc!=SQLITE_OK ) goto finished; 2107 2217 } 2108 2218 fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp); 2109 2219 } 2110 2220 2111 2221 finished: 2112 2222 sqlite3_free(aBuffer); ................................................................................ 2124 2234 ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 2125 2235 ** an SQLite error code is returned. 2126 2236 */ 2127 2237 static int fts3SegmentMerge(Fts3Table *p, int iLevel){ 2128 2238 int i; /* Iterator variable */ 2129 2239 int rc; /* Return code */ 2130 2240 int iIdx; /* Index of new segment */ 2131 - int iNewLevel; /* Level to create new segment at */ 2241 + int iNewLevel = 0; /* Level to create new segment at */ 2132 2242 sqlite3_stmt *pStmt = 0; 2133 2243 SegmentWriter *pWriter = 0; 2134 2244 int nSegment = 0; /* Number of segments being merged */ 2135 2245 Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ 2136 2246 Fts3SegReader *pPending = 0; /* Iterator for pending-terms */ 2137 2247 Fts3SegFilter filter; /* Segment term filter condition */ 2138 2248 ................................................................................ 2413 2523 sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); 2414 2524 sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); 2415 2525 sqlite3_step(pStmt); 2416 2526 *pRC = sqlite3_reset(pStmt); 2417 2527 } 2418 2528 2419 2529 /* 2420 -** Update the 0 record of the %_stat table so that it holds a blob 2421 -** which contains the document count followed by the cumulative 2422 -** document sizes for all columns. 2530 +** Record 0 of the %_stat table contains a blob consisting of N varints, 2531 +** where N is the number of user defined columns in the fts3 table plus 2532 +** two. If nCol is the number of user defined columns, then values of the 2533 +** varints are set as follows: 2534 +** 2535 +** Varint 0: Total number of rows in the table. 2536 +** 2537 +** Varint 1..nCol: For each column, the total number of tokens stored in 2538 +** the column for all rows of the table. 2539 +** 2540 +** Varint 1+nCol: The total size, in bytes, of all text values in all 2541 +** columns of all rows of the table. 2542 +** 2423 2543 */ 2424 2544 static void fts3UpdateDocTotals( 2425 - int *pRC, /* The result code */ 2426 - Fts3Table *p, /* Table being updated */ 2427 - u32 *aSzIns, /* Size increases */ 2428 - u32 *aSzDel, /* Size decreases */ 2429 - int nChng /* Change in the number of documents */ 2545 + int *pRC, /* The result code */ 2546 + Fts3Table *p, /* Table being updated */ 2547 + u32 *aSzIns, /* Size increases */ 2548 + u32 *aSzDel, /* Size decreases */ 2549 + int nChng /* Change in the number of documents */ 2430 2550 ){ 2431 2551 char *pBlob; /* Storage for BLOB written into %_stat */ 2432 2552 int nBlob; /* Size of BLOB written into %_stat */ 2433 2553 u32 *a; /* Array of integers that becomes the BLOB */ 2434 2554 sqlite3_stmt *pStmt; /* Statement for reading and writing */ 2435 2555 int i; /* Loop counter */ 2436 2556 int rc; /* Result code from subfunctions */ 2437 2557 2558 + const int nStat = p->nColumn+2; 2559 + 2438 2560 if( *pRC ) return; 2439 - a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) ); 2561 + a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); 2440 2562 if( a==0 ){ 2441 2563 *pRC = SQLITE_NOMEM; 2442 2564 return; 2443 2565 } 2444 - pBlob = (char*)&a[p->nColumn+1]; 2566 + pBlob = (char*)&a[nStat]; 2445 2567 rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); 2446 2568 if( rc ){ 2447 2569 sqlite3_free(a); 2448 2570 *pRC = rc; 2449 2571 return; 2450 2572 } 2451 2573 if( sqlite3_step(pStmt)==SQLITE_ROW ){ 2452 - fts3DecodeIntArray(p->nColumn+1, a, 2574 + fts3DecodeIntArray(nStat, a, 2453 2575 sqlite3_column_blob(pStmt, 0), 2454 2576 sqlite3_column_bytes(pStmt, 0)); 2455 2577 }else{ 2456 - memset(a, 0, sizeof(u32)*(p->nColumn+1) ); 2578 + memset(a, 0, sizeof(u32)*(nStat) ); 2457 2579 } 2458 2580 sqlite3_reset(pStmt); 2459 2581 if( nChng<0 && a[0]<(u32)(-nChng) ){ 2460 2582 a[0] = 0; 2461 2583 }else{ 2462 2584 a[0] += nChng; 2463 2585 } 2464 - for(i=0; i<p->nColumn; i++){ 2586 + for(i=0; i<p->nColumn+1; i++){ 2465 2587 u32 x = a[i+1]; 2466 2588 if( x+aSzIns[i] < aSzDel[i] ){ 2467 2589 x = 0; 2468 2590 }else{ 2469 2591 x = x + aSzIns[i] - aSzDel[i]; 2470 2592 } 2471 2593 a[i+1] = x; 2472 2594 } 2473 - fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob); 2595 + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); 2474 2596 rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); 2475 2597 if( rc ){ 2476 2598 sqlite3_free(a); 2477 2599 *pRC = rc; 2478 2600 return; 2479 2601 } 2480 2602 sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); ................................................................................ 2512 2634 }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ 2513 2635 p->nMaxPendingData = atoi(&zVal[11]); 2514 2636 rc = SQLITE_OK; 2515 2637 #endif 2516 2638 }else{ 2517 2639 rc = SQLITE_ERROR; 2518 2640 } 2641 + 2642 + sqlite3Fts3SegmentsClose(p); 2643 + return rc; 2644 +} 2645 + 2646 +/* 2647 +** Return the deferred doclist associated with deferred token pDeferred. 2648 +** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already 2649 +** been called to allocate and populate the doclist. 2650 +*/ 2651 +char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ 2652 + if( pDeferred->pList ){ 2653 + *pnByte = pDeferred->pList->nData; 2654 + return pDeferred->pList->aData; 2655 + } 2656 + *pnByte = 0; 2657 + return 0; 2658 +} 2659 + 2660 +/* 2661 +** Helper fucntion for FreeDeferredDoclists(). This function removes all 2662 +** references to deferred doclists from within the tree of Fts3Expr 2663 +** structures headed by 2664 +*/ 2665 +static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ 2666 + if( pExpr ){ 2667 + fts3DeferredDoclistClear(pExpr->pLeft); 2668 + fts3DeferredDoclistClear(pExpr->pRight); 2669 + if( pExpr->isLoaded ){ 2670 + sqlite3_free(pExpr->aDoclist); 2671 + pExpr->isLoaded = 0; 2672 + pExpr->aDoclist = 0; 2673 + pExpr->nDoclist = 0; 2674 + pExpr->pCurrent = 0; 2675 + pExpr->iCurrent = 0; 2676 + } 2677 + } 2678 +} 2679 + 2680 +/* 2681 +** Delete all cached deferred doclists. Deferred doclists are cached 2682 +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. 2683 +*/ 2684 +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ 2685 + Fts3DeferredToken *pDef; 2686 + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ 2687 + sqlite3_free(pDef->pList); 2688 + pDef->pList = 0; 2689 + } 2690 + if( pCsr->pDeferred ){ 2691 + fts3DeferredDoclistClear(pCsr->pExpr); 2692 + } 2693 +} 2694 + 2695 +/* 2696 +** Free all entries in the pCsr->pDeffered list. Entries are added to 2697 +** this list using sqlite3Fts3DeferToken(). 2698 +*/ 2699 +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ 2700 + Fts3DeferredToken *pDef; 2701 + Fts3DeferredToken *pNext; 2702 + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ 2703 + pNext = pDef->pNext; 2704 + sqlite3_free(pDef->pList); 2705 + sqlite3_free(pDef); 2706 + } 2707 + pCsr->pDeferred = 0; 2708 +} 2709 + 2710 +/* 2711 +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list 2712 +** based on the row that pCsr currently points to. 2713 +** 2714 +** A deferred-doclist is like any other doclist with position information 2715 +** included, except that it only contains entries for a single row of the 2716 +** table, not for all rows. 2717 +*/ 2718 +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ 2719 + int rc = SQLITE_OK; /* Return code */ 2720 + if( pCsr->pDeferred ){ 2721 + int i; /* Used to iterate through table columns */ 2722 + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ 2723 + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ 2724 + 2725 + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; 2726 + sqlite3_tokenizer *pT = p->pTokenizer; 2727 + sqlite3_tokenizer_module const *pModule = pT->pModule; 2728 + 2729 + assert( pCsr->isRequireSeek==0 ); 2730 + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); 2731 + 2732 + for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){ 2733 + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); 2734 + sqlite3_tokenizer_cursor *pTC = 0; 2735 + 2736 + rc = pModule->xOpen(pT, zText, -1, &pTC); 2737 + while( rc==SQLITE_OK ){ 2738 + char const *zToken; /* Buffer containing token */ 2739 + int nToken; /* Number of bytes in token */ 2740 + int iDum1, iDum2; /* Dummy variables */ 2741 + int iPos; /* Position of token in zText */ 2742 + 2743 + pTC->pTokenizer = pT; 2744 + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); 2745 + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ 2746 + Fts3PhraseToken *pPT = pDef->pToken; 2747 + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) 2748 + && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken)) 2749 + && (0==memcmp(zToken, pPT->z, pPT->n)) 2750 + ){ 2751 + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); 2752 + } 2753 + } 2754 + } 2755 + if( pTC ) pModule->xClose(pTC); 2756 + if( rc==SQLITE_DONE ) rc = SQLITE_OK; 2757 + } 2758 + 2759 + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ 2760 + if( pDef->pList ){ 2761 + rc = fts3PendingListAppendVarint(&pDef->pList, 0); 2762 + } 2763 + } 2764 + } 2519 2765 2520 2766 return rc; 2521 2767 } 2768 + 2769 +/* 2770 +** Add an entry for token pToken to the pCsr->pDeferred list. 2771 +*/ 2772 +int sqlite3Fts3DeferToken( 2773 + Fts3Cursor *pCsr, /* Fts3 table cursor */ 2774 + Fts3PhraseToken *pToken, /* Token to defer */ 2775 + int iCol /* Column that token must appear in (or -1) */ 2776 +){ 2777 + Fts3DeferredToken *pDeferred; 2778 + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); 2779 + if( !pDeferred ){ 2780 + return SQLITE_NOMEM; 2781 + } 2782 + memset(pDeferred, 0, sizeof(*pDeferred)); 2783 + pDeferred->pToken = pToken; 2784 + pDeferred->pNext = pCsr->pDeferred; 2785 + pDeferred->iCol = iCol; 2786 + pCsr->pDeferred = pDeferred; 2787 + 2788 + assert( pToken->pDeferred==0 ); 2789 + pToken->pDeferred = pDeferred; 2790 + 2791 + return SQLITE_OK; 2792 +} 2793 + 2522 2794 2523 2795 /* 2524 2796 ** This function does the work for the xUpdate method of FTS3 virtual 2525 2797 ** tables. 2526 2798 */ 2527 2799 int sqlite3Fts3UpdateMethod( 2528 2800 sqlite3_vtab *pVtab, /* FTS3 vtab object */ ................................................................................ 2534 2806 int rc = SQLITE_OK; /* Return Code */ 2535 2807 int isRemove = 0; /* True for an UPDATE or DELETE */ 2536 2808 sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ 2537 2809 u32 *aSzIns; /* Sizes of inserted documents */ 2538 2810 u32 *aSzDel; /* Sizes of deleted documents */ 2539 2811 int nChng = 0; /* Net change in number of documents */ 2540 2812 2813 + assert( p->pSegments==0 ); 2541 2814 2542 2815 /* Allocate space to hold the change in document sizes */ 2543 - aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 ); 2816 + aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); 2544 2817 if( aSzIns==0 ) return SQLITE_NOMEM; 2545 - aSzDel = &aSzIns[p->nColumn]; 2546 - memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2); 2818 + aSzDel = &aSzIns[p->nColumn+1]; 2819 + memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); 2547 2820 2548 2821 /* If this is a DELETE or UPDATE operation, remove the old record. */ 2549 2822 if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ 2550 - int isEmpty; 2823 + int isEmpty = 0; 2551 2824 rc = fts3IsEmpty(p, apVal, &isEmpty); 2552 2825 if( rc==SQLITE_OK ){ 2553 2826 if( isEmpty ){ 2554 2827 /* Deleting this row means the whole table is empty. In this case 2555 2828 ** delete the contents of all three tables and throw away any 2556 2829 ** data in the pendingTerms hash table. 2557 2830 */ ................................................................................ 2589 2862 } 2590 2863 2591 2864 if( p->bHasDocsize ){ 2592 2865 fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); 2593 2866 } 2594 2867 2595 2868 sqlite3_free(aSzIns); 2869 + sqlite3Fts3SegmentsClose(p); 2596 2870 return rc; 2597 2871 } 2598 2872 2599 2873 /* 2600 2874 ** Flush any data in the pending-terms hash table to disk. If successful, 2601 2875 ** merge all segments in the database (including the new segment, if 2602 2876 ** there was any data to flush) into a single segment. ................................................................................ 2612 2886 sqlite3Fts3PendingTermsClear(p); 2613 2887 } 2614 2888 }else{ 2615 2889 sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); 2616 2890 sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); 2617 2891 } 2618 2892 } 2893 + sqlite3Fts3SegmentsClose(p); 2619 2894 return rc; 2620 2895 } 2621 2896 2622 2897 #endif
Added ext/fts3/fts3speed.tcl.
1 + 2 + 3 +#-------------------------------------------------------------------------- 4 +# This script contains several sub-programs used to test FTS3/FTS4 5 +# performance. It does not run the queries directly, but generates SQL 6 +# scripts that can be run using the shell tool. 7 +# 8 +# The following cases are tested: 9 +# 10 +# 1. Inserting documents into an FTS3 table. 11 +# 2. Optimizing an FTS3 table (i.e. "INSERT INTO t1 VALUES('optimize')"). 12 +# 3. Deleting documents from an FTS3 table. 13 +# 4. Querying FTS3 tables. 14 +# 15 + 16 +# Number of tokens in vocabulary. And number of tokens in each document. 17 +# 18 +set VOCAB_SIZE 2000 19 +set DOC_SIZE 100 20 + 21 +set NUM_INSERTS 100000 22 +set NUM_SELECTS 1000 23 + 24 +# Force everything in this script to be deterministic. 25 +# 26 +expr {srand(0)} 27 + 28 +proc usage {} { 29 + puts stderr "Usage: $::argv0 <rows> <selects>" 30 + exit -1 31 +} 32 + 33 +proc sql {sql} { 34 + puts $::fd $sql 35 +} 36 + 37 + 38 +# Return a list of $nWord randomly generated tokens each between 2 and 10 39 +# characters in length. 40 +# 41 +proc build_vocab {nWord} { 42 + set ret [list] 43 + set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z] 44 + for {set i 0} {$i<$nWord} {incr i} { 45 + set len [expr {int((rand()*9.0)+2)}] 46 + set term "" 47 + for {set j 0} {$j<$len} {incr j} { 48 + append term [lindex $chars [expr {int(rand()*[llength $chars])}]] 49 + } 50 + lappend ret $term 51 + } 52 + set ret 53 +} 54 + 55 +proc select_term {} { 56 + set n [llength $::vocab] 57 + set t [expr int(rand()*$n*3)] 58 + if {$t>=2*$n} { set t [expr {($t-2*$n)/100}] } 59 + if {$t>=$n} { set t [expr {($t-$n)/10}] } 60 + lindex $::vocab $t 61 +} 62 + 63 +proc select_doc {nTerm} { 64 + set ret [list] 65 + for {set i 0} {$i<$nTerm} {incr i} { 66 + lappend ret [select_term] 67 + } 68 + set ret 69 +} 70 + 71 +proc test_1 {nInsert} { 72 + sql "PRAGMA synchronous = OFF;" 73 + sql "DROP TABLE IF EXISTS t1;" 74 + sql "CREATE VIRTUAL TABLE t1 USING fts4;" 75 + for {set i 0} {$i < $nInsert} {incr i} { 76 + set doc [select_doc $::DOC_SIZE] 77 + sql "INSERT INTO t1 VALUES('$doc');" 78 + } 79 +} 80 + 81 +proc test_2 {} { 82 + sql "INSERT INTO t1(t1) VALUES('optimize');" 83 +} 84 + 85 +proc test_3 {nSelect} { 86 + for {set i 0} {$i < $nSelect} {incr i} { 87 + sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term]';" 88 + } 89 +} 90 + 91 +proc test_4 {nSelect} { 92 + for {set i 0} {$i < $nSelect} {incr i} { 93 + sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term] [select_term]';" 94 + } 95 +} 96 + 97 +if {[llength $argv]!=0} usage 98 + 99 +set ::vocab [build_vocab $::VOCAB_SIZE] 100 + 101 +set ::fd [open fts3speed_insert.sql w] 102 +test_1 $NUM_INSERTS 103 +close $::fd 104 + 105 +set ::fd [open fts3speed_select.sql w] 106 +test_3 $NUM_SELECTS 107 +close $::fd 108 + 109 +set ::fd [open fts3speed_select2.sql w] 110 +test_4 $NUM_SELECTS 111 +close $::fd 112 + 113 +set ::fd [open fts3speed_optimize.sql w] 114 +test_2 115 +close $::fd 116 + 117 +puts "Success. Created files:" 118 +puts " fts3speed_insert.sql" 119 +puts " fts3speed_select.sql" 120 +puts " fts3speed_select2.sql" 121 +puts " fts3speed_optimize.sql" 122 +
Changes to src/btree.c.
8092 8092 ** (stored in BtCursor.aOverflow[]) is allocated and used by function 8093 8093 ** accessPayload() (the worker function for sqlite3BtreeData() and 8094 8094 ** sqlite3BtreePutData()). 8095 8095 */ 8096 8096 void sqlite3BtreeCacheOverflow(BtCursor *pCur){ 8097 8097 assert( cursorHoldsMutex(pCur) ); 8098 8098 assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); 8099 - assert(!pCur->isIncrblobHandle); 8100 - assert(!pCur->aOverflow); 8099 + invalidateOverflowCache(pCur); 8101 8100 pCur->isIncrblobHandle = 1; 8102 8101 } 8103 8102 #endif 8104 8103 8105 8104 /* 8106 8105 ** Set both the "read version" (single byte at byte offset 18) and 8107 8106 ** "write version" (single byte at byte offset 19) fields in the database
Changes to src/sqlite.h.in.
4786 4786 const char *zTable, 4787 4787 const char *zColumn, 4788 4788 sqlite3_int64 iRow, 4789 4789 int flags, 4790 4790 sqlite3_blob **ppBlob 4791 4791 ); 4792 4792 4793 +/* 4794 +** CAPI3REF: Move a BLOB Handle to a New Row 4795 +** 4796 +** This function is used to move an existing blob handle so that it points 4797 +** to a different row of the same database table. The new row is identified 4798 +** by the rowid value passed as the second argument. Only the row can be 4799 +** changed, the database, table and column on which the blob handle is open 4800 +** remain the same. Moving an existing blob handle to a new row can be 4801 +** faster than closing the existing handle and opening a new one. 4802 +** 4803 +** The new row must meet the same criteria as for [sqlite3_blob_open()] - 4804 +** it must exist and there must be either a blob or text value stored in 4805 +** the nominated column. If the new row is not present in the table, or if 4806 +** it does not contain a blob or text value, or if another error occurs, an 4807 +** SQLite error code is returned and the blob handle is considered aborted. 4808 +** All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or 4809 +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return 4810 +** SQLITE_ABORT. 4811 +** 4812 +** This function sets the database handle error code and message. 4813 +*/ 4814 +SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); 4815 + 4793 4816 /* 4794 4817 ** CAPI3REF: Close A BLOB Handle 4795 4818 ** 4796 4819 ** ^Closes an open [BLOB handle]. 4797 4820 ** 4798 4821 ** ^Closing a BLOB shall cause the current transaction to commit 4799 4822 ** if there are no other BLOBs, no pending prepared statements, and the
Changes to src/test1.c.
1585 1585 Tcl_SetObjResult(interp, pRet); 1586 1586 1587 1587 return TCL_OK; 1588 1588 } 1589 1589 #endif 1590 1590 1591 1591 #ifndef SQLITE_OMIT_INCRBLOB 1592 + 1593 +static int blobHandleFromObj( 1594 + Tcl_Interp *interp, 1595 + Tcl_Obj *pObj, 1596 + sqlite3_blob **ppBlob 1597 +){ 1598 + char *z; 1599 + int n; 1600 + 1601 + z = Tcl_GetStringFromObj(pObj, &n); 1602 + if( n==0 ){ 1603 + *ppBlob = 0; 1604 + }else{ 1605 + int notUsed; 1606 + Tcl_Channel channel; 1607 + ClientData instanceData; 1608 + 1609 + channel = Tcl_GetChannel(interp, z, ¬Used); 1610 + if( !channel ) return TCL_ERROR; 1611 + 1612 + Tcl_Flush(channel); 1613 + Tcl_Seek(channel, 0, SEEK_SET); 1614 + 1615 + instanceData = Tcl_GetChannelInstanceData(channel); 1616 + *ppBlob = *((sqlite3_blob **)instanceData); 1617 + } 1618 + 1619 + return TCL_OK; 1620 +} 1621 + 1622 +/* 1623 +** sqlite3_blob_bytes CHANNEL 1624 +*/ 1625 +static int test_blob_bytes( 1626 + ClientData clientData, /* Not used */ 1627 + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ 1628 + int objc, /* Number of arguments */ 1629 + Tcl_Obj *CONST objv[] /* Command arguments */ 1630 +){ 1631 + sqlite3_blob *pBlob; 1632 + int nByte; 1633 + 1634 + if( objc!=2 ){ 1635 + Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); 1636 + return TCL_ERROR; 1637 + } 1638 + 1639 + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; 1640 + nByte = sqlite3_blob_bytes(pBlob); 1641 + Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); 1642 + 1643 + return TCL_OK; 1644 +} 1645 + 1646 +/* 1647 +** sqlite3_blob_close CHANNEL 1648 +*/ 1649 +static int test_blob_close( 1650 + ClientData clientData, /* Not used */ 1651 + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ 1652 + int objc, /* Number of arguments */ 1653 + Tcl_Obj *CONST objv[] /* Command arguments */ 1654 +){ 1655 + sqlite3_blob *pBlob; 1656 + int nByte; 1657 + 1658 + if( objc!=2 ){ 1659 + Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); 1660 + return TCL_ERROR; 1661 + } 1662 + 1663 + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; 1664 + sqlite3_blob_close(pBlob); 1665 + 1666 + return TCL_OK; 1667 +} 1592 1668 1593 1669 /* 1594 1670 ** sqlite3_blob_read CHANNEL OFFSET N 1595 1671 ** 1596 1672 ** This command is used to test the sqlite3_blob_read() in ways that 1597 1673 ** the Tcl channel interface does not. The first argument should 1598 1674 ** be the name of a valid channel created by the [incrblob] method ................................................................................ 1607 1683 */ 1608 1684 static int test_blob_read( 1609 1685 ClientData clientData, /* Not used */ 1610 1686 Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ 1611 1687 int objc, /* Number of arguments */ 1612 1688 Tcl_Obj *CONST objv[] /* Command arguments */ 1613 1689 ){ 1614 - Tcl_Channel channel; 1615 - ClientData instanceData; 1616 1690 sqlite3_blob *pBlob; 1617 - int notUsed; 1618 1691 int nByte; 1619 1692 int iOffset; 1620 1693 unsigned char *zBuf; 1621 1694 int rc; 1622 1695 1623 1696 if( objc!=4 ){ 1624 1697 Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); 1625 1698 return TCL_ERROR; 1626 1699 } 1627 1700 1628 - channel = Tcl_GetChannel(interp, Tcl_GetString(objv[1]), ¬Used); 1629 - if( !channel 1630 - || TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) 1701 + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; 1702 + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) 1631 1703 || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) 1632 - || nByte<0 || iOffset<0 1633 1704 ){ 1634 1705 return TCL_ERROR; 1635 1706 } 1636 1707 1637 - instanceData = Tcl_GetChannelInstanceData(channel); 1638 - pBlob = *((sqlite3_blob **)instanceData); 1639 - 1640 1708 zBuf = (unsigned char *)Tcl_Alloc(nByte); 1641 1709 rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); 1642 1710 if( rc==SQLITE_OK ){ 1643 1711 Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); 1644 1712 }else{ 1645 1713 Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); 1646 1714 } ................................................................................ 1665 1733 */ 1666 1734 static int test_blob_write( 1667 1735 ClientData clientData, /* Not used */ 1668 1736 Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ 1669 1737 int objc, /* Number of arguments */ 1670 1738 Tcl_Obj *CONST objv[] /* Command arguments */ 1671 1739 ){ 1672 - Tcl_Channel channel; 1673 - ClientData instanceData; 1674 1740 sqlite3_blob *pBlob; 1675 - int notUsed; 1676 1741 int iOffset; 1677 1742 int rc; 1678 1743 1679 1744 unsigned char *zBuf; 1680 1745 int nBuf; 1681 1746 1682 1747 if( objc!=4 && objc!=5 ){ 1683 1748 Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET DATA ?NDATA?"); 1684 1749 return TCL_ERROR; 1685 1750 } 1686 1751 1687 - channel = Tcl_GetChannel(interp, Tcl_GetString(objv[1]), ¬Used); 1688 - if( !channel || TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ 1752 + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; 1753 + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ 1689 1754 return TCL_ERROR; 1690 1755 } 1691 1756 1692 - instanceData = Tcl_GetChannelInstanceData(channel); 1693 - pBlob = *((sqlite3_blob **)instanceData); 1694 - 1695 1757 zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); 1696 1758 if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ 1697 1759 return TCL_ERROR; 1698 1760 } 1699 1761 rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); 1700 1762 if( rc!=SQLITE_OK ){ 1701 1763 Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); 1702 1764 } 1703 1765 1704 1766 return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); 1705 1767 } 1768 + 1769 +static int test_blob_reopen( 1770 + ClientData clientData, /* Not used */ 1771 + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ 1772 + int objc, /* Number of arguments */ 1773 + Tcl_Obj *CONST objv[] /* Command arguments */ 1774 +){ 1775 + Tcl_WideInt iRowid; 1776 + sqlite3_blob *pBlob; 1777 + int rc; 1778 + 1779 + unsigned char *zBuf; 1780 + int nBuf; 1781 + 1782 + if( objc!=3 ){ 1783 + Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL ROWID"); 1784 + return TCL_ERROR; 1785 + } 1786 + 1787 + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; 1788 + if( Tcl_GetWideIntFromObj(interp, objv[2], &iRowid) ) return TCL_ERROR; 1789 + 1790 + rc = sqlite3_blob_reopen(pBlob, iRowid); 1791 + if( rc!=SQLITE_OK ){ 1792 + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); 1793 + } 1794 + 1795 + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); 1796 +} 1797 + 1706 1798 #endif 1707 1799 1708 1800 /* 1709 1801 ** Usage: sqlite3_create_collation_v2 DB-HANDLE NAME CMP-PROC DEL-PROC 1710 1802 ** 1711 1803 ** This Tcl proc is used for testing the experimental 1712 1804 ** sqlite3_create_collation_v2() interface. ................................................................................ 5322 5414 { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0}, 5323 5415 #endif 5324 5416 { "sqlite3_libversion_number", test_libversion_number, 0 }, 5325 5417 #ifdef SQLITE_ENABLE_COLUMN_METADATA 5326 5418 { "sqlite3_table_column_metadata", test_table_column_metadata, 0 }, 5327 5419 #endif 5328 5420 #ifndef SQLITE_OMIT_INCRBLOB 5329 - { "sqlite3_blob_read", test_blob_read, 0 }, 5330 - { "sqlite3_blob_write", test_blob_write, 0 }, 5421 + { "sqlite3_blob_read", test_blob_read, 0 }, 5422 + { "sqlite3_blob_write", test_blob_write, 0 }, 5423 + { "sqlite3_blob_reopen", test_blob_reopen, 0 }, 5424 + { "sqlite3_blob_bytes", test_blob_bytes, 0 }, 5425 + { "sqlite3_blob_close", test_blob_close, 0 }, 5331 5426 #endif 5332 5427 { "pcache_stats", test_pcache_stats, 0 }, 5333 5428 #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY 5334 5429 { "sqlite3_unlock_notify", test_unlock_notify, 0 }, 5335 5430 #endif 5336 5431 { "sqlite3_wal_checkpoint", test_wal_checkpoint, 0 }, 5337 5432 { "test_sqlite3_log", test_sqlite3_log, 0 },
Changes to src/vdbeblob.c.
22 22 ** Valid sqlite3_blob* handles point to Incrblob structures. 23 23 */ 24 24 typedef struct Incrblob Incrblob; 25 25 struct Incrblob { 26 26 int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ 27 27 int nByte; /* Size of open blob, in bytes */ 28 28 int iOffset; /* Byte offset of blob in cursor data */ 29 + int iCol; /* Table column this handle is open on */ 29 30 BtCursor *pCsr; /* Cursor pointing at blob row */ 30 31 sqlite3_stmt *pStmt; /* Statement holding cursor open */ 31 32 sqlite3 *db; /* The associated database */ 32 33 }; 33 34 35 + 36 +/* 37 +** This function is used by both blob_open() and blob_reopen(). It seeks 38 +** the b-tree cursor associated with blob handle p to point to row iRow. 39 +** If successful, SQLITE_OK is returned and subsequent calls to 40 +** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. 41 +** 42 +** If an error occurs, or if the specified row does not exist or does not 43 +** contain a value of type TEXT or BLOB in the column nominated when the 44 +** blob handle was opened, then an error code is returned and *pzErr may 45 +** be set to point to a buffer containing an error message. It is the 46 +** responsibility of the caller to free the error message buffer using 47 +** sqlite3DbFree(). 48 +** 49 +** If an error does occur, then the b-tree cursor is closed. All subsequent 50 +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will 51 +** immediately return SQLITE_ABORT. 52 +*/ 53 +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ 54 + int rc; /* Error code */ 55 + char *zErr = 0; /* Error message */ 56 + Vdbe *v = (Vdbe *)p->pStmt; 57 + 58 + /* Set the value of the SQL statements only variable to integer iRow. 59 + ** This is done directly instead of using sqlite3_bind_int64() to avoid 60 + ** triggering asserts related to mutexes. 61 + */ 62 + assert( v->aVar[0].flags&MEM_Int ); 63 + v->aVar[0].u.i = iRow; 64 + 65 + rc = sqlite3_step(p->pStmt); 66 + if( rc==SQLITE_ROW ){ 67 + u32 type = v->apCsr[0]->aType[p->iCol]; 68 + if( type<12 ){ 69 + zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", 70 + type==0?"null": type==7?"real": "integer" 71 + ); 72 + rc = SQLITE_ERROR; 73 + sqlite3_finalize(p->pStmt); 74 + p->pStmt = 0; 75 + }else{ 76 + p->iOffset = v->apCsr[0]->aOffset[p->iCol]; 77 + p->nByte = sqlite3VdbeSerialTypeLen(type); 78 + p->pCsr = v->apCsr[0]->pCursor; 79 + sqlite3BtreeEnterCursor(p->pCsr); 80 + sqlite3BtreeCacheOverflow(p->pCsr); 81 + sqlite3BtreeLeaveCursor(p->pCsr); 82 + } 83 + } 84 + 85 + if( rc==SQLITE_ROW ){ 86 + rc = SQLITE_OK; 87 + }else if( p->pStmt ){ 88 + rc = sqlite3_finalize(p->pStmt); 89 + p->pStmt = 0; 90 + if( rc==SQLITE_OK ){ 91 + zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); 92 + rc = SQLITE_ERROR; 93 + }else{ 94 + zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); 95 + } 96 + } 97 + 98 + assert( rc!=SQLITE_OK || zErr==0 ); 99 + assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); 100 + 101 + *pzErr = zErr; 102 + return rc; 103 +} 104 + 34 105 /* 35 106 ** Open a blob handle. 36 107 */ 37 108 int sqlite3_blob_open( 38 109 sqlite3* db, /* The database connection */ 39 110 const char *zDb, /* The attached database containing the blob */ 40 111 const char *zTable, /* The table containing the blob */ ................................................................................ 67 138 {OP_TableLock, 0, 0, 0}, /* 2: Acquire a read or write lock */ 68 139 69 140 /* One of the following two instructions is replaced by an OP_Noop. */ 70 141 {OP_OpenRead, 0, 0, 0}, /* 3: Open cursor 0 for reading */ 71 142 {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ 72 143 73 144 {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ 74 - {OP_NotExists, 0, 9, 1}, /* 6: Seek the cursor */ 145 + {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ 75 146 {OP_Column, 0, 0, 1}, /* 7 */ 76 147 {OP_ResultRow, 1, 0, 0}, /* 8 */ 77 - {OP_Close, 0, 0, 0}, /* 9 */ 78 - {OP_Halt, 0, 0, 0}, /* 10 */ 148 + {OP_Goto, 0, 5, 0}, /* 9 */ 149 + {OP_Close, 0, 0, 0}, /* 10 */ 150 + {OP_Halt, 0, 0, 0}, /* 11 */ 79 151 }; 80 152 81 - Vdbe *v = 0; 82 153 int rc = SQLITE_OK; 83 154 char *zErr = 0; 84 155 Table *pTab; 85 - Parse *pParse; 156 + Parse *pParse = 0; 157 + Incrblob *pBlob = 0; 86 158 159 + flags = !!flags; /* flags = (flags ? 1 : 0); */ 87 160 *ppBlob = 0; 161 + 88 162 sqlite3_mutex_enter(db->mutex); 163 + 164 + pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); 165 + if( !pBlob ) goto blob_open_out; 89 166 pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); 90 - if( pParse==0 ){ 91 - rc = SQLITE_NOMEM; 92 - goto blob_open_out; 93 - } 167 + if( !pParse ) goto blob_open_out; 168 + 94 169 do { 95 170 memset(pParse, 0, sizeof(Parse)); 96 171 pParse->db = db; 172 + sqlite3DbFree(db, zErr); 173 + zErr = 0; 97 174 98 175 sqlite3BtreeEnterAll(db); 99 176 pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); 100 177 if( pTab && IsVirtual(pTab) ){ 101 178 pTab = 0; 102 179 sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable); 103 180 } ................................................................................ 115 192 } 116 193 rc = SQLITE_ERROR; 117 194 sqlite3BtreeLeaveAll(db); 118 195 goto blob_open_out; 119 196 } 120 197 121 198 /* Now search pTab for the exact column. */ 122 - for(iCol=0; iCol < pTab->nCol; iCol++) { 199 + for(iCol=0; iCol<pTab->nCol; iCol++) { 123 200 if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ 124 201 break; 125 202 } 126 203 } 127 204 if( iCol==pTab->nCol ){ 128 205 sqlite3DbFree(db, zErr); 129 206 zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); ................................................................................ 169 246 zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); 170 247 rc = SQLITE_ERROR; 171 248 sqlite3BtreeLeaveAll(db); 172 249 goto blob_open_out; 173 250 } 174 251 } 175 252 176 - v = sqlite3VdbeCreate(db); 177 - if( v ){ 253 + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db); 254 + assert( pBlob->pStmt || db->mallocFailed ); 255 + if( pBlob->pStmt ){ 256 + Vdbe *v = (Vdbe *)pBlob->pStmt; 178 257 int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); 258 + 179 259 sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); 180 - flags = !!flags; /* flags = (flags ? 1 : 0); */ 260 + 181 261 182 262 /* Configure the OP_Transaction */ 183 263 sqlite3VdbeChangeP1(v, 0, iDb); 184 264 sqlite3VdbeChangeP2(v, 0, flags); 185 265 186 266 /* Configure the OP_VerifyCookie */ 187 267 sqlite3VdbeChangeP1(v, 1, iDb); ................................................................................ 216 296 sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); 217 297 sqlite3VdbeChangeP2(v, 7, pTab->nCol); 218 298 if( !db->mallocFailed ){ 219 299 sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); 220 300 } 221 301 } 222 302 303 + pBlob->flags = flags; 304 + pBlob->iCol = iCol; 305 + pBlob->db = db; 223 306 sqlite3BtreeLeaveAll(db); 224 307 if( db->mallocFailed ){ 225 308 goto blob_open_out; 226 309 } 227 - 228 - sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow); 229 - rc = sqlite3_step((sqlite3_stmt *)v); 230 - if( rc!=SQLITE_ROW ){ 231 - nAttempt++; 232 - rc = sqlite3_finalize((sqlite3_stmt *)v); 233 - sqlite3DbFree(db, zErr); 234 - zErr = sqlite3MPrintf(db, "%s", sqlite3_errmsg(db)); 235 - v = 0; 236 - } 237 - } while( nAttempt<5 && rc==SQLITE_SCHEMA ); 238 - 239 - if( rc==SQLITE_ROW ){ 240 - /* The row-record has been opened successfully. Check that the 241 - ** column in question contains text or a blob. If it contains 242 - ** text, it is up to the caller to get the encoding right. 243 - */ 244 - Incrblob *pBlob; 245 - u32 type = v->apCsr[0]->aType[iCol]; 246 - 247 - if( type<12 ){ 248 - sqlite3DbFree(db, zErr); 249 - zErr = sqlite3MPrintf(db, "cannot open value of type %s", 250 - type==0?"null": type==7?"real": "integer" 251 - ); 252 - rc = SQLITE_ERROR; 253 - goto blob_open_out; 254 - } 255 - pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); 256 - if( db->mallocFailed ){ 257 - sqlite3DbFree(db, pBlob); 258 - goto blob_open_out; 259 - } 260 - pBlob->flags = flags; 261 - pBlob->pCsr = v->apCsr[0]->pCursor; 262 - sqlite3BtreeEnterCursor(pBlob->pCsr); 263 - sqlite3BtreeCacheOverflow(pBlob->pCsr); 264 - sqlite3BtreeLeaveCursor(pBlob->pCsr); 265 - pBlob->pStmt = (sqlite3_stmt *)v; 266 - pBlob->iOffset = v->apCsr[0]->aOffset[iCol]; 267 - pBlob->nByte = sqlite3VdbeSerialTypeLen(type); 268 - pBlob->db = db; 269 - *ppBlob = (sqlite3_blob *)pBlob; 270 - rc = SQLITE_OK; 271 - }else if( rc==SQLITE_OK ){ 272 - sqlite3DbFree(db, zErr); 273 - zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow); 274 - rc = SQLITE_ERROR; 275 - } 310 + sqlite3_bind_int64(pBlob->pStmt, 1, iRow); 311 + rc = blobSeekToRow(pBlob, iRow, &zErr); 312 + } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); 276 313 277 314 blob_open_out: 278 - if( v && (rc!=SQLITE_OK || db->mallocFailed) ){ 279 - sqlite3VdbeFinalize(v); 315 + if( rc==SQLITE_OK && db->mallocFailed==0 ){ 316 + *ppBlob = (sqlite3_blob *)pBlob; 317 + }else{ 318 + if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); 319 + sqlite3DbFree(db, pBlob); 280 320 } 281 321 sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); 282 322 sqlite3DbFree(db, zErr); 283 323 sqlite3StackFree(db, pParse); 284 324 rc = sqlite3ApiExit(db, rc); 285 325 sqlite3_mutex_leave(db->mutex); 286 326 return rc; ................................................................................ 327 367 sqlite3_mutex_enter(db->mutex); 328 368 v = (Vdbe*)p->pStmt; 329 369 330 370 if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ 331 371 /* Request is out of range. Return a transient error. */ 332 372 rc = SQLITE_ERROR; 333 373 sqlite3Error(db, SQLITE_ERROR, 0); 334 - } else if( v==0 ){ 374 + }else if( v==0 ){ 335 375 /* If there is no statement handle, then the blob-handle has 336 376 ** already been invalidated. Return SQLITE_ABORT in this case. 337 377 */ 338 378 rc = SQLITE_ABORT; 339 379 }else{ 340 380 /* Call either BtreeData() or BtreePutData(). If SQLITE_ABORT is 341 381 ** returned, clean-up the statement handle. ................................................................................ 377 417 ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob 378 418 ** so no mutex is required for access. 379 419 */ 380 420 int sqlite3_blob_bytes(sqlite3_blob *pBlob){ 381 421 Incrblob *p = (Incrblob *)pBlob; 382 422 return p ? p->nByte : 0; 383 423 } 424 + 425 +/* 426 +** Move an existing blob handle to point to a different row of the same 427 +** database table. 428 +** 429 +** If an error occurs, or if the specified row does not exist or does not 430 +** contain a blob or text value, then an error code is returned and the 431 +** database handle error code and message set. If this happens, then all 432 +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) 433 +** immediately return SQLITE_ABORT. 434 +*/ 435 +int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ 436 + int rc; 437 + Incrblob *p = (Incrblob *)pBlob; 438 + sqlite3 *db; 439 + 440 + if( p==0 ) return SQLITE_MISUSE_BKPT; 441 + db = p->db; 442 + sqlite3_mutex_enter(db->mutex); 443 + 444 + if( p->pStmt==0 ){ 445 + /* If there is no statement handle, then the blob-handle has 446 + ** already been invalidated. Return SQLITE_ABORT in this case. 447 + */ 448 + rc = SQLITE_ABORT; 449 + }else{ 450 + char *zErr; 451 + rc = blobSeekToRow(p, iRow, &zErr); 452 + if( rc!=SQLITE_OK ){ 453 + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); 454 + sqlite3DbFree(db, zErr); 455 + } 456 + assert( rc!=SQLITE_SCHEMA ); 457 + } 458 + 459 + rc = sqlite3ApiExit(db, rc); 460 + sqlite3_mutex_leave(db->mutex); 461 + return rc; 462 +} 384 463 385 464 #endif /* #ifndef SQLITE_OMIT_INCRBLOB */
Changes to test/fts3ah.test.
1 -# 2006 October 31 (scaaarey) 1 +# 2006 October 31 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 2 5 # 3 -# The author disclaims copyright to this source code. 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 4 9 # 5 10 #************************************************************************* 6 11 # This file implements regression tests for SQLite library. The focus 7 -# here is testing correct handling of excessively long terms. 8 -# 9 -# $Id: fts3ah.test,v 1.1 2007/08/20 17:38:42 shess Exp $ 12 +# here is testing correct handling of very long terms. 10 13 # 11 14 12 15 set testdir [file dirname $argv0] 13 16 source $testdir/tester.tcl 14 17 15 -# If SQLITE_ENABLE_FTS3 is defined, omit this file. 18 +# If SQLITE_ENABLE_FTS3 is not defined, omit this file. 16 19 ifcapable !fts3 { 17 20 finish_test 18 21 return 19 22 } 20 23 21 -# Generate a term of len copies of char. 22 -proc bigterm {char len} { 23 - for {set term ""} {$len>0} {incr len -1} { 24 - append term $char 25 - } 26 - return $term 27 -} 28 - 29 24 # Generate a document of bigterms based on characters from the list 30 25 # chars. 31 26 proc bigtermdoc {chars len} { 32 27 set doc "" 33 28 foreach char $chars { 34 - append doc " " [bigterm $char $len] 29 + append doc " " [string repeat $char $len] 35 30 } 36 31 return $doc 37 32 } 38 33 39 34 set len 5000 40 35 set doc1 [bigtermdoc {a b c d} $len] 41 36 set doc2 [bigtermdoc {b d e f} $len] 42 37 set doc3 [bigtermdoc {a c e} $len] 43 38 44 -set aterm [bigterm a $len] 45 -set bterm [bigterm b $len] 46 -set xterm [bigterm x $len] 39 +set aterm [string repeat a $len] 40 +set bterm [string repeat b $len] 41 +set xterm [string repeat x $len] 47 42 48 43 db eval { 49 44 CREATE VIRTUAL TABLE t1 USING fts3(content); 50 45 INSERT INTO t1 (rowid, content) VALUES(1, $doc1); 51 46 INSERT INTO t1 (rowid, content) VALUES(2, $doc2); 52 47 INSERT INTO t1 (rowid, content) VALUES(3, $doc3); 53 48 } ................................................................................ 57 52 execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something'} 58 53 } {} 59 54 60 55 do_test fts3ah-1.2 { 61 56 execsql {SELECT rowid FROM t1 WHERE t1 MATCH $aterm} 62 57 } {1 3} 63 58 64 -do_test fts3ah-1.2 { 59 +do_test fts3ah-1.3 { 65 60 execsql {SELECT rowid FROM t1 WHERE t1 MATCH $xterm} 66 61 } {} 67 62 68 -do_test fts3ah-1.3 { 63 +do_test fts3ah-1.4 { 69 64 execsql "SELECT rowid FROM t1 WHERE t1 MATCH '$aterm -$xterm'" 70 65 } {1 3} 71 66 72 -do_test fts3ah-1.4 { 67 +do_test fts3ah-1.5 { 73 68 execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"$aterm $bterm\"'" 74 69 } {1} 75 70 76 71 finish_test
Changes to test/fts3ao.test.
18 18 source $testdir/tester.tcl 19 19 20 20 # If SQLITE_ENABLE_FTS3 is not defined, omit this file. 21 21 ifcapable !fts3 { 22 22 finish_test 23 23 return 24 24 } 25 + 26 +set ::testprefix fts3ao 25 27 26 28 #--------------------------------------------------------------------- 27 29 # These tests, fts3ao-1.*, test that ticket #2429 is fixed. 28 30 # 29 31 db eval { 30 32 CREATE VIRTUAL TABLE t1 USING fts3(a, b, c); 31 33 INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one four two'); ................................................................................ 194 196 INSERT INTO t5 VALUES('Down came a jumbuck to drink at that billabong'); 195 197 ALTER TABLE t5 RENAME TO t6; 196 198 INSERT INTO t6 VALUES('Down came the troopers, one, two, three'); 197 199 ROLLBACK; 198 200 SELECT * FROM t5; 199 201 } 200 202 } {{the quick brown fox} {jumped over the} {lazy dog}} 203 + 204 +# Test that it is possible to rename an FTS4 table. Renaming an FTS4 table 205 +# involves renaming the extra %_docsize and %_stat tables. 206 +# 207 +do_execsql_test 5.1 { 208 + CREATE VIRTUAL TABLE t7 USING FTS4; 209 + INSERT INTO t7 VALUES('coined by a German clinician'); 210 + SELECT count(*) FROM sqlite_master WHERE name LIKE 't7%'; 211 + SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%'; 212 +} {6 0} 213 +do_execsql_test 5.2 { 214 + ALTER TABLE t7 RENAME TO t8; 215 + SELECT count(*) FROM sqlite_master WHERE name LIKE 't7%'; 216 + SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%'; 217 +} {0 6} 201 218 202 219 finish_test 203 220
Added test/fts3corrupt.test.
1 +# 2010 October 27 2 +# 3 +# May you do good and not evil. 4 +# May you find forgiveness for yourself and forgive others. 5 +# May you share freely, never taking more than you give. 6 +# 7 +#*********************************************************************** 8 +# Test that the FTS3 extension does not crash when it encounters a 9 +# corrupt data structure on disk. 10 +# 11 + 12 + 13 +set testdir [file dirname $argv0] 14 +source $testdir/tester.tcl 15 + 16 +# If SQLITE_ENABLE_FTS3 is not defined, omit this file. 17 +ifcapable !fts3 { finish_test ; return } 18 + 19 +set ::testprefix fts3corrupt 20 + 21 +do_execsql_test 1.0 { 22 + CREATE VIRTUAL TABLE t1 USING fts3; 23 + INSERT INTO t1 VALUES('hello'); 24 +} {} 25 + 26 +do_test fts3corrupt-1.1 { 27 + set blob [db one {SELECT root from t1_segdir}] 28 + set blob [binary format a7ca* $blob 24 [string range $blob 8 end]] 29 + execsql { UPDATE t1_segdir SET root = $blob } 30 +} {} 31 + 32 +do_test fts3corrupt-1.2 { 33 + foreach w {a b c d e f g h i j k l m n o} { 34 + execsql { INSERT INTO t1 VALUES($w) } 35 + } 36 +} {} 37 + 38 +do_catchsql_test 1.3 { 39 + INSERT INTO t1 VALUES('world'); 40 +} {1 {database disk image is malformed}} 41 + 42 +finish_test 43 +
Changes to test/fts3cov.test.
2 2 # 3 3 # May you do good and not evil. 4 4 # May you find forgiveness for yourself and forgive others. 5 5 # May you share freely, never taking more than you give. 6 6 # 7 7 #*********************************************************************** 8 8 # 9 -# The tests in this file are structural coverage tests. They are designed 10 -# to complement the tests in fts3rnd.test and fts3doc.test. Between them, 11 -# the three files should provide full coverage of the fts3 extension code. 9 +# The tests in this file are structural coverage tests for FTS3. 12 10 # 13 11 14 12 set testdir [file dirname $argv0] 15 13 source $testdir/tester.tcl 16 14 17 15 # If this build does not include FTS3, skip the tests in this file. 18 16 # 19 17 ifcapable !fts3 { finish_test ; return } 20 18 source $testdir/fts3_common.tcl 21 19 source $testdir/malloc_common.tcl 22 20 23 21 set DO_MALLOC_TEST 0 22 +set testprefix fts3cov 24 23 25 24 #-------------------------------------------------------------------------- 26 25 # When it first needs to read a block from the %_segments table, the FTS3 27 26 # module compiles an SQL statement for that purpose. The statement is 28 27 # stored and reused each subsequent time a block is read. This test case 29 28 # tests the effects of an OOM error occuring while compiling the statement. 30 29 # ................................................................................ 78 77 INSERT INTO t1 VALUES('What makes her in the wood so late,'); 79 78 INSERT INTO t1 VALUES('A furlong from the castle gate?'); 80 79 INSERT INTO t1 VALUES('She had dreams all yesternight'); 81 80 INSERT INTO t1 VALUES('Of her own betrothed knight;'); 82 81 INSERT INTO t1 VALUES('And she in the midnight wood will pray'); 83 82 INSERT INTO t1 VALUES('For the weal of her lover that''s far away.'); 84 83 COMMIT; 85 - 84 + } 85 + execsql { 86 86 INSERT INTO t1(t1) VALUES('optimize'); 87 87 SELECT substr(hex(root), 1, 2) FROM t1_segdir; 88 88 } 89 89 } {03} 90 90 91 91 # Test the "missing entry" case: 92 -do_test fts3cov-2.1 { 92 +do_test fts3cov-2.2 { 93 93 set root [db one {SELECT root FROM t1_segdir}] 94 94 read_fts3varint [string range $root 1 end] left_child 95 95 execsql { DELETE FROM t1_segments WHERE blockid = $left_child } 96 96 } {} 97 -do_error_test fts3cov-2.2 { 97 +do_error_test fts3cov-2.3 { 98 98 SELECT * FROM t1 WHERE t1 MATCH 'c*' 99 -} {database disk image is malformed} 99 +} {SQL logic error or missing database} 100 100 101 101 # Test the "replaced with NULL" case: 102 -do_test fts3cov-2.3 { 102 +do_test fts3cov-2.4 { 103 103 execsql { INSERT INTO t1_segments VALUES($left_child, NULL) } 104 104 } {} 105 -do_error_test fts3cov-2.4 { 105 +do_error_test fts3cov-2.5 { 106 106 SELECT * FROM t1 WHERE t1 MATCH 'cloud' 107 -} {database disk image is malformed} 107 +} {SQL logic error or missing database} 108 108 109 109 #-------------------------------------------------------------------------- 110 110 # The following tests are to test the effects of OOM errors while storing 111 111 # terms in the pending-hash table. Specifically, while creating doclist 112 112 # blobs to store in the table. More specifically, to test OOM errors while 113 113 # appending column numbers to doclists. For example, if a doclist consists 114 114 # of: ................................................................................ 365 365 INSERT INTO t13 VALUES('scalar two functions'); 366 366 INSERT INTO t13 VALUES('functions scalar two'); 367 367 } -sqlbody { 368 368 SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two'; 369 369 SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two'; 370 370 SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two'; 371 371 } 372 + 373 +do_execsql_test 14.0 { 374 + CREATE VIRTUAL TABLE t14 USING fts4(a, b); 375 + INSERT INTO t14 VALUES('one two three', 'one three four'); 376 + INSERT INTO t14 VALUES('a b c', 'd e a'); 377 +} 378 +do_execsql_test 14.1 { 379 + SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"' 380 +} {1} 381 +do_execsql_test 14.2 { 382 + SELECT rowid FROM t14 WHERE t14 MATCH '"one four"' 383 +} {} 384 +do_execsql_test 14.3 { 385 + SELECT rowid FROM t14 WHERE t14 MATCH '"e a"' 386 +} {2} 387 +do_execsql_test 14.5 { 388 + SELECT rowid FROM t14 WHERE t14 MATCH '"e b"' 389 +} {} 390 +do_catchsql_test 14.6 { 391 + SELECT rowid FROM t14 WHERE rowid MATCH 'one' 392 +} {1 {unable to use function MATCH in the requested context}} 393 +do_catchsql_test 14.7 { 394 + SELECT rowid FROM t14 WHERE docid MATCH 'one' 395 +} {1 {unable to use function MATCH in the requested context}} 396 + 397 +do_execsql_test 15.0 { 398 + CREATE VIRTUAL TABLE t15 USING fts4(a, b, c); 399 + INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3'); 400 + INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3'); 401 +} 402 +do_execsql_test 15.1 { 403 + SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"' 404 +} {1 2} 405 + 406 +# Test a corruption case. 407 +# 408 +do_execsql_test 16.1 { 409 + CREATE VIRTUAL TABLE t16 USING fts4; 410 + INSERT INTO t16 VALUES('theoretical work to examine the relationship'); 411 + INSERT INTO t16 VALUES('solution of our problems on the invisible'); 412 + DELETE FROM t16_content WHERE rowid = 2; 413 +} 414 +do_catchsql_test 16.2 { 415 + SELECT * FROM t16 WHERE t16 MATCH 'invisible' 416 +} {1 {database disk image is malformed}} 417 + 418 +# And another corruption test case. 419 +# 420 +do_execsql_test 17.1 { 421 + CREATE VIRTUAL TABLE t17 USING fts4; 422 + INSERT INTO t17(content) VALUES('one one one'); 423 + UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300' 424 +} {} 425 +do_catchsql_test 17.2 { 426 + SELECT * FROM t17 WHERE t17 MATCH 'one' 427 +} {1 {database disk image is malformed}} 428 + 429 + 430 + 372 431 373 432 finish_test
Added test/fts3defer.test.
1 +# 2010 October 15 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 + 12 +set testdir [file dirname $argv0] 13 +source $testdir/tester.tcl 14 +source $testdir/malloc_common.tcl 15 + 16 +ifcapable !fts3 { 17 + finish_test 18 + return 19 +} 20 + 21 +set sqlite_fts3_enable_parentheses 1 22 + 23 +set ::testprefix fts3defer 24 + 25 +#-------------------------------------------------------------------------- 26 +# Test cases fts3defer-1.* are the "warm body" cases. The database contains 27 +# one row with 15000 instances of the token "a". This makes the doclist for 28 +# "a" so large that FTS3 will avoid loading it in most cases. 29 +# 30 +# To show this, test cases fts3defer-1.2.* execute a bunch of FTS3 queries 31 +# involving token "a". Then, fts3defer-1.3.* replaces the doclist for token 32 +# "a" with all zeroes and fts3defer-1.4.* repeats the tests from 1.2. If 33 +# the tests still work, we can conclude that the doclist for "a" was not 34 +# used. 35 +# 36 + 37 +set aaa [string repeat "a " 15000] 38 + 39 +do_execsql_test 1.1 { 40 + CREATE VIRTUAL TABLE t1 USING fts4; 41 + BEGIN; 42 + INSERT INTO t1 VALUES('this is a dog'); 43 + INSERT INTO t1 VALUES('an instance of a phrase'); 44 + INSERT INTO t1 VALUES('an instance of a longer phrase'); 45 + INSERT INTO t1 VALUES($aaa); 46 + COMMIT; 47 +} {} 48 + 49 +set tests { 50 + 1 {SELECT rowid FROM t1 WHERE t1 MATCH '"a dog"'} {1} 51 + 2 {SELECT rowid FROM t1 WHERE t1 MATCH '"is a dog"'} {1} 52 + 3 {SELECT rowid FROM t1 WHERE t1 MATCH '"a longer phrase"'} {3} 53 + 4 {SELECT snippet(t1) FROM t1 WHERE t1 MATCH '"a longer phrase"'} 54 + {"an instance of <b>a</b> <b>longer</b> <b>phrase</b>"} 55 + 5 {SELECT rowid FROM t1 WHERE t1 MATCH 'a dog'} {1} 56 +} 57 + 58 +do_select_tests 1.2 $tests 59 + 60 +do_execsql_test 1.3 { 61 + SELECT count(*) FROM t1_segments WHERE length(block)>10000; 62 + UPDATE t1_segments 63 + SET block = zeroblob(length(block)) 64 + WHERE length(block)>10000; 65 +} {1} 66 + 67 +do_select_tests 1.4 $tests 68 + 69 +# Drop the table. It is corrupt now anyhow, so not useful for subsequent tests. 70 +# 71 +do_execsql_test 1.5 { DROP TABLE t1 } 72 + 73 +#-------------------------------------------------------------------------- 74 +# These tests - fts3defer-2.* - are more rigorous. They test that for a 75 +# variety of queries, FTS3 and FTS4 return the same results. And that 76 +# zeroing the very large doclists that FTS4 does not load does not change 77 +# the results. 78 +# 79 +# They use the following pseudo-randomly generated document data. The 80 +# tokens "zm" and "jk" are especially common in this dataset. Additionally, 81 +# two documents are added to the pseudo-random data before it is loaded 82 +# into FTS4 containing 100,000 instances of the "zm" and "jk" tokens. This 83 +# makes the doclists for those tokens so large that FTS4 avoids loading them 84 +# into memory if possible. 85 +# 86 +set data [list] 87 +lappend data [string repeat "zm " 100000] 88 +lappend data [string repeat "jk " 100000] 89 +lappend data {*}{ 90 + "zm zm agmckuiu uhzq nsab jk rrkx duszemmzl hyq jk" 91 + "jk uhzq zm zm rgpzmlnmd zm zk jk jk zm" 92 + "duszemmzl zm jk xldlpy zm jk sbptoa xh jk xldlpy" 93 + "zm xh zm xqf azavwm jk jk trqd rgpzmlnmd jk" 94 + "zm vwq urvysbnykk ubwrfqnbjf zk lsz jk doiwavhwwo jk jk" 95 + "jk xduvfhk orpfawpx zkhdvkw jk mjpavjuhw zm jk duszemmzl zm" 96 + "jk igju jk jk zm hmjf xh zm gwdfhwurx zk" 97 + "vgsld jk jk zm hrlipdm jn zm zsmhnf vgsld duszemmzl" 98 + "gtuiexzsu aayxpmve zm zm zm drir scpgna xh azavwm uhzq" 99 + "farlehdhq hkfoudzftq igju duszemmzl xnxhf ewle zm hrlipdm urvysbnykk kn" 100 + "xnxhf jk jk agmckuiu duszemmzl jk zm zm jk vgsld" 101 + "zm zm zm jk jk urvysbnykk ogttbykvt zm zm jk" 102 + "iasrqgqv zm azavwm zidhxhbtv jk jk mjpavjuhw zm zm ajmvcydy" 103 + "rgpzmlnmd tmt mjpavjuhw xh igju jk azavwm fibokdry vgsld ofm" 104 + "zm jk vgsld jk xh jk csjqxhgj drir jk pmrb" 105 + "xh jk jk zm rrkx duszemmzl mjpavjuhw xldlpy igju zm" 106 + "jk hkfoudzftq zf rrkx wdmy jupk jk zm urvysbnykk npywgdvgz" 107 + "zm jk zm zm zhbrzadb uenvbm aayxpmve urvysbnykk duszemmzl jk" 108 + "uenvbm jk zm fxw xh bdilwmjw mjpavjuhw uv jk zm" 109 + "nk jk bnhc pahlds jk igju dzadnqzprr jk jk jk" 110 + "uhzq uv zm duszemmzl tlqix jk jk xh jk zm" 111 + "jk zm agmckuiu urvysbnykk jk jk zm zm jk jk" 112 + "azavwm mjpavjuhw lsgshn trqd xldlpy ogyavjvv agmckuiu ryvwwhlbc jk jk" 113 + "tmt jk zk zm azavwm ofm acpgim bvgimjik iasrqgqv wuvajhwqz" 114 + "igju ogyavjvv xrbdak rrkx fibokdry zf ujfhmrllq jk zm hxgwvib" 115 + "zm pahlds jk uenvbm aayxpmve iaf hmjf xph vnlyvtkgx zm" 116 + "jk xnxhf igju jk xh jk nvfasfh zm js jk" 117 + "zm zm rwaj igju xr rrkx xnxhf nvfasfh skxbsqzvmt xatbxeqq" 118 + "vgsld zm ujfhmrllq uhzq ogyavjvv nsab azavwm zm vgsld jmfiqhwnjg" 119 + "ymjoym duszemmzl urvysbnykk azavwm jk jmfiqhwnjg bu qcdziqomqk vnlyvtkgx" 120 + "zm nbilqcnz dzadnqzprr xh bkfgzsxn urvysbnykk xrujfzxqf zm zf agmckuiu" 121 + "jk urvysbnykk nvfasfh zf xh zm zm qcdziqomqk qvxtclg wdmy" 122 + "fibokdry jk urvysbnykk jk xr osff zm cvnnsl zm vgsld" 123 + "jk mjpavjuhw hkfoudzftq jk zm xh xqf urvysbnykk jk iasrqgqv" 124 + "jk csjqxhgj duszemmzl iasrqgqv aayxpmve zm brsuoqww jk qpmhtvl wluvgsw" 125 + "jk mj azavwm jk zm jn dzadnqzprr zm jk uhzq" 126 + "zk xqf jupk fxw nbilqcnz zm jk jcpiwj tznlvbfcv nvfasfh" 127 + "jk jcpiwj zm xnxhf zm mjpavjuhw mj drir pa pvjrjlas" 128 + "duszemmzl dzadnqzprr jk swc duszemmzl tmt jk jk pahlds jk" 129 + "zk zm jk zm zm eczkjblu zm hi pmrb jk" 130 + "azavwm zm iz agmckuiu jk sntk jk duszemmzl duszemmzl zm" 131 + "jk zm jk eczkjblu urvysbnykk sk gnl jk ttvgf hmjf" 132 + "jk bnhc jjrxpjkb mjpavjuhw fibokdry igju jk zm zm xh" 133 + "wxe ogttbykvt uhzq xr iaf zf urvysbnykk aayxpmve oacaxgjoo mjpavjuhw" 134 + "gazrt jk ephknonq myjp uenvbm wuvajhwqz jk zm xnxhf nvfasfh" 135 + "zm aayxpmve csjqxhgj xnxhf xr jk aayxpmve xnxhf zm zm" 136 + "sokcyf zm ogyavjvv jk zm fibokdry zm jk igju igju" 137 + "vgsld bvgimjik xuprtlyle jk akmikrqyt jk aayxpmve hkfoudzftq ddjj ithtir" 138 + "zm uhzq ovkyevlgv zk uenvbm csjqxhgj jk vgsld pgybs jk" 139 + "zm agmckuiu zexh fibokdry jk uhzq bu tugflixoex xnxhf sk" 140 + "zm zf uenvbm jk azavwm zm zm agmckuiu zm jk" 141 + "rrkx jk zf jt zm oacaxgjoo fibokdry wdmy igju csjqxhgj" 142 + "hi igju zm jk zidhxhbtv dzadnqzprr jk jk trqd duszemmzl" 143 + "zm zm mjpavjuhw xrbdak qrvbjruc jk qzzqdxq guwq cvnnsl zm" 144 + "ithtir jk jk qcdziqomqk zm farlehdhq zm zm xrbdak jk" 145 + "ixfipk csjqxhgj azavwm sokcyf ttvgf vgsld jk sk xh zk" 146 + "nvfasfh azavwm zm zm zm fxw nvfasfh zk gnl trqd" 147 + "zm fibokdry csjqxhgj ofm dzadnqzprr jk akmikrqyt orpfawpx duszemmzl vwq" 148 + "csjqxhgj jk jk vgsld urvysbnykk jk nxum jk jk nxum" 149 + "zm hkfoudzftq jk ryvwwhlbc mjpavjuhw ephknonq jk zm ogyavjvv zm" 150 + "lwa hi xnxhf qdyerbws zk njtc jk uhzq zm jk" 151 + "trqd zm dzadnqzprr zm urvysbnykk jk lsz jk mjpavjuhw cmnnkna" 152 + "duszemmzl zk jk jk fibokdry jseuhjnzo zm aayxpmve zk jk" 153 + "fibokdry jk sviq qvxtclg wdmy jk doiwavhwwo zexh jk zm" 154 + "jupk zm xh jk mjpavjuhw zm jk nsab npywgdvgz duszemmzl" 155 + "zm igju zm zm nvfasfh eh hkfoudzftq fibokdry fxw xkblf" 156 + "jk zm jk jk zm xh zk abthnzcv zf csjqxhgj" 157 + "zm zm jk nkaotm urvysbnykk sbptoa bq jk ktxdty ubwrfqnbjf" 158 + "nvfasfh aayxpmve xdcuz zm tugflixoex jcpiwj zm mjpavjuhw fibokdry doiwavhwwo" 159 + "iaf jk mjpavjuhw zm duszemmzl jk jk uhzq pahlds fibokdry" 160 + "ddjj zk azavwm jk swc zm gjtexkv jk xh jk" 161 + "igju jk csjqxhgj zm jk dzadnqzprr duszemmzl ulvcbv jk jk" 162 + "jk fibokdry zm csjqxhgj jn zm zm zm zf uhzq" 163 + "duszemmzl jk xkblf zk hrlipdm aayxpmve uenvbm uhzq jk zf" 164 + "dzadnqzprr jk zm zdu nvfasfh zm jk urvysbnykk hmjf jk" 165 + "jk aayxpmve aserrdxm acpgim fibokdry jk drir wxe brsuoqww rrkx" 166 + "uhzq csjqxhgj nvfasfh jk rrkx qbamok trqd uenvbm sntk zm" 167 + "ps azavwm zkhdvkw jk zm jk jk zm csjqxhgj xedlrcfo" 168 + "jk jk ogyavjvv jk zm farlehdhq duszemmzl jk agitgxamxe jk" 169 + "qzzqdxq rwaj jk jk zm xqf jk uenvbm jk zk" 170 + "zm hxgwvib akmikrqyt zf agmckuiu uenvbm bq npywgdvgz azavwm jk" 171 + "zf jmfiqhwnjg js igju zm aayxpmve zm mbxnljomiv csjqxhgj nvfasfh" 172 + "zm jk jk gazrt jk jk lkc jk nvfasfh jk" 173 + "xldlpy orpfawpx zkhdvkw jk zm igju zm urvysbnykk dzadnqzprr mbxnljomiv" 174 + "urvysbnykk jk zk igju zm uenvbm jk zm ithtir jk" 175 + "zm zk zm zf ofm zm xdcuz dzadnqzprr zm vgsld" 176 + "sbptoa jk tugflixoex jk zm zm vgsld zm xh zm" 177 + "uhzq jk zk evvivo vgsld vniqnuynvf agmckuiu jk zm zm" 178 + "zm nvfasfh zm zm zm abthnzcv uenvbm jk zk dzadnqzprr" 179 + "zm azavwm igju qzzqdxq jk xnxhf abthnzcv jk nvfasfh zm" 180 + "qbamok fxw vgsld igju cmnnkna xnxhf vniqnuynvf zk xh zm" 181 + "nvfasfh zk zm mjpavjuhw dzadnqzprr jk jk duszemmzl xldlpy nvfasfh" 182 + "xnxhf sviq nsab npywgdvgz osff vgsld farlehdhq fibokdry wjbkhzsa hhac" 183 + "zm azavwm scpgna jk jk bq jk duszemmzl fibokdry ovkyevlgv" 184 + "csjqxhgj zm jk jk duszemmzl zk xh zm jk zf" 185 + "urvysbnykk dzadnqzprr csjqxhgj mjpavjuhw ubwrfqnbjf nkaotm jk jk zm drir" 186 + "nvfasfh xh igju zm wluvgsw jk zm srwwnezqk ewle ovnq" 187 + "jk nvfasfh eh ktxdty urvysbnykk vgsld zm jk eh uenvbm" 188 + "orpfawpx pahlds jk uhzq hi zm zm zf jk dzadnqzprr" 189 + "srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki" 190 +} 191 + 192 +proc add_empty_records {n} { 193 + execsql BEGIN 194 + for {set i 0} {$i < $n} {incr i} { 195 + execsql { INSERT INTO t1 VALUES('') } 196 + } 197 + execsql COMMIT 198 +} 199 + 200 + 201 +#set e [list] 202 +#foreach d $data {set e [concat $e $d]} 203 +#puts [lsort -unique $e] 204 +#exit 205 + 206 +set zero_long_doclists { 207 + UPDATE t1_segments SET block=zeroblob(length(block)) WHERE length(block)>10000 208 +} 209 + 210 +foreach {tn setup} { 211 + 1 { 212 + set dmt_modes 0 213 + execsql { CREATE VIRTUAL TABLE t1 USING FTS3 } 214 + foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } 215 + } 216 + 2 { 217 + set dmt_modes 0 218 + execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } 219 + foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } 220 + } 221 + 3 { 222 + set dmt_modes {0 1 2} 223 + execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } 224 + foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } 225 + add_empty_records 1000 226 + execsql $zero_long_doclists 227 + } 228 + 4 { 229 + set dmt_modes 0 230 + execsql { CREATE VIRTUAL TABLE t1 USING FTS4 } 231 + foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } } 232 + add_empty_records 1000 233 + execsql "INSERT INTO t1(t1) VALUES('optimize')" 234 + execsql $zero_long_doclists 235 + } 236 +} { 237 + 238 + execsql { DROP TABLE IF EXISTS t1 } 239 + eval $setup 240 + set ::testprefix fts3defer-2.$tn 241 + set DO_MALLOC_TEST 0 242 + 243 + do_execsql_test 0 { 244 + SELECT count(*) FROM t1_segments WHERE length(block)>10000 245 + } {2} 246 + 247 + do_select_test 1.1 { 248 + SELECT rowid FROM t1 WHERE t1 MATCH 'jk xnxhf' 249 + } {13 29 40 47 48 52 63 92} 250 + do_select_test 1.2 { 251 + SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh' 252 + } {100} 253 +if {$tn==3} breakpoint 254 + do_select_test 1.3 { 255 + SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf' 256 + } {7 70 98} 257 + do_select_test 1.4 { 258 + SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl jk' 259 + } {3 5 8 10 13 18 20 23 32 37 41 43 55 60 65 67 72 74 76 81 94 96 97} 260 + do_select_test 1.5 { 261 + SELECT rowid FROM t1 WHERE t1 MATCH 'ubwrfqnbjf jk' 262 + } {7 70 98} 263 + do_select_test 1.6 { 264 + SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf jk jk jk jk' 265 + } {7 70 98} 266 + do_select_test 1.7 { 267 + SELECT rowid FROM t1 WHERE t1 MATCH 'zm xnxhf' 268 + } {12 13 29 30 40 47 48 52 63 92 93} 269 + do_select_test 1.8 { 270 + SELECT rowid FROM t1 WHERE t1 MATCH 'zm eh' 271 + } {68 100} 272 + do_select_test 1.9 { 273 + SELECT rowid FROM t1 WHERE t1 MATCH 'zm ubwrfqnbjf' 274 + } {7 70 98} 275 + do_select_test 1.10 { 276 + SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' 277 + } {10 13 17 31 35 51 58 88 89 90 93 100} 278 + do_select_test 1.11 { 279 + SELECT rowid FROM t1 280 + WHERE t1 MATCH '( 281 + zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR 282 + zk OR zkhdvkw OR zm OR zsmhnf 283 + ) vgsld' 284 + } {10 13 17 31 35 51 58 88 89 90 93 100} 285 + 286 + do_select_test 2.1 { 287 + SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"' 288 + } {3 24 52 53} 289 + do_select_test 2.2 { 290 + SELECT rowid FROM t1 WHERE t1 MATCH '"zm zf"' 291 + } {33 53 75 88 101} 292 + do_select_test 2.3 { 293 + SELECT rowid FROM t1 WHERE t1 MATCH '"zm aayxpmve"' 294 + } {48 65 84} 295 + do_select_test 2.4 { 296 + SELECT rowid FROM t1 WHERE t1 MATCH '"aayxpmve zm"' 297 + } {11 37 84} 298 + do_select_test 2.5 { 299 + SELECT rowid FROM t1 WHERE t1 MATCH '"jk azavwm"' 300 + } {16 53} 301 + do_select_test 2.6 { 302 + SELECT rowid FROM t1 WHERE t1 MATCH '"xh jk jk"' 303 + } {18} 304 + do_select_test 2.7 { 305 + SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld"' 306 + } {13 17} 307 + do_select_test 2.8 { 308 + SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld lkjlkjlkj"' 309 + } {} 310 + 311 + do_select_test 3.1 { 312 + SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm agmckuiu"' 313 + } { 314 + {zm [zm] [agmckuiu] uhzq nsab jk rrkx duszemmzl hyq jk} 315 + {jk [zm] [agmckuiu] urvysbnykk jk jk zm zm jk jk} 316 + {[zm] [agmckuiu] zexh fibokdry jk uhzq bu tugflixoex xnxhf sk} 317 + {zm zf uenvbm jk azavwm zm [zm] [agmckuiu] zm jk} 318 + } 319 + 320 + do_select_test 3.2 { 321 + SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'xnxhf jk' 322 + } { 323 + {[xnxhf] [jk] [jk] agmckuiu duszemmzl [jk] zm zm [jk] vgsld} 324 + {[jk] [xnxhf] igju [jk] xh [jk] nvfasfh zm js [jk]} 325 + {[jk] jcpiwj zm [xnxhf] zm mjpavjuhw mj drir pa pvjrjlas} 326 + {gazrt [jk] ephknonq myjp uenvbm wuvajhwqz [jk] zm [xnxhf] nvfasfh} 327 + {zm aayxpmve csjqxhgj [xnxhf] xr [jk] aayxpmve [xnxhf] zm zm} 328 + {zm agmckuiu zexh fibokdry [jk] uhzq bu tugflixoex [xnxhf] sk} 329 + {lwa hi [xnxhf] qdyerbws zk njtc [jk] uhzq zm [jk]} 330 + {zm azavwm igju qzzqdxq [jk] [xnxhf] abthnzcv [jk] nvfasfh zm} 331 + } 332 + 333 + do_select_test 4.1 { 334 + SELECT offsets(t1) FROM t1 WHERE t1 MATCH '"jk uenvbm"' 335 + } { 336 + {0 0 10 2 0 1 13 6} {0 0 26 2 0 1 29 6} 337 + } 338 + 339 + do_select_test 4.2 { 340 + SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'duszemmzl jk fibokdry' 341 + } { 342 + {0 2 3 8 0 1 36 2 0 0 58 9} 343 + {0 0 0 9 0 1 13 2 0 1 16 2 0 2 19 8 0 1 53 2} 344 + {0 1 4 2 0 0 20 9 0 1 30 2 0 1 33 2 0 2 48 8} 345 + {0 1 17 2 0 1 20 2 0 1 26 2 0 0 29 9 0 2 39 8} 346 + } 347 + 348 + do_select_test 4.3 { 349 + SELECT offsets(t1) FROM t1 350 + WHERE t1 MATCH 'vgsld (hrlipdm OR (aapmve NEAR duszemmzl))' 351 + } {{0 0 0 5 0 1 15 7 0 0 36 5}} 352 + 353 + # The following block of tests runs normally with FTS3 or FTS4 without the 354 + # long doclists zeroed. And with OOM-injection for FTS4 with long doclists 355 + # zeroed. Change this by messing with the [set dmt_modes] commands above. 356 + # 357 + foreach DO_MALLOC_TEST $dmt_modes { 358 + 359 + # Phrase search. 360 + do_select_test 5.$DO_MALLOC_TEST.1 { 361 + SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"' 362 + } {8 15 36 64 67 72} 363 + 364 + # Multiple tokens search. 365 + do_select_test 5.$DO_MALLOC_TEST.2 { 366 + SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl zm' 367 + } {3 5 8 10 12 13 18 20 23 37 43 55 60 65 67 72 74 81 94 96 97} 368 + 369 + # snippet() function with phrase. 370 + do_select_test 5.$DO_MALLOC_TEST.3 { 371 + SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm aayxpmve"' 372 + } { 373 + {[zm] [aayxpmve] csjqxhgj xnxhf xr jk aayxpmve xnxhf zm zm} 374 + {duszemmzl zk jk jk fibokdry jseuhjnzo [zm] [aayxpmve] zk jk} 375 + {zf jmfiqhwnjg js igju [zm] [aayxpmve] zm mbxnljomiv csjqxhgj nvfasfh} 376 + } 377 + 378 + # snippet() function with multiple tokens. 379 + do_select_test 5.$DO_MALLOC_TEST.4 { 380 + SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'zm zhbrzadb' 381 + } { 382 + {[zm] jk [zm] [zm] [zhbrzadb] uenvbm aayxpmve urvysbnykk duszemmzl jk} 383 + } 384 + 385 + # snippet() function with phrase. 386 + do_select_test 5.$DO_MALLOC_TEST.5 { 387 + SELECT offsets(t1) FROM t1 WHERE t1 MATCH '"zm aayxpmve"' 388 + } { 389 + {0 0 0 2 0 1 3 8} {0 0 38 2 0 1 41 8} {0 0 22 2 0 1 25 8} 390 + } 391 + 392 + # snippet() function with multiple tokens. 393 + do_select_test 5.$DO_MALLOC_TEST.6 { 394 + SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'zm zhbrzadb' 395 + } { 396 + {0 0 0 2 0 0 6 2 0 0 9 2 0 1 12 8} 397 + } 398 + 399 + set DO_MALLOC_TEST 0 400 + } 401 + 402 + do_select_test 6.1 { 403 + SELECT rowid FROM t1 404 + WHERE t1 MATCH 'vgsld (hrlipdm OR (aayxpmve duszemmzl))' 405 + } {10} 406 + do_select_test 6.2.1 { 407 + SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk"' 408 + } {8} 409 + do_select_test 6.2.2 { 410 + SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"' 411 + } {15 26 92 96} 412 + do_select_test 6.2.3 { 413 + SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' 414 + } {8 15 26 92 96} 415 +} 416 + 417 +set testprefix fts3defer 418 + 419 +do_execsql_test 3.1 { 420 + CREATE VIRTUAL TABLE x1 USING fts4(a, b); 421 + INSERT INTO x1 VALUES('a b c', 'd e f'); 422 + INSERT INTO x1 SELECT * FROM x1; 423 + INSERT INTO x1 SELECT * FROM x1; 424 + INSERT INTO x1 SELECT * FROM x1; 425 + INSERT INTO x1 SELECT * FROM x1; 426 +} 427 +do_execsql_test 3.2 " 428 + INSERT INTO x1 VALUES( 429 + '[string repeat {d } 3000]', '[string repeat {f } 30000]' 430 + ); 431 + INSERT INTO x1(x1) VALUES('optimize'); 432 +" 433 + 434 +do_execsql_test 3.3 { 435 + SELECT count(*) FROM x1 WHERE x1 MATCH '"d e f"' 436 +} {16} 437 + 438 + 439 +finish_test
Added test/fts3defer2.test.
1 +# 2010 October 23 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 + 13 +set testdir [file dirname $argv0] 14 +source $testdir/tester.tcl 15 +source $testdir/malloc_common.tcl 16 +ifcapable !fts3 { finish_test ; return } 17 + 18 +set testprefix fts3defer2 19 + 20 +proc mit {blob} { 21 + set scan(littleEndian) i* 22 + set scan(bigEndian) I* 23 + binary scan $blob $scan($::tcl_platform(byteOrder)) r 24 + return $r 25 +} 26 +db func mit mit 27 + 28 +#----------------------------------------------------------------------------- 29 +# This block of tests - fts3defer2-1.* - test the interaction of deferred NEAR 30 +# expressions and the snippet, offsets and matchinfo functions. 31 +# 32 +do_execsql_test 1.1.1 { 33 + CREATE VIRTUAL TABLE t1 USING fts4; 34 +} 35 +do_execsql_test 1.1.2 "INSERT INTO t1 VALUES('[string repeat {a } 20000]')" 36 +do_execsql_test 1.1.3 "INSERT INTO t1 VALUES('[string repeat {z } 20000]')" 37 +do_execsql_test 1.1.4 { 38 + INSERT INTO t1 VALUES('a b c d e f a x y'); 39 + INSERT INTO t1 VALUES(''); 40 + INSERT INTO t1 VALUES(''); 41 + INSERT INTO t1 VALUES(''); 42 + INSERT INTO t1 VALUES(''); 43 + INSERT INTO t1 VALUES(''); 44 + INSERT INTO t1(t1) VALUES('optimize'); 45 +} 46 +do_execsql_test 1.1.4 { 47 + SELECT count(*) FROM t1_segments WHERE length(block)>10000; 48 + UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; 49 +} {2} 50 + 51 +do_execsql_test 1.2.1 { 52 + SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; 53 +} {{a b c d e f a x y}} 54 + 55 +do_execsql_test 1.2.2 { 56 + SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1)) 57 + FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; 58 +} [list \ 59 + {a b c d [e] [f] [a] x y} \ 60 + {0 1 8 1 0 0 10 1 0 2 12 1} \ 61 + [list 3 1 1 1 1 1 8 8 1 8 8 8 5001 9] 62 +] 63 + 64 +do_execsql_test 1.2.3 { 65 + SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1)) 66 + FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)'; 67 +} [list \ 68 + {[a] b c d [e] [f] [a] x y} \ 69 + {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ 70 + [list 3 1 1 1 1 1 8 8 2 8 8 8 5001 9] 71 +] 72 + 73 +do_execsql_test 1.3.1 { DROP TABLE t1 } 74 + 75 +#----------------------------------------------------------------------------- 76 +# Test cases fts3defer2-2.* focus specifically on the matchinfo function. 77 +# 78 +do_execsql_test 2.1.1 "CREATE VIRTUAL TABLE t2 USING fts4" 79 +do_execsql_test 2.1.2 "INSERT INTO t2 VALUES('[string repeat {a } 10000]')" 80 +do_execsql_test 2.1.3 "INSERT INTO t2 VALUES('b [string repeat {z } 10000]')" 81 +do_execsql_test 2.1.4 [string repeat "INSERT INTO t2 VALUES('x');" 50] 82 +do_execsql_test 2.1.5 { 83 + INSERT INTO t2 VALUES('a b c d e f g'); 84 + INSERT INTO t2 VALUES('a b c d e f g'); 85 +} 86 +foreach {tn sql} { 87 + 1 {} 88 + 2 { INSERT INTO t2(t2) VALUES('optimize') } 89 + 3 { UPDATE t2_segments SET block = zeroblob(length(block)) 90 + WHERE length(block)>10000; 91 + } 92 +} { 93 + execsql $sql 94 + do_execsql_test 2.2.$tn { 95 + SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'a b'; 96 + } [list \ 97 + [list 2 1 1 54 54 1 3 3 54 372 7] \ 98 + [list 2 1 1 54 54 1 3 3 54 372 7] \ 99 + ] 100 +} 101 + 102 +do_execsql_test 2.3.1 { 103 + CREATE VIRTUAL TABLE t3 USING fts4; 104 + INSERT INTO t3 VALUES('a b c d e f'); 105 + INSERT INTO t3 VALUES('x b c d e f'); 106 + INSERT INTO t3 VALUES('d e f a b c'); 107 + INSERT INTO t3 VALUES('b c d e f'); 108 + INSERT INTO t3 VALUES(''); 109 + INSERT INTO t3 VALUES(''); 110 + INSERT INTO t3 VALUES(''); 111 + INSERT INTO t3 VALUES(''); 112 + INSERT INTO t3 VALUES(''); 113 + INSERT INTO t3 VALUES(''); 114 +} 115 +do_execsql_test 2.3.2 " 116 + INSERT INTO t3 VALUES('f e d c b [string repeat {a } 10000]') 117 +" 118 +foreach {tn sql} { 119 + 1 {} 120 + 2 { INSERT INTO t3(t3) VALUES('optimize') } 121 + 3 { UPDATE t3_segments SET block = zeroblob(length(block)) 122 + WHERE length(block)>10000; 123 + } 124 +} { 125 + execsql $sql 126 + do_execsql_test 2.4.$tn { 127 + SELECT docid, mit(matchinfo(t3)) FROM t3 WHERE t3 MATCH '"a b c"'; 128 + } {1 {1 1 1 4 4 11 912 6} 3 {1 1 1 4 4 11 912 6}} 129 +} 130 + 131 + 132 +finish_test 133 +
Added test/fts3fault.test.
1 +# 2010 June 15 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 + 13 +set testdir [file dirname $argv0] 14 +source $testdir/tester.tcl 15 + 16 +set ::testprefix fts3fault 17 + 18 +# Test error handling in the sqlite3Fts3Init() function. This is the 19 +# function that registers the FTS3 module and various support functions 20 +# with SQLite. 21 +# 22 +do_faultsim_test 1 -body { 23 + sqlite3 db test.db 24 + expr 0 25 +} -test { 26 + catch { db close } 27 +} 28 + 29 +# Test error handling in an "ALTER TABLE ... RENAME TO" statement on an 30 +# FTS3 table. Specifically, test renaming the table within a transaction 31 +# after it has been written to. 32 +# 33 +faultsim_delete_and_reopen 34 +do_execsql_test 2.0 { 35 + CREATE VIRTUAL TABLE t1 USING fts3; 36 + INSERT INTO t1 VALUES('test renaming the table'); 37 + INSERT INTO t1 VALUES(' after it has been written'); 38 +} 39 +do_faultsim_test 2 -prep { 40 + sqlite3 db test.db 41 + execsql { 42 + BEGIN; 43 + INSERT INTO t1 VALUES('registers the FTS3 module'); 44 + INSERT INTO t1 VALUES('various support functions'); 45 + } 46 +} -body { 47 + execsql { ALTER TABLE t1 RENAME TO t2 } 48 +} -test { 49 + faultsim_test_result {0 {}} 50 +} 51 + 52 +# Test error handling in the special case where a single prefix query 53 +# matches terms that reside on a large range of leaf nodes. 54 +# 55 +do_test fts3fault-3.0 { 56 + sqlite3 db test.db 57 + execsql { CREATE VIRTUAL TABLE t3 USING fts4; } 58 + execsql { INSERT INTO t3(t3) VALUES('nodesize=50') } 59 + execsql { BEGIN } 60 + for {set i 0} {$i < 1000} {incr i} { 61 + execsql { INSERT INTO t3 VALUES('aaa' || $i) } 62 + } 63 + execsql { COMMIT } 64 +} {} 65 + 66 +do_faultsim_test 3 -faults oom-transient -prep { 67 + sqlite3 db test.db 68 + execsql { SELECT * FROM t3 WHERE t3 MATCH 'x' } 69 +} -body { 70 + execsql { SELECT count(rowid) FROM t3 WHERE t3 MATCH 'aa*' } 71 +} -test { 72 + faultsim_test_result {0 1000} 73 +} 74 + 75 +do_test fts3fault-4.0 { 76 + faultsim_delete_and_reopen 77 + execsql { 78 + CREATE VIRTUAL TABLE t4 USING fts4; 79 + INSERT INTO t4 VALUES('The British Government called on'); 80 + INSERT INTO t4 VALUES('as pesetas then became much'); 81 + } 82 +} {} 83 +faultsim_save_and_close 84 +do_faultsim_test 4 -prep { 85 + faultsim_restore_and_reopen 86 + execsql { SELECT content FROM t4 } 87 +} -body { 88 + execsql { SELECT optimize(t4) FROM t4 LIMIT 1 } 89 +} -test { 90 + faultsim_test_result {0 {{Index optimized}}} 91 +} 92 + 93 +do_test fts3fault-5.0 { 94 + faultsim_delete_and_reopen 95 + execsql { 96 + CREATE VIRTUAL TABLE t5 USING fts4; 97 + INSERT INTO t5 VALUES('The British Government called on'); 98 + INSERT INTO t5 VALUES('as pesetas then became much'); 99 + } 100 +} {} 101 +faultsim_save_and_close 102 +do_faultsim_test 5 -prep { 103 + faultsim_restore_and_reopen 104 + execsql { 105 + BEGIN; 106 + INSERT INTO t5 VALUES('influential in shaping his future outlook'); 107 + INSERT INTO t5 VALUES('might be acceptable to the British electorate'); 108 + } 109 +} -body { 110 + execsql { SELECT rowid FROM t5 WHERE t5 MATCH 'british' } 111 +} -test { 112 + faultsim_test_result {0 {1 4}} 113 +} 114 + 115 +do_test fts3fault-6.0 { 116 + faultsim_delete_and_reopen 117 + execsql { CREATE VIRTUAL TABLE t6 USING fts4 } 118 +} {} 119 +faultsim_save_and_close 120 +do_faultsim_test 6 -prep { 121 + faultsim_restore_and_reopen 122 + execsql { SELECT rowid FROM t6 } 123 +} -body { 124 + execsql { DROP TABLE t6 } 125 +} -test { 126 + faultsim_test_result {0 {}} 127 +} 128 + 129 +finish_test
Changes to test/fts3malloc.test.
39 39 40 40 41 41 proc normal_list {l} { 42 42 set ret [list] 43 43 foreach elem $l {lappend ret $elem} 44 44 set ret 45 45 } 46 - 47 46 48 47 do_write_test fts3_malloc-1.1 sqlite_master { 49 48 CREATE VIRTUAL TABLE ft1 USING fts3(a, b) 50 49 } 51 50 do_write_test fts3_malloc-1.2 sqlite_master { 52 51 CREATE VIRTUAL TABLE ft2 USING fts3([a], [b]); 53 52 }
Changes to test/fts3query.test.
20 20 # If this build does not include FTS3, skip the tests in this file. 21 21 # 22 22 ifcapable !fts3 { finish_test ; return } 23 23 source $testdir/malloc_common.tcl 24 24 source $testdir/fts3_common.tcl 25 25 set DO_MALLOC_TEST 0 26 26 27 +set testprefix fts3query 28 + 27 29 do_test fts3query-1.1 { 28 30 execsql { 29 31 CREATE VIRTUAL TABLE t1 USING fts3(x); 30 32 BEGIN; 31 33 INSERT INTO t1 VALUES('The source code for SQLite is in the public'); 32 34 } 33 35 } {} ................................................................................ 125 127 do_test fts3query-4.4 { 126 128 eqp "SELECT t1.number FROM t1, bt WHERE t1.number=bt.rowid ORDER BY t1.date" 127 129 } {0 0 {TABLE t1 WITH INDEX i1 ORDER BY} 1 1 {TABLE bt USING PRIMARY KEY}} 128 130 do_test fts3query-4.5 { 129 131 eqp "SELECT t1.number FROM bt, t1 WHERE t1.number=bt.rowid ORDER BY t1.date" 130 132 } {0 1 {TABLE t1 WITH INDEX i1 ORDER BY} 1 0 {TABLE bt USING PRIMARY KEY}} 131 133 134 + 135 +# Test that calling matchinfo() with the wrong number of arguments, or with 136 +# an invalid argument returns an error. 137 +# 138 +do_execsql_test 5.1 { 139 + CREATE VIRTUAL TABLE t2 USING FTS4; 140 + INSERT INTO t2 VALUES('it was the first time in history'); 141 +} 142 +do_select_tests 5.2 -errorformat { 143 + wrong number of arguments to function %s() 144 +} { 145 + 1 "SELECT matchinfo() FROM t2 WHERE t2 MATCH 'history'" matchinfo 146 + 2 "SELECT matchinfo(t2, t2) FROM t2 WHERE t2 MATCH 'history'" matchinfo 147 + 148 + 3 "SELECT snippet(t2, 1, 2, 3, 4, 5, 6) FROM t2 WHERE t2 MATCH 'history'" 149 + snippet 150 +} 151 +do_select_tests 5.3 -errorformat { 152 + illegal first argument to %s 153 +} { 154 + 1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo 155 + 2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'" offsets 156 + 3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'" snippet 157 + 4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'" optimize 158 +} 159 +do_execsql_test 5.4.0 { UPDATE t2_content SET c0content = X'1234' } 160 +do_select_tests 5.4 -errorformat { 161 + illegal first argument to %s 162 +} { 163 + 1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo 164 + 2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'" offsets 165 + 3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'" snippet 166 + 4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'" optimize 167 +} 168 +do_execsql_test 5.5 { DROP TABLE t2 } 169 + 170 +# Test the snippet() function with 1 to 6 arguments. 171 +# 172 +do_execsql_test 6.1 { 173 + CREATE VIRTUAL TABLE t3 USING FTS4(a, b); 174 + INSERT INTO t3 VALUES('no gestures', 'another intriguing discovery by observing the hand gestures (called beats) people make while speaking. Research has shown that such gestures do more than add visual emphasis to our words (many people gesture while they''re on the telephone, for example); it seems they actually help our brains find words'); 175 +} 176 +do_select_tests 6.2 { 177 + 1 "SELECT snippet(t3) FROM t3 WHERE t3 MATCH 'gestures'" 178 + {{<b>...</b>hand <b>gestures</b> (called beats) people make while speaking. Research has shown that such <b>gestures</b> do<b>...</b>}} 179 + 180 + 2 "SELECT snippet(t3, 'XXX') FROM t3 WHERE t3 MATCH 'gestures'" 181 + {{<b>...</b>hand XXXgestures</b> (called beats) people make while speaking. Research has shown that such XXXgestures</b> do<b>...</b>}} 182 + 183 + 3 "SELECT snippet(t3, 'XXX', 'YYY') FROM t3 WHERE t3 MATCH 'gestures'" 184 + {{<b>...</b>hand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY do<b>...</b>}} 185 + 186 + 4 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ') FROM t3 WHERE t3 MATCH 'gestures'" 187 + {{ZZZhand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY doZZZ}} 188 + 189 + 5 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 1) FROM t3 WHERE t3 MATCH 'gestures'" 190 + {{ZZZhand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY doZZZ}} 191 + 192 + 6 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 0) FROM t3 WHERE t3 MATCH 'gestures'" 193 + {{no XXXgesturesYYY}} 194 + 195 + 7 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 1, 5) FROM t3 WHERE t3 MATCH 'gestures'" 196 + {{ZZZthe hand XXXgesturesYYY (called beatsZZZ}} 197 +} 198 + 132 199 133 200 finish_test 134 201
Changes to test/fts3snippet.test.
16 16 17 17 set testdir [file dirname $argv0] 18 18 source $testdir/tester.tcl 19 19 20 20 # If SQLITE_ENABLE_FTS3 is not defined, omit this file. 21 21 ifcapable !fts3 { finish_test ; return } 22 22 source $testdir/fts3_common.tcl 23 -source $testdir/malloc_common.tcl 24 23 25 24 set sqlite_fts3_enable_parentheses 1 26 25 set DO_MALLOC_TEST 0 27 26 28 27 # Transform the list $L to its "normal" form. So that it can be compared to 29 28 # another list with the same set of elements using [string compare]. 30 29 # ................................................................................ 455 454 do_select_test $T.10.5 { 456 455 SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft; 457 456 } {0 blob 0 blob 0 blob} 458 457 do_select_test $T.10.6 { 459 458 SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r 460 459 } {0 blob} 461 460 } 462 - 463 461 464 462 set sqlite_fts3_enable_parentheses 0 465 463 finish_test
Added test/incrblob3.test.
1 +# 2010 October 20 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 +# 13 + 14 +set testdir [file dirname $argv0] 15 +source $testdir/tester.tcl 16 + 17 +sqlite3 db test.db 18 +sqlite3_db_config_lookaside db 0 0 0 19 + 20 +do_execsql_test incrblob3-1.1 { 21 + CREATE TABLE blobs(k INTEGER PRIMARY KEY, v BLOB); 22 + INSERT INTO blobs VALUES(1, zeroblob(100)); 23 + INSERT INTO blobs VALUES(2, zeroblob(100)); 24 +} {} 25 + 26 +# Test the sqlite3_blob_reopen()/read()/write() functions. 27 +# 28 +do_test incrblob3-1.2 { 29 + set ::blob [db incrblob blobs v 1] 30 + puts $::blob "hello world" 31 +} {} 32 + 33 +do_test incrblob3-1.3 { 34 + sqlite3_blob_reopen $::blob 2 35 + puts $::blob "world hello" 36 +} {} 37 + 38 +do_test incrblob3-1.4 { 39 + sqlite3_blob_reopen $::blob 1 40 + gets $::blob 41 +} {hello world} 42 + 43 +do_test incrblob3-1.5 { 44 + sqlite3_blob_reopen $::blob 2 45 + gets $::blob 46 +} {world hello} 47 + 48 +do_test incrblob3-1.6 { close $::blob } {} 49 + 50 +# Test some error conditions. 51 +# 52 +# incrblob3-2.1: Attempting to reopen a row that does not exist. 53 +# incrblob3-2.2: Attempting to reopen a row that does not contain a blob 54 +# or text value. 55 +# 56 +do_test incrblob3-2.1.1 { 57 + set ::blob [db incrblob blobs v 1] 58 + list [catch {sqlite3_blob_reopen $::blob 3} msg] $msg 59 +} {1 SQLITE_ERROR} 60 +do_test incrblob3-2.1.2 { 61 + list [sqlite3_errcode db] [sqlite3_errmsg db] 62 +} {SQLITE_ERROR {no such rowid: 3}} 63 +do_test incrblob3-2.1.3 { 64 + list [catch {sqlite3_blob_reopen $::blob 1} msg] $msg 65 +} {1 SQLITE_ABORT} 66 +do_test incrblob3-2.1.4 { close $::blob } {} 67 + 68 +do_execsql_test incrblob3-2.2.1 { 69 + INSERT INTO blobs VALUES(3, 42); 70 + INSERT INTO blobs VALUES(4, 54.4); 71 + INSERT INTO blobs VALUES(5, NULL); 72 +} 73 +foreach {tn rowid type} { 74 + 1 3 integer 75 + 2 4 real 76 + 3 5 null 77 +} { 78 + do_test incrblob3-2.2.$tn.1 { 79 + set ::blob [db incrblob blobs v 1] 80 + list [catch {sqlite3_blob_reopen $::blob $rowid} msg] $msg 81 + } {1 SQLITE_ERROR} 82 + do_test incrblob3-2.2.$tn.2 { 83 + list [sqlite3_errcode db] [sqlite3_errmsg db] 84 + } "SQLITE_ERROR {cannot open value of type $type}" 85 + 86 + do_test incrblob3-2.2.$tn.3 { 87 + list [catch {sqlite3_blob_reopen $::blob 1} msg] $msg 88 + } {1 SQLITE_ABORT} 89 + do_test incrblob3-2.2.$tn.4 { 90 + list [catch {sqlite3_blob_read $::blob 0 10} msg] $msg 91 + } {1 SQLITE_ABORT} 92 + do_test incrblob3-2.2.$tn.5 { 93 + list [catch {sqlite3_blob_write $::blob 0 "abcd"} msg] $msg 94 + } {1 SQLITE_ABORT} 95 + 96 + do_test incrblob3-2.2.$tn.4 { close $::blob } {} 97 +} 98 + 99 +# Test that passing NULL to sqlite3_blob_XXX() APIs returns SQLITE_MISUSE. 100 +# 101 +# incrblob3-3.1: sqlite3_blob_reopen() 102 +# incrblob3-3.2: sqlite3_blob_read() 103 +# incrblob3-3.3: sqlite3_blob_write() 104 +# incrblob3-3.4: sqlite3_blob_bytes() 105 +# 106 +do_test incrblob3-3.1 { 107 + list [catch {sqlite3_blob_reopen {} 3} msg] $msg 108 +} {1 SQLITE_MISUSE} 109 + 110 +do_test incrblob3-3.2 { 111 + list [catch {sqlite3_blob_read {} 0 10} msg] $msg 112 +} {1 SQLITE_MISUSE} 113 + 114 +do_test incrblob3-3.3 { 115 + list [catch {sqlite3_blob_write {} 0 "abcd"} msg] $msg 116 +} {1 SQLITE_MISUSE} 117 + 118 +do_test incrblob3-3.4 { sqlite3_blob_bytes {} } {0} 119 + 120 +do_test incrblob3-3.5 { sqlite3_blob_close {} } {} 121 + 122 +# Test out-of-range reading and writing 123 +# 124 +do_test incrblob3-4.1 { 125 + set ::blob [db incrblob blobs v 1] 126 + sqlite3_blob_bytes $::blob 127 +} {100} 128 +do_test incrblob3-4.2 { 129 + list [catch { sqlite3_blob_read $::blob -1 10 } msg] $msg 130 +} {1 SQLITE_ERROR} 131 +do_test incrblob3-4.3 { 132 + list [catch { sqlite3_blob_read $::blob 0 -10 } msg] $msg 133 +} {1 SQLITE_ERROR} 134 +do_test incrblob3-4.4 { 135 + list [catch { sqlite3_blob_read $::blob 95 10 } msg] $msg 136 +} {1 SQLITE_ERROR} 137 +do_test incrblob3-4.5 { 138 + list [catch { sqlite3_blob_write $::blob -1 "abcdefghij" 10 } msg] $msg 139 +} {1 SQLITE_ERROR} 140 +do_test incrblob3-4.6 { 141 + list [catch { sqlite3_blob_write $::blob 0 "abcdefghij" -10 } msg] $msg 142 +} {1 SQLITE_ERROR} 143 +do_test incrblob3-4.7 { 144 + list [catch { sqlite3_blob_write $::blob 95 "abcdefghij" } msg] $msg 145 +} {1 SQLITE_ERROR} 146 + 147 +do_test incrblob3-4.8 { close $::blob } {} 148 + 149 +# Test that modifying the row a blob handle points to aborts the blob. 150 +# 151 +do_test incrblob3-5.1 { 152 + set ::blob [db incrblob blobs v 1] 153 + sqlite3_blob_bytes $::blob 154 +} {100} 155 +do_test incrblob3-5.2 { 156 + execsql { UPDATE blobs SET v = '123456789012345678901234567890' WHERE k = 1 } 157 + list [catch { sqlite3_blob_read $::blob 0 10 } msg] $msg 158 +} {1 SQLITE_ABORT} 159 + 160 +# Test various errors that can occur in sqlite3_blob_open(): 161 +# 162 +# 1. Trying to open a virtual table column. 163 +# 2. Trying to open a view column. 164 +# 3. Trying to open a column that does not exist. 165 +# 4. Trying to open a read/write handle on an indexed column. 166 +# 5. Trying to open a read/write handle on the child key of an FK constraint. 167 +# 168 +ifcapable fts3 { 169 + do_test incrblob3-6.1 { 170 + execsql { 171 + CREATE VIRTUAL TABLE ft USING fts3; 172 + INSERT INTO ft VALUES('rules to open a column to which'); 173 + } 174 + 175 + list [catch { db incrblob ft content 1 } msg] $msg 176 + } {1 {cannot open virtual table: ft}} 177 +} 178 +ifcapable view { 179 + do_test incrblob3-6.2 { 180 + execsql { CREATE VIEW v1 AS SELECT * FROM blobs } 181 + list [catch { db incrblob v1 content 1 } msg] $msg 182 + } {1 {cannot open view: v1}} 183 +} 184 + 185 +do_test incrblob3-6.3 { 186 + list [catch { db incrblob blobs content 1 } msg] $msg 187 +} {1 {no such column: "content"}} 188 + 189 +do_test incrblob3-6.4.1 { 190 + execsql { 191 + CREATE TABLE t1(a, b); 192 + CREATE INDEX i1 ON t1(b); 193 + INSERT INTO t1 VALUES(zeroblob(100), zeroblob(100)); 194 + } 195 + list [catch { db incrblob t1 b 1 } msg] $msg 196 +} {1 {cannot open indexed column for writing}} 197 +do_test incrblob3-6.4.2 { 198 + set ::blob [db incrblob t1 a 1] 199 + close $::blob 200 +} {} 201 +do_test incrblob3-6.4.3 { 202 + set ::blob [db incrblob -readonly t1 b 1] 203 + close $::blob 204 +} {} 205 + 206 +do_test incrblob3-6.5.1 { 207 + execsql { 208 + CREATE TABLE p1(a PRIMARY KEY); 209 + CREATE TABLE c1(a, b REFERENCES p1); 210 + PRAGMA foreign_keys = 1; 211 + INSERT INTO p1 VALUES(zeroblob(100)); 212 + INSERT INTO c1 VALUES(zeroblob(100), zeroblob(100)); 213 + } 214 + list [catch { db incrblob c1 b 1 } msg] $msg 215 +} {1 {cannot open foreign key column for writing}} 216 + 217 +do_test incrblob3-6.5.2 { 218 + set ::blob [db incrblob c1 a 1] 219 + close $::blob 220 +} {} 221 +do_test incrblob3-6.5.3 { 222 + set ::blob [db incrblob -readonly c1 b 1] 223 + close $::blob 224 +} {} 225 +do_test incrblob3-6.5.4 { 226 + execsql { PRAGMA foreign_keys = 0 } 227 + set ::blob [db incrblob c1 b 1] 228 + close $::blob 229 +} {} 230 + 231 + 232 +# Test that sqlite3_blob_open() handles transient and persistent schema 233 +# errors correctly. 234 +# 235 +do_test incrblob3-7.1 { 236 + sqlite3 db2 test.db 237 + sqlite3_db_config_lookaside db2 0 0 0 238 + execsql { CREATE TABLE t2(x) } db2 239 + set ::blob [db incrblob blobs v 1] 240 + close $::blob 241 +} {} 242 +db2 close 243 + 244 +testvfs tvfs -default 1 245 +tvfs filter xAccess 246 +tvfs script access_method 247 + 248 +proc access_method {args} { 249 + set schemacookie [hexio_get_int [hexio_read test.db 40 4]] 250 + incr schemacookie 251 + hexio_write test.db 40 [hexio_render_int32 $schemacookie] 252 + 253 + set dbversion [hexio_get_int [hexio_read test.db 24 4]] 254 + incr dbversion 255 + hexio_write test.db 24 [hexio_render_int32 $dbversion] 256 + 257 + return "" 258 +} 259 + 260 +do_test incrblob3-7.2 { 261 + sqlite3 db test.db 262 + sqlite3_db_config_lookaside db 0 0 0 263 + list [catch {db incrblob blobs v 1} msg] $msg 264 +} {1 {database schema has changed}} 265 +db close 266 +tvfs delete 267 + 268 +finish_test 269 +
Added test/incrblobfault.test.
1 +# 2010 October 26 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 +# 13 + 14 +set testdir [file dirname $argv0] 15 +source $testdir/tester.tcl 16 + 17 +set testprefix incrblobfault 18 + 19 +do_execsql_test 1.0 { 20 + CREATE TABLE blob(x INTEGER PRIMARY KEY, v BLOB); 21 + INSERT INTO blob VALUES(1, 'hello world'); 22 + INSERT INTO blob VALUES(2, 'world hello'); 23 + INSERT INTO blob SELECT NULL, v FROM blob; 24 + INSERT INTO blob SELECT NULL, v FROM blob; 25 + INSERT INTO blob SELECT NULL, v FROM blob; 26 + INSERT INTO blob SELECT NULL, v FROM blob; 27 + INSERT INTO blob SELECT NULL, v FROM blob; 28 + INSERT INTO blob SELECT NULL, v FROM blob; 29 + INSERT INTO blob SELECT NULL, v FROM blob; 30 + INSERT INTO blob SELECT NULL, v FROM blob; 31 + INSERT INTO blob SELECT NULL, v FROM blob; 32 + INSERT INTO blob SELECT NULL, v FROM blob; 33 +} 34 + 35 +do_faultsim_test 1 -prep { 36 + sqlite3 db test.db 37 + set ::blob [db incrblob blob v 1] 38 +} -body { 39 + if {[catch {sqlite3_blob_reopen $::blob 1000}]} { 40 + error [sqlite3_errmsg db] 41 + } 42 +} -test { 43 + faultsim_test_result {0 {}} 44 + close $::blob 45 +} 46 + 47 +do_faultsim_test 2 -prep { 48 + sqlite3 db test.db 49 + set ::blob [db incrblob blob v 1] 50 +} -body { 51 + if {[catch {sqlite3_blob_reopen $::blob -1}]} { 52 + error [sqlite3_errmsg db] 53 + } 54 +} -test { 55 + faultsim_test_result {1 {no such rowid: -1}} 56 + close $::blob 57 +} 58 + 59 +do_faultsim_test 3 -prep { 60 + sqlite3 db test.db 61 +} -body { 62 + set ::blob [db incrblob blob v 1] 63 + gets $::blob 64 +} -test { 65 + faultsim_test_result {0 {hello world}} 66 + catch { close $::blob } 67 +} 68 + 69 +finish_test 70 +
Changes to test/malloc_common.tcl.
109 109 proc do_faultsim_test {name args} { 110 110 global FAULTSIM 111 111 112 112 set DEFAULT(-faults) [array names FAULTSIM] 113 113 set DEFAULT(-prep) "" 114 114 set DEFAULT(-body) "" 115 115 set DEFAULT(-test) "" 116 + 117 + fix_testname name 116 118 117 119 array set O [array get DEFAULT] 118 120 array set O $args 119 121 foreach o [array names O] { 120 122 if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" } 121 123 } 122 124 ................................................................................ 522 524 # by parameter $result, or (b) TCL throws an "out of memory" error. 523 525 # 524 526 # If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement 525 527 # is executed just once. In this case the test case passes if the results 526 528 # match the expected results passed via parameter $result. 527 529 # 528 530 proc do_select_test {name sql result} { 529 - uplevel [list doPassiveTest 0 $name $sql [list 0 $result]] 531 + uplevel [list doPassiveTest 0 $name $sql [list 0 [list {*}$result]]] 530 532 } 531 533 532 534 proc do_restart_select_test {name sql result} { 533 535 uplevel [list doPassiveTest 1 $name $sql [list 0 $result]] 534 536 } 535 537 536 538 proc do_error_test {name sql error} { 537 539 uplevel [list doPassiveTest 0 $name $sql [list 1 $error]] 538 540 } 539 541 540 542 proc doPassiveTest {isRestart name sql catchres} { 541 543 if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } 544 + 545 + if {[info exists ::testprefix] 546 + && [string is integer [string range $name 0 0]] 547 + } { 548 + set name $::testprefix.$name 549 + } 542 550 543 551 switch $::DO_MALLOC_TEST { 544 552 0 { # No malloc failures. 545 553 do_test $name [list set {} [uplevel [list catchsql $sql]]] $catchres 546 554 return 547 555 } 548 556 1 { # Simulate transient failures.
Changes to test/permutations.test.
155 155 All multi-threaded tests. 156 156 } -files { 157 157 notify2.test thread001.test thread002.test thread003.test 158 158 thread004.test thread005.test walthread.test 159 159 } 160 160 161 161 test_suite "fts3" -prefix "" -description { 162 - All FTS3 tests except fts3malloc.test and fts3rnd.test. 162 + All FTS3 tests except fts3rnd.test. 163 163 } -files { 164 164 fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test 165 165 fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test 166 166 fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test 167 167 fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test 168 - fts3e.test fts3expr.test fts3expr2.test fts3near.test 169 - fts3query.test fts3snippet.test 168 + fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 169 + fts3near.test fts3query.test fts3shared.test fts3snippet.test 170 + 171 + fts3fault.test fts3malloc.test 170 172 } 171 173 172 174 173 175 lappend ::testsuitelist xxx 174 176 #------------------------------------------------------------------------- 175 177 # Define the coverage related test suites: 176 178 #
Changes to test/tester.tcl.
1371 1371 } 1372 1372 1373 1373 # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set 1374 1374 # to non-zero, then set the global variable $AUTOVACUUM to 1. 1375 1375 set AUTOVACUUM $sqlite_options(default_autovacuum) 1376 1376 1377 1377 source $testdir/thread_common.tcl 1378 +source $testdir/malloc_common.tcl