Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add virtual table module "fts4aux", used to inspect the full-text index of an fts4 table directly. Also add the "compress" and "uncompress" fts4 options. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts4aux |
Files: | files | file ages | folders |
SHA1: |
b010ddcc52889160af2183a33c5f483b |
User & Date: | dan 2011-02-01 16:34:32.732 |
Context
2011-02-01
| ||
17:55 | Fix a case in fts4 where a corrupt %_stat table could lead to a crash. (check-in: 4ade96ce97 user: dan tags: fts4aux) | |
16:34 | Add virtual table module "fts4aux", used to inspect the full-text index of an fts4 table directly. Also add the "compress" and "uncompress" fts4 options. (check-in: b010ddcc52 user: dan tags: fts4aux) | |
00:04 | Version 3.7.5 (check-in: ed759d5a9e user: drh tags: trunk, release, version-3.7.5) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
444 445 446 447 448 449 450 451 452 453 454 455 456 457 | assert( p->pSegments==0 ); /* Free any prepared statements held */ for(i=0; i<SizeofArray(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p->zSegmentsTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); return SQLITE_OK; } | > > | 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 | assert( p->pSegments==0 ); /* Free any prepared statements held */ for(i=0; i<SizeofArray(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p->zSegmentsTbl); sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); return SQLITE_OK; } |
︙ | ︙ | |||
660 661 662 663 664 665 666 667 668 669 670 671 672 673 | zValue = sqlite3_mprintf("%s", &zCsr[1]); if( zValue ){ sqlite3Fts3Dequote(zValue); } *pzValue = zValue; return 1; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 | zValue = sqlite3_mprintf("%s", &zCsr[1]); if( zValue ){ sqlite3Fts3Dequote(zValue); } *pzValue = zValue; return 1; } /* ** Append the output of a printf() style formatting to an existing string. */ static void fts3Appendf( int *pRc, /* IN/OUT: Error code */ char **pz, /* IN/OUT: Pointer to string buffer */ const char *zFormat, /* Printf format string to append */ ... /* Arguments for printf format string */ ){ if( *pRc==SQLITE_OK ){ va_list ap; char *z; va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); if( z && *pz ){ char *z2 = sqlite3_mprintf("%s%s", *pz, z); sqlite3_free(z); z = z2; } if( z==0 ) *pRc = SQLITE_NOMEM; sqlite3_free(*pz); *pz = z; } } /* ** Return a list of comma separated SQL expressions that could be used ** in a SELECT statement such as the following: ** ** SELECT <list of expressions> FROM %_content AS x ... ** ** to return the docid, followed by each column of text data in order ** from left to write. If parameter zFunc is not NULL, then instead of ** being returned directly each column of text data is passed to an SQL ** function named zFunc first. For example, if zFunc is "unzip" and the ** table has the three user-defined columns "a", "b", and "c", the following ** string is returned: ** ** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and ** a NULL pointer is returned). Otherwise, if an OOM error is encountered ** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If ** no error occurs, *pRc is left unmodified. */ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zRet = 0; int i; if( !zFunc ) zFunc = ""; fts3Appendf(pRc, &zRet, "docid"); for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunc, i, p->azColumn[i]); } return zRet; } /* ** Return a list of N comma separated question marks, where N is the number ** of columns in the %_content table (one for the docid plus one for each ** user-defined text column). ** ** If argument zFunc is not NULL, then all but the first question mark ** is preceded by zFunc and an open bracket, and followed by a closed ** bracket. For example, if zFunc is "zip" and the FTS3 table has three ** user-defined text columns, the following string is returned: ** ** "?, zip(?), zip(?), zip(?)" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and ** a NULL pointer is returned). Otherwise, if an OOM error is encountered ** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If ** no error occurs, *pRc is left unmodified. */ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zRet = 0; int i; if( !zFunc ) zFunc = ""; fts3Appendf(pRc, &zRet, "?"); for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunc); } return zRet; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** |
︙ | ︙ | |||
697 698 699 700 701 702 703 704 705 706 707 708 709 710 | int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; | > > > | 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 | int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ char *zCompress = 0; char *zUncompress = 0; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; |
︙ | ︙ | |||
747 748 749 750 751 752 753 754 755 756 757 758 759 760 | if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ bNoDocsize = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); rc = SQLITE_ERROR; } }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; } sqlite3_free(zVal); } | > > > > > > | 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 | if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ bNoDocsize = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); rc = SQLITE_ERROR; } }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ zCompress = zVal; zVal = 0; }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ zUncompress = zVal; zVal = 0; }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; } sqlite3_free(zVal); } |
︙ | ︙ | |||
820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 | memcpy(zCsr, z, n); zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; zCsr += n+1; assert( zCsr <= &((char *)p)[nByte] ); } /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ rc = fts3CreateTables(p); } /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database (see ** function sqlite3Fts3SegReaderCost() for details). */ fts3DatabasePageSize(&rc, p); /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: | > > > > > > > > > | > | 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 | memcpy(zCsr, z, n); zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; zCsr += n+1; assert( zCsr <= &((char *)p)[nByte] ); } if( (zCompress==0)!=(zUncompress==0) ){ char const *zMissing = (zCompress==0 ? "compress" : "uncompress"); rc = SQLITE_ERROR; *pzErr = sqlite3_mprintf("missing %s parameter", zMissing); } p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc); p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); if( rc!=SQLITE_OK ) goto fts3_init_out; /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ rc = fts3CreateTables(p); } /* Figure out the page-size for the database. This is required in order to ** estimate the cost of loading large doclists from the database (see ** function sqlite3Fts3SegReaderCost() for details). */ fts3DatabasePageSize(&rc, p); /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ pTokenizer->pModule->xDestroy(pTokenizer); } |
︙ | ︙ | |||
1931 1932 1933 1934 1935 1936 1937 | if( !*ppOut ) return SQLITE_NOMEM; sqlite3Fts3PutVarint(*ppOut, docid); } return SQLITE_OK; } | < < < < | | > > | > | > > > > > | | > > > > > > > > < > > | > | < | | > > | | < > | > > | > > > | > | > > > > > > > | | > | | > > > > > | > > > > > > > | | < | < > | | | < > < < < | < < | > > > > < | > | | < < | < < < | < < < < | | < < < < < < | < < < | | < | < < | > < < < < < < < < < | < < < < | < < < | < < | < < < < < < < | | > > > > > | 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 | if( !*ppOut ) return SQLITE_NOMEM; sqlite3Fts3PutVarint(*ppOut, docid); } return SQLITE_OK; } int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3SegReaderCursor *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; int iAge = 0; sqlite3_stmt *pStmt = 0; Fts3SegReader *pPending = 0; assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); assert( FTS3_SEGCURSOR_PENDING<0 ); assert( FTS3_SEGCURSOR_ALL<0 ); assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ if( iLevel<0 ){ rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); if( rc==SQLITE_OK && pPending ){ int nByte = (sizeof(Fts3SegReader *) * 16); pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; }else{ pCsr->apSegment[0] = pPending; pCsr->nSegment = 1; pPending = 0; } } } if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); /* If nSegment is a multiple of 16 the array needs to be extended. */ if( (pCsr->nSegment%16)==0 ){ Fts3SegReader **apNew; int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); if( !apNew ){ rc = SQLITE_NOMEM; goto finished; } pCsr->apSegment = apNew; } /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ if( iStartBlock && zTerm ){ sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); if( rc!=SQLITE_OK ) goto finished; if( isPrefix==0 ) iLeavesEndBlock = iStartBlock; } rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] ); if( rc!=SQLITE_OK ) goto finished; pCsr->nSegment++; iAge++; } } finished: rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_DONE ) rc = rc2; sqlite3Fts3SegReaderFree(pPending); return rc; } static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); if( pSegcsr ){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i; int nCost = 0; rc = sqlite3Fts3SegReaderCursor( p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, pSegcsr); for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); } pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; return rc; } static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } /* ** This function retreives the doclist for the specified term (or term ** prefix) from the database. ** ** The returned doclist may be in one of two formats, depending on the ** value of parameter isReqPos. If isReqPos is zero, then the doclist is |
︙ | ︙ | |||
2077 2078 2079 2080 2081 2082 2083 | Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ | | | | | | > > > | > | > > < | | | 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 | Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); tsc.isReqPos = isReqPos; filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; filter.nTerm = pTok->n; rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); while( SQLITE_OK==rc && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ rc = fts3TermSelectCb(p, (void *)&tsc, pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist ); } if( rc==SQLITE_OK ){ rc = fts3TermSelectMerge(&tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ int i; for(i=0; i<SizeofArray(tsc.aaOutput); i++){ sqlite3_free(tsc.aaOutput[i]); } } fts3SegReaderCursorFree(pSegcsr); pTok->pSegcsr = 0; return rc; } /* ** This function counts the total number of docids in the doclist stored ** in buffer aList[], size nList bytes. ** |
︙ | ︙ | |||
2234 2235 2236 2237 2238 2239 2240 | /* If this is an xFilter() evaluation, create a segment-reader for each ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo ** evaluation, only create segment-readers if there are no Fts3DeferredToken ** objects attached to the phrase-tokens. */ for(ii=0; ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; | | | | | 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 | /* If this is an xFilter() evaluation, create a segment-reader for each ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo ** evaluation, only create segment-readers if there are no Fts3DeferredToken ** objects attached to the phrase-tokens. */ for(ii=0; ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pSegcsr==0 ){ if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) ){ rc = fts3TermSegReaderCursor( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); if( rc!=SQLITE_OK ) return rc; } } } for(ii=0; ii<pPhrase->nToken; ii++){ |
︙ | ︙ | |||
2271 2272 2273 2274 2275 2276 2277 | pTok = &pPhrase->aToken[iTok]; }else{ int nMinCost = 0x7FFFFFFF; int jj; /* Find the remaining token with the lowest cost. */ for(jj=0; jj<pPhrase->nToken; jj++){ | | | | | | | 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 | pTok = &pPhrase->aToken[iTok]; }else{ int nMinCost = 0x7FFFFFFF; int jj; /* Find the remaining token with the lowest cost. */ for(jj=0; jj<pPhrase->nToken; jj++){ Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; if( pSegcsr && pSegcsr->nCost<nMinCost ){ iTok = jj; nMinCost = pSegcsr->nCost; } } pTok = &pPhrase->aToken[iTok]; /* This branch is taken if it is determined that loading the doclist ** for the next token would require more IO than loading all documents ** currently identified by doclist pOut/nOut. No further doclists will ** be loaded from the full-text index for this phrase. */ if( nMinCost>nDoc && ii>0 ){ rc = fts3DeferExpression(pCsr, pCsr->pExpr); break; } } if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); }else{ if( pTok->pSegcsr ){ rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); } pTok->bFulltext = 1; } assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); if( rc!=SQLITE_OK ) break; if( isFirst ){ pOut = pList; nOut = nList; if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ nDoc = fts3DoclistCountDocids(1, pOut, nOut); |
︙ | ︙ | |||
2476 2477 2478 2479 2480 2481 2482 | if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; | | | | | 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 | if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pSegcsr==0 ){ rc = fts3TermSegReaderCursor( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); } } }else{ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); if( rc==SQLITE_OK ){ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); |
︙ | ︙ | |||
2502 2503 2504 2505 2506 2507 2508 | */ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ int kk; for(kk=0; kk<pPhrase->nToken; kk++){ | | | | < | < | 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 | */ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ int kk; for(kk=0; kk<pPhrase->nToken; kk++){ fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); pPhrase->aToken[kk].pSegcsr = 0; } } fts3ExprFreeSegReaders(pExpr->pLeft); fts3ExprFreeSegReaders(pExpr->pRight); } } /* ** Return the sum of the costs of all tokens in the expression pExpr. This ** function must be called after Fts3SegReaderArrays have been allocated ** for all tokens using fts3ExprAllocateSegReaders(). */ static int fts3ExprCost(Fts3Expr *pExpr){ int nCost; /* Return value */ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; nCost = 0; for(ii=0; ii<pPhrase->nToken; ii++){ Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; if( pSegcsr ) nCost += pSegcsr->nCost; } }else{ nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); } return nCost; } |
︙ | ︙ | |||
2868 2869 2870 2871 2872 2873 2874 | sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ const char *azSql[] = { | | | | 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 | sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ const char *azSql[] = { "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ "SELECT %s FROM %Q.'%q_content' AS x ", /* full-scan */ }; int rc; /* Return code */ char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; UNUSED_PARAMETER(idxStr); |
︙ | ︙ | |||
2924 2925 2926 2927 2928 2929 2930 | } /* Compile a SELECT statement for this cursor. For a full-table-scan, the ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ | | > | 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 | } /* Compile a SELECT statement for this cursor. For a full-table-scan, the ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; zSql = sqlite3_mprintf(zSql, p->zReadExprlist, p->zDb, p->zName); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); } if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ |
︙ | ︙ | |||
3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 | const sqlite3_tokenizer_module *pPorter = 0; #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; | > > > | 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 | const sqlite3_tokenizer_module *pPorter = 0; #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
103 104 105 106 107 108 109 | typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; | | | 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. ** All other methods receive a pointer to the structure as one of their ** arguments. |
︙ | ︙ | |||
126 127 128 129 130 131 132 133 134 135 136 137 138 139 | sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[24]; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ | > > > | 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[24]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ |
︙ | ︙ | |||
213 214 215 216 217 218 219 | ** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ int bFulltext; /* True if full-text index was used */ | | | 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | ** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ int bFulltext; /* True if full-text index was used */ Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ }; struct Fts3Phrase { /* Variables populated by fts3_expr.c when parsing a MATCH expression */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ |
︙ | ︙ | |||
281 282 283 284 285 286 287 | int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); | < < < < | > > > > > > > | > > > > > > > > > > > > > > > > > > > > | 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 | int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); int sqlite3Fts3SegReaderCursor( Fts3Table *, int, const char *, int, int, Fts3SegReaderCursor *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 /* Type passed as 4th argument to SegmentReaderIterate() */ struct Fts3SegFilter { const char *zTerm; int nTerm; int iCol; int flags; }; struct Fts3SegReaderCursor { /* Used internally by sqlite3Fts3SegReaderXXX() calls */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ int nSegment; /* Size of apSegment array */ int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ /* Cost of running this iterator. Used by fts3.c only. */ int nCost; /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ int nTerm; /* Size of zTerm in bytes */ char *aDoclist; /* Pointer to doclist buffer */ int nDoclist; /* Size of aDoclist[] in bytes */ }; /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); |
︙ | ︙ | |||
351 352 353 354 355 356 357 358 | char **, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); #endif #endif /* _FTSINT_H */ | > > > | 377 378 379 380 381 382 383 384 385 386 387 | char **, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); #endif /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); #endif /* _FTSINT_H */ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
208 209 210 211 212 213 214 | /* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", /* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", /* 2 */ "DELETE FROM %Q.'%q_content'", /* 3 */ "DELETE FROM %Q.'%q_segments'", /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", | | | < < | | < < < < < < < < < | | 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 | /* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", /* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", /* 2 */ "DELETE FROM %Q.'%q_content'", /* 3 */ "DELETE FROM %Q.'%q_segments'", /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", /* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", /* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", /* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); pStmt = p->aStmt[eStmt]; if( !pStmt ){ char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); |
︙ | ︙ | |||
397 398 399 400 401 402 403 | ** ** 0: idx ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ | | > > > | > > > > > > | 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 | ** ** 0: idx ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ int rc; sqlite3_stmt *pStmt = 0; if( iLevel<0 ){ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); }else{ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); } *ppStmt = pStmt; return rc; } /* ** Append a single varint to a PendingList buffer. SQLITE_OK is returned ** if successful, or an SQLite error code otherwise. ** |
︙ | ︙ | |||
1299 1300 1301 1302 1303 1304 1305 | if( isPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; return rc; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 | if( isPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; return rc; } /* ** Compare the entries pointed to by two Fts3SegReader structures. ** Comparison is as follows: ** ** 1) EOF is greater than not EOF. ** ** 2) The current terms (if any) are compared using memcmp(). If one |
︙ | ︙ | |||
1939 1940 1941 1942 1943 1944 1945 | *pisEmpty = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); } return rc; } | < < < < < < < < < < < < < < < < < < < | 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 | *pisEmpty = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); } return rc; } /* ** Set *pnSegment to the total number of segments in the database. Set ** *pnMax to the largest segment level in the database (segment levels ** are stored in the 'level' column of the %_segdir table). ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ |
︙ | ︙ | |||
2016 2017 2018 2019 2020 2021 2022 | rc = sqlite3_reset(pDelete); } } if( rc!=SQLITE_OK ){ return rc; } | > > > > > | < < | 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 | rc = sqlite3_reset(pDelete); } } if( rc!=SQLITE_OK ){ return rc; } if( iLevel==FTS3_SEGCURSOR_ALL ){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ sqlite3Fts3PendingTermsClear(p); }else{ assert( iLevel>=0 ); rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iLevel); sqlite3_step(pDelete); rc = sqlite3_reset(pDelete); } } return rc; } /* ** When this function is called, buffer *ppList (size *pnList bytes) contains |
︙ | ︙ | |||
2073 2074 2075 2076 2077 2078 2079 | p += sqlite3Fts3GetVarint32(p, &iCurrent); } *ppList = pList; *pnList = nList; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < | < < | < < < < < < < | < < < < > | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > | > | | < | | | | | > | | | | | | | 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 | p += sqlite3Fts3GetVarint32(p, &iCurrent); } *ppList = pList; *pnList = nList; } int sqlite3Fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ Fts3SegReaderCursor *pCsr, /* Cursor object */ Fts3SegFilter *pFilter /* Restrictions on range of iteration */ ){ int i; /* Initialize the cursor object */ pCsr->pFilter = pFilter; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ for(i=0; i<pCsr->nSegment; i++){ int nTerm = pFilter->nTerm; const char *zTerm = pFilter->zTerm; Fts3SegReader *pSeg = pCsr->apSegment[i]; do { int rc = fts3SegReaderNext(p, pSeg); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } fts3SegReaderSort( pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); return SQLITE_OK; } int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ Fts3SegReaderCursor *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; if( pCsr->nSegment==0 ) return SQLITE_OK; do { int nMerge; int i; /* Advance the first pCsr->nAdvance entries in the apSegment[] array ** forward. Then sort the list in order of current term again. */ for(i=0; i<pCsr->nAdvance; i++){ rc = fts3SegReaderNext(p, apSegment[i]); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); pCsr->nAdvance = 0; /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ assert( rc==SQLITE_OK ); if( apSegment[0]->aNode==0 ) break; pCsr->nTerm = apSegment[0]->nTerm; pCsr->zTerm = apSegment[0]->zTerm; /* If this is a prefix-search, and if the term that apSegment[0] points ** to does not share a suffix with pFilter->zTerm/nTerm, then all ** required callbacks have been made. In this case exit early. ** ** Similarly, if this is a search for an exact match, and the first term ** of segment apSegment[0] is not a match, exit early. */ if( pFilter->zTerm ){ if( pCsr->nTerm<pFilter->nTerm || (!isPrefix && pCsr->nTerm>pFilter->nTerm) || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) ){ break; } } nMerge = 1; while( nMerge<nSegment && apSegment[nMerge]->aNode && apSegment[nMerge]->nTerm==pCsr->nTerm && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) ){ nMerge++; } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty ){ pCsr->aDoclist = apSegment[0]->aDoclist; pCsr->nDoclist = apSegment[0]->nDoclist; rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ /* The current term of the first nMerge entries in the array ** of Fts3SegReader objects is the same. The doclists must be merged ** and a single term returned with the merged doclist. */ for(i=0; i<nMerge; i++){ fts3SegReaderFirstDocid(apSegment[i]); } fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); while( apSegment[0]->pOffsetList ){ int j; /* Number of segments that share a docid */ |
︙ | ︙ | |||
2238 2239 2240 2241 2242 2243 2244 | if( isColFilter ){ fts3ColumnFilter(pFilter->iCol, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); | | | | | < | | > > | | < > | | > > < | < < < < | > > > > > | | < | > | > > > | < < < < < | < < < > > > > | | < < < < < < | | | > > > | < < < < > | < | < | < > | < | < < < < < < | < < | | < < < | < < < > < > < < < | < < < < < | < < < < < < < | < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 | if( isColFilter ){ fts3ColumnFilter(pFilter->iCol, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ return SQLITE_NOMEM; } pCsr->aBuffer = aNew; } nDoclist += sqlite3Fts3PutVarint( &pCsr->aBuffer[nDoclist], iDocid-iPrev ); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); nDoclist += nList; pCsr->aBuffer[nDoclist++] = '\0'; } } fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; pCsr->nDoclist = nDoclist; rc = SQLITE_ROW; } } pCsr->nAdvance = nMerge; }while( rc==SQLITE_OK ); return rc; } void sqlite3Fts3SegReaderFinish( Fts3SegReaderCursor *pCsr /* Cursor object */ ){ if( pCsr ){ int i; for(i=0; i<pCsr->nSegment; i++){ sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); } sqlite3_free(pCsr->apSegment); sqlite3_free(pCsr->aBuffer); pCsr->nSegment = 0; pCsr->apSegment = 0; pCsr->aBuffer = 0; } } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level ** currently present in the database. ** ** If this function is called with iLevel<0, but there is only one ** segment in the database, SQLITE_DONE is returned immediately. ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ int rc; /* Return code */ int iIdx; /* Index of new segment */ int iNewLevel = 0; /* Level to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically ** greatest segment level currently present in the database. The index ** of the new segment is always 0. */ int nDummy; /* TODO: Remove this */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } iIdx = 0; rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); }else{ /* This call is to merge all segments at level iLevel. Find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ iNewLevel = iLevel+1; rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); assert( iNewLevel>=0 ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ rc = sqlite3Fts3SegReaderStep(p, &csr); if( rc!=SQLITE_ROW ) break; rc = fts3SegWriterAdd(p, &pWriter, 1, csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); } if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); if( rc!=SQLITE_OK ) goto finished; rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: fts3SegWriterFree(pWriter); sqlite3Fts3SegReaderFinish(&csr); return rc; } /* ** Flush the contents of pendingTerms to a level 0 segment. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); } /* ** Encode N integers as varints into a blob. */ static void fts3EncodeIntArray( int N, /* The number of integers to encode */ |
︙ | ︙ | |||
2615 2616 2617 2618 2619 2620 2621 | int rc; /* Return Code */ const char *zVal = (const char *)sqlite3_value_text(pVal); int nVal = sqlite3_value_bytes(pVal); if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ | | | 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 | int rc; /* Return Code */ const char *zVal = (const char *)sqlite3_value_text(pVal); int nVal = sqlite3_value_bytes(pVal); if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; }else{ sqlite3Fts3PendingTermsClear(p); } #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ |
︙ | ︙ | |||
2873 2874 2875 2876 2877 2878 2879 | ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ | | | 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 | ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); if( rc==SQLITE_OK ){ sqlite3Fts3PendingTermsClear(p); } }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); |
︙ | ︙ |
Changes to main.mk.
︙ | ︙ | |||
49 50 51 52 53 54 55 | TCCX += -I$(TOP)/ext/async # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ | | | 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | TCCX += -I$(TOP)/ext/async # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \ notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \ |
︙ | ︙ | |||
183 184 185 186 187 188 189 190 191 192 193 194 195 196 | $(TOP)/ext/fts2/fts2_tokenizer.h \ $(TOP)/ext/fts2/fts2_tokenizer.c \ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ $(TOP)/ext/fts3/fts3_snippet.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ | > | 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | $(TOP)/ext/fts2/fts2_tokenizer.h \ $(TOP)/ext/fts2/fts2_tokenizer.c \ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ $(TOP)/ext/fts3/fts3_snippet.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ |
︙ | ︙ | |||
289 290 291 292 293 294 295 296 297 298 299 300 301 302 | $(TOP)/src/vdbeapi.c \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ $(TOP)/src/where.c \ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c # Header files used by all library source files. # | > | 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 | $(TOP)/src/vdbeapi.c \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ $(TOP)/src/where.c \ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c # Header files used by all library source files. # |
︙ | ︙ | |||
457 458 459 460 461 462 463 464 465 466 467 468 469 470 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c | > > > | 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c fts3_aux.o: $(TOP)/ext/fts3/fts3_aux.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_aux.c fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c |
︙ | ︙ |
Added test/fts3aux1.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | # 2011 January 27 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS3 module. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } set ::testprefix fts3aux1 do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4; INSERT INTO t1 VALUES('one two three four'); INSERT INTO t1 VALUES('three four five six'); INSERT INTO t1 VALUES('one three five seven'); CREATE VIRTUAL TABLE terms USING fts4aux(t1); SELECT * FROM terms; } { five 2 2 four 2 2 one 2 2 seven 1 1 six 1 1 three 3 3 two 1 1 } do_execsql_test 1.2 { INSERT INTO t1 VALUES('one one one three three three'); SELECT * FROM terms; } { five 2 2 four 2 2 one 3 5 seven 1 1 six 1 1 three 4 6 two 1 1 } do_execsql_test 1.3 { DELETE FROM t1; SELECT * FROM terms; } {} do_execsql_test 1.4 { INSERT INTO t1 VALUES('a b a b a b a'); INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; SELECT * FROM terms; } {a 256 1024 b 256 768} finish_test |
Added test/fts3comp1.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | # 2011 January 27 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS3 module. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } set ::testprefix fts3comp1 set next_x 0 proc zip {x} { incr ::next_x set ::strings($::next_x) $x return $::next_x } proc unzip {x} { return $::strings($x) } db func zip zip db func unzip unzip do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts4( a, b, compress=zip, uncompress=unzip ); INSERT INTO t1 VALUES('one two three', 'two four six'); } do_execsql_test 1.1 { SELECT a, b FROM t1; } {{one two three} {two four six}} do_execsql_test 1.2 { SELECT c0a, c1b FROM t1_content; } {1 2} do_execsql_test 1.3 { INSERT INTO t1 VALUES('three six nine', 'four eight twelve'); SELECT a, b FROM t1; } {{one two three} {two four six} {three six nine} {four eight twelve}} do_execsql_test 1.4 { SELECT c0a, c1b FROM t1_content; } {1 2 3 4} do_execsql_test 1.5 { CREATE VIRTUAL TABLE terms USING fts4aux(t1); SELECT * FROM terms; } { eight 1 1 four 2 2 nine 1 1 one 1 1 six 2 2 three 2 2 twelve 1 1 two 1 2 } do_execsql_test 1.6 { DELETE FROM t1 WHERE docid = 1; SELECT * FROM terms; } { eight 1 1 four 1 1 nine 1 1 six 1 1 three 1 1 twelve 1 1 } do_execsql_test 1.7 { SELECT c0a, c1b FROM t1_content; } {3 4} finish_test |
Changes to test/permutations.test.
︙ | ︙ | |||
177 178 179 180 181 182 183 184 185 186 187 188 189 190 | fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3fault.test fts3malloc.test fts3matchinfo.test } lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the coverage related test suites: # | > > | 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3fault.test fts3malloc.test fts3matchinfo.test fts3aux1.test fts3comp1.test } lappend ::testsuitelist xxx #------------------------------------------------------------------------- # Define the coverage related test suites: # |
︙ | ︙ |
Changes to test/trace2.test.
︙ | ︙ | |||
124 125 126 127 128 129 130 | INSERT INTO x1 VALUES('Wind chill values as low as -13'); } do_trace_test 2.2 { INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph'); } { "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" | | < > | 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | INSERT INTO x1 VALUES('Wind chill values as low as -13'); } do_trace_test 2.2 { INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph'); } { "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" "-- SELECT value FROM 'main'.'x1_stat' WHERE id=0" "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } do_trace_test 2.3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC" "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- DELETE FROM 'main'.'x1_segdir'" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } finish_test |