Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | fts2 functions for testing scripts. Backports (5340) from fts3. (CVS 5456) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
4e47394be9dfbf0f9309e55eb6c6a3a5 |
User & Date: | shess 2008-07-22 23:32:28.000 |
Context
2008-07-22
| ||
23:41 | Delete all fts2 index data the table becomes empty. Backports check-in (5413) from fts3. (CVS 5457) (check-in: 4c98179be2 user: shess tags: trunk) | |
23:32 | fts2 functions for testing scripts. Backports (5340) from fts3. (CVS 5456) (check-in: 4e47394be9 user: shess tags: trunk) | |
23:08 | Change prefix search from O(N*M) to O(NlogM). Backports (4599) from fts3. (CVS 5455) (check-in: 3f614453d2 user: shess tags: trunk) | |
Changes
Changes to ext/fts2/fts2.c.
︙ | ︙ | |||
1777 1778 1779 1780 1781 1782 1783 | BLOCK_INSERT_STMT, BLOCK_SELECT_STMT, BLOCK_DELETE_STMT, SEGDIR_MAX_INDEX_STMT, SEGDIR_SET_STMT, | | > | 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 | BLOCK_INSERT_STMT, BLOCK_SELECT_STMT, BLOCK_DELETE_STMT, SEGDIR_MAX_INDEX_STMT, SEGDIR_SET_STMT, SEGDIR_SELECT_LEVEL_STMT, SEGDIR_SPAN_STMT, SEGDIR_DELETE_STMT, SEGDIR_SELECT_SEGMENT_STMT, SEGDIR_SELECT_ALL_STMT, MAX_STMT /* Always at end! */ } fulltext_statement; /* These must exactly match the enum above. */ /* TODO(shess): Is there some risk that a statement will be used in two |
︙ | ︙ | |||
1802 1803 1804 1805 1806 1807 1808 | /* BLOCK_INSERT */ "insert into %_segments values (?)", /* BLOCK_SELECT */ "select block from %_segments where rowid = ?", /* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?", /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", | | > > > > > > > > | | 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 | /* BLOCK_INSERT */ "insert into %_segments values (?)", /* BLOCK_SELECT */ "select block from %_segments where rowid = ?", /* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?", /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", /* SEGDIR_SELECT_LEVEL */ "select start_block, leaves_end_block, root from %_segdir " " where level = ? order by idx", /* SEGDIR_SPAN */ "select min(start_block), max(end_block) from %_segdir " " where level = ? and start_block <> 0", /* SEGDIR_DELETE */ "delete from %_segdir where level = ?", /* NOTE(shess): The first three results of the following two ** statements must match. */ /* SEGDIR_SELECT_SEGMENT */ "select start_block, leaves_end_block, root from %_segdir " " where level = ? and idx = ?", /* SEGDIR_SELECT_ALL */ "select start_block, leaves_end_block, root from %_segdir " " order by level desc, idx asc", }; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. ** All other methods receive a pointer to the structure as one of their |
︙ | ︙ | |||
5069 5070 5071 5072 5073 5074 5075 | /* Initializes pReaders with the segments from level iLevel, returning ** the number of segments in *piReaders. Leaves pReaders in sorted ** order. */ static int leavesReadersInit(fulltext_vtab *v, int iLevel, LeavesReader *pReaders, int *piReaders){ sqlite3_stmt *s; | | | 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 | /* Initializes pReaders with the segments from level iLevel, returning ** the number of segments in *piReaders. Leaves pReaders in sorted ** order. */ static int leavesReadersInit(fulltext_vtab *v, int iLevel, LeavesReader *pReaders, int *piReaders){ sqlite3_stmt *s; int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s); if( rc!=SQLITE_OK ) return rc; rc = sqlite3_bind_int(s, 1, iLevel); if( rc!=SQLITE_OK ) return rc; i = 0; while( (rc = sqlite3_step(s))==SQLITE_ROW ){ |
︙ | ︙ | |||
5607 5608 5609 5610 5611 5612 5613 | dataBufferInit(&doclist, 0); /* Traverse the segments from oldest to newest so that newer doclist ** elements for given docids overwrite older elements. */ while( (rc = sqlite3_step(s))==SQLITE_ROW ){ | | | | 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 | dataBufferInit(&doclist, 0); /* Traverse the segments from oldest to newest so that newer doclist ** elements for given docids overwrite older elements. */ while( (rc = sqlite3_step(s))==SQLITE_ROW ){ const char *pData = sqlite3_column_blob(s, 2); const int nData = sqlite3_column_bytes(s, 2); const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix, &doclist); if( rc!=SQLITE_OK ) goto err; } if( rc==SQLITE_DONE ){ if( doclist.nData!=0 ){ |
︙ | ︙ | |||
5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 | snippetOffsetText(&pCursor->snippet); sqlite3_result_text(pContext, pCursor->snippet.zOffset, pCursor->snippet.nOffset, SQLITE_STATIC); } } /* ** This routine implements the xFindFunction method for the FTS2 ** virtual table. */ static int fulltextFindFunction( sqlite3_vtab *pVtab, int nArg, const char *zName, void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg ){ if( strcmp(zName,"snippet")==0 ){ *pxFunc = snippetFunc; return 1; }else if( strcmp(zName,"offsets")==0 ){ *pxFunc = snippetOffsetsFunc; return 1; } return 0; } /* ** Rename an fts2 table. */ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 | snippetOffsetText(&pCursor->snippet); sqlite3_result_text(pContext, pCursor->snippet.zOffset, pCursor->snippet.nOffset, SQLITE_STATIC); } } #ifdef SQLITE_TEST /* Generate an error of the form "<prefix>: <msg>". If msg is NULL, ** pull the error from the context's db handle. */ static void generateError(sqlite3_context *pContext, const char *prefix, const char *msg){ char buf[512]; if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext)); sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg); sqlite3_result_error(pContext, buf, -1); } /* Helper function to collect the set of terms in the segment into ** pTerms. The segment is defined by the leaf nodes between ** iStartBlockid and iEndBlockid, inclusive, or by the contents of ** pRootData if iStartBlockid is 0 (in which case the entire segment ** fit in a leaf). */ static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s, fts2Hash *pTerms){ const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0); const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1); const char *pRootData = sqlite3_column_blob(s, 2); const int nRootData = sqlite3_column_bytes(s, 2); LeavesReader reader; int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid, pRootData, nRootData, &reader); if( rc!=SQLITE_OK ) return rc; while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){ const char *pTerm = leavesReaderTerm(&reader); const int nTerm = leavesReaderTermBytes(&reader); void *oldValue = sqlite3Fts2HashFind(pTerms, pTerm, nTerm); void *newValue = (void *)((char *)oldValue+1); /* From the comment before sqlite3Fts2HashInsert in fts2_hash.c, ** the data value passed is returned in case of malloc failure. */ if( newValue==sqlite3Fts2HashInsert(pTerms, pTerm, nTerm, newValue) ){ rc = SQLITE_NOMEM; }else{ rc = leavesReaderStep(v, &reader); } } leavesReaderDestroy(&reader); return rc; } /* Helper function to build the result string for dump_terms(). */ static int generateTermsResult(sqlite3_context *pContext, fts2Hash *pTerms){ int iTerm, nTerms, nResultBytes, iByte; char *result; TermData *pData; fts2HashElem *e; /* Iterate pTerms to generate an array of terms in pData for ** sorting. */ nTerms = fts2HashCount(pTerms); assert( nTerms>0 ); pData = sqlite3_malloc(nTerms*sizeof(TermData)); if( pData==NULL ) return SQLITE_NOMEM; nResultBytes = 0; for(iTerm = 0, e = fts2HashFirst(pTerms); e; iTerm++, e = fts2HashNext(e)){ nResultBytes += fts2HashKeysize(e)+1; /* Term plus trailing space */ assert( iTerm<nTerms ); pData[iTerm].pTerm = fts2HashKey(e); pData[iTerm].nTerm = fts2HashKeysize(e); pData[iTerm].pCollector = fts2HashData(e); /* unused */ } assert( iTerm==nTerms ); assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */ result = sqlite3_malloc(nResultBytes); if( result==NULL ){ sqlite3_free(pData); return SQLITE_NOMEM; } if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp); /* Read the terms in order to build the result. */ iByte = 0; for(iTerm=0; iTerm<nTerms; ++iTerm){ memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm); iByte += pData[iTerm].nTerm; result[iByte++] = ' '; } assert( iByte==nResultBytes ); assert( result[nResultBytes-1]==' ' ); result[nResultBytes-1] = '\0'; /* Passes away ownership of result. */ sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free); sqlite3_free(pData); return SQLITE_OK; } /* Implements dump_terms() for use in inspecting the fts2 index from ** tests. TEXT result containing the ordered list of terms joined by ** spaces. dump_terms(t, level, idx) dumps the terms for the segment ** specified by level, idx (in %_segdir), while dump_terms(t) dumps ** all terms in the index. In both cases t is the fts table's magic ** table-named column. */ static void dumpTermsFunc( sqlite3_context *pContext, int argc, sqlite3_value **argv ){ fulltext_cursor *pCursor; if( argc!=3 && argc!=1 ){ generateError(pContext, "dump_terms", "incorrect arguments"); }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ generateError(pContext, "dump_terms", "illegal first argument"); }else{ fulltext_vtab *v; fts2Hash terms; sqlite3_stmt *s = NULL; int rc; memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); v = cursor_vtab(pCursor); /* If passed only the cursor column, get all segments. Otherwise ** get the segment described by the following two arguments. */ if( argc==1 ){ rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s); }else{ rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1])); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2])); } } } if( rc!=SQLITE_OK ){ generateError(pContext, "dump_terms", NULL); return; } /* Collect the terms for each segment. */ sqlite3Fts2HashInit(&terms, FTS2_HASH_STRING, 1); while( (rc = sqlite3_step(s))==SQLITE_ROW ){ rc = collectSegmentTerms(v, s, &terms); if( rc!=SQLITE_OK ) break; } if( rc!=SQLITE_DONE ){ sqlite3_reset(s); generateError(pContext, "dump_terms", NULL); }else{ const int nTerms = fts2HashCount(&terms); if( nTerms>0 ){ rc = generateTermsResult(pContext, &terms); if( rc==SQLITE_NOMEM ){ generateError(pContext, "dump_terms", "out of memory"); }else{ assert( rc==SQLITE_OK ); } }else if( argc==3 ){ /* The specific segment asked for could not be found. */ generateError(pContext, "dump_terms", "segment not found"); }else{ /* No segments found. */ /* TODO(shess): It should be impossible to reach this. This ** case can only happen for an empty table, in which case ** SQLite has no rows to call this function on. */ sqlite3_result_null(pContext); } } sqlite3Fts2HashClear(&terms); } } /* Expand the DL_DEFAULT doclist in pData into a text result in ** pContext. */ static void createDoclistResult(sqlite3_context *pContext, const char *pData, int nData){ DataBuffer dump; DLReader dlReader; assert( pData!=NULL && nData>0 ); dataBufferInit(&dump, 0); dlrInit(&dlReader, DL_DEFAULT, pData, nData); for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){ char buf[256]; PLReader plReader; plrInit(&plReader, &dlReader); if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){ sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader)); dataBufferAppend(&dump, buf, strlen(buf)); }else{ int iColumn = plrColumn(&plReader); sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[", dlrDocid(&dlReader), iColumn); dataBufferAppend(&dump, buf, strlen(buf)); for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){ if( plrColumn(&plReader)!=iColumn ){ iColumn = plrColumn(&plReader); sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn); assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ assert( dump.pData[dump.nData]==' '); dataBufferAppend(&dump, buf, strlen(buf)); } if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){ sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ", plrPosition(&plReader), plrStartOffset(&plReader), plrEndOffset(&plReader)); }else if( DL_DEFAULT==DL_POSITIONS ){ sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader)); }else{ assert( NULL=="Unhandled DL_DEFAULT value"); } dataBufferAppend(&dump, buf, strlen(buf)); } plrDestroy(&plReader); assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ assert( dump.pData[dump.nData]==' '); dataBufferAppend(&dump, "]] ", 3); } } dlrDestroy(&dlReader); assert( dump.nData>0 ); dump.nData--; /* Overwrite trailing space. */ assert( dump.pData[dump.nData]==' '); dump.pData[dump.nData] = '\0'; assert( dump.nData>0 ); /* Passes ownership of dump's buffer to pContext. */ sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free); dump.pData = NULL; dump.nData = dump.nCapacity = 0; } /* Implements dump_doclist() for use in inspecting the fts2 index from ** tests. TEXT result containing a string representation of the ** doclist for the indicated term. dump_doclist(t, term, level, idx) ** dumps the doclist for term from the segment specified by level, idx ** (in %_segdir), while dump_doclist(t, term) dumps the logical ** doclist for the term across all segments. The per-segment doclist ** can contain deletions, while the full-index doclist will not ** (deletions are omitted). ** ** Result formats differ with the setting of DL_DEFAULTS. Examples: ** ** DL_DOCIDS: [1] [3] [7] ** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]] ** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]] ** ** In each case the number after the outer '[' is the docid. In the ** latter two cases, the number before the inner '[' is the column ** associated with the values within. For DL_POSITIONS the numbers ** within are the positions, for DL_POSITIONS_OFFSETS they are the ** position, the start offset, and the end offset. */ static void dumpDoclistFunc( sqlite3_context *pContext, int argc, sqlite3_value **argv ){ fulltext_cursor *pCursor; if( argc!=2 && argc!=4 ){ generateError(pContext, "dump_doclist", "incorrect arguments"); }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ generateError(pContext, "dump_doclist", "illegal first argument"); }else if( sqlite3_value_text(argv[1])==NULL || sqlite3_value_text(argv[1])[0]=='\0' ){ generateError(pContext, "dump_doclist", "empty second argument"); }else{ const char *pTerm = (const char *)sqlite3_value_text(argv[1]); const int nTerm = strlen(pTerm); fulltext_vtab *v; int rc; DataBuffer doclist; memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); v = cursor_vtab(pCursor); dataBufferInit(&doclist, 0); /* termSelect() yields the same logical doclist that queries are ** run against. */ if( argc==2 ){ rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist); }else{ sqlite3_stmt *s = NULL; /* Get our specific segment's information. */ rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2])); if( rc==SQLITE_OK ){ rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3])); } } if( rc==SQLITE_OK ){ rc = sqlite3_step(s); if( rc==SQLITE_DONE ){ dataBufferDestroy(&doclist); generateError(pContext, "dump_doclist", "segment not found"); return; } /* Found a segment, load it into doclist. */ if( rc==SQLITE_ROW ){ const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1); const char *pData = sqlite3_column_blob(s, 2); const int nData = sqlite3_column_bytes(s, 2); /* loadSegment() is used by termSelect() to load each ** segment's data. */ rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0, &doclist); if( rc==SQLITE_OK ){ rc = sqlite3_step(s); /* Should not have more than one matching segment. */ if( rc!=SQLITE_DONE ){ sqlite3_reset(s); dataBufferDestroy(&doclist); generateError(pContext, "dump_doclist", "invalid segdir"); return; } rc = SQLITE_OK; } } } sqlite3_reset(s); } if( rc==SQLITE_OK ){ if( doclist.nData>0 ){ createDoclistResult(pContext, doclist.pData, doclist.nData); }else{ /* TODO(shess): This can happen if the term is not present, or ** if all instances of the term have been deleted and this is ** an all-index dump. It may be interesting to distinguish ** these cases. */ sqlite3_result_text(pContext, "", 0, SQLITE_STATIC); } }else if( rc==SQLITE_NOMEM ){ /* Handle out-of-memory cases specially because if they are ** generated in fts2 code they may not be reflected in the db ** handle. */ /* TODO(shess): Handle this more comprehensively. ** sqlite3ErrStr() has what I need, but is internal. */ generateError(pContext, "dump_doclist", "out of memory"); }else{ generateError(pContext, "dump_doclist", NULL); } dataBufferDestroy(&doclist); } } #endif /* ** This routine implements the xFindFunction method for the FTS2 ** virtual table. */ static int fulltextFindFunction( sqlite3_vtab *pVtab, int nArg, const char *zName, void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg ){ if( strcmp(zName,"snippet")==0 ){ *pxFunc = snippetFunc; return 1; }else if( strcmp(zName,"offsets")==0 ){ *pxFunc = snippetOffsetsFunc; return 1; #ifdef SQLITE_TEST /* NOTE(shess): These functions are present only for testing ** purposes. No particular effort is made to optimize their ** execution or how they build their results. */ }else if( strcmp(zName,"dump_terms")==0 ){ /* fprintf(stderr, "Found dump_terms\n"); */ *pxFunc = dumpTermsFunc; return 1; }else if( strcmp(zName,"dump_doclist")==0 ){ /* fprintf(stderr, "Found dump_doclist\n"); */ *pxFunc = dumpDoclistFunc; return 1; #endif } return 0; } /* ** Rename an fts2 table. */ |
︙ | ︙ | |||
6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 | ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) ){ return sqlite3_create_module_v2( db, "fts2", &fts2Module, (void *)pHash, hashDestroy ); } /* An error has occured. Delete the hash table and return the error code. */ | > > > > | 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 | ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts2InitHashTable(db, pHash, "fts2_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1)) #ifdef SQLITE_TEST && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1)) #endif ){ return sqlite3_create_module_v2( db, "fts2", &fts2Module, (void *)pHash, hashDestroy ); } /* An error has occured. Delete the hash table and return the error code. */ |
︙ | ︙ |
Added test/fts2p.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 | # 2008 June 26 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file exercises some new testing functions in the FTS2 module, # and then uses them to do some basic tests that FTS2 is internally # working as expected. # # $Id: fts2p.test,v 1.1 2008/07/22 23:32:28 shess Exp $ # set testdir [file dirname $argv0] source $testdir/tester.tcl # If SQLITE_ENABLE_FTS2 is not defined, omit this file. ifcapable !fts2 { finish_test return } #************************************************************************* # Probe to see if support for these functions is compiled in. # TODO(shess): Change main.mk to do the right thing and remove this test. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts2(c); INSERT INTO t1 (rowid, c) VALUES (1, 'x'); } set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1} set r {1 {unable to use function dump_terms in the requested context}} if {[catchsql $s]==$r} { finish_test return } #************************************************************************* # Test that the new functions give appropriate errors. do_test fts2p-0.0 { catchsql { SELECT dump_terms(t1, 1) FROM t1 LIMIT 1; } } {1 {dump_terms: incorrect arguments}} do_test fts2p-0.1 { catchsql { SELECT dump_terms(t1, 0, 0, 0) FROM t1 LIMIT 1; } } {1 {dump_terms: incorrect arguments}} do_test fts2p-0.2 { catchsql { SELECT dump_terms(1, t1) FROM t1 LIMIT 1; } } {1 {unable to use function dump_terms in the requested context}} do_test fts2p-0.3 { catchsql { SELECT dump_terms(t1, 16, 16) FROM t1 LIMIT 1; } } {1 {dump_terms: segment not found}} do_test fts2p-0.4 { catchsql { SELECT dump_doclist(t1) FROM t1 LIMIT 1; } } {1 {dump_doclist: incorrect arguments}} do_test fts2p-0.5 { catchsql { SELECT dump_doclist(t1, NULL) FROM t1 LIMIT 1; } } {1 {dump_doclist: empty second argument}} do_test fts2p-0.6 { catchsql { SELECT dump_doclist(t1, '') FROM t1 LIMIT 1; } } {1 {dump_doclist: empty second argument}} do_test fts2p-0.7 { catchsql { SELECT dump_doclist(t1, 'a', 0) FROM t1 LIMIT 1; } } {1 {dump_doclist: incorrect arguments}} do_test fts2p-0.8 { catchsql { SELECT dump_doclist(t1, 'a', 0, 0, 0) FROM t1 LIMIT 1; } } {1 {dump_doclist: incorrect arguments}} do_test fts2p-0.9 { catchsql { SELECT dump_doclist(t1, 'a', 16, 16) FROM t1 LIMIT 1; } } {1 {dump_doclist: segment not found}} #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { # TODO(shess): Figure out why uplevel in do_test can't catch # $level and $index directly. set ::level $level set ::index $index do_test $test.terms { execsql { SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1; } } [list $terms] } proc check_terms_all {test terms} { do_test $test.terms { execsql { SELECT dump_terms(t1) FROM t1 LIMIT 1; } } [list $terms] } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { # TODO(shess): Again, why can't the non-:: versions work? set ::term $term set ::level $level set ::index $index do_test $test { execsql { SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1; } } [list $doclist] } proc check_doclist_all {test term doclist} { set ::term $term do_test $test { execsql { SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1; } } [list $doclist] } #************************************************************************* # Test the segments resulting from straight-forward inserts. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts2(c); INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test'); } # Check for expected segments and expected matches. do_test fts2p-1.0.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0 0 1 0 2} do_test fts2p-1.0.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid; } } [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \ {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \ {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}] # Check the specifics of the segments constructed. # Logical view of entire index. check_terms_all fts2p-1.0.1 {a is test that this was} check_doclist_all fts2p-1.0.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]} check_doclist_all fts2p-1.0.1.2 is {[1 0[1]] [3 0[1]]} check_doclist_all fts2p-1.0.1.3 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist_all fts2p-1.0.1.4 that {[2 0[0]]} check_doclist_all fts2p-1.0.1.5 this {[1 0[0]] [3 0[0]]} check_doclist_all fts2p-1.0.1.6 was {[2 0[1]]} # Segment 0,0 check_terms fts2p-1.0.2 0 0 {a is test this} check_doclist fts2p-1.0.2.1 0 0 a {[1 0[2]]} check_doclist fts2p-1.0.2.2 0 0 is {[1 0[1]]} check_doclist fts2p-1.0.2.3 0 0 test {[1 0[3]]} check_doclist fts2p-1.0.2.4 0 0 this {[1 0[0]]} # Segment 0,1 check_terms fts2p-1.0.3 0 1 {a test that was} check_doclist fts2p-1.0.3.1 0 1 a {[2 0[2]]} check_doclist fts2p-1.0.3.2 0 1 test {[2 0[3]]} check_doclist fts2p-1.0.3.3 0 1 that {[2 0[0]]} check_doclist fts2p-1.0.3.4 0 1 was {[2 0[1]]} # Segment 0,2 check_terms fts2p-1.0.4 0 2 {a is test this} check_doclist fts2p-1.0.4.1 0 2 a {[3 0[2]]} check_doclist fts2p-1.0.4.2 0 2 is {[3 0[1]]} check_doclist fts2p-1.0.4.3 0 2 test {[3 0[3]]} check_doclist fts2p-1.0.4.4 0 2 this {[3 0[0]]} #************************************************************************* # Test the segments resulting from inserts followed by a delete. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts2(c); INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test'); DELETE FROM t1 WHERE rowid = 1; } do_test fts2p-1.1.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0 0 1 0 2 0 3} do_test fts2p-1.1.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid; } } {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}} check_terms_all fts2p-1.1.1 {a is test that this was} check_doclist_all fts2p-1.1.1.1 a {[2 0[2]] [3 0[2]]} check_doclist_all fts2p-1.1.1.2 is {[3 0[1]]} check_doclist_all fts2p-1.1.1.3 test {[2 0[3]] [3 0[3]]} check_doclist_all fts2p-1.1.1.4 that {[2 0[0]]} check_doclist_all fts2p-1.1.1.5 this {[3 0[0]]} check_doclist_all fts2p-1.1.1.6 was {[2 0[1]]} check_terms fts2p-1.1.2 0 0 {a is test this} check_doclist fts2p-1.1.2.1 0 0 a {[1 0[2]]} check_doclist fts2p-1.1.2.2 0 0 is {[1 0[1]]} check_doclist fts2p-1.1.2.3 0 0 test {[1 0[3]]} check_doclist fts2p-1.1.2.4 0 0 this {[1 0[0]]} check_terms fts2p-1.1.3 0 1 {a test that was} check_doclist fts2p-1.1.3.1 0 1 a {[2 0[2]]} check_doclist fts2p-1.1.3.2 0 1 test {[2 0[3]]} check_doclist fts2p-1.1.3.3 0 1 that {[2 0[0]]} check_doclist fts2p-1.1.3.4 0 1 was {[2 0[1]]} check_terms fts2p-1.1.4 0 2 {a is test this} check_doclist fts2p-1.1.4.1 0 2 a {[3 0[2]]} check_doclist fts2p-1.1.4.2 0 2 is {[3 0[1]]} check_doclist fts2p-1.1.4.3 0 2 test {[3 0[3]]} check_doclist fts2p-1.1.4.4 0 2 this {[3 0[0]]} check_terms fts2p-1.1.5 0 3 {a is test this} check_doclist fts2p-1.1.5.1 0 3 a {[1]} check_doclist fts2p-1.1.5.2 0 3 is {[1]} check_doclist fts2p-1.1.5.3 0 3 test {[1]} check_doclist fts2p-1.1.5.4 0 3 this {[1]} #************************************************************************* # Test results when all references to certain tokens are deleted. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts2(c); INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test'); DELETE FROM t1 WHERE rowid IN (1,3); } # Still 4 segments because 0,3 will contain deletes for rowid 1 and 3. do_test fts2p-1.2.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0 0 1 0 2 0 3} do_test fts2p-1.2.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid; } } {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}} check_terms_all fts2p-1.2.1 {a is test that this was} check_doclist_all fts2p-1.2.1.1 a {[2 0[2]]} check_doclist_all fts2p-1.2.1.2 is {} check_doclist_all fts2p-1.2.1.3 test {[2 0[3]]} check_doclist_all fts2p-1.2.1.4 that {[2 0[0]]} check_doclist_all fts2p-1.2.1.5 this {} check_doclist_all fts2p-1.2.1.6 was {[2 0[1]]} check_terms fts2p-1.2.2 0 0 {a is test this} check_doclist fts2p-1.2.2.1 0 0 a {[1 0[2]]} check_doclist fts2p-1.2.2.2 0 0 is {[1 0[1]]} check_doclist fts2p-1.2.2.3 0 0 test {[1 0[3]]} check_doclist fts2p-1.2.2.4 0 0 this {[1 0[0]]} check_terms fts2p-1.2.3 0 1 {a test that was} check_doclist fts2p-1.2.3.1 0 1 a {[2 0[2]]} check_doclist fts2p-1.2.3.2 0 1 test {[2 0[3]]} check_doclist fts2p-1.2.3.3 0 1 that {[2 0[0]]} check_doclist fts2p-1.2.3.4 0 1 was {[2 0[1]]} check_terms fts2p-1.2.4 0 2 {a is test this} check_doclist fts2p-1.2.4.1 0 2 a {[3 0[2]]} check_doclist fts2p-1.2.4.2 0 2 is {[3 0[1]]} check_doclist fts2p-1.2.4.3 0 2 test {[3 0[3]]} check_doclist fts2p-1.2.4.4 0 2 this {[3 0[0]]} check_terms fts2p-1.2.5 0 3 {a is test this} check_doclist fts2p-1.2.5.1 0 3 a {[1] [3]} check_doclist fts2p-1.2.5.2 0 3 is {[1] [3]} check_doclist fts2p-1.2.5.3 0 3 test {[1] [3]} check_doclist fts2p-1.2.5.4 0 3 this {[1] [3]} #************************************************************************* # Test results when everything is optimized manually. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts2(c); INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test'); INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test'); INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test'); DELETE FROM t1 WHERE rowid IN (1,3); DROP TABLE IF EXISTS t1old; ALTER TABLE t1 RENAME TO t1old; CREATE VIRTUAL TABLE t1 USING fts2(c); INSERT INTO t1 (rowid, c) SELECT rowid, c FROM t1old; DROP TABLE t1old; } # Should be a single optimal segment with the same logical results. do_test fts2p-1.3.segments { execsql { SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {0 0} do_test fts2p-1.3.matches { execsql { SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid; } } {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}} check_terms_all fts2p-1.3.1 {a test that was} check_doclist_all fts2p-1.3.1.1 a {[2 0[2]]} check_doclist_all fts2p-1.3.1.2 test {[2 0[3]]} check_doclist_all fts2p-1.3.1.3 that {[2 0[0]]} check_doclist_all fts2p-1.3.1.4 was {[2 0[1]]} check_terms fts2p-1.3.2 0 0 {a test that was} check_doclist fts2p-1.3.2.1 0 0 a {[2 0[2]]} check_doclist fts2p-1.3.2.2 0 0 test {[2 0[3]]} check_doclist fts2p-1.3.2.3 0 0 that {[2 0[0]]} check_doclist fts2p-1.3.2.4 0 0 was {[2 0[1]]} finish_test |