Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix problems with the matchinfo functions used by the bm25(f) implementation. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
739cd82fca631bc54a5cf99e8746c0ac |
User & Date: | dan 2013-01-14 17:05:09.416 |
Context
2013-01-15
| ||
17:39 | Fix a typo in lsmusr.wiki. check-in: ace47a5829 user: dan tags: trunk | |
2013-01-14
| ||
17:05 | Fix problems with the matchinfo functions used by the bm25(f) implementation. check-in: 739cd82fca user: dan tags: trunk | |
2013-01-13
| ||
05:30 | Avoid using keyword "near" as a variable name in fts5.c. check-in: 278cfaeb70 user: dan tags: trunk | |
Changes
Changes to src/fts5.c.
︙ | ︙ | |||
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 | TokenizeTerm *pTerm = 0; /* TODO: Error here if iStream is out of range */ if( nToken>p->nMax ) p->nMax = nToken; if( iStream>=p->nStream ){ int nOld = p->nStream; int nNew = 4; while( nNew<=iStream ) nNew = nNew*2; | > > > | | | > > > > > > > > > | | 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 | TokenizeTerm *pTerm = 0; /* TODO: Error here if iStream is out of range */ if( nToken>p->nMax ) p->nMax = nToken; if( iStream>=p->nStream ){ int iCol; int nOld = p->nStream; int nNew = 4; i64 *aNew; while( nNew<=iStream ) nNew = nNew*2; aNew = sqlite4DbMallocZero(db, nNew*p->nCol*sizeof(i64)); if( aNew==0 ) goto tokenize_cb_out; for(iCol=0; iCol<p->nCol; iCol++){ int iStr; for(iStr=0; iStr<nOld; iStr++){ aNew[nNew*iCol + iStr] = p->aSz[nOld*iCol + iStr]; } } sqlite4DbFree(db, p->aSz); p->aSz = aNew; p->nStream = nNew; } p->aSz[p->iCol * p->nStream + iStream]++; pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken); if( pTerm==0 ){ /* Size the initial allocation so that it fits in the lookaside buffer */ int nAlloc = sizeof(TokenizeTerm) + nToken + 32; pTerm = sqlite4DbMallocZero(p->db, nAlloc); |
︙ | ︙ | |||
1425 1426 1427 1428 1429 1430 1431 | iOff += sqlite4PutVarint(&a[iOff], nRow); } iOff += sqlite4PutVarint(&a[iOff], pSz->nStream); for(iCol=0; iCol<pSz->nCol; iCol++){ int i; for(i=0; i<pSz->nStream; i++){ | | | 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 | iOff += sqlite4PutVarint(&a[iOff], nRow); } iOff += sqlite4PutVarint(&a[iOff], pSz->nStream); for(iCol=0; iCol<pSz->nCol; iCol++){ int i; for(i=0; i<pSz->nStream; i++){ iOff += sqlite4PutVarint(&a[iOff], pSz->aSz[iCol*pSz->nStream+i]); } } return sqlite4KVStoreReplace(p, aKey, nKey, a, iOff); } static int fts5CsrLoadGlobal(Fts5Cursor *pCsr){ |
︙ | ︙ | |||
1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 | int iCol; for(iCol=0; iCol<pSz->nCol; iCol++){ int iStr; i64 *aIn = &sCtx.aSz[iCol * sCtx.nStream]; i64 *aOut = &pSz->aSz[iCol * pSz->nStream]; for(iStr=0; iStr<sCtx.nStream; iStr++){ aOut[iStr] += (aIn[iStr] * (bDel?-1:1)); } } nRow += (bDel?-1:1); rc = fts5StoreSizeRecord(pStore, aKey, nKey, pSz, nRow, &aKey[nKey]); } sqlite4DbFree(db, pSz); | > | 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 | int iCol; for(iCol=0; iCol<pSz->nCol; iCol++){ int iStr; i64 *aIn = &sCtx.aSz[iCol * sCtx.nStream]; i64 *aOut = &pSz->aSz[iCol * pSz->nStream]; for(iStr=0; iStr<sCtx.nStream; iStr++){ aOut[iStr] += (aIn[iStr] * (bDel?-1:1)); assert( iStr==0 || aOut[iStr]==0 ); } } nRow += (bDel?-1:1); rc = fts5StoreSizeRecord(pStore, aKey, nKey, pSz, nRow, &aKey[nKey]); } sqlite4DbFree(db, pSz); |
︙ | ︙ | |||
2866 2867 2868 2869 2870 2871 2872 | }else{ Fts5Info *pInfo = pCsr->pInfo; rc = pInfo->pTokenizer->xTokenize(p, pInfo->p, zText, nText, x); } return rc; } | | > > > > | > > | | > | | < | | | > | 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 | }else{ Fts5Info *pInfo = pCsr->pInfo; rc = pInfo->pTokenizer->xTokenize(p, pInfo->p, zText, nText, x); } return rc; } static Fts5Str *fts5FindStr( const u8 *aPk, int nPk, Fts5ExprNode *p, int *piStr ){ Fts5Str *pRet = 0; if( p->eType==TOKEN_PRIMITIVE ){ int iStr = *piStr; if( iStr<p->pPhrase->nStr && iStr>=0 && p->nPk==nPk && 0==memcmp(p->aPk, aPk, nPk) ){ pRet = &p->pPhrase->aStr[iStr]; }else{ *piStr = iStr - p->pPhrase->nStr; } }else{ pRet = fts5FindStr(aPk, nPk, p->pLeft, piStr); if( pRet==0 ) pRet = fts5FindStr(aPk, nPk, p->pRight, piStr); } return pRet; } int sqlite4_mi_match_count( sqlite4_context *pCtx, int iC, int iS, int iPhrase, int *pnMatch ){ int rc = SQLITE4_OK; Fts5Cursor *pCsr = pCtx->pFts; if( pCsr ){ Fts5ExprNode *pRoot = pCsr->pExpr->pRoot; int nMatch = 0; Fts5Str *pStr; int iCopy = iPhrase; InstanceList sList; pStr = fts5FindStr(pRoot->aPk, pRoot->nPk, pRoot, &iCopy); if( pStr ){ fts5InstanceListInit(pStr->aList, pStr->nList, &sList); while( 0==fts5InstanceListNext(&sList) ){ if( (iC<0 || sList.iCol==iC) && (iS<0 || sList.iStream==iS) ) nMatch++; } } *pnMatch = nMatch; }else{ rc = SQLITE4_MISUSE; } return rc; } |
︙ | ︙ | |||
2977 2978 2979 2980 2981 2982 2983 | if( pNode->eType==TOKEN_PRIMITIVE ){ Fts5Phrase *pPhrase = pNode->pPhrase; int iStr = *piStr; rc = fts5ExprAdvance(db, pNode, 1); while( rc==SQLITE4_OK && pNode->aPk ){ | < | 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 | if( pNode->eType==TOKEN_PRIMITIVE ){ Fts5Phrase *pPhrase = pNode->pPhrase; int iStr = *piStr; rc = fts5ExprAdvance(db, pNode, 1); while( rc==SQLITE4_OK && pNode->aPk ){ int i; for(i=0; i<pPhrase->nStr; i++){ int *anRow = &pCsr->anRow[(iStr+i) * pInfo->nCol * nStream]; int *anRowC = &pCsr->anRowC[(iStr+i) * pInfo->nCol]; int *anRowS = &pCsr->anRowS[(iStr+i) * nStream]; int *pnRowCS = &pCsr->anRowCS[iStr+i]; fts5StrLoadRowcounts( |
︙ | ︙ |
Changes to src/fts5func.c.
︙ | ︙ | |||
40 41 42 43 44 45 46 | static int fts5SimpleDestroy(sqlite4_tokenizer *p){ return SQLITE4_OK; } typedef struct Fts5RankCtx Fts5RankCtx; struct Fts5RankCtx { sqlite4 *db; | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | > | > > > > | > | > > > | > > > > > | > | | > > > > > > > | < < < < > > > | < < > | > > > > | > > > | > > > > | < < | > > > > > > > > | > > | > | < > | | < < < | | > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > < | | | < < > | 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 | static int fts5SimpleDestroy(sqlite4_tokenizer *p){ return SQLITE4_OK; } typedef struct Fts5RankCtx Fts5RankCtx; struct Fts5RankCtx { sqlite4 *db; double *aAvgdl; /* Average document size of each field */ int nPhrase; /* Number of phrases in query */ double *aIdf; /* IDF weights for each phrase in query */ }; static void fts5RankFreeCtx(void *pCtx){ if( pCtx ){ Fts5RankCtx *p = (Fts5RankCtx *)pCtx; sqlite4DbFree(p->db, p); } } #define BM25_EXPLAIN 0x01 #define BM25_FCOLUMNS 0x02 #define BM25_FSTREAMS 0x04 static int fts5GetSizeFreqScale( sqlite4_context *pCtx, int nArg, sqlite4_value **apArg,/* Function arguments */ int bm25mask, /* bm25 configuration mask */ int iPhrase, /* Phrase number */ int iField, /* Field number */ int *pnSize, /* OUT: Size of field in tokens */ int *pnFreq, /* OUT: Occurences of phrase in field */ double *pdScale /* OUT: Scale to use with this field */ ){ int rc; double scale = 1.0; int nSize = 0; int nFreq = 0; if( bm25mask & BM25_FCOLUMNS ){ rc = sqlite4_mi_match_count(pCtx, iField, -1, iPhrase, &nFreq); if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, iField, -1, &nSize); if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]); }else if( bm25mask & BM25_FSTREAMS ){ rc = sqlite4_mi_match_count(pCtx, -1, iField, iPhrase, &nFreq); if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, iField, &nSize); if( nArg>iField ) scale = sqlite4_value_double(apArg[iField]); }else{ rc = sqlite4_mi_match_count(pCtx, -1, -1, iPhrase, &nFreq); if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, -1, &nSize); } *pnSize = nSize; *pnFreq = nFreq; *pdScale = scale; return rc; } /* ** A BM25(F) based ranking function for fts5. ** ** This is based on the information in the Robertson/Zaragoza paper ** referenced above. As there is no way to provide relevance feedback ** IDF weights (equation 3.3 in R/Z) are used instead of RSJ for each phrase. ** The rest of the implementation is as presented in equations 3.19-21. ** ** R and Z observe that the experimental evidence suggests that reasonable ** values for free parameters "b" and "k1" are often in the ranges ** (0.5 < b < 0.8) and (1.2 < k1 < 2), although the optimal values depend ** on the nature of both the documents and queries. The implementation ** below sets each parameter to the midpoint of the suggested range. */ static void fts5Rank(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){ const double b = 0.65; const double k1 = 1.6; sqlite4 *db = sqlite4_context_db_handle(pCtx); int rc = SQLITE4_OK; /* Error code */ Fts5RankCtx *p; /* Structure to store reusable values */ int i; /* Used to iterate through phrases */ double rank = 0.0; /* UDF return value */ int bExplain; /* True to run in explain mode */ char *zExplain = 0; /* String to return in explain mode */ int nField = 1; /* Number of fields in collection */ int bm25mask = SQLITE4_PTR_TO_INT(sqlite4_user_data(pCtx)); bExplain = (bm25mask & BM25_EXPLAIN); if( bm25mask & BM25_FCOLUMNS ) sqlite4_mi_column_count(pCtx, &nField); if( bm25mask & BM25_FSTREAMS ) sqlite4_mi_stream_count(pCtx, &nField); p = sqlite4_get_auxdata(pCtx, 0); if( p==0 ){ int nPhrase; /* Number of phrases in query expression */ int nByte; /* Number of bytes of data to allocate */ sqlite4_mi_phrase_count(pCtx, &nPhrase); nByte = sizeof(Fts5RankCtx) + (nPhrase+nField) * sizeof(double); p = (Fts5RankCtx *)sqlite4DbMallocZero(db, nByte); sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx); p = sqlite4_get_auxdata(pCtx, 0); if( !p ){ rc = SQLITE4_NOMEM; }else{ int N; /* Total number of docs in collection */ int ni; /* Number of docs with phrase i */ p->db = db; p->nPhrase = nPhrase; p->aIdf = (double *)&p[1]; p->aAvgdl = &p->aIdf[nPhrase]; /* Determine the IDF weight for each phrase in the query. */ rc = sqlite4_mi_total_rows(pCtx, &N); for(i=0; rc==SQLITE4_OK && i<nPhrase; i++){ rc = sqlite4_mi_row_count(pCtx, -1, -1, i, &ni); if( rc==SQLITE4_OK ){ assert( ni<=N ); p->aIdf[i] = log((0.5 + N - ni) / (0.5 + ni)); } } /* Determine the average document length. For bm25f, determine the ** average length of each field. */ if( rc==SQLITE4_OK ){ int iField; for(iField=0; iField<nField; iField++){ int nTotal; if( bm25mask & BM25_FCOLUMNS ){ rc = sqlite4_mi_total_size(pCtx, iField, -1, &nTotal); }else if( bm25mask & BM25_FSTREAMS ){ rc = sqlite4_mi_total_size(pCtx, -1, iField, &nTotal); }else{ rc = sqlite4_mi_total_size(pCtx, -1, -1, &nTotal); } if( rc==SQLITE4_OK ){ p->aAvgdl[iField] = (double)nTotal / (double)N; } } } } } if( bExplain ){ int iField; zExplain = sqlite4MAppendf( db, zExplain, "%s<table><tr><th>Stream<th>Scale<th>avgsl<th>sl", zExplain ); for(i=0; i<p->nPhrase; i++){ zExplain = sqlite4MAppendf( db, zExplain, "%s<th>tf</span><sub>%d</sub>", zExplain, i ); } for(iField=0; rc==SQLITE4_OK && iField<nField; iField++){ int dl, tf; double scale; rc = fts5GetSizeFreqScale( pCtx, nArg, apArg, bm25mask, 0, iField, &dl, &tf, &scale ); zExplain = sqlite4MAppendf( db, zExplain, "%s<tr><td>%d<td>%.2f<td>%.2f<td>%d", zExplain, iField, scale, p->aAvgdl[iField], dl ); for(i=0; rc==SQLITE4_OK && i<p->nPhrase; i++){ rc = fts5GetSizeFreqScale( pCtx, nArg, apArg, bm25mask, i, iField, &dl, &tf, &scale ); zExplain = sqlite4MAppendf(db, zExplain, "%s<td>%d", zExplain, tf); } } zExplain = sqlite4MAppendf( db, zExplain, "%s</table><table><tr><th>Phrase<th>IDF", zExplain ); for(i=0; i<nField; i++){ zExplain = sqlite4MAppendf( db, zExplain, "%s<th><span style=text-decoration:overline>" "tf</span><sub>s%d</sub>", zExplain, i ); } zExplain = sqlite4MAppendf(db, zExplain, "%s<th>rank", zExplain); } for(i=0; rc==SQLITE4_OK && i<p->nPhrase; i++){ int iField; double tfns = 0.0; /* Sum of tfn for all fields */ double prank; /* Contribution to rank of this phrase */ if( bExplain ){ zExplain = sqlite4MAppendf( db, zExplain, "%s<tr><td>%d<td>%.2f", zExplain, i, p->aIdf[i] ); } for(iField = 0; iField<nField; iField++){ double scale = 1.0; int tf; /* Count of phrase i in row (term freq.) */ double tfn; /* Normalized term frequency */ int dl; /* Tokens in this row (document length) */ double B; /* B from formula 3.20 */ /* Set variable tf to the total number of occurrences of phrase iPhrase ** in this row/field. And dl to the number of tokens in the current ** row/field. */ rc = fts5GetSizeFreqScale( pCtx, nArg, apArg, bm25mask, i, iField, &dl, &tf, &scale ); B = (1.0 - b) + b * (double)dl / p->aAvgdl[iField]; /* 3.20 */ tfn = scale * (double)tf / B; tfns += tfn; /* 3.19 */ if( bExplain ){ zExplain = sqlite4MAppendf(db, zExplain, "%s<td>%.2f", zExplain, tfn); } } prank = p->aIdf[i] * tfns / (k1 + tfns); /* 3.21 */ if( bExplain ){ zExplain = sqlite4MAppendf(db, zExplain, "%s<td>%.2f", zExplain, prank); } /* Add it to the overall rank */ rank += prank; } if( rc==SQLITE4_OK ){ if( bExplain ){ zExplain = sqlite4MAppendf( db, zExplain, "%s</table><b>overall rank=%.2f</b>", zExplain, rank ); sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT); }else{ sqlite4_result_double(pCtx, rank); } }else{ sqlite4_result_error_code(pCtx, rc); } sqlite4DbFree(db, zExplain); } typedef struct Snippet Snippet; typedef struct SnippetText SnippetText; struct Snippet { int iCol; |
︙ | ︙ | |||
531 532 533 534 535 536 537 538 539 540 541 542 | sqlite4_free(pEnv, aBuf); return SQLITE4_OK; } int sqlite4InitFts5Func(sqlite4 *db){ int rc; sqlite4_env *pEnv = sqlite4_db_env(db); rc = sqlite4_create_tokenizer(db, "simple", (void *)pEnv, fts5SimpleCreate, fts5SimpleTokenize, fts5SimpleDestroy ); | > > > > > > > > > > > > > | > > | | | > > | < < < | | < < | 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 | sqlite4_free(pEnv, aBuf); return SQLITE4_OK; } int sqlite4InitFts5Func(sqlite4 *db){ int rc; int i; sqlite4_env *pEnv = sqlite4_db_env(db); struct RankFunction { const char *zName; int mask; } aRank[] = { { "rank", 0 }, { "erank", BM25_EXPLAIN }, { "rankc", BM25_FCOLUMNS }, { "erankc", BM25_FCOLUMNS|BM25_EXPLAIN }, { "ranks", BM25_FSTREAMS }, { "eranks", BM25_FSTREAMS|BM25_EXPLAIN } }; rc = sqlite4_create_tokenizer(db, "simple", (void *)pEnv, fts5SimpleCreate, fts5SimpleTokenize, fts5SimpleDestroy ); if( rc==SQLITE4_OK ){ rc = sqlite4_create_mi_function( db, "snippet", -1, SQLITE4_UTF8, 0, fts5Snippet, 0); } for(i=0; rc==SQLITE4_OK && i<ArraySize(aRank); i++){ void *p = SQLITE4_INT_TO_PTR(aRank[i].mask); const char *z = aRank[i].zName; rc = sqlite4_create_mi_function(db, z, -1, SQLITE4_UTF8, p, fts5Rank, 0); } return rc; } |