/ Check-in [deb80eac]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes for the matchinfo() function related to FTS4 common token handling.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | experimental
Files: files | file ages | folders
SHA1: deb80eac9112d21835dfd3cee08ed8f09d975bf7
User & Date: dan 2010-10-23 19:07:30
Context
2010-10-25
09:01
Add test for matchinfo when a phrase includes some common tokens. check-in: 80a54ebc user: dan tags: experimental
2010-10-23
19:07
Fixes for the matchinfo() function related to FTS4 common token handling. check-in: deb80eac user: dan tags: experimental
2010-10-22
19:03
Add new test file fts3defer2.test. check-in: 5a4d5bfc user: dan tags: experimental
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  2104   2104       rc = fts3DeferExpression(pCsr, pExpr->pLeft);
  2105   2105       if( rc==SQLITE_OK ){
  2106   2106         rc = fts3DeferExpression(pCsr, pExpr->pRight);
  2107   2107       }
  2108   2108       if( pExpr->eType==FTSQUERY_PHRASE ){
  2109   2109         int iCol = pExpr->pPhrase->iColumn;
  2110   2110         int i;
  2111         -      pExpr->bDeferred = 1;
  2112   2111         for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
  2113   2112           Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
  2114   2113           if( pToken->pDeferred==0 ){
  2115   2114             rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
  2116   2115           }
  2117   2116         }
  2118   2117       }
................................................................................
  2166   2165     char *pOut = 0;
  2167   2166     int nOut = 0;
  2168   2167     int rc = SQLITE_OK;
  2169   2168     int ii;
  2170   2169     int iCol = pPhrase->iColumn;
  2171   2170     int isTermPos = (pPhrase->nToken>1 || isReqPos);
  2172   2171     Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         2172  +  int isFirst = 1;
  2173   2173   
  2174   2174     int iPrevTok = 0;
  2175   2175     int nDoc = 0;
  2176   2176   
  2177   2177     /* If this is an xFilter() evaluation, create a segment-reader for each
  2178         -  ** phrase token. Or, if this is an xNest() or snippet/offsets/matchinfo
         2178  +  ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
  2179   2179     ** evaluation, only create segment-readers if there are no Fts3DeferredToken
  2180   2180     ** objects attached to the phrase-tokens.
  2181   2181     */
  2182   2182     for(ii=0; ii<pPhrase->nToken; ii++){
  2183   2183       Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
  2184         -    if( pTok->pArray==0 && (pCsr->doDeferred==0 || pTok->pDeferred==0) ){
  2185         -      rc = fts3TermSegReaderArray(
  2186         -          pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
  2187         -      );
  2188         -      if( rc!=SQLITE_OK ) return rc;
         2184  +    if( pTok->pArray==0 ){
         2185  +      if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
         2186  +       || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) 
         2187  +       || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) 
         2188  +      ){
         2189  +        rc = fts3TermSegReaderArray(
         2190  +            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
         2191  +        );
         2192  +        if( rc!=SQLITE_OK ) return rc;
         2193  +      }
  2189   2194       }
  2190   2195     }
  2191   2196   
  2192   2197     for(ii=0; ii<pPhrase->nToken; ii++){
  2193   2198       Fts3PhraseToken *pTok;        /* Token to find doclist for */
  2194   2199       int iTok;                     /* The token being queried this iteration */
  2195   2200       char *pList;                  /* Pointer to token doclist */
  2196   2201       int nList;                    /* Size of buffer at pList */
  2197   2202   
  2198   2203       /* Select a token to process. If this is an xFilter() call, then tokens 
  2199   2204       ** are processed in order from least to most costly. Otherwise, tokens 
  2200   2205       ** are processed in the order in which they occur in the phrase.
  2201   2206       */
  2202         -    if( pCsr->doDeferred || isReqPos ){
         2207  +    if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
         2208  +      assert( isReqPos );
         2209  +      iTok = ii;
         2210  +      pTok = &pPhrase->aToken[iTok];
         2211  +      if( pTok->bFulltext==0 ) continue;
         2212  +    }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
  2203   2213         iTok = ii;
  2204   2214         pTok = &pPhrase->aToken[iTok];
  2205   2215       }else{
  2206   2216         int nMinCost = 0x7FFFFFFF;
  2207   2217         int jj;
  2208   2218   
  2209   2219         /* Find the remaining token with the lowest cost. */
................................................................................
  2223   2233         */
  2224   2234         if( nMinCost>nDoc && ii>0 ){
  2225   2235           rc = fts3DeferExpression(pCsr, pCsr->pExpr);
  2226   2236           break;
  2227   2237         }
  2228   2238       }
  2229   2239   
  2230         -    if( pCsr->doDeferred && pTok->pDeferred ){
         2240  +    if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
  2231   2241         rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
  2232   2242       }else{
  2233   2243         assert( pTok->pArray );
  2234   2244         rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
         2245  +      pTok->bFulltext = 1;
  2235   2246       }
  2236         -    assert( rc!=SQLITE_OK || pCsr->doDeferred || pTok->pArray==0 );
         2247  +    assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 );
  2237   2248       if( rc!=SQLITE_OK ) break;
  2238   2249   
  2239         -    if( ii==0 ){
         2250  +    if( isFirst ){
  2240   2251         pOut = pList;
  2241   2252         nOut = nList;
  2242         -      if( pCsr->doDeferred==0 && pPhrase->nToken>1 ){
         2253  +      if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
  2243   2254           nDoc = fts3DoclistCountDocids(1, pOut, nOut);
  2244   2255         }
         2256  +      isFirst = 0;
  2245   2257       }else{
  2246   2258         /* Merge the new term list and the current output. */
  2247   2259         char *aLeft, *aRight;
  2248   2260         int nLeft, nRight;
  2249   2261         int nDist;
  2250   2262         int mt;
  2251   2263   
................................................................................
  2278   2290       }
  2279   2291       assert( nOut==0 || pOut!=0 );
  2280   2292   
  2281   2293       iPrevTok = iTok;
  2282   2294     }
  2283   2295   
  2284   2296     if( rc==SQLITE_OK ){
  2285         -    if( ii!=pPhrase->nToken ){
  2286         -      assert( pCsr->doDeferred==0 && isReqPos==0 );
         2297  +    if( ii!=pPhrase->nToken && pCsr->eEvalmode==FTS3_EVAL_FILTER ){
         2298  +      assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
  2287   2299         fts3DoclistStripPositions(pOut, &nOut);
  2288   2300       }
  2289   2301       *paOut = pOut;
  2290   2302       *pnOut = nOut;
  2291   2303     }else{
  2292   2304       sqlite3_free(pOut);
  2293   2305     }
................................................................................
  2394   2406   static int fts3ExprAllocateSegReaders(
  2395   2407     Fts3Cursor *pCsr,               /* FTS3 table */
  2396   2408     Fts3Expr *pExpr,                /* Expression to create seg-readers for */
  2397   2409     int *pnExpr                     /* OUT: Number of AND'd expressions */
  2398   2410   ){
  2399   2411     int rc = SQLITE_OK;             /* Return code */
  2400   2412   
  2401         -  if( pCsr->doDeferred ) return SQLITE_OK;
         2413  +  assert( pCsr->eEvalmode!=FTS3_EVAL_MATCHINFO );
         2414  +  if( pCsr->eEvalmode==FTS3_EVAL_NEXT ) return SQLITE_OK;
  2402   2415     if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
  2403   2416       (*pnExpr)++;
  2404   2417       pnExpr = 0;
  2405   2418     }
  2406   2419   
  2407   2420     if( pExpr->eType==FTSQUERY_PHRASE ){
  2408   2421       Fts3Phrase *pPhrase = pExpr->pPhrase;
................................................................................
  2555   2568   
  2556   2569       if( pExpr->eType==FTSQUERY_PHRASE ){
  2557   2570         rc = fts3PhraseSelect(p, pExpr->pPhrase,
  2558   2571             isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
  2559   2572             paOut, pnOut
  2560   2573         );
  2561   2574         fts3ExprFreeSegReaders(pExpr);
  2562         -    }else if( p->doDeferred==0 && pExpr->eType==FTSQUERY_AND ){
         2575  +    }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
  2563   2576         ExprAndCost *aExpr = 0;     /* Array of AND'd expressions and costs */
  2564   2577         int nExpr = 0;              /* Size of aExpr[] */
  2565   2578         char *aRet = 0;             /* Doclist to return to caller */
  2566   2579         int nRet = 0;               /* Length of aRet[] in bytes */
  2567   2580         int nDoc = 0x7FFFFFFF;
  2568   2581   
  2569   2582         assert( !isReqPos );
................................................................................
  2624   2637         char *aRight;
  2625   2638         int nLeft;
  2626   2639         int nRight;
  2627   2640   
  2628   2641         assert( pExpr->eType==FTSQUERY_NEAR 
  2629   2642              || pExpr->eType==FTSQUERY_OR
  2630   2643              || pExpr->eType==FTSQUERY_NOT
  2631         -           || (pExpr->eType==FTSQUERY_AND && p->doDeferred)
         2644  +           || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
  2632   2645         );
  2633   2646   
  2634   2647         if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
  2635   2648          && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
  2636   2649         ){
  2637   2650           switch( pExpr->eType ){
  2638   2651             case FTSQUERY_NEAR: {
................................................................................
  2721   2734       if( rc==SQLITE_OK ){
  2722   2735         sqlite3Fts3FreeDeferredDoclists(pCsr);
  2723   2736         rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
  2724   2737       }
  2725   2738       if( rc==SQLITE_OK ){
  2726   2739         char *a = 0;
  2727   2740         int n = 0;
  2728         -      pCsr->doDeferred = 1;
  2729   2741         rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
  2730         -      pCsr->doDeferred = 0;
  2731   2742         assert( n>=0 );
  2732   2743         *pbRes = (n>0);
  2733   2744         sqlite3_free(a);
  2734   2745       }
  2735   2746     }
  2736   2747     return rc;
  2737   2748   }
................................................................................
  2748   2759   ** subsequently to determine whether or not an EOF was hit.
  2749   2760   */
  2750   2761   static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
  2751   2762     int res;
  2752   2763     int rc = SQLITE_OK;             /* Return code */
  2753   2764     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  2754   2765   
         2766  +  pCsr->eEvalmode = FTS3_EVAL_NEXT;
  2755   2767     do {
  2756   2768       if( pCsr->aDoclist==0 ){
  2757   2769         if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
  2758   2770           pCsr->isEof = 1;
  2759   2771           rc = sqlite3_reset(pCsr->pStmt);
  2760   2772           break;
  2761   2773         }
................................................................................
  2999   3011   ** Load the doclist associated with expression pExpr to pExpr->aDoclist.
  3000   3012   ** The loaded doclist contains positions as well as the document ids.
  3001   3013   ** This is used by the matchinfo(), snippet() and offsets() auxillary
  3002   3014   ** functions.
  3003   3015   */
  3004   3016   int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
  3005   3017     int rc;
  3006         -  pCsr->doDeferred = 1;
         3018  +  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
         3019  +  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
  3007   3020     rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
  3008         -  pCsr->doDeferred = 0;
         3021  +  return rc;
         3022  +}
         3023  +
         3024  +int sqlite3Fts3ExprLoadFtDoclist(
         3025  +  Fts3Cursor *pCsr, 
         3026  +  Fts3Expr *pExpr,
         3027  +  char **paDoclist,
         3028  +  int *pnDoclist
         3029  +){
         3030  +  int rc;
         3031  +  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
         3032  +  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
         3033  +  pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
         3034  +  rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
         3035  +  pCsr->eEvalmode = FTS3_EVAL_NEXT;
  3009   3036     return rc;
  3010   3037   }
  3011   3038   
  3012   3039   /*
  3013   3040   ** After ExprLoadDoclist() (see above) has been called, this function is
  3014   3041   ** used to iterate/search through the position lists that make up the doclist
  3015   3042   ** stored in pExpr->aDoclist.

Changes to ext/fts3/fts3Int.h.

   159    159     Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
   160    160     sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
   161    161     char *pNextId;                  /* Pointer into the body of aDoclist */
   162    162     char *aDoclist;                 /* List of docids for full-text queries */
   163    163     int nDoclist;                   /* Size of buffer at aDoclist */
   164    164     int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
   165    165     u32 *aMatchinfo;                /* Information about most recent match */
   166         -  int doDeferred;
          166  +  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
   167    167     int nRowAvg;                    /* Average size of database rows, in pages */
   168    168   };
          169  +
          170  +#define FTS3_EVAL_FILTER    0
          171  +#define FTS3_EVAL_NEXT      1
          172  +#define FTS3_EVAL_MATCHINFO 2
   169    173   
   170    174   /*
   171    175   ** The Fts3Cursor.eSearch member is always set to one of the following.
   172    176   ** Actualy, Fts3Cursor.eSearch can be greater than or equal to
   173    177   ** FTS3_FULLTEXT_SEARCH.  If so, then Fts3Cursor.eSearch - 2 is the index
   174    178   ** of the column to be searched.  For example, in
   175    179   **
................................................................................
   187    191   #define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */
   188    192   
   189    193   /*
   190    194   ** A "phrase" is a sequence of one or more tokens that must match in
   191    195   ** sequence.  A single token is the base case and the most common case.
   192    196   ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
   193    197   ** nToken will be the number of tokens in the string.
          198  +**
          199  +** The nDocMatch and nMatch variables contain data that may be used by the
          200  +** matchinfo() function. They are populated when the full-text index is 
          201  +** queried for hits on the phrase. If one or more tokens in the phrase
          202  +** are deferred, the nDocMatch and nMatch variables are populated based
          203  +** on the assumption that the 
   194    204   */
   195    205   struct Fts3PhraseToken {
   196    206     char *z;                        /* Text of the token */
   197    207     int n;                          /* Number of bytes in buffer z */
   198    208     int isPrefix;                   /* True if token ends with a "*" character */
          209  +  int bFulltext;                  /* True if full-text index was used */
   199    210     Fts3SegReaderArray *pArray;     /* Segment-reader for this token */
   200    211     Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
   201    212   };
   202    213   
   203    214   struct Fts3Phrase {
          215  +  /* Variables populated by fts3_expr.c when parsing a MATCH expression */
   204    216     int nToken;                /* Number of tokens in the phrase */
   205    217     int iColumn;               /* Index of column this phrase must match */
   206    218     int isNot;                 /* Phrase prefixed by unary not (-) operator */
   207    219     Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
   208    220   };
   209    221   
   210    222   /*
................................................................................
   224    236     int eType;                 /* One of the FTSQUERY_XXX values defined below */
   225    237     int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
   226    238     Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
   227    239     Fts3Expr *pLeft;           /* Left operand */
   228    240     Fts3Expr *pRight;          /* Right operand */
   229    241     Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
   230    242   
   231         -  int bDeferred;
   232         -
   233    243     int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
   234    244     char *aDoclist;            /* Buffer containing doclist */
   235    245     int nDoclist;              /* Size of aDoclist in bytes */
   236    246   
   237    247     sqlite3_int64 iCurrent;
   238    248     char *pCurrent;
   239    249   };
................................................................................
   303    313   int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
   304    314   int sqlite3Fts3GetVarint32(const char *, int *);
   305    315   int sqlite3Fts3VarintLen(sqlite3_uint64);
   306    316   void sqlite3Fts3Dequote(char *);
   307    317   
   308    318   char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int);
   309    319   int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
          320  +int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
   310    321   int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
   311    322   
   312    323   /* fts3_tokenizer.c */
   313    324   const char *sqlite3Fts3NextToken(const char *, int *);
   314    325   int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
   315    326   int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, 
   316    327     const char *, sqlite3_tokenizer **, const char **, char **

Changes to ext/fts3/fts3_expr.c.

   101    101   ** is defined to accept an argument of type char, and always returns 0 for
   102    102   ** any values that fall outside of the range of the unsigned char type (i.e.
   103    103   ** negative values).
   104    104   */
   105    105   static int fts3isspace(char c){
   106    106     return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
   107    107   }
          108  +
          109  +/*
          110  +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful,
          111  +** zero the memory before returning a pointer to it. If unsuccessful, 
          112  +** return NULL.
          113  +*/
          114  +static void *fts3MallocZero(int nByte){
          115  +  void *pRet = sqlite3_malloc(nByte);
          116  +  if( pRet ) memset(pRet, 0, nByte);
          117  +  return pRet;
          118  +}
          119  +
   108    120   
   109    121   /*
   110    122   ** Extract the next token from buffer z (length n) using the tokenizer
   111    123   ** and other information (column names etc.) in pParse. Create an Fts3Expr
   112    124   ** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
   113    125   ** single token and set *ppExpr to point to it. If the end of the buffer is
   114    126   ** reached before a token is found, set *ppExpr to zero. It is the
................................................................................
   139    151       int nByte;                               /* total space to allocate */
   140    152   
   141    153       pCursor->pTokenizer = pTokenizer;
   142    154       rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
   143    155   
   144    156       if( rc==SQLITE_OK ){
   145    157         nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
   146         -      pRet = (Fts3Expr *)sqlite3_malloc(nByte);
          158  +      pRet = (Fts3Expr *)fts3MallocZero(nByte);
   147    159         if( !pRet ){
   148    160           rc = SQLITE_NOMEM;
   149    161         }else{
   150         -        memset(pRet, 0, nByte);
   151    162           pRet->eType = FTSQUERY_PHRASE;
   152    163           pRet->pPhrase = (Fts3Phrase *)&pRet[1];
   153    164           pRet->pPhrase->nToken = 1;
   154    165           pRet->pPhrase->iColumn = iCol;
   155    166           pRet->pPhrase->aToken[0].n = nToken;
   156    167           pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
   157    168           memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
................................................................................
   229    240             goto no_mem;
   230    241           }
   231    242           if( ii==0 ){
   232    243             memset(p, 0, nByte);
   233    244             p->pPhrase = (Fts3Phrase *)&p[1];
   234    245           }
   235    246           p->pPhrase = (Fts3Phrase *)&p[1];
          247  +        memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
   236    248           p->pPhrase->nToken = ii+1;
   237    249           p->pPhrase->aToken[ii].n = nToken;
   238         -        p->pPhrase->aToken[ii].pDeferred = 0;
   239         -        p->pPhrase->aToken[ii].pArray = 0;
   240    250           memcpy(&zTemp[nTemp], zToken, nToken);
   241    251           nTemp += nToken;
   242    252           if( iEnd<nInput && zInput[iEnd]=='*' ){
   243    253             p->pPhrase->aToken[ii].isPrefix = 1;
   244    254           }else{
   245    255             p->pPhrase->aToken[ii].isPrefix = 0;
   246    256           }
................................................................................
   370    380         ** the next byte must contain either whitespace, an open or close
   371    381         ** parenthesis, a quote character, or EOF. 
   372    382         */
   373    383         cNext = zInput[nKey];
   374    384         if( fts3isspace(cNext) 
   375    385          || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
   376    386         ){
   377         -        pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
          387  +        pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
   378    388           if( !pRet ){
   379    389             return SQLITE_NOMEM;
   380    390           }
   381         -        memset(pRet, 0, sizeof(Fts3Expr));
   382    391           pRet->eType = pKey->eType;
   383    392           pRet->nNear = nNear;
   384    393           *ppExpr = pRet;
   385    394           *pnConsumed = (int)((zInput - z) + nKey);
   386    395           return SQLITE_OK;
   387    396         }
   388    397   
................................................................................
   550    559       if( rc==SQLITE_OK ){
   551    560         int isPhrase;
   552    561   
   553    562         if( !sqlite3_fts3_enable_parentheses 
   554    563          && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot 
   555    564         ){
   556    565           /* Create an implicit NOT operator. */
   557         -        Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr));
          566  +        Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
   558    567           if( !pNot ){
   559    568             sqlite3Fts3ExprFree(p);
   560    569             rc = SQLITE_NOMEM;
   561    570             goto exprparse_out;
   562    571           }
   563         -        memset(pNot, 0, sizeof(Fts3Expr));
   564    572           pNot->eType = FTSQUERY_NOT;
   565    573           pNot->pRight = p;
   566    574           if( pNotBranch ){
   567    575             pNot->pLeft = pNotBranch;
   568    576           }
   569    577           pNotBranch = pNot;
   570    578           p = pPrev;
................................................................................
   584    592             goto exprparse_out;
   585    593           }
   586    594     
   587    595           if( isPhrase && !isRequirePhrase ){
   588    596             /* Insert an implicit AND operator. */
   589    597             Fts3Expr *pAnd;
   590    598             assert( pRet && pPrev );
   591         -          pAnd = sqlite3_malloc(sizeof(Fts3Expr));
          599  +          pAnd = fts3MallocZero(sizeof(Fts3Expr));
   592    600             if( !pAnd ){
   593    601               sqlite3Fts3ExprFree(p);
   594    602               rc = SQLITE_NOMEM;
   595    603               goto exprparse_out;
   596    604             }
   597         -          memset(pAnd, 0, sizeof(Fts3Expr));
   598    605             pAnd->eType = FTSQUERY_AND;
   599    606             insertBinaryOperator(&pRet, pPrev, pAnd);
   600    607             pPrev = pAnd;
   601    608           }
   602    609   
   603    610           /* This test catches attempts to make either operand of a NEAR
   604    611           ** operator something other than a phrase. For example, either of

Changes to ext/fts3/fts3_snippet.c.

   264    264     sCtx.pCsr = pCsr;
   265    265     rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx);
   266    266     if( rc==SQLITE_OK ){
   267    267       (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
   268    268     }
   269    269     if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   270    270     if( pnToken ) *pnToken = sCtx.nToken;
   271         -  sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab);
   272    271     return rc;
   273    272   }
   274    273   
   275    274   /*
   276    275   ** Advance the position list iterator specified by the first two 
   277    276   ** arguments so that it points to the first element with a value greater
   278    277   ** than or equal to parameter iNext.
................................................................................
   789    788   */
   790    789   static int fts3ExprGlobalMatchinfoCb(
   791    790     Fts3Expr *pExpr,                /* Phrase expression node */
   792    791     int iPhrase,                    /* Phrase number (numbered from zero) */
   793    792     void *pCtx                      /* Pointer to MatchInfo structure */
   794    793   ){
   795    794     MatchInfo *p = (MatchInfo *)pCtx;
   796         -  char *pCsr;
          795  +  Fts3Cursor *pCsr = p->pCursor;
          796  +  char *pIter;
   797    797     char *pEnd;
          798  +  char *pFree = 0;
   798    799     const int iStart = 2 + (iPhrase * p->nCol * 3) + 1;
   799    800   
   800    801     assert( pExpr->isLoaded );
          802  +  assert( pExpr->eType==FTSQUERY_PHRASE );
          803  +
          804  +  if( pCsr->pDeferred ){
          805  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
          806  +    int ii;
          807  +    for(ii=0; ii<pPhrase->nToken; ii++){
          808  +      if( pPhrase->aToken[ii].bFulltext ) break;
          809  +    }
          810  +    if( ii<pPhrase->nToken ){
          811  +      int nFree = 0;
          812  +      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
          813  +      if( rc!=SQLITE_OK ) return rc;
          814  +      pIter = pFree;
          815  +      pEnd = &pFree[nFree];
          816  +    }else{
          817  +      int nDoc = p->aMatchinfo[2 + 3*p->nCol*p->aMatchinfo[0]];
          818  +      for(ii=0; ii<p->nCol; ii++){
          819  +        p->aMatchinfo[iStart + ii*3] = nDoc;
          820  +        p->aMatchinfo[iStart + ii*3 + 1] = nDoc;
          821  +      }
          822  +      return SQLITE_OK;
          823  +    }
          824  +  }else{
          825  +    pIter = pExpr->aDoclist;
          826  +    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
          827  +  }
   801    828   
   802    829     /* Fill in the global hit count matrix row for this phrase. */
   803         -  pCsr = pExpr->aDoclist;
   804         -  pEnd = &pExpr->aDoclist[pExpr->nDoclist];
   805         -  while( pCsr<pEnd ){
   806         -    while( *pCsr++ & 0x80 );      /* Skip past docid. */
   807         -    fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 1);
          830  +  while( pIter<pEnd ){
          831  +    while( *pIter++ & 0x80 );      /* Skip past docid. */
          832  +    fts3LoadColumnlistCounts(&pIter, &p->aMatchinfo[iStart], 1);
   808    833     }
   809    834   
          835  +  sqlite3_free(pFree);
   810    836     return SQLITE_OK;
   811    837   }
   812    838   
   813    839   /*
   814    840   ** fts3ExprIterate() callback used to collect the "local" matchinfo stats
   815    841   ** for a single query. The "local" stats are those elements of the matchinfo
   816    842   ** array that are different for each row returned by the query.
................................................................................
   871    897   
   872    898       sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
   873    899       if( !sInfo.aMatchinfo ){ 
   874    900         return SQLITE_NOMEM;
   875    901       }
   876    902       memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo);
   877    903   
   878         -
   879    904       /* First element of match-info is the number of phrases in the query */
   880    905       sInfo.aMatchinfo[0] = nPhrase;
   881    906       sInfo.aMatchinfo[1] = sInfo.nCol;
   882         -    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
   883    907       if( pTab->bHasDocsize ){
   884    908         int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
   885    909         rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]);
   886    910       }
          911  +    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
   887    912       pCsr->aMatchinfo = sInfo.aMatchinfo;
   888    913       pCsr->isMatchinfoNeeded = 1;
   889    914     }
   890    915   
   891    916     sInfo.aMatchinfo = pCsr->aMatchinfo;
   892    917     if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
   893    918       (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo);
................................................................................
   989   1014     for(i=0; i<nSnippet && rc==SQLITE_OK; i++){
   990   1015       rc = fts3SnippetText(pCsr, &aSnippet[i], 
   991   1016           i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
   992   1017       );
   993   1018     }
   994   1019   
   995   1020    snippet_out:
         1021  +  sqlite3Fts3SegmentsClose(pTab);
   996   1022     if( rc!=SQLITE_OK ){
   997   1023       sqlite3_result_error_code(pCtx, rc);
   998   1024       sqlite3_free(res.z);
   999   1025     }else{
  1000   1026       sqlite3_result_text(pCtx, res.z, -1, sqlite3_free);
  1001   1027     }
  1002   1028   }
................................................................................
  1168   1194       pMod->xClose(pC);
  1169   1195       if( rc!=SQLITE_OK ) goto offsets_out;
  1170   1196     }
  1171   1197   
  1172   1198    offsets_out:
  1173   1199     sqlite3_free(sCtx.aTerm);
  1174   1200     assert( rc!=SQLITE_DONE );
         1201  +  sqlite3Fts3SegmentsClose(pTab);
  1175   1202     if( rc!=SQLITE_OK ){
  1176   1203       sqlite3_result_error_code(pCtx,  rc);
  1177   1204       sqlite3_free(res.z);
  1178   1205     }else{
  1179   1206       sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free);
  1180   1207     }
  1181   1208     return;
................................................................................
  1187   1214   void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){
  1188   1215     int rc;
  1189   1216     if( !pCsr->pExpr ){
  1190   1217       sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
  1191   1218       return;
  1192   1219     }
  1193   1220     rc = fts3GetMatchinfo(pCsr);
         1221  +  sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab );
  1194   1222     if( rc!=SQLITE_OK ){
  1195   1223       sqlite3_result_error_code(pContext, rc);
  1196   1224     }else{
  1197   1225       Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
  1198   1226       int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3);
  1199   1227       if( pTab->bHasDocsize ){
  1200   1228         n += sizeof(u32)*(1 + 2*pTab->nColumn);
  1201   1229       }
  1202   1230       sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
  1203   1231     }
  1204   1232   }
  1205   1233   
  1206   1234   #endif

Changes to ext/fts3/fts3_write.c.

  2640   2640   ** references to deferred doclists from within the tree of Fts3Expr 
  2641   2641   ** structures headed by 
  2642   2642   */
  2643   2643   static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
  2644   2644     if( pExpr ){
  2645   2645       fts3DeferredDoclistClear(pExpr->pLeft);
  2646   2646       fts3DeferredDoclistClear(pExpr->pRight);
  2647         -    if( pExpr->bDeferred && pExpr->isLoaded ){
         2647  +    if( pExpr->isLoaded ){
  2648   2648         sqlite3_free(pExpr->aDoclist);
  2649   2649         pExpr->isLoaded = 0;
  2650   2650         pExpr->aDoclist = 0;
  2651   2651         pExpr->nDoclist = 0;
  2652   2652         pExpr->pCurrent = 0;
  2653   2653         pExpr->iCurrent = 0;
  2654   2654       }
................................................................................
  2661   2661   */
  2662   2662   void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
  2663   2663     Fts3DeferredToken *pDef;
  2664   2664     for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
  2665   2665       sqlite3_free(pDef->pList);
  2666   2666       pDef->pList = 0;
  2667   2667     }
  2668         -  fts3DeferredDoclistClear(pCsr->pExpr);
         2668  +  if( pCsr->pDeferred ){
         2669  +    fts3DeferredDoclistClear(pCsr->pExpr);
         2670  +  }
  2669   2671   }
  2670   2672   
  2671   2673   /*
  2672   2674   ** Free all entries in the pCsr->pDeffered list. Entries are added to 
  2673   2675   ** this list using sqlite3Fts3DeferToken().
  2674   2676   */
  2675   2677   void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){

Changes to test/fts3defer2.test.

    53     53   
    54     54   do_execsql_test 1.2.2 {
    55     55     SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
    56     56     FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
    57     57   } [list                              \
    58     58      {a b c d [e] [f] [a] x y}         \
    59     59      {0 1 8 1 0 0 10 1 0 2 12 1}       \
    60         -   [list 3 1   1 1 1   1 1 1   1 1 1   3 13336 9]
           60  +   [list 3 1   1 1 1   1 3 3   1 3 3   3 13336 9]
    61     61   ]
    62     62   
    63     63   do_execsql_test 1.2.3 {
    64     64     SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
    65     65     FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
    66     66   } [list                                 \
    67     67      {[a] b c d [e] [f] [a] x y}          \
    68     68      {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
    69         -   [list 3 1   1 1 1   1 1 1   2 2 1   3 13336 9]
           69  +   [list 3 1   1 1 1   1 3 3   2 3 3   3 13336 9]
    70     70   ]
    71     71   
    72     72   do_execsql_test 1.3.1 { DROP TABLE t1 }
    73     73   
    74     74   #-----------------------------------------------------------------------------
    75     75   # Test cases fts3defer2-2.* focus specifically on the matchinfo function.
    76     76   # 
    77     77   do_execsql_test 2.1.1 {
    78     78     CREATE VIRTUAL TABLE t2 USING fts4;
    79     79   }
    80         -do_execsql_test 2.1.2 "INSERT INTO t2 VALUES('[string repeat {a } 20000]')"
    81         -do_execsql_test 2.1.3 "INSERT INTO t2 VALUES('[string repeat {z } 20000]')"
    82         -do_execsql_test 2.1.4 {
           80  +do_execsql_test 2.1.2 "INSERT INTO t2 VALUES('[string repeat {a } 10000]')"
           81  +do_execsql_test 2.1.3 "INSERT INTO t2 VALUES('b [string repeat {z } 10000]')"
           82  +do_execsql_test 2.1.4 [string repeat "INSERT INTO t2 VALUES('x');" 50]
           83  +do_execsql_test 2.1.5 {
    83     84     INSERT INTO t2 VALUES('a b c d e f g');
    84     85     INSERT INTO t2 VALUES('a b c d e f g');
    85     86   }
    86     87   
    87     88   foreach {tn sql} {
    88     89     1 {}
    89     90     2 { INSERT INTO t2(t2) VALUES('optimize') }
    90     91     3 { UPDATE t2_segments SET block = zeroblob(length(block)) 
    91     92         WHERE length(block)>10000;
    92     93     }
    93     94   } {
    94         -if {$tn==3} break
    95     95     execsql $sql
    96     96     do_execsql_test 2.2.$tn.1 {
    97     97       SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'a b';
    98     98     } [list                                          \
    99         -    [list 2 1  1 20002 3  1 2 2  4 10004 7]        \
   100         -    [list 2 1  1 20002 3  1 2 2  4 10004 7]        \
           99  +    [list 2 1  1 54 54  1 3 3  54 372 7]        \
          100  +    [list 2 1  1 54 54  1 3 3  54 372 7]        \
   101    101     ]
   102    102   }
   103    103   
   104    104   
   105    105   finish_test
   106    106