/ Check-in [84097a4c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Minor changes made while planning a larger change.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-prefix-search
Files: files | file ages | folders
SHA1: 84097a4c759b1d65890af885f137d3cb16eef584
User & Date: dan 2011-05-28 15:57:40
Context
2011-06-02
19:57
Changes to improve performance and support LIMIT clauses on fts3 tables. This branch is unstable for now. check-in: 28149a78 user: dan tags: fts3-prefix-search
2011-05-28
15:57
Minor changes made while planning a larger change. check-in: 84097a4c user: dan tags: fts3-prefix-search
2011-05-25
19:17
If a prefix index of size N is not present, use a prefix index of size N+1 along with the terms index for queries for prefixes of length N. check-in: cc83991c user: dan tags: fts3-prefix-search
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  2763   2763   **
  2764   2764   ** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
  2765   2765   ** have their respective doclists (including position information) loaded
  2766   2766   ** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
  2767   2767   ** each doclist that are not within nNear tokens of a corresponding entry
  2768   2768   ** in the other doclist.
  2769   2769   */
  2770         -int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
         2770  +int sqlite3Fts3ExprNearTrim(Fts3Expr *pELeft, Fts3Expr *pERight, int nNear){
  2771   2771     int rc;                         /* Return code */
         2772  +  Fts3Phrase *pLeft = pELeft->pPhrase;
         2773  +  Fts3Phrase *pRight = pERight->pPhrase;
  2772   2774   
  2773         -  assert( pLeft->eType==FTSQUERY_PHRASE );
  2774         -  assert( pRight->eType==FTSQUERY_PHRASE );
         2775  +  assert( pELeft->eType==FTSQUERY_PHRASE && pLeft );
         2776  +  assert( pERight->eType==FTSQUERY_PHRASE && pRight );
  2775   2777     assert( pLeft->isLoaded && pRight->isLoaded );
  2776   2778   
  2777   2779     if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
  2778   2780       sqlite3_free(pLeft->aDoclist);
  2779   2781       sqlite3_free(pRight->aDoclist);
  2780   2782       pRight->aDoclist = 0;
  2781   2783       pLeft->aDoclist = 0;
  2782   2784       rc = SQLITE_OK;
  2783   2785     }else{
  2784   2786       char *aOut;                   /* Buffer in which to assemble new doclist */
  2785   2787       int nOut;                     /* Size of buffer aOut in bytes */
  2786   2788   
  2787   2789       rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
  2788         -        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
  2789         -        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
         2790  +        pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist,
         2791  +        pRight->nToken, pRight->aDoclist, pRight->nDoclist,
  2790   2792           &aOut, &nOut
  2791   2793       );
  2792   2794       if( rc!=SQLITE_OK ) return rc;
  2793   2795       sqlite3_free(pRight->aDoclist);
  2794   2796       pRight->aDoclist = aOut;
  2795   2797       pRight->nDoclist = nOut;
  2796   2798   
  2797   2799       rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
  2798         -        pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
  2799         -        pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
         2800  +        pRight->nToken, pRight->aDoclist, pRight->nDoclist,
         2801  +        pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist,
  2800   2802           &aOut, &nOut
  2801   2803       );
  2802   2804       sqlite3_free(pLeft->aDoclist);
  2803   2805       pLeft->aDoclist = aOut;
  2804   2806       pLeft->nDoclist = nOut;
  2805   2807     }
  2806   2808     return rc;
................................................................................
  3455   3457   ** Load the doclist associated with expression pExpr to pExpr->aDoclist.
  3456   3458   ** The loaded doclist contains positions as well as the document ids.
  3457   3459   ** This is used by the matchinfo(), snippet() and offsets() auxillary
  3458   3460   ** functions.
  3459   3461   */
  3460   3462   int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
  3461   3463     int rc;
  3462         -  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
         3464  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
         3465  +  assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase );
  3463   3466     assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
  3464         -  rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
         3467  +  rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1);
  3465   3468     return rc;
  3466   3469   }
  3467   3470   
         3471  +/*
         3472  +** TODO: This is something to do with matchinfo(). Similar to
         3473  +** sqlite3ExprLoadDoclists() but slightly different.
         3474  +**
         3475  +** UPDATE: Only used when there are deferred tokens.
         3476  +*/
  3468   3477   int sqlite3Fts3ExprLoadFtDoclist(
  3469   3478     Fts3Cursor *pCsr, 
  3470   3479     Fts3Expr *pExpr,
  3471   3480     char **paDoclist,
  3472   3481     int *pnDoclist
  3473   3482   ){
  3474   3483     int rc;
................................................................................
  3506   3515   */
  3507   3516   char *sqlite3Fts3FindPositions(
  3508   3517     Fts3Cursor *pCursor,            /* Associate FTS3 cursor */
  3509   3518     Fts3Expr *pExpr,                /* Access this expressions doclist */
  3510   3519     sqlite3_int64 iDocid,           /* Docid associated with requested pos-list */
  3511   3520     int iCol                        /* Column of requested pos-list */
  3512   3521   ){
  3513         -  assert( pExpr->isLoaded );
  3514         -  if( pExpr->aDoclist ){
  3515         -    char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
         3522  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
         3523  +  assert( pPhrase->isLoaded );
         3524  +
         3525  +  if( pPhrase->aDoclist ){
         3526  +    char *pEnd = &pPhrase->aDoclist[pPhrase->nDoclist];
  3516   3527       char *pCsr;
  3517   3528   
  3518         -    if( pExpr->pCurrent==0 ){
         3529  +    if( pPhrase->pCurrent==0 ){
  3519   3530         if( pCursor->desc==0 ){
  3520         -        pExpr->pCurrent = pExpr->aDoclist;
  3521         -        pExpr->iCurrent = 0;
  3522         -        fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent);
         3531  +        pPhrase->pCurrent = pPhrase->aDoclist;
         3532  +        pPhrase->iCurrent = 0;
         3533  +        fts3GetDeltaVarint(&pPhrase->pCurrent, &pPhrase->iCurrent);
  3523   3534         }else{
  3524         -        pCsr = pExpr->aDoclist;
         3535  +        pCsr = pPhrase->aDoclist;
  3525   3536           while( pCsr<pEnd ){
  3526         -          fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
         3537  +          fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent);
  3527   3538             fts3PoslistCopy(0, &pCsr);
  3528   3539           }
  3529         -        fts3ReversePoslist(pExpr->aDoclist, &pCsr);
  3530         -        pExpr->pCurrent = pCsr;
         3540  +        fts3ReversePoslist(pPhrase->aDoclist, &pCsr);
         3541  +        pPhrase->pCurrent = pCsr;
  3531   3542         }
  3532   3543       }
  3533         -    pCsr = pExpr->pCurrent;
         3544  +    pCsr = pPhrase->pCurrent;
  3534   3545       assert( pCsr );
  3535   3546   
  3536   3547       while( (pCursor->desc==0 && pCsr<pEnd) 
  3537         -        || (pCursor->desc && pCsr>pExpr->aDoclist) 
         3548  +        || (pCursor->desc && pCsr>pPhrase->aDoclist) 
  3538   3549       ){
  3539         -      if( pCursor->desc==0 && pExpr->iCurrent<iDocid ){
         3550  +      if( pCursor->desc==0 && pPhrase->iCurrent<iDocid ){
  3540   3551           fts3PoslistCopy(0, &pCsr);
  3541   3552           if( pCsr<pEnd ){
  3542         -          fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
         3553  +          fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent);
  3543   3554           }
  3544         -        pExpr->pCurrent = pCsr;
  3545         -      }else if( pCursor->desc && pExpr->iCurrent>iDocid ){
  3546         -        fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent);
  3547         -        fts3ReversePoslist(pExpr->aDoclist, &pCsr);
  3548         -        pExpr->pCurrent = pCsr;
         3555  +        pPhrase->pCurrent = pCsr;
         3556  +      }else if( pCursor->desc && pPhrase->iCurrent>iDocid ){
         3557  +        fts3GetReverseDeltaVarint(&pCsr, pPhrase->aDoclist, &pPhrase->iCurrent);
         3558  +        fts3ReversePoslist(pPhrase->aDoclist, &pCsr);
         3559  +        pPhrase->pCurrent = pCsr;
  3549   3560         }else{
  3550         -        if( pExpr->iCurrent==iDocid ){
         3561  +        if( pPhrase->iCurrent==iDocid ){
  3551   3562             int iThis = 0;
  3552   3563             if( iCol<0 ){
  3553   3564               /* If iCol is negative, return a pointer to the start of the
  3554   3565               ** position-list (instead of a pointer to the start of a list
  3555   3566               ** of offsets associated with a specific column).
  3556   3567               */
  3557   3568               return pCsr;

Changes to ext/fts3/fts3Int.h.

   264    264   #define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */
   265    265   
   266    266   /*
   267    267   ** A "phrase" is a sequence of one or more tokens that must match in
   268    268   ** sequence.  A single token is the base case and the most common case.
   269    269   ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
   270    270   ** nToken will be the number of tokens in the string.
   271         -**
   272         -** The nDocMatch and nMatch variables contain data that may be used by the
   273         -** matchinfo() function. They are populated when the full-text index is 
   274         -** queried for hits on the phrase. If one or more tokens in the phrase
   275         -** are deferred, the nDocMatch and nMatch variables are populated based
   276         -** on the assumption that the 
   277    271   */
   278    272   struct Fts3PhraseToken {
   279    273     char *z;                        /* Text of the token */
   280    274     int n;                          /* Number of bytes in buffer z */
   281    275     int isPrefix;                   /* True if token ends with a "*" character */
          276  +
          277  +  /* Variables above this point are populated when the expression is
          278  +  ** parsed (by code in fts3_expr.c). Below this point the variables are
          279  +  ** used when evaluating the expression. */
          280  +
   282    281     int bFulltext;                  /* True if full-text index was used */
   283    282     Fts3SegReaderCursor *pSegcsr;   /* Segment-reader for this token */
   284    283     Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
   285    284   };
   286    285   
   287    286   struct Fts3Phrase {
   288    287     /* Variables populated by fts3_expr.c when parsing a MATCH expression */
   289    288     int nToken;                /* Number of tokens in the phrase */
   290    289     int iColumn;               /* Index of column this phrase must match */
   291         -  int isNot;                 /* Phrase prefixed by unary not (-) operator */
          290  +
          291  +  int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
          292  +  char *aDoclist;            /* Buffer containing doclist */
          293  +  int nDoclist;              /* Size of aDoclist in bytes */
          294  +  sqlite3_int64 iCurrent;
          295  +  char *pCurrent;
          296  +
   292    297     Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
   293    298   };
   294    299   
   295    300   /*
   296    301   ** A tree of these objects forms the RHS of a MATCH operator.
   297    302   **
   298         -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
   299         -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, 
   300         -** containing the results of the NEAR or phrase query in FTS3 doclist
   301         -** format. As usual, the initial "Length" field found in doclists stored
   302         -** on disk is omitted from this buffer.
          303  +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist 
          304  +** points to a malloced buffer, size nDoclist bytes, containing the results 
          305  +** of this phrase query in FTS3 doclist format. As usual, the initial 
          306  +** "Length" field found in doclists stored on disk is omitted from this 
          307  +** buffer.
   303    308   **
   304    309   ** Variable pCurrent always points to the start of a docid field within
   305    310   ** aDoclist. Since the doclist is usually scanned in docid order, this can
   306    311   ** be used to accelerate seeking to the required docid within the doclist.
   307    312   */
   308    313   struct Fts3Expr {
   309    314     int eType;                 /* One of the FTSQUERY_XXX values defined below */
   310    315     int nNear;                 /* Valid if eType==FTSQUERY_NEAR */
   311    316     Fts3Expr *pParent;         /* pParent->pLeft==this or pParent->pRight==this */
   312    317     Fts3Expr *pLeft;           /* Left operand */
   313    318     Fts3Expr *pRight;          /* Right operand */
   314    319     Fts3Phrase *pPhrase;       /* Valid if eType==FTSQUERY_PHRASE */
   315         -
   316         -  int isLoaded;              /* True if aDoclist/nDoclist are initialized. */
   317         -  char *aDoclist;            /* Buffer containing doclist */
   318         -  int nDoclist;              /* Size of aDoclist in bytes */
   319         -
   320         -  sqlite3_int64 iCurrent;
   321         -  char *pCurrent;
   322    320   };
   323    321   
   324    322   /*
   325    323   ** Candidate values for Fts3Query.eType. Note that the order of the first
   326    324   ** four values is in order of precedence when parsing expressions. For 
   327    325   ** example, the following:
   328    326   **

Changes to ext/fts3/fts3_expr.c.

    77     77   */
    78     78   #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
    79     79   
    80     80   #include "fts3Int.h"
    81     81   #include <string.h>
    82     82   #include <assert.h>
    83     83   
           84  +/*
           85  +** isNot:
           86  +**   This variable is used by function getNextNode(). When getNextNode() is
           87  +**   called, it sets ParseContext.isNot to true if the 'next node' is a 
           88  +**   FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
           89  +**   FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
           90  +**   zero.
           91  +*/
    84     92   typedef struct ParseContext ParseContext;
    85     93   struct ParseContext {
    86     94     sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
    87     95     const char **azCol;                 /* Array of column names for fts3 table */
    88     96     int nCol;                           /* Number of entries in azCol[] */
    89     97     int iDefaultCol;                    /* Default column to query */
           98  +  int isNot;                          /* True if getNextNode() sees a unary - */
    90     99     sqlite3_context *pCtx;              /* Write error message here */
    91    100     int nNest;                          /* Number of nested brackets */
    92    101   };
    93    102   
    94    103   /*
    95    104   ** This function is equivalent to the standard isspace() function. 
    96    105   **
................................................................................
   168    177           memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
   169    178   
   170    179           if( iEnd<n && z[iEnd]=='*' ){
   171    180             pRet->pPhrase->aToken[0].isPrefix = 1;
   172    181             iEnd++;
   173    182           }
   174    183           if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
   175         -          pRet->pPhrase->isNot = 1;
          184  +          pParse->isNot = 1;
   176    185           }
   177    186         }
   178    187         nConsumed = iEnd;
   179    188       }
   180    189   
   181    190       pModule->xClose(pCursor);
   182    191     }
................................................................................
   220    229     sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
   221    230     int rc;
   222    231     Fts3Expr *p = 0;
   223    232     sqlite3_tokenizer_cursor *pCursor = 0;
   224    233     char *zTemp = 0;
   225    234     int nTemp = 0;
   226    235   
          236  +  const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
          237  +  int nToken = 0;
          238  +
          239  +  /* The final Fts3Expr data structure, including the Fts3Phrase,
          240  +  ** Fts3PhraseToken structures token buffers are all stored as a single 
          241  +  ** allocation so that the expression can be freed with a single call to
          242  +  ** sqlite3_free(). Setting this up requires a two pass approach.
          243  +  **
          244  +  ** The first pass, in the block below, uses a tokenizer cursor to iterate
          245  +  ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
          246  +  ** to assemble data in two dynamic buffers:
          247  +  **
          248  +  **   Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
          249  +  **             structure, followed by the array of Fts3PhraseToken 
          250  +  **             structures. This pass only populates the Fts3PhraseToken array.
          251  +  **
          252  +  **   Buffer zTemp: Contains copies of all tokens.
          253  +  **
          254  +  ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
          255  +  ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
          256  +  ** structures.
          257  +  */
   227    258     rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
   228    259     if( rc==SQLITE_OK ){
   229    260       int ii;
   230    261       pCursor->pTokenizer = pTokenizer;
   231    262       for(ii=0; rc==SQLITE_OK; ii++){
   232         -      const char *zToken;
   233         -      int nToken, iBegin, iEnd, iPos;
   234         -      rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
          263  +      const char *zByte;
          264  +      int nByte, iBegin, iEnd, iPos;
          265  +      rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
   235    266         if( rc==SQLITE_OK ){
   236         -        int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   237         -        p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
   238         -        zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
   239         -        if( !p || !zTemp ){
   240         -          goto no_mem;
   241         -        }
   242         -        if( ii==0 ){
   243         -          memset(p, 0, nByte);
   244         -          p->pPhrase = (Fts3Phrase *)&p[1];
   245         -        }
   246         -        p->pPhrase = (Fts3Phrase *)&p[1];
   247         -        memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
   248         -        p->pPhrase->nToken = ii+1;
   249         -        p->pPhrase->aToken[ii].n = nToken;
   250         -        memcpy(&zTemp[nTemp], zToken, nToken);
   251         -        nTemp += nToken;
   252         -        if( iEnd<nInput && zInput[iEnd]=='*' ){
   253         -          p->pPhrase->aToken[ii].isPrefix = 1;
   254         -        }else{
   255         -          p->pPhrase->aToken[ii].isPrefix = 0;
   256         -        }
          267  +        Fts3PhraseToken *pToken;
          268  +
          269  +        p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
          270  +        if( !p ) goto no_mem;
          271  +
          272  +        zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
          273  +        if( !zTemp ) goto no_mem;
          274  +
          275  +        assert( nToken==ii );
          276  +        pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
          277  +        memset(pToken, 0, sizeof(Fts3PhraseToken));
          278  +
          279  +        memcpy(&zTemp[nTemp], zByte, nByte);
          280  +        nTemp += nByte;
          281  +
          282  +        pToken->n = nByte;
          283  +        pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
          284  +        nToken = ii+1;
   257    285         }
   258    286       }
   259    287   
   260    288       pModule->xClose(pCursor);
   261    289       pCursor = 0;
   262    290     }
   263    291   
   264    292     if( rc==SQLITE_DONE ){
   265    293       int jj;
   266         -    char *zNew = NULL;
   267         -    int nNew = 0;
   268         -    int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   269         -    nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
   270         -    p = fts3ReallocOrFree(p, nByte + nTemp);
   271         -    if( !p ){
   272         -      goto no_mem;
   273         -    }
   274         -    if( zTemp ){
   275         -      zNew = &(((char *)p)[nByte]);
   276         -      memcpy(zNew, zTemp, nTemp);
   277         -    }else{
   278         -      memset(p, 0, nByte+nTemp);
   279         -    }
          294  +    char *zBuf = 0;
          295  +
          296  +    p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
          297  +    if( !p ) goto no_mem;
          298  +    memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
          299  +    p->eType = FTSQUERY_PHRASE;
   280    300       p->pPhrase = (Fts3Phrase *)&p[1];
          301  +    p->pPhrase->iColumn = pParse->iDefaultCol;
          302  +    p->pPhrase->nToken = nToken;
          303  +
          304  +    zBuf = (char *)&p->pPhrase->aToken[nToken];
          305  +    memcpy(zBuf, zTemp, nTemp);
          306  +    sqlite3_free(zTemp);
          307  +
   281    308       for(jj=0; jj<p->pPhrase->nToken; jj++){
   282         -      p->pPhrase->aToken[jj].z = &zNew[nNew];
   283         -      nNew += p->pPhrase->aToken[jj].n;
          309  +      p->pPhrase->aToken[jj].z = zBuf;
          310  +      zBuf += p->pPhrase->aToken[jj].n;
   284    311       }
   285         -    sqlite3_free(zTemp);
   286         -    p->eType = FTSQUERY_PHRASE;
   287         -    p->pPhrase->iColumn = pParse->iDefaultCol;
   288    312       rc = SQLITE_OK;
   289    313     }
   290    314   
   291    315     *ppExpr = p;
   292    316     return rc;
   293    317   no_mem:
   294    318   
................................................................................
   336    360     int iCol;
   337    361     int iColLen;
   338    362     int rc;
   339    363     Fts3Expr *pRet = 0;
   340    364   
   341    365     const char *zInput = z;
   342    366     int nInput = n;
          367  +
          368  +  pParse->isNot = 0;
   343    369   
   344    370     /* Skip over any whitespace before checking for a keyword, an open or
   345    371     ** close bracket, or a quoted string. 
   346    372     */
   347    373     while( nInput>0 && fts3isspace(*zInput) ){
   348    374       nInput--;
   349    375       zInput++;
................................................................................
   555    581       Fts3Expr *p = 0;
   556    582       int nByte = 0;
   557    583       rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
   558    584       if( rc==SQLITE_OK ){
   559    585         int isPhrase;
   560    586   
   561    587         if( !sqlite3_fts3_enable_parentheses 
   562         -       && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot 
          588  +       && p->eType==FTSQUERY_PHRASE && pParse->isNot 
   563    589         ){
   564    590           /* Create an implicit NOT operator. */
   565    591           Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
   566    592           if( !pNot ){
   567    593             sqlite3Fts3ExprFree(p);
   568    594             rc = SQLITE_NOMEM;
   569    595             goto exprparse_out;
................................................................................
   573    599           if( pNotBranch ){
   574    600             pNot->pLeft = pNotBranch;
   575    601           }
   576    602           pNotBranch = pNot;
   577    603           p = pPrev;
   578    604         }else{
   579    605           int eType = p->eType;
   580         -        assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
   581    606           isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
   582    607   
   583    608           /* The isRequirePhrase variable is set to true if a phrase or
   584    609           ** an expression contained in parenthesis is required. If a
   585    610           ** binary operator (AND, OR, NOT or NEAR) is encounted when
   586    611           ** isRequirePhrase is set, this is a syntax error.
   587    612           */
................................................................................
   736    761   }
   737    762   
   738    763   /*
   739    764   ** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
   740    765   */
   741    766   void sqlite3Fts3ExprFree(Fts3Expr *p){
   742    767     if( p ){
          768  +    assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
   743    769       sqlite3Fts3ExprFree(p->pLeft);
   744    770       sqlite3Fts3ExprFree(p->pRight);
   745         -    sqlite3_free(p->aDoclist);
          771  +    if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist);
   746    772       sqlite3_free(p);
   747    773     }
   748    774   }
   749    775   
   750    776   /****************************************************************************
   751    777   *****************************************************************************
   752    778   ** Everything after this point is just test code.
................................................................................
   796    822   */
   797    823   static char *exprToString(Fts3Expr *pExpr, char *zBuf){
   798    824     switch( pExpr->eType ){
   799    825       case FTSQUERY_PHRASE: {
   800    826         Fts3Phrase *pPhrase = pExpr->pPhrase;
   801    827         int i;
   802    828         zBuf = sqlite3_mprintf(
   803         -          "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
          829  +          "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
   804    830         for(i=0; zBuf && i<pPhrase->nToken; i++){
   805    831           zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, 
   806    832               pPhrase->aToken[i].n, pPhrase->aToken[i].z,
   807    833               (pPhrase->aToken[i].isPrefix?"+":"")
   808    834           );
   809    835         }
   810    836         return zBuf;

Changes to ext/fts3/fts3_snippet.c.

   224    224   /*
   225    225   ** This is an fts3ExprIterate() callback used while loading the doclists
   226    226   ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
   227    227   ** fts3ExprLoadDoclists().
   228    228   */
   229    229   static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   230    230     int rc = SQLITE_OK;
          231  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
   231    232     LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
   232    233   
   233    234     UNUSED_PARAMETER(iPhrase);
   234    235   
   235    236     p->nPhrase++;
   236         -  p->nToken += pExpr->pPhrase->nToken;
          237  +  p->nToken += pPhrase->nToken;
   237    238   
   238         -  if( pExpr->isLoaded==0 ){
          239  +  if( pPhrase->isLoaded==0 ){
   239    240       rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
   240         -    pExpr->isLoaded = 1;
          241  +    pPhrase->isLoaded = 1;
   241    242       if( rc==SQLITE_OK ){
   242    243         rc = fts3ExprNearTrim(pExpr);
   243    244       }
   244    245     }
   245    246   
   246    247     return rc;
   247    248   }
................................................................................
   822    823   static int fts3ExprGlobalHitsCb(
   823    824     Fts3Expr *pExpr,                /* Phrase expression node */
   824    825     int iPhrase,                    /* Phrase number (numbered from zero) */
   825    826     void *pCtx                      /* Pointer to MatchInfo structure */
   826    827   ){
   827    828     MatchInfo *p = (MatchInfo *)pCtx;
   828    829     Fts3Cursor *pCsr = p->pCursor;
          830  +  Fts3Phrase *pPhrase = pExpr->pPhrase; 
   829    831     char *pIter;
   830    832     char *pEnd;
   831    833     char *pFree = 0;
   832    834     u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
   833    835   
   834         -  assert( pExpr->isLoaded );
   835         -  assert( pExpr->eType==FTSQUERY_PHRASE );
          836  +  assert( pPhrase->isLoaded );
   836    837   
   837    838     if( pCsr->pDeferred ){
   838         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
   839    839       int ii;
   840    840       for(ii=0; ii<pPhrase->nToken; ii++){
   841    841         if( pPhrase->aToken[ii].bFulltext ) break;
   842    842       }
   843    843       if( ii<pPhrase->nToken ){
   844    844         int nFree = 0;
   845    845         int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
................................................................................
   851    851         for(iCol=0; iCol<p->nCol; iCol++){
   852    852           aOut[iCol*3 + 1] = (u32)p->nDoc;
   853    853           aOut[iCol*3 + 2] = (u32)p->nDoc;
   854    854         }
   855    855         return SQLITE_OK;
   856    856       }
   857    857     }else{
   858         -    pIter = pExpr->aDoclist;
   859         -    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
          858  +    pIter = pPhrase->aDoclist;
          859  +    pEnd = &pPhrase->aDoclist[pPhrase->nDoclist];
   860    860     }
   861    861   
   862    862     /* Fill in the global hit count matrix row for this phrase. */
   863    863     while( pIter<pEnd ){
   864    864       while( *pIter++ & 0x80 );      /* Skip past docid. */
   865    865       fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
   866    866     }
................................................................................
   881    881   ){
   882    882     MatchInfo *p = (MatchInfo *)pCtx;
   883    883     int iStart = iPhrase * p->nCol * 3;
   884    884     int i;
   885    885   
   886    886     for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
   887    887   
   888         -  if( pExpr->aDoclist ){
          888  +  if( pExpr->pPhrase->aDoclist ){
   889    889       char *pCsr;
   890    890   
   891    891       pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
   892    892       if( pCsr ){
   893    893         fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
   894    894       }
   895    895     }

Changes to ext/fts3/fts3_write.c.

  2638   2638   /*
  2639   2639   ** Helper fucntion for FreeDeferredDoclists(). This function removes all
  2640   2640   ** references to deferred doclists from within the tree of Fts3Expr 
  2641   2641   ** structures headed by 
  2642   2642   */
  2643   2643   static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
  2644   2644     if( pExpr ){
         2645  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
  2645   2646       fts3DeferredDoclistClear(pExpr->pLeft);
  2646   2647       fts3DeferredDoclistClear(pExpr->pRight);
  2647         -    if( pExpr->isLoaded ){
  2648         -      sqlite3_free(pExpr->aDoclist);
  2649         -      pExpr->isLoaded = 0;
  2650         -      pExpr->aDoclist = 0;
  2651         -      pExpr->nDoclist = 0;
  2652         -      pExpr->pCurrent = 0;
  2653         -      pExpr->iCurrent = 0;
         2648  +    if( pPhrase ){
         2649  +      assert( pExpr->eType==FTSQUERY_PHRASE );
         2650  +      sqlite3_free(pPhrase->aDoclist);
         2651  +      pPhrase->isLoaded = 0;
         2652  +      pPhrase->aDoclist = 0;
         2653  +      pPhrase->nDoclist = 0;
         2654  +      pPhrase->pCurrent = 0;
         2655  +      pPhrase->iCurrent = 0;
  2654   2656       }
  2655   2657     }
  2656   2658   }
  2657   2659   
  2658   2660   /*
  2659   2661   ** Delete all cached deferred doclists. Deferred doclists are cached
  2660   2662   ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.

Changes to test/hook.test.

   270    270         SELECT * FROM t1 EXCEPT SELECT * FROM t3;
   271    271         SELECT * FROM t1 ORDER BY b;
   272    272         SELECT * FROM t1 GROUP BY b;
   273    273       }
   274    274       set ::update_hook
   275    275     } [list]
   276    276   }
          277  +
          278  +do_test hook-4.4 {
          279  +  execsql {
          280  +    CREATE TABLE t4(a UNIQUE, b);
          281  +    INSERT INTO t4 VALUES(1, 'a');
          282  +    INSERT INTO t4 VALUES(2, 'b');
          283  +  }
          284  +  set ::update_hook [list]
          285  +  execsql {
          286  +    REPLACE INTO t4 VALUES(1, 'c');
          287  +  }
          288  +  set ::update_hook
          289  +} [list INSERT main t4 3 ]
          290  +do_execsql_test hook-4.4.1 {
          291  +  SELECT * FROM t4 ORDER BY a;
          292  +} {1 c 2 b}
          293  +do_test hook-4.4.2 {
          294  +  set ::update_hook [list]
          295  +  execsql {
          296  +    PRAGMA recursive_triggers = on;
          297  +    REPLACE INTO t4 VALUES(1, 'd');
          298  +  }
          299  +  set ::update_hook
          300  +} [list INSERT main t4 4 ]
          301  +do_execsql_test hook-4.4.3 {
          302  +  SELECT * FROM t4 ORDER BY a;
          303  +} {1 d 2 b}
          304  +
   277    305   db update_hook {}
   278    306   #
   279    307   #----------------------------------------------------------------------------
   280    308   
   281    309   #----------------------------------------------------------------------------
   282    310   # Test the rollback-hook. The rollback-hook is a bit more complicated than
   283    311   # either the commit or update hooks because a rollback can happen