/ Check-in [d464a7b1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fixes to fts5 snippet() function.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:d464a7b18d212720a4f4e2fa4e204f1aca99c837
User & Date: dan 2016-08-27 18:34:06
Context
2016-08-27
18:35
Fix some comments in sqlite3session.h. No changes to code. check-in: 78cd64e2 user: dan tags: trunk
18:34
Fixes to fts5 snippet() function. check-in: d464a7b1 user: dan tags: trunk
14:05
Fix the extra comments (added with -DSQLITE_ENABLE_EXPLAIN_COMMENTS) so that the sense of <, <=, >, and >= tests is correct and so that the SQLITE_STOREP2 version is shown correctly. These changes are already in the rowvalue branch but are added here since they are technically unrelated to rowvalue. check-in: 4d43c469 user: drh tags: trunk
2016-08-24
18:50
Fix a bug in the fts5 snippet function causing it to return text containing zero phrase instances. Closed-Leaf check-in: b174ed2b user: dan tags: fts5-snippet-bias
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_aux.c.

   241    241     if( rc!=SQLITE_OK ){
   242    242       sqlite3_result_error_code(pCtx, rc);
   243    243     }
   244    244   }
   245    245   /*
   246    246   ** End of highlight() implementation.
   247    247   **************************************************************************/
          248  +
          249  +/*
          250  +** Context object passed to the fts5SentenceFinderCb() function.
          251  +*/
          252  +typedef struct Fts5SFinder Fts5SFinder;
          253  +struct Fts5SFinder {
          254  +  int iPos;                       /* Current token position */
          255  +  int nFirstAlloc;                /* Allocated size of aFirst[] */
          256  +  int nFirst;                     /* Number of entries in aFirst[] */
          257  +  int *aFirst;                    /* Array of first token in each sentence */
          258  +  const char *zDoc;               /* Document being tokenized */
          259  +};
          260  +
          261  +/*
          262  +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
          263  +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
          264  +** error occurs.
          265  +*/
          266  +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
          267  +  if( p->nFirstAlloc==p->nFirst ){
          268  +    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
          269  +    int *aNew;
          270  +
          271  +    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
          272  +    if( aNew==0 ) return SQLITE_NOMEM;
          273  +    p->aFirst = aNew;
          274  +    p->nFirstAlloc = nNew;
          275  +  }
          276  +  p->aFirst[p->nFirst++] = iAdd;
          277  +  return SQLITE_OK;
          278  +}
          279  +
          280  +/*
          281  +** This function is an xTokenize() callback used by the auxiliary snippet()
          282  +** function. Its job is to identify tokens that are the first in a sentence.
          283  +** For each such token, an entry is added to the SFinder.aFirst[] array.
          284  +*/
          285  +static int fts5SentenceFinderCb(
          286  +  void *pContext,                 /* Pointer to HighlightContext object */
          287  +  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
          288  +  const char *pToken,             /* Buffer containing token */
          289  +  int nToken,                     /* Size of token in bytes */
          290  +  int iStartOff,                  /* Start offset of token */
          291  +  int iEndOff                     /* End offset of token */
          292  +){
          293  +  int rc = SQLITE_OK;
          294  +
          295  +  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
          296  +    Fts5SFinder *p = (Fts5SFinder*)pContext;
          297  +    if( p->iPos>0 ){
          298  +      int i;
          299  +      char c = 0;
          300  +      for(i=iStartOff-1; i>=0; i--){
          301  +        c = p->zDoc[i];
          302  +        if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
          303  +      }
          304  +      if( i!=iStartOff-1 && (c=='.' || c==':') ){
          305  +        rc = fts5SentenceFinderAdd(p, p->iPos);
          306  +      }
          307  +    }else{
          308  +      rc = fts5SentenceFinderAdd(p, 0);
          309  +    }
          310  +    p->iPos++;
          311  +  }
          312  +  return rc;
          313  +}
          314  +
          315  +static int fts5SnippetScore(
          316  +  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
          317  +  Fts5Context *pFts,              /* First arg to pass to pApi functions */
          318  +  int nDocsize,                   /* Size of column in tokens */
          319  +  unsigned char *aSeen,           /* Array with one element per query phrase */
          320  +  int iCol,                       /* Column to score */
          321  +  int iPos,                       /* Starting offset to score */
          322  +  int nToken,                     /* Max tokens per snippet */
          323  +  int *pnScore,                   /* OUT: Score */
          324  +  int *piPos                      /* OUT: Adjusted offset */
          325  +){
          326  +  int rc;
          327  +  int i;
          328  +  int ip = 0;
          329  +  int ic = 0;
          330  +  int iOff = 0;
          331  +  int iFirst = -1;
          332  +  int nInst;
          333  +  int nScore = 0;
          334  +  int iLast = 0;
          335  +
          336  +  rc = pApi->xInstCount(pFts, &nInst);
          337  +  for(i=0; i<nInst && rc==SQLITE_OK; i++){
          338  +    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
          339  +    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
          340  +      nScore += (aSeen[ip] ? 1 : 1000);
          341  +      aSeen[ip] = 1;
          342  +      if( iFirst<0 ) iFirst = iOff;
          343  +      iLast = iOff + pApi->xPhraseSize(pFts, ip);
          344  +    }
          345  +  }
          346  +
          347  +  *pnScore = nScore;
          348  +  if( piPos ){
          349  +    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
          350  +    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
          351  +    if( iAdj<0 ) iAdj = 0;
          352  +    *piPos = iAdj;
          353  +  }
          354  +
          355  +  return rc;
          356  +}
   248    357   
   249    358   /*
   250    359   ** Implementation of snippet() function.
   251    360   */
   252    361   static void fts5SnippetFunction(
   253    362     const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   254    363     Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................
   263    372     int nToken;                     /* 5th argument to snippet() */
   264    373     int nInst = 0;                  /* Number of instance matches this row */
   265    374     int i;                          /* Used to iterate through instances */
   266    375     int nPhrase;                    /* Number of phrases in query */
   267    376     unsigned char *aSeen;           /* Array of "seen instance" flags */
   268    377     int iBestCol;                   /* Column containing best snippet */
   269    378     int iBestStart = 0;             /* First token of best snippet */
   270         -  int iBestLast;                  /* Last token of best snippet */
   271    379     int nBestScore = 0;             /* Score of best snippet */
   272    380     int nColSize = 0;               /* Total size of iBestCol in tokens */
          381  +  Fts5SFinder sFinder;            /* Used to find the beginnings of sentences */
          382  +  int nCol;
   273    383   
   274    384     if( nVal!=5 ){
   275    385       const char *zErr = "wrong number of arguments to function snippet()";
   276    386       sqlite3_result_error(pCtx, zErr, -1);
   277    387       return;
   278    388     }
   279    389   
          390  +  nCol = pApi->xColumnCount(pFts);
   280    391     memset(&ctx, 0, sizeof(HighlightContext));
   281    392     iCol = sqlite3_value_int(apVal[0]);
   282    393     ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   283    394     ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   284    395     zEllips = (const char*)sqlite3_value_text(apVal[3]);
   285    396     nToken = sqlite3_value_int(apVal[4]);
   286         -  iBestLast = nToken-1;
   287    397   
   288    398     iBestCol = (iCol>=0 ? iCol : 0);
   289    399     nPhrase = pApi->xPhraseCount(pFts);
   290    400     aSeen = sqlite3_malloc(nPhrase);
   291    401     if( aSeen==0 ){
   292    402       rc = SQLITE_NOMEM;
   293    403     }
   294         -
   295    404     if( rc==SQLITE_OK ){
   296    405       rc = pApi->xInstCount(pFts, &nInst);
   297    406     }
   298         -  for(i=0; rc==SQLITE_OK && i<nInst; i++){
   299         -    int ip, iSnippetCol, iStart;
   300         -    memset(aSeen, 0, nPhrase);
   301         -    rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
   302         -    if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
   303         -      int nScore = 1000;
   304         -      int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
   305         -      int j;
   306         -      aSeen[ip] = 1;
   307         -
   308         -      for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
   309         -        int ic; int io; int iFinal;
   310         -        rc = pApi->xInst(pFts, j, &ip, &ic, &io);
   311         -        iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
   312         -        if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
   313         -          nScore += aSeen[ip] ? 1000 : 1;
   314         -          aSeen[ip] = 1;
   315         -          if( iFinal>iLast ) iLast = iFinal;
          407  +
          408  +  memset(&sFinder, 0, sizeof(Fts5SFinder));
          409  +  for(i=0; i<nCol; i++){
          410  +    if( iCol<0 || iCol==i ){
          411  +      int nDoc;
          412  +      int nDocsize;
          413  +      int ii;
          414  +      sFinder.iPos = 0;
          415  +      sFinder.nFirst = 0;
          416  +      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
          417  +      if( rc!=SQLITE_OK ) break;
          418  +      rc = pApi->xTokenize(pFts, 
          419  +          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
          420  +      );
          421  +      if( rc!=SQLITE_OK ) break;
          422  +      rc = pApi->xColumnSize(pFts, i, &nDocsize);
          423  +      if( rc!=SQLITE_OK ) break;
          424  +
          425  +      for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
          426  +        int ip, ic, io;
          427  +        int iAdj;
          428  +        int nScore;
          429  +        int jj;
          430  +
          431  +        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
          432  +        if( ic!=i || rc!=SQLITE_OK ) continue;
          433  +        memset(aSeen, 0, nPhrase);
          434  +        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
          435  +            io, nToken, &nScore, &iAdj
          436  +        );
          437  +        if( rc==SQLITE_OK && nScore>nBestScore ){
          438  +          nBestScore = nScore;
          439  +          iBestCol = i;
          440  +          iBestStart = iAdj;
          441  +          nColSize = nDocsize;
   316    442           }
   317         -      }
   318    443   
   319         -      if( rc==SQLITE_OK && nScore>nBestScore ){
   320         -        iBestCol = iSnippetCol;
   321         -        iBestStart = iStart;
   322         -        iBestLast = iLast;
   323         -        nBestScore = nScore;
          444  +        if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
          445  +          for(jj=0; jj<(sFinder.nFirst-1); jj++){
          446  +            if( sFinder.aFirst[jj+1]>io ) break;
          447  +          }
          448  +
          449  +          if( sFinder.aFirst[jj]<io ){
          450  +            int nScore;
          451  +            memset(aSeen, 0, nPhrase);
          452  +            rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
          453  +              sFinder.aFirst[jj], nToken, &nScore, 0
          454  +            );
          455  +
          456  +            nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
          457  +            if( rc==SQLITE_OK && nScore>nBestScore ){
          458  +              nBestScore = nScore;
          459  +              iBestCol = i;
          460  +              iBestStart = sFinder.aFirst[jj];
          461  +              nColSize = nDocsize;
          462  +            }
          463  +          }
          464  +        }
   324    465         }
   325    466       }
   326    467     }
   327    468   
   328    469     if( rc==SQLITE_OK ){
   329         -    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
          470  +    rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
   330    471     }
   331         -  if( rc==SQLITE_OK ){
   332         -    rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
          472  +  if( rc==SQLITE_OK && nColSize==0 ){
          473  +    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
   333    474     }
   334    475     if( ctx.zIn ){
   335    476       if( rc==SQLITE_OK ){
   336    477         rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
   337    478       }
   338    479   
   339         -    if( (iBestStart+nToken-1)>iBestLast ){
   340         -      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
   341         -    }
   342         -    if( iBestStart+nToken>nColSize ){
   343         -      iBestStart = nColSize - nToken;
   344         -    }
   345         -    if( iBestStart<0 ) iBestStart = 0;
   346         -
   347    480       ctx.iRangeStart = iBestStart;
   348    481       ctx.iRangeEnd = iBestStart + nToken - 1;
   349    482   
   350    483       if( iBestStart>0 ){
   351    484         fts5HighlightAppend(&rc, &ctx, zEllips, -1);
   352    485       }
   353    486   
................................................................................
   361    494         rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
   362    495       }
   363    496       if( ctx.iRangeEnd>=(nColSize-1) ){
   364    497         fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
   365    498       }else{
   366    499         fts5HighlightAppend(&rc, &ctx, zEllips, -1);
   367    500       }
   368         -
   369         -    if( rc==SQLITE_OK ){
   370         -      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
   371         -    }else{
   372         -      sqlite3_result_error_code(pCtx, rc);
   373         -    }
   374         -    sqlite3_free(ctx.zOut);
          501  +  }
          502  +  if( rc==SQLITE_OK ){
          503  +    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
          504  +  }else{
          505  +    sqlite3_result_error_code(pCtx, rc);
   375    506     }
          507  +  sqlite3_free(ctx.zOut);
   376    508     sqlite3_free(aSeen);
          509  +  sqlite3_free(sFinder.aFirst);
   377    510   }
   378    511   
   379    512   /************************************************************************/
   380    513   
   381    514   /*
   382    515   ** The first time the bm25() function is called for a query, an instance
   383    516   ** of the following structure is allocated and populated.

Changes to ext/fts5/test/fts5af.test.

    68     68     1.6 {o o o o o X o} {o o o o o [X] o}
    69     69     1.7 {o o o o o o X} {o o o o o o [X]}
    70     70   
    71     71     2.1 {X o o o o o o o} {[X] o o o o o o...}
    72     72     2.2 {o X o o o o o o} {o [X] o o o o o...}
    73     73     2.3 {o o X o o o o o} {o o [X] o o o o...}
    74     74     2.4 {o o o X o o o o} {o o o [X] o o o...}
    75         -  2.5 {o o o o X o o o} {...o o o [X] o o o}
    76         -  2.6 {o o o o o X o o} {...o o o o [X] o o}
    77         -  2.7 {o o o o o o X o} {...o o o o o [X] o}
           75  +  2.5 {o o o o X o o o} {o o o o [X] o o...}
           76  +  2.6 {o o o o o X o o} {o o o o o [X] o...}
           77  +  2.7 {o o o o o o X o} {o o o o o o [X]...}
    78     78     2.8 {o o o o o o o X} {...o o o o o o [X]}
           79  +
           80  +  2.9  {o o o o o o o X o}       {...o o o o o [X] o}
           81  +  2.10 {o o o o o o o X o o}     {...o o o o [X] o o}
           82  +  2.11 {o o o o o o o X o o o}   {...o o o [X] o o o}
           83  +  2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
           84  +
    79     85   
    80     86     3.1 {X o o o o o o o o} {[X] o o o o o o...}
    81     87     3.2 {o X o o o o o o o} {o [X] o o o o o...}
    82     88     3.3 {o o X o o o o o o} {o o [X] o o o o...}
    83     89     3.4 {o o o X o o o o o} {o o o [X] o o o...}
    84         -  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
    85         -  3.6 {o o o o o X o o o} {...o o o [X] o o o}
    86         -  3.7 {o o o o o o X o o} {...o o o o [X] o o}
    87         -  3.8 {o o o o o o o X o} {...o o o o o [X] o}
    88         -  3.9 {o o o o o o o o X} {...o o o o o o [X]}
           90  +
           91  +  3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
           92  +  3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
           93  +  3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
           94  +  3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
           95  +  3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}
    89     96   
    90     97     4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
    91         -  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
    92         -  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
           98  +  4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
           99  +  4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}
    93    100   
    94    101     5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
    95         -  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
    96         -  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
    97         -  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
          102  +  5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
          103  +  5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
          104  +  5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}
    98    105   
    99    106     6.1 {X o o o X o o o} {[X] o o o [X] o o...}
   100    107     6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
   101         -  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
   102         -  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
   103         -  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
          108  +  6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
          109  +  6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
          110  +  6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}
   104    111   
   105    112     7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
   106    113     7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
   107         -  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
   108         -  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
   109         -  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
   110         -  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
          114  +  7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
          115  +  7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
          116  +  7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
          117  +  7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
   111    118   
   112         -  8.1 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
          119  +  8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
   113    120         {...o o [X] [X] [X] o o...}
          121  +  8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 
          122  +      {...o o [X] o o o o...}
          123  +  8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
          124  +      {o o o o [X] o o...}
   114    125   } {
   115    126     do_snippet_test 1.$tn $doc X $res
   116    127   }
   117    128   
   118    129   if {[detail_is_full]} {
   119    130     foreach {tn doc res} {
   120    131       1.1 {X Y o o o o o} {[X Y] o o o o o}
................................................................................
   123    134       1.4 {o o o X Y o o} {o o o [X Y] o o}
   124    135       1.5 {o o o o X Y o} {o o o o [X Y] o}
   125    136       1.6 {o o o o o X Y} {o o o o o [X Y]}
   126    137   
   127    138       2.1 {X Y o o o o o o} {[X Y] o o o o o...}
   128    139       2.2 {o X Y o o o o o} {o [X Y] o o o o...}
   129    140       2.3 {o o X Y o o o o} {o o [X Y] o o o...}
   130         -    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
   131         -    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
   132         -    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
   133         -    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
          141  +    2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
          142  +    2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
          143  +    2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
          144  +    2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}
   134    145   
   135    146       3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
   136    147       3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
   137    148       3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
   138         -    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
   139         -    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
   140         -    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
   141         -    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
   142         -    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
          149  +    3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
          150  +    3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
          151  +    3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
          152  +    3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
          153  +    3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
   143    154     } {
   144    155       do_snippet_test 2.$tn $doc "X + Y" $res
   145    156     }
   146    157   }
   147    158   
          159  +do_execsql_test 4.0 {
          160  +  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
          161  +  INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
          162  +  SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
          163  +} {
          164  +  {1 2 3 4 5...}
          165  +}
          166  +
          167  +do_execsql_test 5.0 {
          168  +  CREATE VIRTUAL TABLE p1 USING fts5(a, b);
          169  +  INSERT INTO p1 VALUES(
          170  +    'x a a a a a a a a a a',
          171  +    'a a a a a a a a a a a a a a a a a a a x'
          172  +  );
          173  +}
          174  +do_execsql_test 5.1 {
          175  +  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
          176  +} {{[x] a a a a a...}}
          177  +
   148    178   } ;# foreach_detail_mode 
   149    179   
   150    180   finish_test
   151    181   

Changes to ext/fts5/test/fts5unicode2.test.

   156    156        the maximum x value.
   157    157     }
   158    158     3 "ROW" {
   159    159        ...returns the value of y on the same [row] that contains 
   160    160        the maximum x value.
   161    161     }
   162    162     4 "rollback" {
   163         -     ...[ROLLBACK]. Instead, the pending statement
   164         -     will return SQLITE_ABORT upon next access after the [ROLLBACK].
          163  +     Pending statements no longer block [ROLLBACK]. Instead, the pending
          164  +     statement will return SQLITE_ABORT upon...
   165    165     }
   166    166     5 "rOllback" {
   167         -     ...[ROLLBACK]. Instead, the pending statement
   168         -     will return SQLITE_ABORT upon next access after the [ROLLBACK].
          167  +     Pending statements no longer block [ROLLBACK]. Instead, the pending
          168  +     statement will return SQLITE_ABORT upon...
   169    169     }
   170    170     6 "lang*" {
   171    171        Added support for the FTS4 [languageid] option.
   172    172     }
   173    173   } {
   174    174     do_test 2.$tn {
   175    175       set q [mapdoc $query]

Changes to src/main.c.

  2951   2951   
  2952   2952     /* Register all built-in functions, but do not attempt to read the
  2953   2953     ** database schema yet. This is delayed until the first time the database
  2954   2954     ** is accessed.
  2955   2955     */
  2956   2956     sqlite3Error(db, SQLITE_OK);
  2957   2957     sqlite3RegisterPerConnectionBuiltinFunctions(db);
         2958  +  rc = sqlite3_errcode(db);
         2959  +
         2960  +#ifdef SQLITE_ENABLE_FTS5
         2961  +  /* Register any built-in FTS5 module before loading the automatic
         2962  +  ** extensions. This allows automatic extensions to register FTS5 
         2963  +  ** tokenizers and auxiliary functions.  */
         2964  +  if( !db->mallocFailed && rc==SQLITE_OK ){
         2965  +    rc = sqlite3Fts5Init(db);
         2966  +  }
         2967  +#endif
  2958   2968   
  2959   2969     /* Load automatic extensions - extensions that have been registered
  2960   2970     ** using the sqlite3_automatic_extension() API.
  2961   2971     */
  2962         -  rc = sqlite3_errcode(db);
  2963   2972     if( rc==SQLITE_OK ){
  2964   2973       sqlite3AutoLoadExtensions(db);
  2965   2974       rc = sqlite3_errcode(db);
  2966   2975       if( rc!=SQLITE_OK ){
  2967   2976         goto opendb_out;
  2968   2977       }
  2969   2978     }
................................................................................
  2982   2991     }
  2983   2992   #endif
  2984   2993   
  2985   2994   #ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */
  2986   2995     if( !db->mallocFailed && rc==SQLITE_OK ){
  2987   2996       rc = sqlite3Fts3Init(db);
  2988   2997     }
  2989         -#endif
  2990         -
  2991         -#ifdef SQLITE_ENABLE_FTS5
  2992         -  if( !db->mallocFailed && rc==SQLITE_OK ){
  2993         -    rc = sqlite3Fts5Init(db);
  2994         -  }
  2995   2998   #endif
  2996   2999   
  2997   3000   #ifdef SQLITE_ENABLE_ICU
  2998   3001     if( !db->mallocFailed && rc==SQLITE_OK ){
  2999   3002       rc = sqlite3IcuInit(db);
  3000   3003     }
  3001   3004   #endif