/ Check-in [988164cf]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge experimental fts3/fts4 changes with trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 988164cf485300fb3d189fd1453c23c48e737e24
User & Date: dan 2010-10-27 18:10:00
Context
2010-10-27
19:08
Avoid trying to allocate a negative number of bytes of memory in the test wrapper for sqlite3_blob_read(). check-in: 739b5d9a user: dan tags: trunk
18:10
Merge experimental fts3/fts4 changes with trunk. check-in: 988164cf user: dan tags: trunk
16:52
Fix a buffer overread in fts3 that can occur if the database is corrupt. Closed-Leaf check-in: 84194c41 user: dan tags: experimental
15:36
Fix a memory leak in the update_hook method of the TCL interface. check-in: 1d17e3dc user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

   437    437   ** The xDisconnect() virtual table method.
   438    438   */
   439    439   static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
   440    440     Fts3Table *p = (Fts3Table *)pVtab;
   441    441     int i;
   442    442   
   443    443     assert( p->nPendingData==0 );
          444  +  assert( p->pSegments==0 );
   444    445   
   445    446     /* Free any prepared statements held */
   446    447     for(i=0; i<SizeofArray(p->aStmt); i++){
   447    448       sqlite3_finalize(p->aStmt[i]);
   448    449     }
   449         -  for(i=0; i<p->nLeavesStmt; i++){
   450         -    sqlite3_finalize(p->aLeavesStmt[i]);
   451         -  }
   452         -  sqlite3_free(p->zSelectLeaves);
   453         -  sqlite3_free(p->aLeavesStmt);
          450  +  sqlite3_free(p->zSegmentsTbl);
   454    451   
   455    452     /* Invoke the tokenizer destructor to free the tokenizer. */
   456    453     p->pTokenizer->pModule->xDestroy(p->pTokenizer);
   457    454   
   458    455     sqlite3_free(p);
   459    456     return SQLITE_OK;
   460    457   }
   461    458   
   462    459   /*
   463    460   ** Construct one or more SQL statements from the format string given
   464         -** and then evaluate those statements.  The success code is writting
          461  +** and then evaluate those statements. The success code is written
   465    462   ** into *pRc.
   466    463   **
   467    464   ** If *pRc is initially non-zero then this routine is a no-op.
   468    465   */
   469    466   static void fts3DbExec(
   470    467     int *pRc,              /* Success code */
   471    468     sqlite3 *db,           /* Database in which to run SQL */
................................................................................
   509    506   }
   510    507   
   511    508   
   512    509   /*
   513    510   ** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
   514    511   ** passed as the first argument. This is done as part of the xConnect()
   515    512   ** and xCreate() methods.
   516         -*/
   517         -static int fts3DeclareVtab(Fts3Table *p){
   518         -  int i;                          /* Iterator variable */
   519         -  int rc;                         /* Return code */
   520         -  char *zSql;                     /* SQL statement passed to declare_vtab() */
   521         -  char *zCols;                    /* List of user defined columns */
   522         -
   523         -  /* Create a list of user columns for the virtual table */
   524         -  zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
   525         -  for(i=1; zCols && i<p->nColumn; i++){
   526         -    zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
   527         -  }
   528         -
   529         -  /* Create the whole "CREATE TABLE" statement to pass to SQLite */
   530         -  zSql = sqlite3_mprintf(
   531         -      "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
   532         -  );
   533         -
   534         -  if( !zCols || !zSql ){
   535         -    rc = SQLITE_NOMEM;
   536         -  }else{
   537         -    rc = sqlite3_declare_vtab(p->db, zSql);
   538         -  }
   539         -
   540         -  sqlite3_free(zSql);
   541         -  sqlite3_free(zCols);
   542         -  return rc;
          513  +**
          514  +** If *pRc is non-zero when this function is called, it is a no-op. 
          515  +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
          516  +** before returning.
          517  +*/
          518  +static void fts3DeclareVtab(int *pRc, Fts3Table *p){
          519  +  if( *pRc==SQLITE_OK ){
          520  +    int i;                        /* Iterator variable */
          521  +    int rc;                       /* Return code */
          522  +    char *zSql;                   /* SQL statement passed to declare_vtab() */
          523  +    char *zCols;                  /* List of user defined columns */
          524  +
          525  +    /* Create a list of user columns for the virtual table */
          526  +    zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
          527  +    for(i=1; zCols && i<p->nColumn; i++){
          528  +      zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
          529  +    }
          530  +
          531  +    /* Create the whole "CREATE TABLE" statement to pass to SQLite */
          532  +    zSql = sqlite3_mprintf(
          533  +        "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
          534  +    );
          535  +    if( !zCols || !zSql ){
          536  +      rc = SQLITE_NOMEM;
          537  +    }else{
          538  +      rc = sqlite3_declare_vtab(p->db, zSql);
          539  +    }
          540  +
          541  +    sqlite3_free(zSql);
          542  +    sqlite3_free(zCols);
          543  +    *pRc = rc;
          544  +  }
   543    545   }
   544    546   
   545    547   /*
   546    548   ** Create the backing store tables (%_content, %_segments and %_segdir)
   547    549   ** required by the FTS3 table passed as the only argument. This is done
   548    550   ** as part of the vtab xCreate() method.
   549    551   **
................................................................................
   554    556   static int fts3CreateTables(Fts3Table *p){
   555    557     int rc = SQLITE_OK;             /* Return code */
   556    558     int i;                          /* Iterator variable */
   557    559     char *zContentCols;             /* Columns of %_content table */
   558    560     sqlite3 *db = p->db;            /* The database connection */
   559    561   
   560    562     /* Create a list of user columns for the content table */
   561         -  if( p->bHasContent ){
   562         -    zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
   563         -    for(i=0; zContentCols && i<p->nColumn; i++){
   564         -      char *z = p->azColumn[i];
   565         -      zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
   566         -    }
   567         -    if( zContentCols==0 ) rc = SQLITE_NOMEM;
          563  +  zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
          564  +  for(i=0; zContentCols && i<p->nColumn; i++){
          565  +    char *z = p->azColumn[i];
          566  +    zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
          567  +  }
          568  +  if( zContentCols==0 ) rc = SQLITE_NOMEM;
   568    569   
   569         -    /* Create the content table */
   570         -    fts3DbExec(&rc, db, 
   571         -       "CREATE TABLE %Q.'%q_content'(%s)",
   572         -       p->zDb, p->zName, zContentCols
   573         -    );
   574         -    sqlite3_free(zContentCols);
   575         -  }
          570  +  /* Create the content table */
          571  +  fts3DbExec(&rc, db, 
          572  +     "CREATE TABLE %Q.'%q_content'(%s)",
          573  +     p->zDb, p->zName, zContentCols
          574  +  );
          575  +  sqlite3_free(zContentCols);
   576    576     /* Create other tables */
   577    577     fts3DbExec(&rc, db, 
   578    578         "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
   579    579         p->zDb, p->zName
   580    580     );
   581    581     fts3DbExec(&rc, db, 
   582    582         "CREATE TABLE %Q.'%q_segdir'("
................................................................................
   634    634       zDb, zName, zSuffix
   635    635     );    
   636    636     rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0);
   637    637     sqlite3_free(zSql);
   638    638     *pResult = (u8)(res & 0xff);
   639    639     if( rc!=SQLITE_ABORT ) *pRc = rc;
   640    640   }
          641  +
          642  +/*
          643  +** Store the current database page-size in bytes in p->nPgsz.
          644  +**
          645  +** If *pRc is non-zero when this function is called, it is a no-op. 
          646  +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc
          647  +** before returning.
          648  +*/
          649  +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){
          650  +  if( *pRc==SQLITE_OK ){
          651  +    int rc;                       /* Return code */
          652  +    char *zSql;                   /* SQL text "PRAGMA %Q.page_size" */
          653  +    sqlite3_stmt *pStmt;          /* Compiled "PRAGMA %Q.page_size" statement */
          654  +  
          655  +    zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb);
          656  +    if( !zSql ){
          657  +      rc = SQLITE_NOMEM;
          658  +    }else{
          659  +      rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
          660  +      if( rc==SQLITE_OK ){
          661  +        sqlite3_step(pStmt);
          662  +        p->nPgsz = sqlite3_column_int(pStmt, 0);
          663  +        rc = sqlite3_finalize(pStmt);
          664  +      }
          665  +    }
          666  +    assert( p->nPgsz>0 || rc!=SQLITE_OK );
          667  +    sqlite3_free(zSql);
          668  +    *pRc = rc;
          669  +  }
          670  +}
   641    671   
   642    672   /*
   643    673   ** This function is the implementation of both the xConnect and xCreate
   644    674   ** methods of the FTS3 virtual table.
   645    675   **
   646    676   ** The argv[] array contains the following:
   647    677   **
................................................................................
   759    789       p->bHasDocsize = argv[0][3]=='4';
   760    790       rc = fts3CreateTables(p);
   761    791     }else{
   762    792       rc = SQLITE_OK;
   763    793       fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent);
   764    794       fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize);
   765    795     }
   766         -  if( rc!=SQLITE_OK ) goto fts3_init_out;
   767    796   
   768         -  rc = fts3DeclareVtab(p);
   769         -  if( rc!=SQLITE_OK ) goto fts3_init_out;
          797  +  /* Figure out the page-size for the database. This is required in order to
          798  +  ** estimate the cost of loading large doclists from the database (see 
          799  +  ** function sqlite3Fts3SegReaderCost() for details).
          800  +  */
          801  +  fts3DatabasePageSize(&rc, p);
   770    802   
   771         -  *ppVTab = &p->base;
          803  +  /* Declare the table schema to SQLite. */
          804  +  fts3DeclareVtab(&rc, p);
   772    805   
   773    806   fts3_init_out:
   774    807     assert( p || (pTokenizer && rc!=SQLITE_OK) );
   775    808     if( rc!=SQLITE_OK ){
   776    809       if( p ){
   777    810         fts3DisconnectMethod((sqlite3_vtab *)p);
   778    811       }else{
   779    812         pTokenizer->pModule->xDestroy(pTokenizer);
   780    813       }
          814  +  }else{
          815  +    *ppVTab = &p->base;
   781    816     }
   782    817     return rc;
   783    818   }
   784    819   
   785    820   /*
   786    821   ** The xConnect() and xCreate() methods for the virtual table. All the
   787    822   ** work is done in function fts3InitVtab().
................................................................................
   885    920     return SQLITE_OK;
   886    921   }
   887    922   
   888    923   /*
   889    924   ** Close the cursor.  For additional information see the documentation
   890    925   ** on the xClose method of the virtual table interface.
   891    926   */
   892         -static int fulltextClose(sqlite3_vtab_cursor *pCursor){
          927  +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
   893    928     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
          929  +  assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
   894    930     sqlite3_finalize(pCsr->pStmt);
   895    931     sqlite3Fts3ExprFree(pCsr->pExpr);
          932  +  sqlite3Fts3FreeDeferredTokens(pCsr);
   896    933     sqlite3_free(pCsr->aDoclist);
   897    934     sqlite3_free(pCsr->aMatchinfo);
   898    935     sqlite3_free(pCsr);
   899    936     return SQLITE_OK;
   900    937   }
   901    938   
   902    939   /*
................................................................................
   927    964       }
   928    965     }else{
   929    966       return SQLITE_OK;
   930    967     }
   931    968   }
   932    969   
   933    970   /*
   934         -** Advance the cursor to the next row in the %_content table that
   935         -** matches the search criteria.  For a MATCH search, this will be
   936         -** the next row that matches.  For a full-table scan, this will be
   937         -** simply the next row in the %_content table.  For a docid lookup,
   938         -** this routine simply sets the EOF flag.
          971  +** This function is used to process a single interior node when searching
          972  +** a b-tree for a term or term prefix. The node data is passed to this 
          973  +** function via the zNode/nNode parameters. The term to search for is
          974  +** passed in zTerm/nTerm.
   939    975   **
   940         -** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
   941         -** even if we reach end-of-file.  The fts3EofMethod() will be called
   942         -** subsequently to determine whether or not an EOF was hit.
          976  +** If piFirst is not NULL, then this function sets *piFirst to the blockid
          977  +** of the child node that heads the sub-tree that may contain the term.
          978  +**
          979  +** If piLast is not NULL, then *piLast is set to the right-most child node
          980  +** that heads a sub-tree that may contain a term for which zTerm/nTerm is
          981  +** a prefix.
          982  +**
          983  +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
   943    984   */
   944         -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
          985  +static int fts3ScanInteriorNode(
          986  +  Fts3Table *p,                   /* Virtual table handle */
          987  +  const char *zTerm,              /* Term to select leaves for */
          988  +  int nTerm,                      /* Size of term zTerm in bytes */
          989  +  const char *zNode,              /* Buffer containing segment interior node */
          990  +  int nNode,                      /* Size of buffer at zNode */
          991  +  sqlite3_int64 *piFirst,         /* OUT: Selected child node */
          992  +  sqlite3_int64 *piLast           /* OUT: Selected child node */
          993  +){
   945    994     int rc = SQLITE_OK;             /* Return code */
   946         -  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
   947         -
   948         -  if( pCsr->aDoclist==0 ){
   949         -    if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
   950         -      pCsr->isEof = 1;
   951         -      rc = sqlite3_reset(pCsr->pStmt);
   952         -    }
   953         -  }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
   954         -    pCsr->isEof = 1;
   955         -  }else{
   956         -    sqlite3_reset(pCsr->pStmt);
   957         -    fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
   958         -    pCsr->isRequireSeek = 1;
   959         -    pCsr->isMatchinfoNeeded = 1;
   960         -  }
   961         -  return rc;
   962         -}
   963         -
   964         -
   965         -/*
   966         -** The buffer pointed to by argument zNode (size nNode bytes) contains the
   967         -** root node of a b-tree segment. The segment is guaranteed to be at least
   968         -** one level high (i.e. the root node is not also a leaf). If successful,
   969         -** this function locates the leaf node of the segment that may contain the 
   970         -** term specified by arguments zTerm and nTerm and writes its block number 
   971         -** to *piLeaf.
   972         -**
   973         -** It is possible that the returned leaf node does not contain the specified
   974         -** term. However, if the segment does contain said term, it is stored on
   975         -** the identified leaf node. Because this function only inspects interior
   976         -** segment nodes (and never loads leaf nodes into memory), it is not possible
   977         -** to be sure.
          995  +  const char *zCsr = zNode;       /* Cursor to iterate through node */
          996  +  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
          997  +  char *zBuffer = 0;              /* Buffer to load terms into */
          998  +  int nAlloc = 0;                 /* Size of allocated buffer */
          999  +  int isFirstTerm = 1;            /* True when processing first term on page */
         1000  +  sqlite3_int64 iChild;           /* Block id of child node to descend to */
         1001  +
         1002  +  /* Skip over the 'height' varint that occurs at the start of every 
         1003  +  ** interior node. Then load the blockid of the left-child of the b-tree
         1004  +  ** node into variable iChild.  */
         1005  +  zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
         1006  +  zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
         1007  +  
         1008  +  while( zCsr<zEnd && (piFirst || piLast) ){
         1009  +    int cmp;                      /* memcmp() result */
         1010  +    int nSuffix;                  /* Size of term suffix */
         1011  +    int nPrefix = 0;              /* Size of term prefix */
         1012  +    int nBuffer;                  /* Total term size */
         1013  +  
         1014  +    /* Load the next term on the node into zBuffer. Use realloc() to expand
         1015  +    ** the size of zBuffer if required.  */
         1016  +    if( !isFirstTerm ){
         1017  +      zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
         1018  +    }
         1019  +    isFirstTerm = 0;
         1020  +    zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
         1021  +    if( nPrefix+nSuffix>nAlloc ){
         1022  +      char *zNew;
         1023  +      nAlloc = (nPrefix+nSuffix) * 2;
         1024  +      zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
         1025  +      if( !zNew ){
         1026  +        sqlite3_free(zBuffer);
         1027  +        return SQLITE_NOMEM;
         1028  +      }
         1029  +      zBuffer = zNew;
         1030  +    }
         1031  +    memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
         1032  +    nBuffer = nPrefix + nSuffix;
         1033  +    zCsr += nSuffix;
         1034  +
         1035  +    /* Compare the term we are searching for with the term just loaded from
         1036  +    ** the interior node. If the specified term is greater than or equal
         1037  +    ** to the term from the interior node, then all terms on the sub-tree 
         1038  +    ** headed by node iChild are smaller than zTerm. No need to search 
         1039  +    ** iChild.
         1040  +    **
         1041  +    ** If the interior node term is larger than the specified term, then
         1042  +    ** the tree headed by iChild may contain the specified term.
         1043  +    */
         1044  +    cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
         1045  +    if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
         1046  +      *piFirst = iChild;
         1047  +      piFirst = 0;
         1048  +    }
         1049  +
         1050  +    if( piLast && cmp<0 ){
         1051  +      *piLast = iChild;
         1052  +      piLast = 0;
         1053  +    }
         1054  +
         1055  +    iChild++;
         1056  +  };
         1057  +
         1058  +  if( piFirst ) *piFirst = iChild;
         1059  +  if( piLast ) *piLast = iChild;
         1060  +
         1061  +  sqlite3_free(zBuffer);
         1062  +  return rc;
         1063  +}
         1064  +
         1065  +
         1066  +/*
         1067  +** The buffer pointed to by argument zNode (size nNode bytes) contains an
         1068  +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes)
         1069  +** contains a term. This function searches the sub-tree headed by the zNode
         1070  +** node for the range of leaf nodes that may contain the specified term
         1071  +** or terms for which the specified term is a prefix.
         1072  +**
         1073  +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the 
         1074  +** left-most leaf node in the tree that may contain the specified term.
         1075  +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the
         1076  +** right-most leaf node that may contain a term for which the specified
         1077  +** term is a prefix.
         1078  +**
         1079  +** It is possible that the range of returned leaf nodes does not contain 
         1080  +** the specified term or any terms for which it is a prefix. However, if the 
         1081  +** segment does contain any such terms, they are stored within the identified
         1082  +** range. Because this function only inspects interior segment nodes (and
         1083  +** never loads leaf nodes into memory), it is not possible to be sure.
   978   1084   **
   979   1085   ** If an error occurs, an error code other than SQLITE_OK is returned.
   980   1086   */ 
   981   1087   static int fts3SelectLeaf(
   982   1088     Fts3Table *p,                   /* Virtual table handle */
   983   1089     const char *zTerm,              /* Term to select leaves for */
   984   1090     int nTerm,                      /* Size of term zTerm in bytes */
   985   1091     const char *zNode,              /* Buffer containing segment interior node */
   986   1092     int nNode,                      /* Size of buffer at zNode */
   987         -  sqlite3_int64 *piLeaf           /* Selected leaf node */
   988         -){
   989         -  int rc = SQLITE_OK;             /* Return code */
   990         -  const char *zCsr = zNode;       /* Cursor to iterate through node */
   991         -  const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
   992         -  char *zBuffer = 0;              /* Buffer to load terms into */
   993         -  int nAlloc = 0;                 /* Size of allocated buffer */
   994         -
   995         -  while( 1 ){
   996         -    int isFirstTerm = 1;          /* True when processing first term on page */
   997         -    int iHeight;                  /* Height of this node in tree */
   998         -    sqlite3_int64 iChild;         /* Block id of child node to descend to */
   999         -    int nBlock;                   /* Size of child node in bytes */
  1000         -
  1001         -    zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
  1002         -    zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
  1003         -  
  1004         -    while( zCsr<zEnd ){
  1005         -      int cmp;                    /* memcmp() result */
  1006         -      int nSuffix;                /* Size of term suffix */
  1007         -      int nPrefix = 0;            /* Size of term prefix */
  1008         -      int nBuffer;                /* Total term size */
  1009         -  
  1010         -      /* Load the next term on the node into zBuffer */
  1011         -      if( !isFirstTerm ){
  1012         -        zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
  1013         -      }
  1014         -      isFirstTerm = 0;
  1015         -      zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
  1016         -      if( nPrefix+nSuffix>nAlloc ){
  1017         -        char *zNew;
  1018         -        nAlloc = (nPrefix+nSuffix) * 2;
  1019         -        zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
  1020         -        if( !zNew ){
  1021         -          sqlite3_free(zBuffer);
  1022         -          return SQLITE_NOMEM;
  1023         -        }
  1024         -        zBuffer = zNew;
  1025         -      }
  1026         -      memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
  1027         -      nBuffer = nPrefix + nSuffix;
  1028         -      zCsr += nSuffix;
  1029         -  
  1030         -      /* Compare the term we are searching for with the term just loaded from
  1031         -      ** the interior node. If the specified term is greater than or equal
  1032         -      ** to the term from the interior node, then all terms on the sub-tree 
  1033         -      ** headed by node iChild are smaller than zTerm. No need to search 
  1034         -      ** iChild.
  1035         -      **
  1036         -      ** If the interior node term is larger than the specified term, then
  1037         -      ** the tree headed by iChild may contain the specified term.
  1038         -      */
  1039         -      cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
  1040         -      if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
  1041         -      iChild++;
  1042         -    };
  1043         -
  1044         -    /* If (iHeight==1), the children of this interior node are leaves. The
  1045         -    ** specified term may be present on leaf node iChild.
  1046         -    */
  1047         -    if( iHeight==1 ){
  1048         -      *piLeaf = iChild;
  1049         -      break;
  1050         -    }
  1051         -
  1052         -    /* Descend to interior node iChild. */
  1053         -    rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
  1054         -    if( rc!=SQLITE_OK ) break;
  1055         -    zEnd = &zCsr[nBlock];
  1056         -  }
  1057         -  sqlite3_free(zBuffer);
         1093  +  sqlite3_int64 *piLeaf,          /* Selected leaf node */
         1094  +  sqlite3_int64 *piLeaf2          /* Selected leaf node */
         1095  +){
         1096  +  int rc;                         /* Return code */
         1097  +  int iHeight;                    /* Height of this node in tree */
         1098  +
         1099  +  assert( piLeaf || piLeaf2 );
         1100  +
         1101  +  sqlite3Fts3GetVarint32(zNode, &iHeight);
         1102  +  rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
         1103  +  assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
         1104  +
         1105  +  if( rc==SQLITE_OK && iHeight>1 ){
         1106  +    char *zBlob = 0;              /* Blob read from %_segments table */
         1107  +    int nBlob;                    /* Size of zBlob in bytes */
         1108  +
         1109  +    if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
         1110  +      rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
         1111  +      if( rc==SQLITE_OK ){
         1112  +        rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
         1113  +      }
         1114  +      sqlite3_free(zBlob);
         1115  +      piLeaf = 0;
         1116  +      zBlob = 0;
         1117  +    }
         1118  +
         1119  +    if( rc==SQLITE_OK ){
         1120  +      rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
         1121  +    }
         1122  +    if( rc==SQLITE_OK ){
         1123  +      rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
         1124  +    }
         1125  +    sqlite3_free(zBlob);
         1126  +  }
         1127  +
  1058   1128     return rc;
  1059   1129   }
  1060   1130   
  1061   1131   /*
  1062   1132   ** This function is used to create delta-encoded serialized lists of FTS3 
  1063   1133   ** varints. Each call to this function appends a single varint to a list.
  1064   1134   */
................................................................................
  1282   1352     *pp = p;
  1283   1353     *pp1 = p1 + 1;
  1284   1354     *pp2 = p2 + 1;
  1285   1355   }
  1286   1356   
  1287   1357   /*
  1288   1358   ** nToken==1 searches for adjacent positions.
         1359  +**
         1360  +** This function is used to merge two position lists into one. When it is
         1361  +** called, *pp1 and *pp2 must both point to position lists. A position-list is
         1362  +** the part of a doclist that follows each document id. For example, if a row
         1363  +** contains:
         1364  +**
         1365  +**     'a b c'|'x y z'|'a b b a'
         1366  +**
         1367  +** Then the position list for this row for token 'b' would consist of:
         1368  +**
         1369  +**     0x02 0x01 0x02 0x03 0x03 0x00
         1370  +**
         1371  +** When this function returns, both *pp1 and *pp2 are left pointing to the
         1372  +** byte following the 0x00 terminator of their respective position lists.
         1373  +**
         1374  +** If isSaveLeft is 0, an entry is added to the output position list for 
         1375  +** each position in *pp2 for which there exists one or more positions in
         1376  +** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e.
         1377  +** when the *pp1 token appears before the *pp2 token, but not more than nToken
         1378  +** slots before it.
  1289   1379   */
  1290   1380   static int fts3PoslistPhraseMerge(
  1291         -  char **pp,                      /* Output buffer */
         1381  +  char **pp,                      /* IN/OUT: Preallocated output buffer */
  1292   1382     int nToken,                     /* Maximum difference in token positions */
  1293   1383     int isSaveLeft,                 /* Save the left position */
  1294         -  char **pp1,                     /* Left input list */
  1295         -  char **pp2                      /* Right input list */
         1384  +  int isExact,                    /* If *pp1 is exactly nTokens before *pp2 */
         1385  +  char **pp1,                     /* IN/OUT: Left input list */
         1386  +  char **pp2                      /* IN/OUT: Right input list */
  1296   1387   ){
  1297   1388     char *p = (pp ? *pp : 0);
  1298   1389     char *p1 = *pp1;
  1299   1390     char *p2 = *pp2;
  1300         -
  1301   1391     int iCol1 = 0;
  1302   1392     int iCol2 = 0;
         1393  +
         1394  +  /* Never set both isSaveLeft and isExact for the same invocation. */
         1395  +  assert( isSaveLeft==0 || isExact==0 );
         1396  +
  1303   1397     assert( *p1!=0 && *p2!=0 );
  1304   1398     if( *p1==POS_COLUMN ){ 
  1305   1399       p1++;
  1306   1400       p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
  1307   1401     }
  1308   1402     if( *p2==POS_COLUMN ){ 
  1309   1403       p2++;
................................................................................
  1324   1418   
  1325   1419         assert( *p1!=POS_END && *p1!=POS_COLUMN );
  1326   1420         assert( *p2!=POS_END && *p2!=POS_COLUMN );
  1327   1421         fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
  1328   1422         fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
  1329   1423   
  1330   1424         while( 1 ){
  1331         -        if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
         1425  +        if( iPos2==iPos1+nToken 
         1426  +         || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) 
         1427  +        ){
  1332   1428             sqlite3_int64 iSave;
  1333   1429             if( !pp ){
  1334   1430               fts3PoslistCopy(0, &p2);
  1335   1431               fts3PoslistCopy(0, &p1);
  1336   1432               *pp1 = p1;
  1337   1433               *pp2 = p2;
  1338   1434               return 1;
................................................................................
  1407   1503     char **pp1,                     /* IN/OUT: Left input list */
  1408   1504     char **pp2                      /* IN/OUT: Right input list */
  1409   1505   ){
  1410   1506     char *p1 = *pp1;
  1411   1507     char *p2 = *pp2;
  1412   1508   
  1413   1509     if( !pp ){
  1414         -    if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
         1510  +    if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
  1415   1511       *pp1 = p1;
  1416   1512       *pp2 = p2;
  1417         -    return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
         1513  +    return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
  1418   1514     }else{
  1419   1515       char *pTmp1 = aTmp;
  1420   1516       char *pTmp2;
  1421   1517       char *aTmp2;
  1422   1518       int res = 1;
  1423   1519   
  1424         -    fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
         1520  +    fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
  1425   1521       aTmp2 = pTmp2 = pTmp1;
  1426   1522       *pp1 = p1;
  1427   1523       *pp2 = p2;
  1428         -    fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
         1524  +    fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
  1429   1525       if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
  1430   1526         fts3PoslistMerge(pp, &aTmp, &aTmp2);
  1431   1527       }else if( pTmp1!=aTmp ){
  1432   1528         fts3PoslistCopy(pp, &aTmp);
  1433   1529       }else if( pTmp2!=aTmp2 ){
  1434   1530         fts3PoslistCopy(pp, &aTmp2);
  1435   1531       }else{
................................................................................
  1467   1563     int nParam1,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
  1468   1564     int nParam2,                    /* Used by MERGE_NEAR and MERGE_POS_NEAR */
  1469   1565     char *aBuffer,                  /* Pre-allocated output buffer */
  1470   1566     int *pnBuffer,                  /* OUT: Bytes written to aBuffer */
  1471   1567     char *a1,                       /* Buffer containing first doclist */
  1472   1568     int n1,                         /* Size of buffer a1 */
  1473   1569     char *a2,                       /* Buffer containing second doclist */
  1474         -  int n2                          /* Size of buffer a2 */
         1570  +  int n2,                         /* Size of buffer a2 */
         1571  +  int *pnDoc                      /* OUT: Number of docids in output */
  1475   1572   ){
  1476   1573     sqlite3_int64 i1 = 0;
  1477   1574     sqlite3_int64 i2 = 0;
  1478   1575     sqlite3_int64 iPrev = 0;
  1479   1576   
  1480   1577     char *p = aBuffer;
  1481   1578     char *p1 = a1;
  1482   1579     char *p2 = a2;
  1483   1580     char *pEnd1 = &a1[n1];
  1484   1581     char *pEnd2 = &a2[n2];
         1582  +  int nDoc = 0;
  1485   1583   
  1486   1584     assert( mergetype==MERGE_OR     || mergetype==MERGE_POS_OR 
  1487   1585          || mergetype==MERGE_AND    || mergetype==MERGE_NOT
  1488   1586          || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
  1489   1587          || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  1490   1588     );
  1491   1589   
................................................................................
  1521   1619   
  1522   1620       case MERGE_AND:
  1523   1621         while( p1 && p2 ){
  1524   1622           if( i1==i2 ){
  1525   1623             fts3PutDeltaVarint(&p, &iPrev, i1);
  1526   1624             fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1527   1625             fts3GetDeltaVarint2(&p2, pEnd2, &i2);
         1626  +          nDoc++;
  1528   1627           }else if( i1<i2 ){
  1529   1628             fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1530   1629           }else{
  1531   1630             fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1532   1631           }
  1533   1632         }
  1534   1633         break;
................................................................................
  1551   1650       case MERGE_PHRASE: {
  1552   1651         char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
  1553   1652         while( p1 && p2 ){
  1554   1653           if( i1==i2 ){
  1555   1654             char *pSave = p;
  1556   1655             sqlite3_int64 iPrevSave = iPrev;
  1557   1656             fts3PutDeltaVarint(&p, &iPrev, i1);
  1558         -          if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
         1657  +          if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
  1559   1658               p = pSave;
  1560   1659               iPrev = iPrevSave;
         1660  +          }else{
         1661  +            nDoc++;
  1561   1662             }
  1562   1663             fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1563   1664             fts3GetDeltaVarint2(&p2, pEnd2, &i2);
  1564   1665           }else if( i1<i2 ){
  1565   1666             fts3PoslistCopy(0, &p1);
  1566   1667             fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  1567   1668           }else{
................................................................................
  1606   1707           }
  1607   1708         }
  1608   1709         sqlite3_free(aTmp);
  1609   1710         break;
  1610   1711       }
  1611   1712     }
  1612   1713   
         1714  +  if( pnDoc ) *pnDoc = nDoc;
  1613   1715     *pnBuffer = (int)(p-aBuffer);
  1614   1716     return SQLITE_OK;
  1615   1717   }
  1616   1718   
  1617   1719   /* 
  1618   1720   ** A pointer to an instance of this structure is used as the context 
  1619   1721   ** argument to sqlite3Fts3SegReaderIterate()
................................................................................
  1644   1746     ** into a single doclist.
  1645   1747     */
  1646   1748     for(i=0; i<SizeofArray(pTS->aaOutput); i++){
  1647   1749       if( pTS->aaOutput[i] ){
  1648   1750         if( !aOut ){
  1649   1751           aOut = pTS->aaOutput[i];
  1650   1752           nOut = pTS->anOutput[i];
  1651         -        pTS->aaOutput[0] = 0;
         1753  +        pTS->aaOutput[i] = 0;
  1652   1754         }else{
  1653   1755           int nNew = nOut + pTS->anOutput[i];
  1654   1756           char *aNew = sqlite3_malloc(nNew);
  1655   1757           if( !aNew ){
  1656   1758             sqlite3_free(aOut);
  1657   1759             return SQLITE_NOMEM;
  1658   1760           }
  1659   1761           fts3DoclistMerge(mergetype, 0, 0,
  1660         -            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
         1762  +            aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
  1661   1763           );
  1662   1764           sqlite3_free(pTS->aaOutput[i]);
  1663   1765           sqlite3_free(aOut);
  1664   1766           pTS->aaOutput[i] = 0;
  1665   1767           aOut = aNew;
  1666   1768           nOut = nNew;
  1667   1769         }
................................................................................
  1724   1826         aNew = sqlite3_malloc(nNew);
  1725   1827         if( !aNew ){
  1726   1828           if( aMerge!=aDoclist ){
  1727   1829             sqlite3_free(aMerge);
  1728   1830           }
  1729   1831           return SQLITE_NOMEM;
  1730   1832         }
  1731         -      fts3DoclistMerge(mergetype, 0, 0,
  1732         -          aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
         1833  +      fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, 
         1834  +          pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
  1733   1835         );
  1734   1836   
  1735   1837         if( iOut>0 ) sqlite3_free(aMerge);
  1736   1838         sqlite3_free(pTS->aaOutput[iOut]);
  1737   1839         pTS->aaOutput[iOut] = 0;
  1738   1840   
  1739   1841         aMerge = aNew;
................................................................................
  1742   1844           pTS->aaOutput[iOut] = aMerge;
  1743   1845           pTS->anOutput[iOut] = nMerge;
  1744   1846         }
  1745   1847       }
  1746   1848     }
  1747   1849     return SQLITE_OK;
  1748   1850   }
         1851  +
         1852  +static int fts3DeferredTermSelect(
         1853  +  Fts3DeferredToken *pToken,      /* Phrase token */
         1854  +  int isTermPos,                  /* True to include positions */
         1855  +  int *pnOut,                     /* OUT: Size of list */
         1856  +  char **ppOut                    /* OUT: Body of list */
         1857  +){
         1858  +  char *aSource;
         1859  +  int nSource;
         1860  +
         1861  +  aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
         1862  +  if( !aSource ){
         1863  +    *pnOut = 0;
         1864  +    *ppOut = 0;
         1865  +  }else if( isTermPos ){
         1866  +    *ppOut = sqlite3_malloc(nSource);
         1867  +    if( !*ppOut ) return SQLITE_NOMEM;
         1868  +    memcpy(*ppOut, aSource, nSource);
         1869  +    *pnOut = nSource;
         1870  +  }else{
         1871  +    sqlite3_int64 docid;
         1872  +    *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
         1873  +    *ppOut = sqlite3_malloc(*pnOut);
         1874  +    if( !*ppOut ) return SQLITE_NOMEM;
         1875  +    sqlite3Fts3PutVarint(*ppOut, docid);
         1876  +  }
         1877  +
         1878  +  return SQLITE_OK;
         1879  +}
         1880  +
         1881  +/*
         1882  +** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects.
         1883  +** Elements are added to the array using fts3SegReaderArrayAdd(). 
         1884  +*/
         1885  +struct Fts3SegReaderArray {
         1886  +  int nSegment;                   /* Number of valid entries in apSegment[] */
         1887  +  int nAlloc;                     /* Allocated size of apSegment[] */
         1888  +  int nCost;                      /* The cost of executing SegReaderIterate() */
         1889  +  Fts3SegReader *apSegment[1];    /* Array of seg-reader objects */
         1890  +};
         1891  +
         1892  +
         1893  +/*
         1894  +** Free an Fts3SegReaderArray object. Also free all seg-readers in the
         1895  +** array (using sqlite3Fts3SegReaderFree()).
         1896  +*/
         1897  +static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){
         1898  +  if( pArray ){
         1899  +    int i;
         1900  +    for(i=0; i<pArray->nSegment; i++){
         1901  +      sqlite3Fts3SegReaderFree(0, pArray->apSegment[i]);
         1902  +    }
         1903  +    sqlite3_free(pArray);
         1904  +  }
         1905  +}
         1906  +
         1907  +static int fts3SegReaderArrayAdd(
         1908  +  Fts3SegReaderArray **ppArray, 
         1909  +  Fts3SegReader *pNew
         1910  +){
         1911  +  Fts3SegReaderArray *pArray = *ppArray;
         1912  +
         1913  +  if( !pArray || pArray->nAlloc==pArray->nSegment ){
         1914  +    int nNew = (pArray ? pArray->nAlloc+16 : 16);
         1915  +    pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, 
         1916  +        sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*)
         1917  +    );
         1918  +    if( !pArray ){
         1919  +      sqlite3Fts3SegReaderFree(0, pNew);
         1920  +      return SQLITE_NOMEM;
         1921  +    }
         1922  +    if( nNew==16 ){
         1923  +      pArray->nSegment = 0;
         1924  +      pArray->nCost = 0;
         1925  +    }
         1926  +    pArray->nAlloc = nNew;
         1927  +    *ppArray = pArray;
         1928  +  }
         1929  +
         1930  +  pArray->apSegment[pArray->nSegment++] = pNew;
         1931  +  return SQLITE_OK;
         1932  +}
         1933  +
         1934  +static int fts3TermSegReaderArray(
         1935  +  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
         1936  +  const char *zTerm,              /* Term to query for */
         1937  +  int nTerm,                      /* Size of zTerm in bytes */
         1938  +  int isPrefix,                   /* True for a prefix search */
         1939  +  Fts3SegReaderArray **ppArray    /* OUT: Allocated seg-reader array */
         1940  +){
         1941  +  Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         1942  +  int rc;                         /* Return code */
         1943  +  Fts3SegReaderArray *pArray = 0; /* Array object to build */
         1944  +  Fts3SegReader *pReader = 0;     /* Seg-reader to add to pArray */ 
         1945  +  sqlite3_stmt *pStmt = 0;        /* SQL statement to scan %_segdir table */
         1946  +  int iAge = 0;                   /* Used to assign ages to segments */
         1947  +
         1948  +  /* Allocate a seg-reader to scan the pending terms, if any. */
         1949  +  rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader);
         1950  +  if( rc==SQLITE_OK && pReader ) {
         1951  +    rc = fts3SegReaderArrayAdd(&pArray, pReader);
         1952  +  }
         1953  +
         1954  +  /* Loop through the entire %_segdir table. For each segment, create a
         1955  +  ** Fts3SegReader to iterate through the subset of the segment leaves
         1956  +  ** that may contain a term that matches zTerm/nTerm. For non-prefix
         1957  +  ** searches, this is always a single leaf. For prefix searches, this
         1958  +  ** may be a contiguous block of leaves.
         1959  +  */
         1960  +  if( rc==SQLITE_OK ){
         1961  +    rc = sqlite3Fts3AllSegdirs(p, &pStmt);
         1962  +  }
         1963  +  while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
         1964  +    Fts3SegReader *pNew = 0;
         1965  +    int nRoot = sqlite3_column_bytes(pStmt, 4);
         1966  +    char const *zRoot = sqlite3_column_blob(pStmt, 4);
         1967  +    if( sqlite3_column_int64(pStmt, 1)==0 ){
         1968  +      /* The entire segment is stored on the root node (which must be a
         1969  +      ** leaf). Do not bother inspecting any data in this case, just
         1970  +      ** create a Fts3SegReader to scan the single leaf. 
         1971  +      */
         1972  +      rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
         1973  +    }else{
         1974  +      sqlite3_int64 i1;           /* First leaf that may contain zTerm */
         1975  +      sqlite3_int64 i2;           /* Final leaf that may contain zTerm */
         1976  +      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
         1977  +      if( isPrefix==0 ) i2 = i1;
         1978  +      if( rc==SQLITE_OK ){
         1979  +        rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
         1980  +      }
         1981  +    }
         1982  +    assert( (pNew==0)==(rc!=SQLITE_OK) );
         1983  +
         1984  +    /* If a new Fts3SegReader was allocated, add it to the array. */
         1985  +    if( rc==SQLITE_OK ){
         1986  +      rc = fts3SegReaderArrayAdd(&pArray, pNew);
         1987  +    }
         1988  +    if( rc==SQLITE_OK ){
         1989  +      rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost);
         1990  +    }
         1991  +    iAge++;
         1992  +  }
         1993  +
         1994  +  if( rc==SQLITE_DONE ){
         1995  +    rc = sqlite3_reset(pStmt);
         1996  +  }else{
         1997  +    sqlite3_reset(pStmt);
         1998  +  }
         1999  +  if( rc!=SQLITE_OK ){
         2000  +    fts3SegReaderArrayFree(pArray);
         2001  +    pArray = 0;
         2002  +  }
         2003  +  *ppArray = pArray;
         2004  +  return rc;
         2005  +}
  1749   2006   
  1750   2007   /*
  1751   2008   ** This function retreives the doclist for the specified term (or term
  1752   2009   ** prefix) from the database. 
  1753   2010   **
  1754   2011   ** The returned doclist may be in one of two formats, depending on the 
  1755   2012   ** value of parameter isReqPos. If isReqPos is zero, then the doclist is
................................................................................
  1756   2013   ** a sorted list of delta-compressed docids (a bare doclist). If isReqPos
  1757   2014   ** is non-zero, then the returned list is in the same format as is stored 
  1758   2015   ** in the database without the found length specifier at the start of on-disk
  1759   2016   ** doclists.
  1760   2017   */
  1761   2018   static int fts3TermSelect(
  1762   2019     Fts3Table *p,                   /* Virtual table handle */
         2020  +  Fts3PhraseToken *pTok,          /* Token to query for */
  1763   2021     int iColumn,                    /* Column to query (or -ve for all columns) */
  1764         -  const char *zTerm,              /* Term to query for */
  1765         -  int nTerm,                      /* Size of zTerm in bytes */
  1766         -  int isPrefix,                   /* True for a prefix search */
  1767   2022     int isReqPos,                   /* True to include position lists in output */
  1768   2023     int *pnOut,                     /* OUT: Size of buffer at *ppOut */
  1769   2024     char **ppOut                    /* OUT: Malloced result buffer */
  1770   2025   ){
  1771         -  int i;
  1772         -  TermSelect tsc;
  1773         -  Fts3SegFilter filter;           /* Segment term filter configuration */
  1774         -  Fts3SegReader **apSegment;      /* Array of segments to read data from */
  1775         -  int nSegment = 0;               /* Size of apSegment array */
  1776         -  int nAlloc = 16;                /* Allocated size of segment array */
  1777   2026     int rc;                         /* Return code */
  1778         -  sqlite3_stmt *pStmt = 0;        /* SQL statement to scan %_segdir table */
  1779         -  int iAge = 0;                   /* Used to assign ages to segments */
         2027  +  Fts3SegReaderArray *pArray;     /* Seg-reader array for this term */
         2028  +  TermSelect tsc;               /* Context object for fts3TermSelectCb() */
         2029  +  Fts3SegFilter filter;         /* Segment term filter configuration */
  1780   2030   
  1781         -  apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc);
  1782         -  if( !apSegment ) return SQLITE_NOMEM;
  1783         -  rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]);
  1784         -  if( rc!=SQLITE_OK ) goto finished;
  1785         -  if( apSegment[0] ){
  1786         -    nSegment = 1;
  1787         -  }
  1788         -
  1789         -  /* Loop through the entire %_segdir table. For each segment, create a
  1790         -  ** Fts3SegReader to iterate through the subset of the segment leaves
  1791         -  ** that may contain a term that matches zTerm/nTerm. For non-prefix
  1792         -  ** searches, this is always a single leaf. For prefix searches, this
  1793         -  ** may be a contiguous block of leaves.
  1794         -  **
  1795         -  ** The code in this loop does not actually load any leaves into memory
  1796         -  ** (unless the root node happens to be a leaf). It simply examines the
  1797         -  ** b-tree structure to determine which leaves need to be inspected.
  1798         -  */
  1799         -  rc = sqlite3Fts3AllSegdirs(p, &pStmt);
  1800         -  while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
  1801         -    Fts3SegReader *pNew = 0;
  1802         -    int nRoot = sqlite3_column_bytes(pStmt, 4);
  1803         -    char const *zRoot = sqlite3_column_blob(pStmt, 4);
  1804         -    if( sqlite3_column_int64(pStmt, 1)==0 ){
  1805         -      /* The entire segment is stored on the root node (which must be a
  1806         -      ** leaf). Do not bother inspecting any data in this case, just
  1807         -      ** create a Fts3SegReader to scan the single leaf. 
  1808         -      */
  1809         -      rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
  1810         -    }else{
  1811         -      int rc2;                    /* Return value of sqlite3Fts3ReadBlock() */
  1812         -      sqlite3_int64 i1;           /* Blockid of leaf that may contain zTerm */
  1813         -      rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
  1814         -      if( rc==SQLITE_OK ){
  1815         -        sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
  1816         -        rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
  1817         -      }
  1818         -
  1819         -      /* The following call to ReadBlock() serves to reset the SQL statement
  1820         -      ** used to retrieve blocks of data from the %_segments table. If it is
  1821         -      ** not reset here, then it may remain classified as an active statement 
  1822         -      ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands 
  1823         -      ** failing.
  1824         -      */ 
  1825         -      rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0);
  1826         -      if( rc==SQLITE_OK ){
  1827         -        rc = rc2;
  1828         -      }
  1829         -    }
  1830         -    iAge++;
  1831         -
  1832         -    /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
  1833         -    assert( pNew!=0 || rc!=SQLITE_OK );
  1834         -    if( pNew ){
  1835         -      if( nSegment==nAlloc ){
  1836         -        Fts3SegReader **pArray;
  1837         -        nAlloc += 16;
  1838         -        pArray = (Fts3SegReader **)sqlite3_realloc(
  1839         -            apSegment, nAlloc*sizeof(Fts3SegReader *)
  1840         -        );
  1841         -        if( !pArray ){
  1842         -          sqlite3Fts3SegReaderFree(p, pNew);
  1843         -          rc = SQLITE_NOMEM;
  1844         -          goto finished;
  1845         -        }
  1846         -        apSegment = pArray;
  1847         -      }
  1848         -      apSegment[nSegment++] = pNew;
  1849         -    }
  1850         -  }
  1851         -  if( rc!=SQLITE_DONE ){
  1852         -    assert( rc!=SQLITE_OK );
  1853         -    goto finished;
  1854         -  }
  1855         -
         2031  +  pArray = pTok->pArray;
  1856   2032     memset(&tsc, 0, sizeof(TermSelect));
  1857   2033     tsc.isReqPos = isReqPos;
  1858   2034   
  1859   2035     filter.flags = FTS3_SEGMENT_IGNORE_EMPTY 
  1860         -        | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
         2036  +        | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
  1861   2037           | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
  1862   2038           | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
  1863   2039     filter.iCol = iColumn;
  1864         -  filter.zTerm = zTerm;
  1865         -  filter.nTerm = nTerm;
         2040  +  filter.zTerm = pTok->z;
         2041  +  filter.nTerm = pTok->n;
  1866   2042   
  1867         -  rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
  1868         -      fts3TermSelectCb, (void *)&tsc
         2043  +  rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, 
         2044  +      &filter, fts3TermSelectCb, (void *)&tsc
  1869   2045     );
  1870   2046     if( rc==SQLITE_OK ){
  1871   2047       rc = fts3TermSelectMerge(&tsc);
  1872   2048     }
  1873   2049   
  1874   2050     if( rc==SQLITE_OK ){
  1875   2051       *ppOut = tsc.aaOutput[0];
  1876   2052       *pnOut = tsc.anOutput[0];
  1877   2053     }else{
         2054  +    int i;
  1878   2055       for(i=0; i<SizeofArray(tsc.aaOutput); i++){
  1879   2056         sqlite3_free(tsc.aaOutput[i]);
  1880   2057       }
  1881   2058     }
  1882   2059   
  1883         -finished:
  1884         -  sqlite3_reset(pStmt);
  1885         -  for(i=0; i<nSegment; i++){
  1886         -    sqlite3Fts3SegReaderFree(p, apSegment[i]);
  1887         -  }
  1888         -  sqlite3_free(apSegment);
  1889         -  return rc;
  1890         -}
  1891         -
         2060  +  fts3SegReaderArrayFree(pArray);
         2061  +  pTok->pArray = 0;
         2062  +  return rc;
         2063  +}
         2064  +
         2065  +/*
         2066  +** This function counts the total number of docids in the doclist stored
         2067  +** in buffer aList[], size nList bytes.
         2068  +**
         2069  +** If the isPoslist argument is true, then it is assumed that the doclist
         2070  +** contains a position-list following each docid. Otherwise, it is assumed
         2071  +** that the doclist is simply a list of docids stored as delta encoded 
         2072  +** varints.
         2073  +*/
         2074  +static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
         2075  +  int nDoc = 0;                   /* Return value */
         2076  +  if( aList ){
         2077  +    char *aEnd = &aList[nList];   /* Pointer to one byte after EOF */
         2078  +    char *p = aList;              /* Cursor */
         2079  +    if( !isPoslist ){
         2080  +      /* The number of docids in the list is the same as the number of 
         2081  +      ** varints. In FTS3 a varint consists of a single byte with the 0x80 
         2082  +      ** bit cleared and zero or more bytes with the 0x80 bit set. So to
         2083  +      ** count the varints in the buffer, just count the number of bytes
         2084  +      ** with the 0x80 bit clear.  */
         2085  +      while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
         2086  +    }else{
         2087  +      while( p<aEnd ){
         2088  +        nDoc++;
         2089  +        while( (*p++)&0x80 );     /* Skip docid varint */
         2090  +        fts3PoslistCopy(0, &p);   /* Skip over position list */
         2091  +      }
         2092  +    }
         2093  +  }
         2094  +
         2095  +  return nDoc;
         2096  +}
         2097  +
         2098  +/*
         2099  +** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
         2100  +*/
         2101  +static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
         2102  +  int rc = SQLITE_OK;
         2103  +  if( pExpr ){
         2104  +    rc = fts3DeferExpression(pCsr, pExpr->pLeft);
         2105  +    if( rc==SQLITE_OK ){
         2106  +      rc = fts3DeferExpression(pCsr, pExpr->pRight);
         2107  +    }
         2108  +    if( pExpr->eType==FTSQUERY_PHRASE ){
         2109  +      int iCol = pExpr->pPhrase->iColumn;
         2110  +      int i;
         2111  +      for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
         2112  +        Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
         2113  +        if( pToken->pDeferred==0 ){
         2114  +          rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
         2115  +        }
         2116  +      }
         2117  +    }
         2118  +  }
         2119  +  return rc;
         2120  +}
         2121  +
         2122  +/*
         2123  +** This function removes the position information from a doclist. When
         2124  +** called, buffer aList (size *pnList bytes) contains a doclist that includes
         2125  +** position information. This function removes the position information so
         2126  +** that aList contains only docids, and adjusts *pnList to reflect the new
         2127  +** (possibly reduced) size of the doclist.
         2128  +*/
         2129  +static void fts3DoclistStripPositions(
         2130  +  char *aList,                    /* IN/OUT: Buffer containing doclist */
         2131  +  int *pnList                     /* IN/OUT: Size of doclist in bytes */
         2132  +){
         2133  +  if( aList ){
         2134  +    char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
         2135  +    char *p = aList;              /* Input cursor */
         2136  +    char *pOut = aList;           /* Output cursor */
         2137  +  
         2138  +    while( p<aEnd ){
         2139  +      sqlite3_int64 delta;
         2140  +      p += sqlite3Fts3GetVarint(p, &delta);
         2141  +      fts3PoslistCopy(0, &p);
         2142  +      pOut += sqlite3Fts3PutVarint(pOut, delta);
         2143  +    }
         2144  +
         2145  +    *pnList = (pOut - aList);
         2146  +  }
         2147  +}
  1892   2148   
  1893   2149   /* 
  1894   2150   ** Return a DocList corresponding to the phrase *pPhrase.
         2151  +**
         2152  +** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
         2153  +** then no tokens in the phrase were looked up in the full-text index. This
         2154  +** is only possible when this function is called from within xFilter(). The
         2155  +** caller should assume that all documents match the phrase. The actual
         2156  +** filtering will take place in xNext().
  1895   2157   */
  1896   2158   static int fts3PhraseSelect(
  1897         -  Fts3Table *p,                   /* Virtual table handle */
         2159  +  Fts3Cursor *pCsr,               /* Virtual table cursor handle */
  1898   2160     Fts3Phrase *pPhrase,            /* Phrase to return a doclist for */
  1899   2161     int isReqPos,                   /* True if output should contain positions */
  1900   2162     char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  1901   2163     int *pnOut                      /* OUT: Size of buffer at *paOut */
  1902   2164   ){
  1903   2165     char *pOut = 0;
  1904   2166     int nOut = 0;
  1905   2167     int rc = SQLITE_OK;
  1906   2168     int ii;
  1907   2169     int iCol = pPhrase->iColumn;
  1908   2170     int isTermPos = (pPhrase->nToken>1 || isReqPos);
         2171  +  Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         2172  +  int isFirst = 1;
         2173  +
         2174  +  int iPrevTok = 0;
         2175  +  int nDoc = 0;
         2176  +
         2177  +  /* If this is an xFilter() evaluation, create a segment-reader for each
         2178  +  ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
         2179  +  ** evaluation, only create segment-readers if there are no Fts3DeferredToken
         2180  +  ** objects attached to the phrase-tokens.
         2181  +  */
         2182  +  for(ii=0; ii<pPhrase->nToken; ii++){
         2183  +    Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
         2184  +    if( pTok->pArray==0 ){
         2185  +      if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
         2186  +       || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) 
         2187  +       || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) 
         2188  +      ){
         2189  +        rc = fts3TermSegReaderArray(
         2190  +            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
         2191  +        );
         2192  +        if( rc!=SQLITE_OK ) return rc;
         2193  +      }
         2194  +    }
         2195  +  }
  1909   2196   
  1910   2197     for(ii=0; ii<pPhrase->nToken; ii++){
  1911         -    struct PhraseToken *pTok = &pPhrase->aToken[ii];
  1912         -    char *z = pTok->z;            /* Next token of the phrase */
  1913         -    int n = pTok->n;              /* Size of z in bytes */
  1914         -    int isPrefix = pTok->isPrefix;/* True if token is a prefix */
         2198  +    Fts3PhraseToken *pTok;        /* Token to find doclist for */
         2199  +    int iTok;                     /* The token being queried this iteration */
  1915   2200       char *pList;                  /* Pointer to token doclist */
  1916   2201       int nList;                    /* Size of buffer at pList */
  1917   2202   
  1918         -    rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
         2203  +    /* Select a token to process. If this is an xFilter() call, then tokens 
         2204  +    ** are processed in order from least to most costly. Otherwise, tokens 
         2205  +    ** are processed in the order in which they occur in the phrase.
         2206  +    */
         2207  +    if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
         2208  +      assert( isReqPos );
         2209  +      iTok = ii;
         2210  +      pTok = &pPhrase->aToken[iTok];
         2211  +      if( pTok->bFulltext==0 ) continue;
         2212  +    }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
         2213  +      iTok = ii;
         2214  +      pTok = &pPhrase->aToken[iTok];
         2215  +    }else{
         2216  +      int nMinCost = 0x7FFFFFFF;
         2217  +      int jj;
         2218  +
         2219  +      /* Find the remaining token with the lowest cost. */
         2220  +      for(jj=0; jj<pPhrase->nToken; jj++){
         2221  +        Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray;
         2222  +        if( pArray && pArray->nCost<nMinCost ){
         2223  +          iTok = jj;
         2224  +          nMinCost = pArray->nCost;
         2225  +        }
         2226  +      }
         2227  +      pTok = &pPhrase->aToken[iTok];
         2228  +
         2229  +      /* This branch is taken if it is determined that loading the doclist
         2230  +      ** for the next token would require more IO than loading all documents
         2231  +      ** currently identified by doclist pOut/nOut. No further doclists will
         2232  +      ** be loaded from the full-text index for this phrase.
         2233  +      */
         2234  +      if( nMinCost>nDoc && ii>0 ){
         2235  +        rc = fts3DeferExpression(pCsr, pCsr->pExpr);
         2236  +        break;
         2237  +      }
         2238  +    }
         2239  +
         2240  +    if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
         2241  +      rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
         2242  +    }else{
         2243  +      assert( pTok->pArray );
         2244  +      rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
         2245  +      pTok->bFulltext = 1;
         2246  +    }
         2247  +    assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 );
  1919   2248       if( rc!=SQLITE_OK ) break;
  1920   2249   
  1921         -    if( ii==0 ){
         2250  +    if( isFirst ){
  1922   2251         pOut = pList;
  1923   2252         nOut = nList;
         2253  +      if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
         2254  +        nDoc = fts3DoclistCountDocids(1, pOut, nOut);
         2255  +      }
         2256  +      isFirst = 0;
         2257  +      iPrevTok = iTok;
  1924   2258       }else{
  1925         -      /* Merge the new term list and the current output. If this is the
  1926         -      ** last term in the phrase, and positions are not required in the
  1927         -      ** output of this function, the positions can be dropped as part
  1928         -      ** of this merge. Either way, the result of this merge will be
  1929         -      ** smaller than nList bytes. The code in fts3DoclistMerge() is written
  1930         -      ** so that it is safe to use pList as the output as well as an input
  1931         -      ** in this case.
         2259  +      /* Merge the new term list and the current output. */
         2260  +      char *aLeft, *aRight;
         2261  +      int nLeft, nRight;
         2262  +      int nDist;
         2263  +      int mt;
         2264  +
         2265  +      /* If this is the final token of the phrase, and positions were not
         2266  +      ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
         2267  +      ** This drops the position information from the output list.
  1932   2268         */
  1933         -      int mergetype = MERGE_POS_PHRASE;
  1934         -      if( ii==pPhrase->nToken-1 && !isReqPos ){
  1935         -        mergetype = MERGE_PHRASE;
         2269  +      mt = MERGE_POS_PHRASE;
         2270  +      if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
         2271  +
         2272  +      assert( iPrevTok!=iTok );
         2273  +      if( iPrevTok<iTok ){
         2274  +        aLeft = pOut;
         2275  +        nLeft = nOut;
         2276  +        aRight = pList;
         2277  +        nRight = nList;
         2278  +        nDist = iTok-iPrevTok;
         2279  +        iPrevTok = iTok;
         2280  +      }else{
         2281  +        aRight = pOut;
         2282  +        nRight = nOut;
         2283  +        aLeft = pList;
         2284  +        nLeft = nList;
         2285  +        nDist = iPrevTok-iTok;
  1936   2286         }
  1937         -      fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
  1938         -      sqlite3_free(pOut);
  1939         -      pOut = pList;
         2287  +      pOut = aRight;
         2288  +      fts3DoclistMerge(
         2289  +          mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
         2290  +      );
         2291  +      sqlite3_free(aLeft);
  1940   2292       }
  1941   2293       assert( nOut==0 || pOut!=0 );
  1942   2294     }
  1943   2295   
  1944   2296     if( rc==SQLITE_OK ){
         2297  +    if( ii!=pPhrase->nToken ){
         2298  +      assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
         2299  +      fts3DoclistStripPositions(pOut, &nOut);
         2300  +    }
  1945   2301       *paOut = pOut;
  1946   2302       *pnOut = nOut;
  1947   2303     }else{
  1948   2304       sqlite3_free(pOut);
  1949   2305     }
  1950   2306     return rc;
  1951   2307   }
  1952   2308   
         2309  +/*
         2310  +** This function merges two doclists according to the requirements of a
         2311  +** NEAR operator.
         2312  +**
         2313  +** Both input doclists must include position information. The output doclist 
         2314  +** includes position information if the first argument to this function
         2315  +** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
         2316  +*/
  1953   2317   static int fts3NearMerge(
  1954   2318     int mergetype,                  /* MERGE_POS_NEAR or MERGE_NEAR */
  1955   2319     int nNear,                      /* Parameter to NEAR operator */
  1956   2320     int nTokenLeft,                 /* Number of tokens in LHS phrase arg */
  1957   2321     char *aLeft,                    /* Doclist for LHS (incl. positions) */
  1958   2322     int nLeft,                      /* Size of LHS doclist in bytes */
  1959   2323     int nTokenRight,                /* As nTokenLeft */
  1960   2324     char *aRight,                   /* As aLeft */
  1961   2325     int nRight,                     /* As nRight */
  1962   2326     char **paOut,                   /* OUT: Results of merge (malloced) */
  1963   2327     int *pnOut                      /* OUT: Sized of output buffer */
  1964   2328   ){
  1965         -  char *aOut;
  1966         -  int rc;
         2329  +  char *aOut;                     /* Buffer to write output doclist to */
         2330  +  int rc;                         /* Return code */
  1967   2331   
  1968   2332     assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
  1969   2333   
  1970   2334     aOut = sqlite3_malloc(nLeft+nRight+1);
  1971   2335     if( aOut==0 ){
  1972   2336       rc = SQLITE_NOMEM;
  1973   2337     }else{
  1974   2338       rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, 
  1975         -      aOut, pnOut, aLeft, nLeft, aRight, nRight
         2339  +      aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
  1976   2340       );
  1977   2341       if( rc!=SQLITE_OK ){
  1978   2342         sqlite3_free(aOut);
  1979   2343         aOut = 0;
  1980   2344       }
  1981   2345     }
  1982   2346   
  1983   2347     *paOut = aOut;
  1984   2348     return rc;
  1985   2349   }
  1986   2350   
         2351  +/*
         2352  +** This function is used as part of the processing for the snippet() and
         2353  +** offsets() functions.
         2354  +**
         2355  +** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
         2356  +** have their respective doclists (including position information) loaded
         2357  +** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
         2358  +** each doclist that are not within nNear tokens of a corresponding entry
         2359  +** in the other doclist.
         2360  +*/
  1987   2361   int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
  1988         -  int rc;
         2362  +  int rc;                         /* Return code */
         2363  +
         2364  +  assert( pLeft->eType==FTSQUERY_PHRASE );
         2365  +  assert( pRight->eType==FTSQUERY_PHRASE );
         2366  +  assert( pLeft->isLoaded && pRight->isLoaded );
         2367  +
  1989   2368     if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
  1990   2369       sqlite3_free(pLeft->aDoclist);
  1991   2370       sqlite3_free(pRight->aDoclist);
  1992   2371       pRight->aDoclist = 0;
  1993   2372       pLeft->aDoclist = 0;
  1994   2373       rc = SQLITE_OK;
  1995   2374     }else{
  1996         -    char *aOut;
  1997         -    int nOut;
         2375  +    char *aOut;                   /* Buffer in which to assemble new doclist */
         2376  +    int nOut;                     /* Size of buffer aOut in bytes */
  1998   2377   
  1999   2378       rc = fts3NearMerge(MERGE_POS_NEAR, nNear, 
  2000   2379           pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
  2001   2380           pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
  2002   2381           &aOut, &nOut
  2003   2382       );
  2004   2383       if( rc!=SQLITE_OK ) return rc;
................................................................................
  2014   2393       sqlite3_free(pLeft->aDoclist);
  2015   2394       pLeft->aDoclist = aOut;
  2016   2395       pLeft->nDoclist = nOut;
  2017   2396     }
  2018   2397     return rc;
  2019   2398   }
  2020   2399   
  2021         -/*
  2022         -** Evaluate the full-text expression pExpr against fts3 table pTab. Store
  2023         -** the resulting doclist in *paOut and *pnOut.  This routine mallocs for
  2024         -** the space needed to store the output.  The caller is responsible for
         2400  +
         2401  +/*
         2402  +** Allocate an Fts3SegReaderArray for each token in the expression pExpr. 
         2403  +** The allocated objects are stored in the Fts3PhraseToken.pArray member
         2404  +** variables of each token structure.
         2405  +*/
         2406  +static int fts3ExprAllocateSegReaders(
         2407  +  Fts3Cursor *pCsr,               /* FTS3 table */
         2408  +  Fts3Expr *pExpr,                /* Expression to create seg-readers for */
         2409  +  int *pnExpr                     /* OUT: Number of AND'd expressions */
         2410  +){
         2411  +  int rc = SQLITE_OK;             /* Return code */
         2412  +
         2413  +  assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
         2414  +  if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
         2415  +    (*pnExpr)++;
         2416  +    pnExpr = 0;
         2417  +  }
         2418  +
         2419  +  if( pExpr->eType==FTSQUERY_PHRASE ){
         2420  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
         2421  +    int ii;
         2422  +
         2423  +    for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
         2424  +      Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
         2425  +      if( pTok->pArray==0 ){
         2426  +        rc = fts3TermSegReaderArray(
         2427  +            pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray
         2428  +        );
         2429  +      }
         2430  +    }
         2431  +  }else{ 
         2432  +    rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
         2433  +    if( rc==SQLITE_OK ){
         2434  +      rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
         2435  +    }
         2436  +  }
         2437  +  return rc;
         2438  +}
         2439  +
         2440  +/*
         2441  +** Free the Fts3SegReaderArray objects associated with each token in the
         2442  +** expression pExpr. In other words, this function frees the resources
         2443  +** allocated by fts3ExprAllocateSegReaders().
         2444  +*/
         2445  +static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
         2446  +  if( pExpr ){
         2447  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
         2448  +    if( pPhrase ){
         2449  +      int kk;
         2450  +      for(kk=0; kk<pPhrase->nToken; kk++){
         2451  +        fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray);
         2452  +        pPhrase->aToken[kk].pArray = 0;
         2453  +      }
         2454  +    }
         2455  +    fts3ExprFreeSegReaders(pExpr->pLeft);
         2456  +    fts3ExprFreeSegReaders(pExpr->pRight);
         2457  +  }
         2458  +}
         2459  +
         2460  +/*
         2461  +** Return the sum of the costs of all tokens in the expression pExpr. This
         2462  +** function must be called after Fts3SegReaderArrays have been allocated
         2463  +** for all tokens using fts3ExprAllocateSegReaders().
         2464  +*/
         2465  +int fts3ExprCost(Fts3Expr *pExpr){
         2466  +  int nCost;                      /* Return value */
         2467  +  if( pExpr->eType==FTSQUERY_PHRASE ){
         2468  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
         2469  +    int ii;
         2470  +    nCost = 0;
         2471  +    for(ii=0; ii<pPhrase->nToken; ii++){
         2472  +      nCost += pPhrase->aToken[ii].pArray->nCost;
         2473  +    }
         2474  +  }else{
         2475  +    nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
         2476  +  }
         2477  +  return nCost;
         2478  +}
         2479  +
         2480  +/*
         2481  +** The following is a helper function (and type) for fts3EvalExpr(). It
         2482  +** must be called after Fts3SegReaders have been allocated for every token
         2483  +** in the expression. See the context it is called from in fts3EvalExpr()
         2484  +** for further explanation.
         2485  +*/
         2486  +typedef struct ExprAndCost ExprAndCost;
         2487  +struct ExprAndCost {
         2488  +  Fts3Expr *pExpr;
         2489  +  int nCost;
         2490  +};
         2491  +static void fts3ExprAssignCosts(
         2492  +  Fts3Expr *pExpr,                /* Expression to create seg-readers for */
         2493  +  ExprAndCost **ppExprCost        /* OUT: Write to *ppExprCost */
         2494  +){
         2495  +  if( pExpr->eType==FTSQUERY_AND ){
         2496  +    fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
         2497  +    fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
         2498  +  }else{
         2499  +    (*ppExprCost)->pExpr = pExpr;
         2500  +    (*ppExprCost)->nCost = fts3ExprCost(pExpr);;
         2501  +    (*ppExprCost)++;
         2502  +  }
         2503  +}
         2504  +
         2505  +/*
         2506  +** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
         2507  +** the resulting doclist in *paOut and *pnOut. This routine mallocs for
         2508  +** the space needed to store the output. The caller is responsible for
  2025   2509   ** freeing the space when it has finished.
         2510  +**
         2511  +** This function is called in two distinct contexts:
         2512  +**
         2513  +**   * From within the virtual table xFilter() method. In this case, the
         2514  +**     output doclist contains entries for all rows in the table, based on
         2515  +**     data read from the full-text index.
         2516  +**
         2517  +**     In this case, if the query expression contains one or more tokens that 
         2518  +**     are very common, then the returned doclist may contain a superset of 
         2519  +**     the documents that actually match the expression.
         2520  +**
         2521  +**   * From within the virtual table xNext() method. This call is only made
         2522  +**     if the call from within xFilter() found that there were very common 
         2523  +**     tokens in the query expression and did return a superset of the 
         2524  +**     matching documents. In this case the returned doclist contains only
         2525  +**     entries that correspond to the current row of the table. Instead of
         2526  +**     reading the data for each token from the full-text index, the data is
         2527  +**     already available in-memory in the Fts3PhraseToken.pDeferred structures.
         2528  +**     See fts3EvalDeferred() for how it gets there.
         2529  +**
         2530  +** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
         2531  +** required) Fts3Cursor.doDeferred==1.
         2532  +**
         2533  +** If the SQLite invokes the snippet(), offsets() or matchinfo() function
         2534  +** as part of a SELECT on an FTS3 table, this function is called on each
         2535  +** individual phrase expression in the query. If there were very common tokens
         2536  +** found in the xFilter() call, then this function is called once for phrase
         2537  +** for each row visited, and the returned doclist contains entries for the
         2538  +** current row only. Otherwise, if there were no very common tokens, then this
         2539  +** function is called once only for each phrase in the query and the returned
         2540  +** doclist contains entries for all rows of the table.
         2541  +**
         2542  +** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
         2543  +** result of a snippet(), offsets() or matchinfo() invocation.
  2026   2544   */
  2027         -static int evalFts3Expr(
  2028         -  Fts3Table *p,                   /* Virtual table handle */
         2545  +static int fts3EvalExpr(
         2546  +  Fts3Cursor *p,                  /* Virtual table cursor handle */
  2029   2547     Fts3Expr *pExpr,                /* Parsed fts3 expression */
  2030   2548     char **paOut,                   /* OUT: Pointer to malloc'd result buffer */
  2031   2549     int *pnOut,                     /* OUT: Size of buffer at *paOut */
  2032   2550     int isReqPos                    /* Require positions in output buffer */
  2033   2551   ){
  2034   2552     int rc = SQLITE_OK;             /* Return code */
  2035   2553   
  2036   2554     /* Zero the output parameters. */
  2037   2555     *paOut = 0;
  2038   2556     *pnOut = 0;
  2039   2557   
  2040   2558     if( pExpr ){
  2041         -    assert( pExpr->eType==FTSQUERY_PHRASE 
  2042         -         || pExpr->eType==FTSQUERY_NEAR 
  2043         -         || isReqPos==0
         2559  +    assert( pExpr->eType==FTSQUERY_NEAR   || pExpr->eType==FTSQUERY_OR     
         2560  +         || pExpr->eType==FTSQUERY_AND    || pExpr->eType==FTSQUERY_NOT
         2561  +         || pExpr->eType==FTSQUERY_PHRASE
  2044   2562       );
         2563  +    assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
         2564  +
  2045   2565       if( pExpr->eType==FTSQUERY_PHRASE ){
  2046         -      rc = fts3PhraseSelect(p, pExpr->pPhrase, 
         2566  +      rc = fts3PhraseSelect(p, pExpr->pPhrase,
  2047   2567             isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
  2048   2568             paOut, pnOut
  2049   2569         );
         2570  +      fts3ExprFreeSegReaders(pExpr);
         2571  +    }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
         2572  +      ExprAndCost *aExpr = 0;     /* Array of AND'd expressions and costs */
         2573  +      int nExpr = 0;              /* Size of aExpr[] */
         2574  +      char *aRet = 0;             /* Doclist to return to caller */
         2575  +      int nRet = 0;               /* Length of aRet[] in bytes */
         2576  +      int nDoc = 0x7FFFFFFF;
         2577  +
         2578  +      assert( !isReqPos );
         2579  +
         2580  +      rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
         2581  +      if( rc==SQLITE_OK ){
         2582  +        assert( nExpr>1 );
         2583  +        aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
         2584  +        if( !aExpr ) rc = SQLITE_NOMEM;
         2585  +      }
         2586  +      if( rc==SQLITE_OK ){
         2587  +        int ii;                   /* Used to iterate through expressions */
         2588  +
         2589  +        fts3ExprAssignCosts(pExpr, &aExpr);
         2590  +        aExpr -= nExpr;
         2591  +        for(ii=0; ii<nExpr; ii++){
         2592  +          char *aNew;
         2593  +          int nNew;
         2594  +          int jj;
         2595  +          ExprAndCost *pBest = 0;
         2596  +  
         2597  +          for(jj=0; jj<nExpr; jj++){
         2598  +            ExprAndCost *pCand = &aExpr[jj];
         2599  +            if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
         2600  +              pBest = pCand;
         2601  +            }
         2602  +          }
         2603  +  
         2604  +          if( pBest->nCost>nDoc ){
         2605  +            rc = fts3DeferExpression(p, p->pExpr);
         2606  +            break;
         2607  +          }else{
         2608  +            rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
         2609  +            if( rc!=SQLITE_OK ) break;
         2610  +            pBest->pExpr = 0;
         2611  +            if( ii==0 ){
         2612  +              aRet = aNew;
         2613  +              nRet = nNew;
         2614  +              nDoc = fts3DoclistCountDocids(0, aRet, nRet);
         2615  +            }else{
         2616  +              fts3DoclistMerge(
         2617  +                  MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
         2618  +              );
         2619  +              sqlite3_free(aNew);
         2620  +            }
         2621  +          }
         2622  +        }
         2623  +      }
         2624  +
         2625  +      *paOut = aRet;
         2626  +      *pnOut = nRet;
         2627  +      sqlite3_free(aExpr);
         2628  +      fts3ExprFreeSegReaders(pExpr);
         2629  +
  2050   2630       }else{
  2051   2631         char *aLeft;
  2052   2632         char *aRight;
  2053   2633         int nLeft;
  2054   2634         int nRight;
  2055   2635   
  2056         -      if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
  2057         -       && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
         2636  +      assert( pExpr->eType==FTSQUERY_NEAR 
         2637  +           || pExpr->eType==FTSQUERY_OR
         2638  +           || pExpr->eType==FTSQUERY_NOT
         2639  +           || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
         2640  +      );
         2641  +
         2642  +      if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
         2643  +       && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
  2058   2644         ){
  2059         -        assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR     
  2060         -            || pExpr->eType==FTSQUERY_AND  || pExpr->eType==FTSQUERY_NOT
  2061         -        );
  2062   2645           switch( pExpr->eType ){
  2063   2646             case FTSQUERY_NEAR: {
  2064   2647               Fts3Expr *pLeft;
  2065   2648               Fts3Expr *pRight;
  2066         -            int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR;
  2067         -           
         2649  +            int mergetype = MERGE_NEAR;
  2068   2650               if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
  2069   2651                 mergetype = MERGE_POS_NEAR;
  2070   2652               }
  2071   2653               pLeft = pExpr->pLeft;
  2072   2654               while( pLeft->eType==FTSQUERY_NEAR ){ 
  2073   2655                 pLeft=pLeft->pRight;
  2074   2656               }
................................................................................
  2089   2671               /* Allocate a buffer for the output. The maximum size is the
  2090   2672               ** sum of the sizes of the two input buffers. The +1 term is
  2091   2673               ** so that a buffer of zero bytes is never allocated - this can
  2092   2674               ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
  2093   2675               */
  2094   2676               char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
  2095   2677               rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
  2096         -                aLeft, nLeft, aRight, nRight
         2678  +                aLeft, nLeft, aRight, nRight, 0
  2097   2679               );
  2098   2680               *paOut = aBuffer;
  2099   2681               sqlite3_free(aLeft);
  2100   2682               break;
  2101   2683             }
  2102   2684   
  2103   2685             default: {
  2104   2686               assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
  2105   2687               fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
  2106         -                aLeft, nLeft, aRight, nRight
         2688  +                aLeft, nLeft, aRight, nRight, 0
  2107   2689               );
  2108   2690               *paOut = aLeft;
  2109   2691               break;
  2110   2692             }
  2111   2693           }
  2112   2694         }
  2113   2695         sqlite3_free(aRight);
  2114   2696       }
  2115   2697     }
  2116   2698   
         2699  +  return rc;
         2700  +}
         2701  +
         2702  +/*
         2703  +** This function is called from within xNext() for each row visited by
         2704  +** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
         2705  +** was able to determine the exact set of matching rows, this function sets
         2706  +** *pbRes to true and returns SQLITE_IO immediately.
         2707  +**
         2708  +** Otherwise, if evaluating the query expression within xFilter() returned a
         2709  +** superset of the matching documents instead of an exact set (this happens
         2710  +** when the query includes very common tokens and it is deemed too expensive to
         2711  +** load their doclists from disk), this function tests if the current row
         2712  +** really does match the FTS3 query.
         2713  +**
         2714  +** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
         2715  +** is returned and *pbRes is set to true if the current row matches the
         2716  +** FTS3 query (and should be included in the results returned to SQLite), or
         2717  +** false otherwise.
         2718  +*/
         2719  +static int fts3EvalDeferred(
         2720  +  Fts3Cursor *pCsr,               /* FTS3 cursor pointing at row to test */
         2721  +  int *pbRes                      /* OUT: Set to true if row is a match */
         2722  +){
         2723  +  int rc = SQLITE_OK;
         2724  +  if( pCsr->pDeferred==0 ){
         2725  +    *pbRes = 1;
         2726  +  }else{
         2727  +    rc = fts3CursorSeek(0, pCsr);
         2728  +    if( rc==SQLITE_OK ){
         2729  +      sqlite3Fts3FreeDeferredDoclists(pCsr);
         2730  +      rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
         2731  +    }
         2732  +    if( rc==SQLITE_OK ){
         2733  +      char *a = 0;
         2734  +      int n = 0;
         2735  +      rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
         2736  +      assert( n>=0 );
         2737  +      *pbRes = (n>0);
         2738  +      sqlite3_free(a);
         2739  +    }
         2740  +  }
         2741  +  return rc;
         2742  +}
         2743  +
         2744  +/*
         2745  +** Advance the cursor to the next row in the %_content table that
         2746  +** matches the search criteria.  For a MATCH search, this will be
         2747  +** the next row that matches. For a full-table scan, this will be
         2748  +** simply the next row in the %_content table.  For a docid lookup,
         2749  +** this routine simply sets the EOF flag.
         2750  +**
         2751  +** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
         2752  +** even if we reach end-of-file.  The fts3EofMethod() will be called
         2753  +** subsequently to determine whether or not an EOF was hit.
         2754  +*/
         2755  +static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
         2756  +  int res;
         2757  +  int rc = SQLITE_OK;             /* Return code */
         2758  +  Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
         2759  +
         2760  +  pCsr->eEvalmode = FTS3_EVAL_NEXT;
         2761  +  do {
         2762  +    if( pCsr->aDoclist==0 ){
         2763  +      if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
         2764  +        pCsr->isEof = 1;
         2765  +        rc = sqlite3_reset(pCsr->pStmt);
         2766  +        break;
         2767  +      }
         2768  +      pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
         2769  +    }else{
         2770  +      if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
         2771  +        pCsr->isEof = 1;
         2772  +        break;
         2773  +      }
         2774  +      sqlite3_reset(pCsr->pStmt);
         2775  +      fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
         2776  +      pCsr->isRequireSeek = 1;
         2777  +      pCsr->isMatchinfoNeeded = 1;
         2778  +    }
         2779  +  }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
         2780  +
  2117   2781     return rc;
  2118   2782   }
  2119   2783   
  2120   2784   /*
  2121   2785   ** This is the xFilter interface for the virtual table.  See
  2122   2786   ** the virtual table xFilter method documentation for additional
  2123   2787   ** information.
................................................................................
  2129   2793   ** in the %_content table.
  2130   2794   **
  2131   2795   ** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index.  The
  2132   2796   ** column on the left-hand side of the MATCH operator is column
  2133   2797   ** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed.  argv[0] is the right-hand
  2134   2798   ** side of the MATCH operator.
  2135   2799   */
  2136         -/* TODO(shess) Upgrade the cursor initialization and destruction to
  2137         -** account for fts3FilterMethod() being called multiple times on the
  2138         -** same cursor. The current solution is very fragile. Apply fix to
  2139         -** fts3 as appropriate.
  2140         -*/
  2141   2800   static int fts3FilterMethod(
  2142   2801     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  2143   2802     int idxNum,                     /* Strategy index */
  2144   2803     const char *idxStr,             /* Unused */
  2145   2804     int nVal,                       /* Number of elements in apVal */
  2146   2805     sqlite3_value **apVal           /* Arguments for the indexing scheme */
  2147   2806   ){
................................................................................
  2156   2815   
  2157   2816     UNUSED_PARAMETER(idxStr);
  2158   2817     UNUSED_PARAMETER(nVal);
  2159   2818   
  2160   2819     assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
  2161   2820     assert( nVal==0 || nVal==1 );
  2162   2821     assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
         2822  +  assert( p->pSegments==0 );
  2163   2823   
  2164   2824     /* In case the cursor has been used before, clear it now. */
  2165   2825     sqlite3_finalize(pCsr->pStmt);
  2166   2826     sqlite3_free(pCsr->aDoclist);
  2167   2827     sqlite3Fts3ExprFree(pCsr->pExpr);
  2168   2828     memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
  2169   2829   
  2170         -  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  2171         -  ** statement loops through all rows of the %_content table. For a
  2172         -  ** full-text query or docid lookup, the statement retrieves a single
  2173         -  ** row by docid.
  2174         -  */
  2175         -  zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
  2176         -  if( !zSql ){
  2177         -    rc = SQLITE_NOMEM;
  2178         -  }else{
  2179         -    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
  2180         -    sqlite3_free(zSql);
  2181         -  }
  2182         -  if( rc!=SQLITE_OK ) return rc;
  2183         -  pCsr->eSearch = (i16)idxNum;
  2184         -
  2185         -  if( idxNum==FTS3_DOCID_SEARCH ){
  2186         -    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
  2187         -  }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
         2830  +  if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
  2188   2831       int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
  2189   2832       const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
  2190   2833   
  2191   2834       if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  2192   2835         return SQLITE_NOMEM;
  2193   2836       }
  2194   2837   
................................................................................
  2202   2845         }
  2203   2846         return rc;
  2204   2847       }
  2205   2848   
  2206   2849       rc = sqlite3Fts3ReadLock(p);
  2207   2850       if( rc!=SQLITE_OK ) return rc;
  2208   2851   
  2209         -    rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
         2852  +    rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
         2853  +    sqlite3Fts3SegmentsClose(p);
         2854  +    if( rc!=SQLITE_OK ) return rc;
  2210   2855       pCsr->pNextId = pCsr->aDoclist;
  2211   2856       pCsr->iPrevId = 0;
  2212   2857     }
         2858  +
         2859  +  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
         2860  +  ** statement loops through all rows of the %_content table. For a
         2861  +  ** full-text query or docid lookup, the statement retrieves a single
         2862  +  ** row by docid.
         2863  +  */
         2864  +  zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
         2865  +  if( !zSql ){
         2866  +    rc = SQLITE_NOMEM;
         2867  +  }else{
         2868  +    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
         2869  +    sqlite3_free(zSql);
         2870  +  }
         2871  +  if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
         2872  +    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
         2873  +  }
         2874  +  pCsr->eSearch = (i16)idxNum;
  2213   2875   
  2214   2876     if( rc!=SQLITE_OK ) return rc;
  2215   2877     return fts3NextMethod(pCursor);
  2216   2878   }
  2217   2879   
  2218   2880   /* 
  2219   2881   ** This is the xEof method of the virtual table. SQLite calls this 
................................................................................
  2230   2892   ** rowid should be written to *pRowid.
  2231   2893   */
  2232   2894   static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
  2233   2895     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  2234   2896     if( pCsr->aDoclist ){
  2235   2897       *pRowid = pCsr->iPrevId;
  2236   2898     }else{
         2899  +    /* This branch runs if the query is implemented using a full-table scan
         2900  +    ** (not using the full-text index). In this case grab the rowid from the
         2901  +    ** SELECT statement.
         2902  +    */
         2903  +    assert( pCsr->isRequireSeek==0 );
  2237   2904       *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
  2238   2905     }
  2239   2906     return SQLITE_OK;
  2240   2907   }
  2241   2908   
  2242   2909   /* 
  2243   2910   ** This is the xColumn method, called by SQLite to request a value from
................................................................................
  2292   2959   }
  2293   2960   
  2294   2961   /*
  2295   2962   ** Implementation of xSync() method. Flush the contents of the pending-terms
  2296   2963   ** hash-table to the database.
  2297   2964   */
  2298   2965   static int fts3SyncMethod(sqlite3_vtab *pVtab){
  2299         -  return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
         2966  +  int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
         2967  +  sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
         2968  +  return rc;
  2300   2969   }
  2301   2970   
  2302   2971   /*
  2303   2972   ** Implementation of xBegin() method. This is a no-op.
  2304   2973   */
  2305   2974   static int fts3BeginMethod(sqlite3_vtab *pVtab){
  2306   2975     UNUSED_PARAMETER(pVtab);
................................................................................
  2330   2999   
  2331   3000   /*
  2332   3001   ** Load the doclist associated with expression pExpr to pExpr->aDoclist.
  2333   3002   ** The loaded doclist contains positions as well as the document ids.
  2334   3003   ** This is used by the matchinfo(), snippet() and offsets() auxillary
  2335   3004   ** functions.
  2336   3005   */
  2337         -int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){
  2338         -  return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
         3006  +int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
         3007  +  int rc;
         3008  +  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
         3009  +  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
         3010  +  rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
         3011  +  return rc;
         3012  +}
         3013  +
         3014  +int sqlite3Fts3ExprLoadFtDoclist(
         3015  +  Fts3Cursor *pCsr, 
         3016  +  Fts3Expr *pExpr,
         3017  +  char **paDoclist,
         3018  +  int *pnDoclist
         3019  +){
         3020  +  int rc;
         3021  +  assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
         3022  +  assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
         3023  +  pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
         3024  +  rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
         3025  +  pCsr->eEvalmode = FTS3_EVAL_NEXT;
         3026  +  return rc;
  2339   3027   }
  2340   3028   
  2341   3029   /*
  2342   3030   ** After ExprLoadDoclist() (see above) has been called, this function is
  2343   3031   ** used to iterate/search through the position lists that make up the doclist
  2344   3032   ** stored in pExpr->aDoclist.
  2345   3033   */
................................................................................
  2397   3085   ** message is written to context pContext and SQLITE_ERROR returned. The
  2398   3086   ** string passed via zFunc is used as part of the error message.
  2399   3087   */
  2400   3088   static int fts3FunctionArg(
  2401   3089     sqlite3_context *pContext,      /* SQL function call context */
  2402   3090     const char *zFunc,              /* Function name */
  2403   3091     sqlite3_value *pVal,            /* argv[0] passed to function */
  2404         -  Fts3Cursor **ppCsr         /* OUT: Store cursor handle here */
         3092  +  Fts3Cursor **ppCsr              /* OUT: Store cursor handle here */
  2405   3093   ){
  2406   3094     Fts3Cursor *pRet;
  2407   3095     if( sqlite3_value_type(pVal)!=SQLITE_BLOB 
  2408   3096      || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
  2409   3097     ){
  2410   3098       char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
  2411   3099       sqlite3_result_error(pContext, zErr, -1);
................................................................................
  2523   3211   */
  2524   3212   static void fts3MatchinfoFunc(
  2525   3213     sqlite3_context *pContext,      /* SQLite function call context */
  2526   3214     int nVal,                       /* Size of argument array */
  2527   3215     sqlite3_value **apVal           /* Array of arguments */
  2528   3216   ){
  2529   3217     Fts3Cursor *pCsr;               /* Cursor handle passed through apVal[0] */
  2530         -
  2531         -  if( nVal!=1 ){
  2532         -    sqlite3_result_error(pContext,
  2533         -        "wrong number of arguments to function matchinfo()", -1);
  2534         -    return;
  2535         -  }
  2536         -
         3218  +  assert( nVal==1 );
  2537   3219     if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
  2538   3220       sqlite3Fts3Matchinfo(pContext, pCsr);
  2539   3221     }
  2540   3222   }
  2541   3223   
  2542   3224   /*
  2543   3225   ** This routine implements the xFindFunction method for the FTS3
................................................................................
  2592   3274       return rc;
  2593   3275     }
  2594   3276   
  2595   3277     fts3DbExec(&rc, db,
  2596   3278       "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
  2597   3279       p->zDb, p->zName, zName
  2598   3280     );
  2599         -  if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
  2600   3281     if( p->bHasDocsize ){
  2601   3282       fts3DbExec(&rc, db,
  2602   3283         "ALTER TABLE %Q.'%q_docsize'  RENAME TO '%q_docsize';",
  2603   3284         p->zDb, p->zName, zName
  2604   3285       );
  2605   3286       fts3DbExec(&rc, db,
  2606   3287         "ALTER TABLE %Q.'%q_stat'  RENAME TO '%q_stat';",
................................................................................
  2622   3303     /* iVersion      */ 0,
  2623   3304     /* xCreate       */ fts3CreateMethod,
  2624   3305     /* xConnect      */ fts3ConnectMethod,
  2625   3306     /* xBestIndex    */ fts3BestIndexMethod,
  2626   3307     /* xDisconnect   */ fts3DisconnectMethod,
  2627   3308     /* xDestroy      */ fts3DestroyMethod,
  2628   3309     /* xOpen         */ fts3OpenMethod,
  2629         -  /* xClose        */ fulltextClose,
         3310  +  /* xClose        */ fts3CloseMethod,
  2630   3311     /* xFilter       */ fts3FilterMethod,
  2631   3312     /* xNext         */ fts3NextMethod,
  2632   3313     /* xEof          */ fts3EofMethod,
  2633   3314     /* xColumn       */ fts3ColumnMethod,
  2634   3315     /* xRowid        */ fts3RowidMethod,
  2635   3316     /* xUpdate       */ fts3UpdateMethod,
  2636   3317     /* xBegin        */ fts3BeginMethod,
................................................................................
  2649   3330   static void hashDestroy(void *p){
  2650   3331     Fts3Hash *pHash = (Fts3Hash *)p;
  2651   3332     sqlite3Fts3HashClear(pHash);
  2652   3333     sqlite3_free(pHash);
  2653   3334   }
  2654   3335   
  2655   3336   /*
  2656         -** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
  2657         -** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
  2658         -** two forward declarations are for functions declared in these files
  2659         -** used to retrieve the respective implementations.
         3337  +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are 
         3338  +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
         3339  +** respectively. The following three forward declarations are for functions
         3340  +** declared in these files used to retrieve the respective implementations.
  2660   3341   **
  2661   3342   ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
  2662   3343   ** to by the argument to point to the "simple" tokenizer implementation.
  2663         -** Function ...PorterTokenizerModule() sets *pModule to point to the
  2664         -** porter tokenizer/stemmer implementation.
         3344  +** And so on.
  2665   3345   */
  2666   3346   void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
  2667   3347   void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
         3348  +#ifdef SQLITE_ENABLE_ICU
  2668   3349   void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
         3350  +#endif
  2669   3351   
  2670   3352   /*
  2671   3353   ** Initialise the fts3 extension. If this extension is built as part
  2672   3354   ** of the sqlite library, then this function is called directly by
  2673   3355   ** SQLite. If fts3 is built as a dynamically loadable extension, this
  2674   3356   ** function is called by the sqlite3_extension_init() entry point.
  2675   3357   */
................................................................................
  2717   3399     ** the two scalar functions. If this is successful, register the
  2718   3400     ** module with sqlite.
  2719   3401     */
  2720   3402     if( SQLITE_OK==rc 
  2721   3403      && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
  2722   3404      && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
  2723   3405      && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
  2724         -   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1))
         3406  +   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
  2725   3407      && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
  2726   3408     ){
  2727   3409       rc = sqlite3_create_module_v2(
  2728   3410           db, "fts3", &fts3Module, (void *)pHash, hashDestroy
  2729   3411       );
  2730   3412       if( rc==SQLITE_OK ){
  2731   3413         rc = sqlite3_create_module_v2(

Changes to ext/fts3/fts3Int.h.

    73     73   ** amalgamation.
    74     74   */
    75     75   #ifndef SQLITE_AMALGAMATION
    76     76   /*
    77     77   ** Macros indicating that conditional expressions are always true or
    78     78   ** false.
    79     79   */
           80  +#ifdef SQLITE_COVERAGE_TEST
           81  +# define ALWAYS(x) (1)
           82  +# define NEVER(X)  (0)
           83  +#else
    80     84   # define ALWAYS(x) (x)
    81     85   # define NEVER(X)  (x)
           86  +#endif
           87  +
    82     88   /*
    83     89   ** Internal types used by SQLite.
    84     90   */
    85     91   typedef unsigned char u8;         /* 1-byte (or larger) unsigned integer */
    86     92   typedef short int i16;            /* 2-byte (or larger) signed integer */
    87     93   typedef unsigned int u32;         /* 4-byte unsigned integer */
    88     94   typedef sqlite3_uint64 u64;       /* 8-byte unsigned integer */
................................................................................
    92     98   #define UNUSED_PARAMETER(x) (void)(x)
    93     99   #endif
    94    100   
    95    101   typedef struct Fts3Table Fts3Table;
    96    102   typedef struct Fts3Cursor Fts3Cursor;
    97    103   typedef struct Fts3Expr Fts3Expr;
    98    104   typedef struct Fts3Phrase Fts3Phrase;
    99         -typedef struct Fts3SegReader Fts3SegReader;
          105  +typedef struct Fts3PhraseToken Fts3PhraseToken;
          106  +
   100    107   typedef struct Fts3SegFilter Fts3SegFilter;
          108  +typedef struct Fts3DeferredToken Fts3DeferredToken;
          109  +typedef struct Fts3SegReader Fts3SegReader;
          110  +typedef struct Fts3SegReaderArray Fts3SegReaderArray;
   101    111   
   102    112   /*
   103    113   ** A connection to a fulltext index is an instance of the following
   104    114   ** structure. The xCreate and xConnect methods create an instance
   105    115   ** of this structure and xDestroy and xDisconnect free that instance.
   106    116   ** All other methods receive a pointer to the structure as one of their
   107    117   ** arguments.
................................................................................
   114    124     int nColumn;                    /* number of named columns in virtual table */
   115    125     char **azColumn;                /* column names.  malloced */
   116    126     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   117    127   
   118    128     /* Precompiled statements used by the implementation. Each of these 
   119    129     ** statements is run and reset within a single virtual table API call. 
   120    130     */
   121         -  sqlite3_stmt *aStmt[25];
   122         -
   123         -  /* Pointer to string containing the SQL:
   124         -  **
   125         -  ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ? 
   126         -  **    ORDER BY blockid"
   127         -  */
   128         -  char *zSelectLeaves;
   129         -  int nLeavesStmt;                /* Valid statements in aLeavesStmt */
   130         -  int nLeavesTotal;               /* Total number of prepared leaves stmts */
   131         -  int nLeavesAlloc;               /* Allocated size of aLeavesStmt */
   132         -  sqlite3_stmt **aLeavesStmt;     /* Array of prepared zSelectLeaves stmts */
          131  +  sqlite3_stmt *aStmt[24];
   133    132   
   134    133     int nNodeSize;                  /* Soft limit for node size */
   135    134     u8 bHasContent;                 /* True if %_content table exists */
   136    135     u8 bHasDocsize;                 /* True if %_docsize table exists */
          136  +  int nPgsz;                      /* Page size for host database */
          137  +  char *zSegmentsTbl;             /* Name of %_segments table */
          138  +  sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
   137    139   
   138    140     /* The following hash table is used to buffer pending index updates during
   139    141     ** transactions. Variable nPendingData estimates the memory size of the 
   140    142     ** pending data, including hash table overhead, but not malloc overhead. 
   141    143     ** When nPendingData exceeds nMaxPendingData, the buffer is flushed 
   142    144     ** automatically. Variable iPrevDocid is the docid of the most recently
   143    145     ** inserted record.
................................................................................
   156    158   struct Fts3Cursor {
   157    159     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
   158    160     i16 eSearch;                    /* Search strategy (see below) */
   159    161     u8 isEof;                       /* True if at End Of Results */
   160    162     u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
   161    163     sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
   162    164     Fts3Expr *pExpr;                /* Parsed MATCH query string */
          165  +  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
   163    166     sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
   164    167     char *pNextId;                  /* Pointer into the body of aDoclist */
   165    168     char *aDoclist;                 /* List of docids for full-text queries */
   166    169     int nDoclist;                   /* Size of buffer at aDoclist */
   167    170     int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
   168    171     u32 *aMatchinfo;                /* Information about most recent match */
          172  +  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
          173  +  int nRowAvg;                    /* Average size of database rows, in pages */
   169    174   };
          175  +
          176  +#define FTS3_EVAL_FILTER    0
          177  +#define FTS3_EVAL_NEXT      1
          178  +#define FTS3_EVAL_MATCHINFO 2
   170    179   
   171    180   /*
   172    181   ** The Fts3Cursor.eSearch member is always set to one of the following.
   173    182   ** Actualy, Fts3Cursor.eSearch can be greater than or equal to
   174    183   ** FTS3_FULLTEXT_SEARCH.  If so, then Fts3Cursor.eSearch - 2 is the index
   175    184   ** of the column to be searched.  For example, in
   176    185   **
................................................................................
   186    195   #define FTS3_FULLSCAN_SEARCH 0    /* Linear scan of %_content table */
   187    196   #define FTS3_DOCID_SEARCH    1    /* Lookup by rowid on %_content table */
   188    197   #define FTS3_FULLTEXT_SEARCH 2    /* Full-text index search */
   189    198   
   190    199   /*
   191    200   ** A "phrase" is a sequence of one or more tokens that must match in
   192    201   ** sequence.  A single token is the base case and the most common case.
   193         -** For a sequence of tokens contained in "...", nToken will be the number
   194         -** of tokens in the string.
          202  +** For a sequence of tokens contained in double-quotes (i.e. "one two three")
          203  +** nToken will be the number of tokens in the string.
          204  +**
          205  +** The nDocMatch and nMatch variables contain data that may be used by the
          206  +** matchinfo() function. They are populated when the full-text index is 
          207  +** queried for hits on the phrase. If one or more tokens in the phrase
          208  +** are deferred, the nDocMatch and nMatch variables are populated based
          209  +** on the assumption that the 
   195    210   */
          211  +struct Fts3PhraseToken {
          212  +  char *z;                        /* Text of the token */
          213  +  int n;                          /* Number of bytes in buffer z */
          214  +  int isPrefix;                   /* True if token ends with a "*" character */
          215  +  int bFulltext;                  /* True if full-text index was used */
          216  +  Fts3SegReaderArray *pArray;     /* Segment-reader for this token */
          217  +  Fts3DeferredToken *pDeferred;   /* Deferred token object for this token */
          218  +};
          219  +
   196    220   struct Fts3Phrase {
          221  +  /* Variables populated by fts3_expr.c when parsing a MATCH expression */
   197    222     int nToken;                /* Number of tokens in the phrase */
   198    223     int iColumn;               /* Index of column this phrase must match */
   199    224     int isNot;                 /* Phrase prefixed by unary not (-) operator */
   200         -  struct PhraseToken {
   201         -    char *z;                 /* Text of the token */
   202         -    int n;                   /* Number of bytes in buffer pointed to by z */
   203         -    int isPrefix;            /* True if token ends in with a "*" character */
   204         -  } aToken[1];               /* One entry for each token in the phrase */
          225  +  Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
   205    226   };
   206    227   
   207    228   /*
   208    229   ** A tree of these objects forms the RHS of a MATCH operator.
   209    230   **
   210    231   ** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
   211    232   ** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, 
................................................................................
   247    268   #define FTSQUERY_NEAR   1
   248    269   #define FTSQUERY_NOT    2
   249    270   #define FTSQUERY_AND    3
   250    271   #define FTSQUERY_OR     4
   251    272   #define FTSQUERY_PHRASE 5
   252    273   
   253    274   
   254         -/* fts3_init.c */
   255         -int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *);
   256         -int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, 
   257         -                        sqlite3_vtab **, char **);
   258         -
   259    275   /* fts3_write.c */
   260    276   int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
   261    277   int sqlite3Fts3PendingTermsFlush(Fts3Table *);
   262    278   void sqlite3Fts3PendingTermsClear(Fts3Table *);
   263    279   int sqlite3Fts3Optimize(Fts3Table *);
   264    280   int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
   265    281     sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
   266    282   int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
   267    283   void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
   268    284   int sqlite3Fts3SegReaderIterate(
   269    285     Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
   270    286     int (*)(Fts3Table *, void *, char *, int, char *, int),  void *
   271    287   );
   272         -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
          288  +int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
   273    289   int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
   274    290   int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
   275    291   int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
   276    292   int sqlite3Fts3ReadLock(Fts3Table *);
          293  +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
          294  +
          295  +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
          296  +int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
          297  +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
          298  +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
          299  +char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
          300  +
          301  +void sqlite3Fts3SegmentsClose(Fts3Table *);
   277    302   
   278    303   /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
   279    304   #define FTS3_SEGMENT_REQUIRE_POS   0x00000001
   280    305   #define FTS3_SEGMENT_IGNORE_EMPTY  0x00000002
   281    306   #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
   282    307   #define FTS3_SEGMENT_PREFIX        0x00000008
   283    308   
................................................................................
   293    318   int sqlite3Fts3PutVarint(char *, sqlite3_int64);
   294    319   int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
   295    320   int sqlite3Fts3GetVarint32(const char *, int *);
   296    321   int sqlite3Fts3VarintLen(sqlite3_uint64);
   297    322   void sqlite3Fts3Dequote(char *);
   298    323   
   299    324   char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int);
   300         -int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *);
          325  +int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
          326  +int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
   301    327   int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
   302    328   
   303    329   /* fts3_tokenizer.c */
   304    330   const char *sqlite3Fts3NextToken(const char *, int *);
   305    331   int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
   306    332   int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, 
   307    333     const char *, sqlite3_tokenizer **, const char **, char **

Changes to ext/fts3/fts3_expr.c.

   101    101   ** is defined to accept an argument of type char, and always returns 0 for
   102    102   ** any values that fall outside of the range of the unsigned char type (i.e.
   103    103   ** negative values).
   104    104   */
   105    105   static int fts3isspace(char c){
   106    106     return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
   107    107   }
          108  +
          109  +/*
          110  +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful,
          111  +** zero the memory before returning a pointer to it. If unsuccessful, 
          112  +** return NULL.
          113  +*/
          114  +static void *fts3MallocZero(int nByte){
          115  +  void *pRet = sqlite3_malloc(nByte);
          116  +  if( pRet ) memset(pRet, 0, nByte);
          117  +  return pRet;
          118  +}
          119  +
   108    120   
   109    121   /*
   110    122   ** Extract the next token from buffer z (length n) using the tokenizer
   111    123   ** and other information (column names etc.) in pParse. Create an Fts3Expr
   112    124   ** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
   113    125   ** single token and set *ppExpr to point to it. If the end of the buffer is
   114    126   ** reached before a token is found, set *ppExpr to zero. It is the
................................................................................
   139    151       int nByte;                               /* total space to allocate */
   140    152   
   141    153       pCursor->pTokenizer = pTokenizer;
   142    154       rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
   143    155   
   144    156       if( rc==SQLITE_OK ){
   145    157         nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
   146         -      pRet = (Fts3Expr *)sqlite3_malloc(nByte);
          158  +      pRet = (Fts3Expr *)fts3MallocZero(nByte);
   147    159         if( !pRet ){
   148    160           rc = SQLITE_NOMEM;
   149    161         }else{
   150         -        memset(pRet, 0, nByte);
   151    162           pRet->eType = FTSQUERY_PHRASE;
   152    163           pRet->pPhrase = (Fts3Phrase *)&pRet[1];
   153    164           pRet->pPhrase->nToken = 1;
   154    165           pRet->pPhrase->iColumn = iCol;
   155    166           pRet->pPhrase->aToken[0].n = nToken;
   156    167           pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
   157    168           memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
................................................................................
   219    230       pCursor->pTokenizer = pTokenizer;
   220    231       for(ii=0; rc==SQLITE_OK; ii++){
   221    232         const char *zToken;
   222    233         int nToken, iBegin, iEnd, iPos;
   223    234         rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
   224    235         if( rc==SQLITE_OK ){
   225    236           int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   226         -        p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
          237  +        p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
   227    238           zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
   228    239           if( !p || !zTemp ){
   229    240             goto no_mem;
   230    241           }
   231    242           if( ii==0 ){
   232    243             memset(p, 0, nByte);
   233    244             p->pPhrase = (Fts3Phrase *)&p[1];
   234    245           }
   235    246           p->pPhrase = (Fts3Phrase *)&p[1];
          247  +        memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
   236    248           p->pPhrase->nToken = ii+1;
   237    249           p->pPhrase->aToken[ii].n = nToken;
   238    250           memcpy(&zTemp[nTemp], zToken, nToken);
   239    251           nTemp += nToken;
   240    252           if( iEnd<nInput && zInput[iEnd]=='*' ){
   241    253             p->pPhrase->aToken[ii].isPrefix = 1;
   242    254           }else{
................................................................................
   250    262     }
   251    263   
   252    264     if( rc==SQLITE_DONE ){
   253    265       int jj;
   254    266       char *zNew = NULL;
   255    267       int nNew = 0;
   256    268       int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
   257         -    nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
          269  +    nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
   258    270       p = fts3ReallocOrFree(p, nByte + nTemp);
   259    271       if( !p ){
   260    272         goto no_mem;
   261    273       }
   262    274       if( zTemp ){
   263    275         zNew = &(((char *)p)[nByte]);
   264    276         memcpy(zNew, zTemp, nTemp);
................................................................................
   368    380         ** the next byte must contain either whitespace, an open or close
   369    381         ** parenthesis, a quote character, or EOF. 
   370    382         */
   371    383         cNext = zInput[nKey];
   372    384         if( fts3isspace(cNext) 
   373    385          || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
   374    386         ){
   375         -        pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
          387  +        pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr));
   376    388           if( !pRet ){
   377    389             return SQLITE_NOMEM;
   378    390           }
   379         -        memset(pRet, 0, sizeof(Fts3Expr));
   380    391           pRet->eType = pKey->eType;
   381    392           pRet->nNear = nNear;
   382    393           *ppExpr = pRet;
   383    394           *pnConsumed = (int)((zInput - z) + nKey);
   384    395           return SQLITE_OK;
   385    396         }
   386    397   
................................................................................
   548    559       if( rc==SQLITE_OK ){
   549    560         int isPhrase;
   550    561   
   551    562         if( !sqlite3_fts3_enable_parentheses 
   552    563          && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot 
   553    564         ){
   554    565           /* Create an implicit NOT operator. */
   555         -        Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr));
          566  +        Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
   556    567           if( !pNot ){
   557    568             sqlite3Fts3ExprFree(p);
   558    569             rc = SQLITE_NOMEM;
   559    570             goto exprparse_out;
   560    571           }
   561         -        memset(pNot, 0, sizeof(Fts3Expr));
   562    572           pNot->eType = FTSQUERY_NOT;
   563    573           pNot->pRight = p;
   564    574           if( pNotBranch ){
   565    575             pNot->pLeft = pNotBranch;
   566    576           }
   567    577           pNotBranch = pNot;
   568    578           p = pPrev;
................................................................................
   582    592             goto exprparse_out;
   583    593           }
   584    594     
   585    595           if( isPhrase && !isRequirePhrase ){
   586    596             /* Insert an implicit AND operator. */
   587    597             Fts3Expr *pAnd;
   588    598             assert( pRet && pPrev );
   589         -          pAnd = sqlite3_malloc(sizeof(Fts3Expr));
          599  +          pAnd = fts3MallocZero(sizeof(Fts3Expr));
   590    600             if( !pAnd ){
   591    601               sqlite3Fts3ExprFree(p);
   592    602               rc = SQLITE_NOMEM;
   593    603               goto exprparse_out;
   594    604             }
   595         -          memset(pAnd, 0, sizeof(Fts3Expr));
   596    605             pAnd->eType = FTSQUERY_AND;
   597    606             insertBinaryOperator(&pRet, pPrev, pAnd);
   598    607             pPrev = pAnd;
   599    608           }
   600    609   
   601    610           /* This test catches attempts to make either operand of a NEAR
   602    611           ** operator something other than a phrase. For example, either of

Changes to ext/fts3/fts3_snippet.c.

    20     20   
    21     21   /*
    22     22   ** Used as an fts3ExprIterate() context when loading phrase doclists to
    23     23   ** Fts3Expr.aDoclist[]/nDoclist.
    24     24   */
    25     25   typedef struct LoadDoclistCtx LoadDoclistCtx;
    26     26   struct LoadDoclistCtx {
    27         -  Fts3Table *pTab;                /* FTS3 Table */
           27  +  Fts3Cursor *pCsr;               /* FTS3 Cursor */
    28     28     int nPhrase;                    /* Number of phrases seen so far */
    29     29     int nToken;                     /* Number of tokens seen so far */
    30     30   };
    31     31   
    32     32   /*
    33     33   ** The following types are used as part of the implementation of the 
    34     34   ** fts3BestSnippet() routine.
................................................................................
   214    214   
   215    215     UNUSED_PARAMETER(iPhrase);
   216    216   
   217    217     p->nPhrase++;
   218    218     p->nToken += pExpr->pPhrase->nToken;
   219    219   
   220    220     if( pExpr->isLoaded==0 ){
   221         -    rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr);
          221  +    rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
   222    222       pExpr->isLoaded = 1;
   223    223       if( rc==SQLITE_OK ){
   224    224         rc = fts3ExprNearTrim(pExpr);
   225    225       }
   226    226     }
   227    227   
   228    228     return rc;
................................................................................
   257    257   static int fts3ExprLoadDoclists(
   258    258     Fts3Cursor *pCsr,               /* Fts3 cursor for current query */
   259    259     int *pnPhrase,                  /* OUT: Number of phrases in query */
   260    260     int *pnToken                    /* OUT: Number of tokens in query */
   261    261   ){
   262    262     int rc;                         /* Return Code */
   263    263     LoadDoclistCtx sCtx = {0,0,0};  /* Context for fts3ExprIterate() */
   264         -  sCtx.pTab = (Fts3Table *)pCsr->base.pVtab;
          264  +  sCtx.pCsr = pCsr;
   265    265     rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx);
   266    266     if( rc==SQLITE_OK ){
   267    267       (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
   268    268     }
   269    269     if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   270    270     if( pnToken ) *pnToken = sCtx.nToken;
   271    271     return rc;
................................................................................
   788    788   */
   789    789   static int fts3ExprGlobalMatchinfoCb(
   790    790     Fts3Expr *pExpr,                /* Phrase expression node */
   791    791     int iPhrase,                    /* Phrase number (numbered from zero) */
   792    792     void *pCtx                      /* Pointer to MatchInfo structure */
   793    793   ){
   794    794     MatchInfo *p = (MatchInfo *)pCtx;
   795         -  char *pCsr;
          795  +  Fts3Cursor *pCsr = p->pCursor;
          796  +  char *pIter;
   796    797     char *pEnd;
          798  +  char *pFree = 0;
   797    799     const int iStart = 2 + (iPhrase * p->nCol * 3) + 1;
   798    800   
   799    801     assert( pExpr->isLoaded );
          802  +  assert( pExpr->eType==FTSQUERY_PHRASE );
          803  +
          804  +  if( pCsr->pDeferred ){
          805  +    Fts3Phrase *pPhrase = pExpr->pPhrase;
          806  +    int ii;
          807  +    for(ii=0; ii<pPhrase->nToken; ii++){
          808  +      if( pPhrase->aToken[ii].bFulltext ) break;
          809  +    }
          810  +    if( ii<pPhrase->nToken ){
          811  +      int nFree = 0;
          812  +      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
          813  +      if( rc!=SQLITE_OK ) return rc;
          814  +      pIter = pFree;
          815  +      pEnd = &pFree[nFree];
          816  +    }else{
          817  +      int nDoc = p->aMatchinfo[2 + 3*p->nCol*p->aMatchinfo[0]];
          818  +      for(ii=0; ii<p->nCol; ii++){
          819  +        p->aMatchinfo[iStart + ii*3] = nDoc;
          820  +        p->aMatchinfo[iStart + ii*3 + 1] = nDoc;
          821  +      }
          822  +      return SQLITE_OK;
          823  +    }
          824  +  }else{
          825  +    pIter = pExpr->aDoclist;
          826  +    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
          827  +  }
   800    828   
   801    829     /* Fill in the global hit count matrix row for this phrase. */
   802         -  pCsr = pExpr->aDoclist;
   803         -  pEnd = &pExpr->aDoclist[pExpr->nDoclist];
   804         -  while( pCsr<pEnd ){
   805         -    while( *pCsr++ & 0x80 );      /* Skip past docid. */
   806         -    fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 1);
          830  +  while( pIter<pEnd ){
          831  +    while( *pIter++ & 0x80 );      /* Skip past docid. */
          832  +    fts3LoadColumnlistCounts(&pIter, &p->aMatchinfo[iStart], 1);
   807    833     }
   808    834   
          835  +  sqlite3_free(pFree);
   809    836     return SQLITE_OK;
   810    837   }
   811    838   
   812    839   /*
   813    840   ** fts3ExprIterate() callback used to collect the "local" matchinfo stats
   814    841   ** for a single query. The "local" stats are those elements of the matchinfo
   815    842   ** array that are different for each row returned by the query.
................................................................................
   870    897   
   871    898       sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
   872    899       if( !sInfo.aMatchinfo ){ 
   873    900         return SQLITE_NOMEM;
   874    901       }
   875    902       memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo);
   876    903   
   877         -
   878    904       /* First element of match-info is the number of phrases in the query */
   879    905       sInfo.aMatchinfo[0] = nPhrase;
   880    906       sInfo.aMatchinfo[1] = sInfo.nCol;
   881         -    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
   882    907       if( pTab->bHasDocsize ){
   883    908         int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
   884    909         rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]);
   885    910       }
          911  +    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
   886    912       pCsr->aMatchinfo = sInfo.aMatchinfo;
   887    913       pCsr->isMatchinfoNeeded = 1;
   888    914     }
   889    915   
   890    916     sInfo.aMatchinfo = pCsr->aMatchinfo;
   891    917     if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
   892    918       (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo);
................................................................................
   988   1014     for(i=0; i<nSnippet && rc==SQLITE_OK; i++){
   989   1015       rc = fts3SnippetText(pCsr, &aSnippet[i], 
   990   1016           i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
   991   1017       );
   992   1018     }
   993   1019   
   994   1020    snippet_out:
         1021  +  sqlite3Fts3SegmentsClose(pTab);
   995   1022     if( rc!=SQLITE_OK ){
   996   1023       sqlite3_result_error_code(pCtx, rc);
   997   1024       sqlite3_free(res.z);
   998   1025     }else{
   999   1026       sqlite3_result_text(pCtx, res.z, -1, sqlite3_free);
  1000   1027     }
  1001   1028   }
................................................................................
  1167   1194       pMod->xClose(pC);
  1168   1195       if( rc!=SQLITE_OK ) goto offsets_out;
  1169   1196     }
  1170   1197   
  1171   1198    offsets_out:
  1172   1199     sqlite3_free(sCtx.aTerm);
  1173   1200     assert( rc!=SQLITE_DONE );
         1201  +  sqlite3Fts3SegmentsClose(pTab);
  1174   1202     if( rc!=SQLITE_OK ){
  1175   1203       sqlite3_result_error_code(pCtx,  rc);
  1176   1204       sqlite3_free(res.z);
  1177   1205     }else{
  1178   1206       sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free);
  1179   1207     }
  1180   1208     return;
................................................................................
  1186   1214   void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){
  1187   1215     int rc;
  1188   1216     if( !pCsr->pExpr ){
  1189   1217       sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
  1190   1218       return;
  1191   1219     }
  1192   1220     rc = fts3GetMatchinfo(pCsr);
         1221  +  sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab );
  1193   1222     if( rc!=SQLITE_OK ){
  1194   1223       sqlite3_result_error_code(pContext, rc);
  1195   1224     }else{
  1196   1225       Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
  1197   1226       int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3);
  1198   1227       if( pTab->bHasDocsize ){
  1199   1228         n += sizeof(u32)*(1 + 2*pTab->nColumn);
  1200   1229       }
  1201   1230       sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
  1202   1231     }
  1203   1232   }
  1204   1233   
  1205   1234   #endif

Changes to ext/fts3/fts3_write.c.

    38     38     char *aData;
    39     39     int nSpace;
    40     40     sqlite3_int64 iLastDocid;
    41     41     sqlite3_int64 iLastCol;
    42     42     sqlite3_int64 iLastPos;
    43     43   };
    44     44   
           45  +
           46  +/*
           47  +** Each cursor has a (possibly empty) linked list of the following objects.
           48  +*/
           49  +struct Fts3DeferredToken {
           50  +  Fts3PhraseToken *pToken;        /* Pointer to corresponding expr token */
           51  +  int iCol;                       /* Column token must occur in */
           52  +  Fts3DeferredToken *pNext;       /* Next in list of deferred tokens */
           53  +  PendingList *pList;             /* Doclist is assembled here */
           54  +};
           55  +
    45     56   /*
    46     57   ** An instance of this structure is used to iterate through the terms on
    47     58   ** a contiguous set of segment b-tree leaf nodes. Although the details of
    48     59   ** this structure are only manipulated by code in this file, opaque handles
    49     60   ** of type Fts3SegReader* are also used by code in fts3.c to iterate through
    50     61   ** terms when querying the full-text index. See functions:
    51     62   **
    52     63   **   sqlite3Fts3SegReaderNew()
    53     64   **   sqlite3Fts3SegReaderFree()
           65  +**   sqlite3Fts3SegReaderCost()
    54     66   **   sqlite3Fts3SegReaderIterate()
    55     67   **
    56     68   ** Methods used to manipulate Fts3SegReader structures:
    57     69   **
    58     70   **   fts3SegReaderNext()
    59     71   **   fts3SegReaderFirstDocid()
    60     72   **   fts3SegReaderNextDocid()
    61     73   */
    62     74   struct Fts3SegReader {
    63     75     int iIdx;                       /* Index within level, or 0x7FFFFFFF for PT */
    64         -  sqlite3_int64 iStartBlock;
    65         -  sqlite3_int64 iEndBlock;
    66         -  sqlite3_stmt *pStmt;            /* SQL Statement to access leaf nodes */
           76  +
           77  +  sqlite3_int64 iStartBlock;      /* Rowid of first leaf block to traverse */
           78  +  sqlite3_int64 iLeafEndBlock;    /* Rowid of final leaf block to traverse */
           79  +  sqlite3_int64 iEndBlock;        /* Rowid of final block in segment (or 0) */
           80  +  sqlite3_int64 iCurrentBlock;    /* Current leaf block (or 0) */
           81  +
    67     82     char *aNode;                    /* Pointer to node data (or NULL) */
    68     83     int nNode;                      /* Size of buffer at aNode (or 0) */
    69         -  int nTermAlloc;                 /* Allocated size of zTerm buffer */
    70     84     Fts3HashElem **ppNextElem;
    71     85   
    72     86     /* Variables set by fts3SegReaderNext(). These may be read directly
    73     87     ** by the caller. They are valid from the time SegmentReaderNew() returns
    74     88     ** until SegmentReaderNext() returns something other than SQLITE_OK
    75     89     ** (i.e. SQLITE_DONE).
    76     90     */
    77     91     int nTerm;                      /* Number of bytes in current term */
    78     92     char *zTerm;                    /* Pointer to current term */
           93  +  int nTermAlloc;                 /* Allocated size of zTerm buffer */
    79     94     char *aDoclist;                 /* Pointer to doclist of current entry */
    80     95     int nDoclist;                   /* Size of doclist in current entry */
    81     96   
    82     97     /* The following variables are used to iterate through the current doclist */
    83     98     char *pOffsetList;
    84     99     sqlite3_int64 iDocid;
    85    100   };
    86    101   
    87    102   #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
          103  +#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
    88    104   
    89    105   /*
    90    106   ** An instance of this structure is used to create a segment b-tree in the
    91    107   ** database. The internal details of this type are only accessed by the
    92    108   ** following functions:
    93    109   **
    94    110   **   fts3SegWriterAdd()
................................................................................
   149    165   #define SQL_SELECT_LEVEL              12
   150    166   #define SQL_SELECT_ALL_LEVEL          13
   151    167   #define SQL_SELECT_LEVEL_COUNT        14
   152    168   #define SQL_SELECT_SEGDIR_COUNT_MAX   15
   153    169   #define SQL_DELETE_SEGDIR_BY_LEVEL    16
   154    170   #define SQL_DELETE_SEGMENTS_RANGE     17
   155    171   #define SQL_CONTENT_INSERT            18
   156         -#define SQL_GET_BLOCK                 19
   157         -#define SQL_DELETE_DOCSIZE            20
   158         -#define SQL_REPLACE_DOCSIZE           21
   159         -#define SQL_SELECT_DOCSIZE            22
   160         -#define SQL_SELECT_DOCTOTAL           23
   161         -#define SQL_REPLACE_DOCTOTAL          24
          172  +#define SQL_DELETE_DOCSIZE            19
          173  +#define SQL_REPLACE_DOCSIZE           20
          174  +#define SQL_SELECT_DOCSIZE            21
          175  +#define SQL_SELECT_DOCTOTAL           22
          176  +#define SQL_REPLACE_DOCTOTAL          23
   162    177   
   163    178   /*
   164    179   ** This function is used to obtain an SQLite prepared statement handle
   165    180   ** for the statement identified by the second argument. If successful,
   166    181   ** *pp is set to the requested statement handle and SQLITE_OK returned.
   167    182   ** Otherwise, an SQLite error code is returned and *pp is set to 0.
   168    183   **
................................................................................
   199    214   
   200    215   /* 14 */  "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
   201    216   /* 15 */  "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
   202    217   
   203    218   /* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
   204    219   /* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
   205    220   /* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%z)",
   206         -/* 19 */  "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
   207         -/* 20 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
   208         -/* 21 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
   209         -/* 22 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
   210         -/* 23 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
   211         -/* 24 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
          221  +/* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
          222  +/* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
          223  +/* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
          224  +/* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
          225  +/* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
   212    226     };
   213    227     int rc = SQLITE_OK;
   214    228     sqlite3_stmt *pStmt;
   215    229   
   216    230     assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
   217    231     assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
   218    232     
................................................................................
   279    293       sqlite3_step(pStmt);
   280    294       rc = sqlite3_reset(pStmt);
   281    295     }
   282    296     *pRC = rc;
   283    297   }
   284    298   
   285    299   
   286         -/*
   287         -** Read a single block from the %_segments table. If the specified block
   288         -** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO 
   289         -** etc.) occurs, return the appropriate SQLite error code.
   290         -**
   291         -** Otherwise, if successful, set *pzBlock to point to a buffer containing
   292         -** the block read from the database, and *pnBlock to the size of the read
   293         -** block in bytes.
   294         -**
   295         -** WARNING: The returned buffer is only valid until the next call to 
   296         -** sqlite3Fts3ReadBlock().
   297         -*/
   298         -int sqlite3Fts3ReadBlock(
   299         -  Fts3Table *p,
   300         -  sqlite3_int64 iBlock,
   301         -  char const **pzBlock,
   302         -  int *pnBlock
   303         -){
   304         -  sqlite3_stmt *pStmt;
   305         -  int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
   306         -  if( rc!=SQLITE_OK ) return rc;
   307         -  sqlite3_reset(pStmt);
   308         -
   309         -  if( pzBlock ){
   310         -    sqlite3_bind_int64(pStmt, 1, iBlock);
   311         -    rc = sqlite3_step(pStmt); 
   312         -    if( rc!=SQLITE_ROW ){
   313         -      return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc);
   314         -    }
   315         -  
   316         -    *pnBlock = sqlite3_column_bytes(pStmt, 0);
   317         -    *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
   318         -    if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
   319         -      return SQLITE_CORRUPT;
   320         -    }
   321         -  }
   322         -  return SQLITE_OK;
   323         -}
   324         -
   325    300   /*
   326    301   ** This function ensures that the caller has obtained a shared-cache
   327    302   ** table-lock on the %_content table. This is required before reading
   328    303   ** data from the fts3 table. If this lock is not acquired first, then
   329    304   ** the caller may end up holding read-locks on the %_segments and %_segdir
   330    305   ** tables, but no read-lock on the %_content table. If this happens 
   331    306   ** a second connection will be able to write to the fts3 table, but
................................................................................
   486    461   ** Tokenize the nul-terminated string zText and add all tokens to the
   487    462   ** pending-terms hash-table. The docid used is that currently stored in
   488    463   ** p->iPrevDocid, and the column is specified by argument iCol.
   489    464   **
   490    465   ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
   491    466   */
   492    467   static int fts3PendingTermsAdd(
   493         -  Fts3Table *p,          /* FTS table into which text will be inserted */
   494         -  const char *zText,     /* Text of document to be inseted */
   495         -  int iCol,              /* Column number into which text is inserted */
   496         -  u32 *pnWord            /* OUT: Number of tokens inserted */
          468  +  Fts3Table *p,                   /* Table into which text will be inserted */
          469  +  const char *zText,              /* Text of document to be inserted */
          470  +  int iCol,                       /* Column into which text is being inserted */
          471  +  u32 *pnWord                     /* OUT: Number of tokens inserted */
   497    472   ){
   498    473     int rc;
   499    474     int iStart;
   500    475     int iEnd;
   501    476     int iPos;
   502    477     int nWord = 0;
   503    478   
................................................................................
   574    549       int rc = sqlite3Fts3PendingTermsFlush(p);
   575    550       if( rc!=SQLITE_OK ) return rc;
   576    551     }
   577    552     p->iPrevDocid = iDocid;
   578    553     return SQLITE_OK;
   579    554   }
   580    555   
          556  +/*
          557  +** Discard the contents of the pending-terms hash table. 
          558  +*/
   581    559   void sqlite3Fts3PendingTermsClear(Fts3Table *p){
   582    560     Fts3HashElem *pElem;
   583    561     for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
   584    562       sqlite3_free(fts3HashData(pElem));
   585    563     }
   586    564     fts3HashClear(&p->pendingTerms);
   587    565     p->nPendingData = 0;
................................................................................
   601    579       const char *zText = (const char *)sqlite3_value_text(apVal[i]);
   602    580       if( zText ){
   603    581         int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
   604    582         if( rc!=SQLITE_OK ){
   605    583           return rc;
   606    584         }
   607    585       }
          586  +    aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
   608    587     }
   609    588     return SQLITE_OK;
   610    589   }
   611    590   
   612    591   /*
   613    592   ** This function is called by the xUpdate() method for an INSERT operation.
   614    593   ** The apVal parameter is passed a copy of the apVal argument passed by
................................................................................
   698    677   }
   699    678   
   700    679   /*
   701    680   ** The first element in the apVal[] array is assumed to contain the docid
   702    681   ** (an integer) of a row about to be deleted. Remove all terms from the
   703    682   ** full-text index.
   704    683   */
   705         -static void fts3DeleteTerms(
          684  +static void fts3DeleteTerms( 
   706    685     int *pRC,               /* Result code */
   707    686     Fts3Table *p,           /* The FTS table to delete from */
   708    687     sqlite3_value **apVal,  /* apVal[] contains the docid to be deleted */
   709    688     u32 *aSz                /* Sizes of deleted document written here */
   710    689   ){
   711    690     int rc;
   712    691     sqlite3_stmt *pSelect;
................................................................................
   720    699           const char *zText = (const char *)sqlite3_column_text(pSelect, i);
   721    700           rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]);
   722    701           if( rc!=SQLITE_OK ){
   723    702             sqlite3_reset(pSelect);
   724    703             *pRC = rc;
   725    704             return;
   726    705           }
          706  +        aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
   727    707         }
   728    708       }
   729    709       rc = sqlite3_reset(pSelect);
   730    710     }else{
   731    711       sqlite3_reset(pSelect);
   732    712     }
   733    713     *pRC = rc;
................................................................................
   781    761       }else{
   782    762         *piIdx = iNext;
   783    763       }
   784    764     }
   785    765   
   786    766     return rc;
   787    767   }
          768  +
          769  +/*
          770  +** The %_segments table is declared as follows:
          771  +**
          772  +**   CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
          773  +**
          774  +** This function reads data from a single row of the %_segments table. The
          775  +** specific row is identified by the iBlockid parameter. If paBlob is not
          776  +** NULL, then a buffer is allocated using sqlite3_malloc() and populated
          777  +** with the contents of the blob stored in the "block" column of the 
          778  +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set
          779  +** to the size of the blob in bytes before returning.
          780  +**
          781  +** If an error occurs, or the table does not contain the specified row,
          782  +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If
          783  +** paBlob is non-NULL, then it is the responsibility of the caller to
          784  +** eventually free the returned buffer.
          785  +**
          786  +** This function may leave an open sqlite3_blob* handle in the
          787  +** Fts3Table.pSegments variable. This handle is reused by subsequent calls
          788  +** to this function. The handle may be closed by calling the
          789  +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy
          790  +** performance improvement, but the blob handle should always be closed
          791  +** before control is returned to the user (to prevent a lock being held
          792  +** on the database file for longer than necessary). Thus, any virtual table
          793  +** method (xFilter etc.) that may directly or indirectly call this function
          794  +** must call sqlite3Fts3SegmentsClose() before returning.
          795  +*/
          796  +int sqlite3Fts3ReadBlock(
          797  +  Fts3Table *p,                   /* FTS3 table handle */
          798  +  sqlite3_int64 iBlockid,         /* Access the row with blockid=$iBlockid */
          799  +  char **paBlob,                  /* OUT: Blob data in malloc'd buffer */
          800  +  int *pnBlob                     /* OUT: Size of blob data */
          801  +){
          802  +  int rc;                         /* Return code */
          803  +
          804  +  /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
          805  +  assert( pnBlob);
          806  +
          807  +  if( p->pSegments ){
          808  +    rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
          809  +  }else{
          810  +    if( 0==p->zSegmentsTbl ){
          811  +      p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
          812  +      if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
          813  +    }
          814  +    rc = sqlite3_blob_open(
          815  +       p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments
          816  +    );
          817  +  }
          818  +
          819  +  if( rc==SQLITE_OK ){
          820  +    int nByte = sqlite3_blob_bytes(p->pSegments);
          821  +    if( paBlob ){
          822  +      char *aByte = sqlite3_malloc(nByte);
          823  +      if( !aByte ){
          824  +        rc = SQLITE_NOMEM;
          825  +      }else{
          826  +        rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
          827  +        if( rc!=SQLITE_OK ){
          828  +          sqlite3_free(aByte);
          829  +          aByte = 0;
          830  +        }
          831  +      }
          832  +      *paBlob = aByte;
          833  +    }
          834  +    *pnBlob = nByte;
          835  +  }
          836  +
          837  +  return rc;
          838  +}
          839  +
          840  +/*
          841  +** Close the blob handle at p->pSegments, if it is open. See comments above
          842  +** the sqlite3Fts3ReadBlock() function for details.
          843  +*/
          844  +void sqlite3Fts3SegmentsClose(Fts3Table *p){
          845  +  sqlite3_blob_close(p->pSegments);
          846  +  p->pSegments = 0;
          847  +}
   788    848   
   789    849   /*
   790    850   ** Move the iterator passed as the first argument to the next term in the
   791    851   ** segment. If successful, SQLITE_OK is returned. If there is no next term,
   792    852   ** SQLITE_DONE. Otherwise, an SQLite error code.
   793    853   */
   794         -static int fts3SegReaderNext(Fts3SegReader *pReader){
          854  +static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
   795    855     char *pNext;                    /* Cursor variable */
   796    856     int nPrefix;                    /* Number of bytes in term prefix */
   797    857     int nSuffix;                    /* Number of bytes in term suffix */
   798    858   
   799    859     if( !pReader->aDoclist ){
   800    860       pNext = pReader->aNode;
   801    861     }else{
   802    862       pNext = &pReader->aDoclist[pReader->nDoclist];
   803    863     }
   804    864   
   805    865     if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
   806         -    int rc;
          866  +    int rc;                       /* Return code from Fts3ReadBlock() */
          867  +
   807    868       if( fts3SegReaderIsPending(pReader) ){
   808    869         Fts3HashElem *pElem = *(pReader->ppNextElem);
   809    870         if( pElem==0 ){
   810    871           pReader->aNode = 0;
   811    872         }else{
   812    873           PendingList *pList = (PendingList *)fts3HashData(pElem);
   813    874           pReader->zTerm = (char *)fts3HashKey(pElem);
................................................................................
   815    876           pReader->nNode = pReader->nDoclist = pList->nData + 1;
   816    877           pReader->aNode = pReader->aDoclist = pList->aData;
   817    878           pReader->ppNextElem++;
   818    879           assert( pReader->aNode );
   819    880         }
   820    881         return SQLITE_OK;
   821    882       }
   822         -    if( !pReader->pStmt ){
   823         -      pReader->aNode = 0;
          883  +
          884  +    if( !fts3SegReaderIsRootOnly(pReader) ){
          885  +      sqlite3_free(pReader->aNode);
          886  +    }
          887  +    pReader->aNode = 0;
          888  +
          889  +    /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf 
          890  +    ** blocks have already been traversed.  */
          891  +    assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
          892  +    if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
   824    893         return SQLITE_OK;
   825    894       }
   826         -    rc = sqlite3_step(pReader->pStmt);
   827         -    if( rc!=SQLITE_ROW ){
   828         -      pReader->aNode = 0;
   829         -      return (rc==SQLITE_DONE ? SQLITE_OK : rc);
   830         -    }
   831         -    pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
   832         -    pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
          895  +
          896  +    rc = sqlite3Fts3ReadBlock(
          897  +        p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
          898  +    );
          899  +    if( rc!=SQLITE_OK ) return rc;
   833    900       pNext = pReader->aNode;
   834    901     }
   835    902     
   836    903     pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
   837    904     pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
   838    905   
   839    906     if( nPrefix+nSuffix>pReader->nTermAlloc ){
................................................................................
   845    912       pReader->zTerm = zNew;
   846    913       pReader->nTermAlloc = nNew;
   847    914     }
   848    915     memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
   849    916     pReader->nTerm = nPrefix+nSuffix;
   850    917     pNext += nSuffix;
   851    918     pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
   852         -  assert( pNext<&pReader->aNode[pReader->nNode] );
   853    919     pReader->aDoclist = pNext;
   854    920     pReader->pOffsetList = 0;
          921  +
          922  +  /* Check that the doclist does not appear to extend past the end of the
          923  +  ** b-tree node. And that the final byte of the doclist is either an 0x00 
          924  +  ** or 0x01. If either of these statements is untrue, then the data structure 
          925  +  ** is corrupt.
          926  +  */
          927  +  if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] 
          928  +   || (pReader->aDoclist[pReader->nDoclist-1]&0xFE)!=0
          929  +  ){
          930  +    return SQLITE_CORRUPT;
          931  +  }
   855    932     return SQLITE_OK;
   856    933   }
   857    934   
   858    935   /*
   859    936   ** Set the SegReader to point to the first docid in the doclist associated
   860    937   ** with the current term.
   861    938   */
................................................................................
   909    986       pReader->pOffsetList = 0;
   910    987     }else{
   911    988       sqlite3_int64 iDelta;
   912    989       pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
   913    990       pReader->iDocid += iDelta;
   914    991     }
   915    992   }
          993  +
          994  +/*
          995  +** This function is called to estimate the amount of data that will be 
          996  +** loaded from the disk If SegReaderIterate() is called on this seg-reader,
          997  +** in units of average document size.
          998  +** 
          999  +** This can be used as follows: If the caller has a small doclist that 
         1000  +** contains references to N documents, and is considering merging it with
         1001  +** a large doclist (size X "average documents"), it may opt not to load
         1002  +** the large doclist if X>N.
         1003  +*/
         1004  +int sqlite3Fts3SegReaderCost(
         1005  +  Fts3Cursor *pCsr,               /* FTS3 cursor handle */
         1006  +  Fts3SegReader *pReader,         /* Segment-reader handle */
         1007  +  int *pnCost                     /* IN/OUT: Number of bytes read */
         1008  +){
         1009  +  Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
         1010  +  int rc = SQLITE_OK;             /* Return code */
         1011  +  int nCost = 0;                  /* Cost in bytes to return */
         1012  +  int pgsz = p->nPgsz;            /* Database page size */
         1013  +
         1014  +  /* If this seg-reader is reading the pending-terms table, or if all data
         1015  +  ** for the segment is stored on the root page of the b-tree, then the cost
         1016  +  ** is zero. In this case all required data is already in main memory.
         1017  +  */
         1018  +  if( p->bHasDocsize 
         1019  +   && !fts3SegReaderIsPending(pReader) 
         1020  +   && !fts3SegReaderIsRootOnly(pReader) 
         1021  +  ){
         1022  +    int nBlob = 0;
         1023  +    sqlite3_int64 iBlock;
         1024  +
         1025  +    if( pCsr->nRowAvg==0 ){
         1026  +      /* The average document size, which is required to calculate the cost
         1027  +      ** of each doclist, has not yet been determined. Read the required 
         1028  +      ** data from the %_stat table to calculate it.
         1029  +      **
         1030  +      ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 
         1031  +      ** varints, where nCol is the number of columns in the FTS3 table.
         1032  +      ** The first varint is the number of documents currently stored in
         1033  +      ** the table. The following nCol varints contain the total amount of
         1034  +      ** data stored in all rows of each column of the table, from left
         1035  +      ** to right.
         1036  +      */
         1037  +      sqlite3_stmt *pStmt;
         1038  +      rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
         1039  +      if( rc ) return rc;
         1040  +      if( sqlite3_step(pStmt)==SQLITE_ROW ){
         1041  +        sqlite3_int64 nDoc = 0;
         1042  +        sqlite3_int64 nByte = 0;
         1043  +        const char *a = sqlite3_column_blob(pStmt, 0);
         1044  +        if( a ){
         1045  +          const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
         1046  +          a += sqlite3Fts3GetVarint(a, &nDoc);
         1047  +          while( a<pEnd ){
         1048  +            a += sqlite3Fts3GetVarint(a, &nByte);
         1049  +          }
         1050  +        }
         1051  +
         1052  +        pCsr->nRowAvg = (((nByte / nDoc) + pgsz - 1) / pgsz);
         1053  +      }
         1054  +      rc = sqlite3_reset(pStmt);
         1055  +      if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
         1056  +    }
         1057  +
         1058  +    /* Assume that a blob flows over onto overflow pages if it is larger
         1059  +    ** than (pgsz-35) bytes in size (the file-format documentation
         1060  +    ** confirms this).
         1061  +    */
         1062  +    for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
         1063  +      rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
         1064  +      if( rc!=SQLITE_OK ) break;
         1065  +      if( (nBlob+35)>pgsz ){
         1066  +        int nOvfl = (nBlob + 34)/pgsz;
         1067  +        nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
         1068  +      }
         1069  +    }
         1070  +  }
         1071  +
         1072  +  *pnCost += nCost;
         1073  +  return rc;
         1074  +}
   916   1075   
   917   1076   /*
   918   1077   ** Free all allocations associated with the iterator passed as the 
   919   1078   ** second argument.
   920   1079   */
   921   1080   void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
   922         -  if( pReader ){
   923         -    if( pReader->pStmt ){
   924         -      /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
   925         -      ** so that it can be reused when required by another query.
   926         -      */
   927         -      assert( p->nLeavesStmt<p->nLeavesTotal );
   928         -      sqlite3_reset(pReader->pStmt);
   929         -      p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
         1081  +  if( pReader && !fts3SegReaderIsPending(pReader) ){
         1082  +    sqlite3_free(pReader->zTerm);
         1083  +    if( !fts3SegReaderIsRootOnly(pReader) ){
         1084  +      sqlite3_free(pReader->aNode);
   930   1085       }
   931         -    if( !fts3SegReaderIsPending(pReader) ){
   932         -      sqlite3_free(pReader->zTerm);
   933         -    }
   934         -    sqlite3_free(pReader);
   935   1086     }
         1087  +  sqlite3_free(pReader);
   936   1088   }
   937   1089   
   938   1090   /*
   939   1091   ** Allocate a new SegReader object.
   940   1092   */
   941   1093   int sqlite3Fts3SegReaderNew(
   942   1094     Fts3Table *p,                   /* Virtual table handle */
................................................................................
   948   1100     int nRoot,                      /* Size of buffer containing root node */
   949   1101     Fts3SegReader **ppReader        /* OUT: Allocated Fts3SegReader */
   950   1102   ){
   951   1103     int rc = SQLITE_OK;             /* Return code */
   952   1104     Fts3SegReader *pReader;         /* Newly allocated SegReader object */
   953   1105     int nExtra = 0;                 /* Bytes to allocate segment root node */
   954   1106   
         1107  +  assert( iStartLeaf<=iEndLeaf );
   955   1108     if( iStartLeaf==0 ){
   956   1109       nExtra = nRoot;
   957   1110     }
   958   1111   
   959   1112     pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
   960   1113     if( !pReader ){
   961   1114       return SQLITE_NOMEM;
   962   1115     }
   963   1116     memset(pReader, 0, sizeof(Fts3SegReader));
   964         -  pReader->iStartBlock = iStartLeaf;
   965   1117     pReader->iIdx = iAge;
         1118  +  pReader->iStartBlock = iStartLeaf;
         1119  +  pReader->iLeafEndBlock = iEndLeaf;
   966   1120     pReader->iEndBlock = iEndBlock;
   967   1121   
   968   1122     if( nExtra ){
   969   1123       /* The entire segment is stored in the root node. */
   970   1124       pReader->aNode = (char *)&pReader[1];
   971   1125       pReader->nNode = nRoot;
   972   1126       memcpy(pReader->aNode, zRoot, nRoot);
   973   1127     }else{
   974         -    /* If the text of the SQL statement to iterate through a contiguous
   975         -    ** set of entries in the %_segments table has not yet been composed,
   976         -    ** compose it now.
   977         -    */
   978         -    if( !p->zSelectLeaves ){
   979         -      p->zSelectLeaves = sqlite3_mprintf(
   980         -          "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
   981         -          "ORDER BY blockid", p->zDb, p->zName
   982         -      );
   983         -      if( !p->zSelectLeaves ){
   984         -        rc = SQLITE_NOMEM;
   985         -        goto finished;
   986         -      }
   987         -    }
         1128  +    pReader->iCurrentBlock = iStartLeaf-1;
         1129  +  }
         1130  +  rc = fts3SegReaderNext(p, pReader);
   988   1131   
   989         -    /* If there are no free statements in the aLeavesStmt[] array, prepare
   990         -    ** a new statement now. Otherwise, reuse a prepared statement from
   991         -    ** aLeavesStmt[].
   992         -    */
   993         -    if( p->nLeavesStmt==0 ){
   994         -      if( p->nLeavesTotal==p->nLeavesAlloc ){
   995         -        int nNew = p->nLeavesAlloc + 16;
   996         -        sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
   997         -            p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
   998         -        );
   999         -        if( !aNew ){
  1000         -          rc = SQLITE_NOMEM;
  1001         -          goto finished;
  1002         -        }
  1003         -        p->nLeavesAlloc = nNew;
  1004         -        p->aLeavesStmt = aNew;
  1005         -      }
  1006         -      rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
  1007         -      if( rc!=SQLITE_OK ){
  1008         -        goto finished;
  1009         -      }
  1010         -      p->nLeavesTotal++;
  1011         -    }else{
  1012         -      pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
  1013         -    }
  1014         -
  1015         -    /* Bind the start and end leaf blockids to the prepared SQL statement. */
  1016         -    sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
  1017         -    sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
  1018         -  }
  1019         -  rc = fts3SegReaderNext(pReader);
  1020         -
  1021         - finished:
  1022   1132     if( rc==SQLITE_OK ){
  1023   1133       *ppReader = pReader;
  1024   1134     }else{
  1025   1135       sqlite3Fts3SegReaderFree(p, pReader);
  1026   1136     }
  1027   1137     return rc;
  1028   1138   }
................................................................................
  1109   1219       if( !pReader ){
  1110   1220         rc = SQLITE_NOMEM;
  1111   1221       }else{
  1112   1222         memset(pReader, 0, nByte);
  1113   1223         pReader->iIdx = 0x7FFFFFFF;
  1114   1224         pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
  1115   1225         memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
  1116         -      fts3SegReaderNext(pReader);
         1226  +      fts3SegReaderNext(p, pReader);
  1117   1227       }
  1118   1228     }
  1119   1229   
  1120   1230     if( isPrefix ){
  1121   1231       sqlite3_free(aElem);
  1122   1232     }
  1123   1233     *ppReader = pReader;
................................................................................
  1351   1461   }
  1352   1462   
  1353   1463   /*
  1354   1464   ** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
  1355   1465   ** (according to memcmp) than the previous term.
  1356   1466   */
  1357   1467   static int fts3NodeAddTerm(
  1358         -  Fts3Table *p,               /* Virtual table handle */
         1468  +  Fts3Table *p,                   /* Virtual table handle */
  1359   1469     SegmentNode **ppTree,           /* IN/OUT: SegmentNode handle */ 
  1360   1470     int isCopyTerm,                 /* True if zTerm/nTerm is transient */
  1361   1471     const char *zTerm,              /* Pointer to buffer containing term */
  1362   1472     int nTerm                       /* Size of term in bytes */
  1363   1473   ){
  1364   1474     SegmentNode *pTree = *ppTree;
  1365   1475     int rc;
................................................................................
  1987   2097     */
  1988   2098     if( pFilter->zTerm ){
  1989   2099       int nTerm = pFilter->nTerm;
  1990   2100       const char *zTerm = pFilter->zTerm;
  1991   2101       for(i=0; i<nSegment; i++){
  1992   2102         Fts3SegReader *pSeg = apSegment[i];
  1993   2103         while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
  1994         -        rc = fts3SegReaderNext(pSeg);
         2104  +        rc = fts3SegReaderNext(p, pSeg);
  1995   2105           if( rc!=SQLITE_OK ) goto finished; }
  1996   2106       }
  1997   2107     }
  1998   2108   
  1999   2109     fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
  2000   2110     while( apSegment[0]->aNode ){
  2001   2111       int nTerm = apSegment[0]->nTerm;
................................................................................
  2098   2208       ** term (if such a term exists in the index) has already been made.
  2099   2209       */
  2100   2210       if( pFilter->zTerm && !isPrefix ){
  2101   2211         goto finished;
  2102   2212       }
  2103   2213   
  2104   2214       for(i=0; i<nMerge; i++){
  2105         -      rc = fts3SegReaderNext(apSegment[i]);
         2215  +      rc = fts3SegReaderNext(p, apSegment[i]);
  2106   2216         if( rc!=SQLITE_OK ) goto finished;
  2107   2217       }
  2108   2218       fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
  2109   2219     }
  2110   2220   
  2111   2221    finished:
  2112   2222     sqlite3_free(aBuffer);
................................................................................
  2124   2234   ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, 
  2125   2235   ** an SQLite error code is returned.
  2126   2236   */
  2127   2237   static int fts3SegmentMerge(Fts3Table *p, int iLevel){
  2128   2238     int i;                          /* Iterator variable */
  2129   2239     int rc;                         /* Return code */
  2130   2240     int iIdx;                       /* Index of new segment */
  2131         -  int iNewLevel;                  /* Level to create new segment at */
         2241  +  int iNewLevel = 0;              /* Level to create new segment at */
  2132   2242     sqlite3_stmt *pStmt = 0;
  2133   2243     SegmentWriter *pWriter = 0;
  2134   2244     int nSegment = 0;               /* Number of segments being merged */
  2135   2245     Fts3SegReader **apSegment = 0;  /* Array of Segment iterators */
  2136   2246     Fts3SegReader *pPending = 0;    /* Iterator for pending-terms */
  2137   2247     Fts3SegFilter filter;           /* Segment term filter condition */
  2138   2248   
................................................................................
  2413   2523     sqlite3_bind_int64(pStmt, 1, p->iPrevDocid);
  2414   2524     sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free);
  2415   2525     sqlite3_step(pStmt);
  2416   2526     *pRC = sqlite3_reset(pStmt);
  2417   2527   }
  2418   2528   
  2419   2529   /*
  2420         -** Update the 0 record of the %_stat table so that it holds a blob
  2421         -** which contains the document count followed by the cumulative
  2422         -** document sizes for all columns.
         2530  +** Record 0 of the %_stat table contains a blob consisting of N varints,
         2531  +** where N is the number of user defined columns in the fts3 table plus
         2532  +** two. If nCol is the number of user defined columns, then values of the 
         2533  +** varints are set as follows:
         2534  +**
         2535  +**   Varint 0:       Total number of rows in the table.
         2536  +**
         2537  +**   Varint 1..nCol: For each column, the total number of tokens stored in
         2538  +**                   the column for all rows of the table.
         2539  +**
         2540  +**   Varint 1+nCol:  The total size, in bytes, of all text values in all
         2541  +**                   columns of all rows of the table.
         2542  +**
  2423   2543   */
  2424   2544   static void fts3UpdateDocTotals(
  2425         -  int *pRC,       /* The result code */
  2426         -  Fts3Table *p,   /* Table being updated */
  2427         -  u32 *aSzIns,    /* Size increases */
  2428         -  u32 *aSzDel,    /* Size decreases */
  2429         -  int nChng       /* Change in the number of documents */
         2545  +  int *pRC,                       /* The result code */
         2546  +  Fts3Table *p,                   /* Table being updated */
         2547  +  u32 *aSzIns,                    /* Size increases */
         2548  +  u32 *aSzDel,                    /* Size decreases */
         2549  +  int nChng                       /* Change in the number of documents */
  2430   2550   ){
  2431   2551     char *pBlob;             /* Storage for BLOB written into %_stat */
  2432   2552     int nBlob;               /* Size of BLOB written into %_stat */
  2433   2553     u32 *a;                  /* Array of integers that becomes the BLOB */
  2434   2554     sqlite3_stmt *pStmt;     /* Statement for reading and writing */
  2435   2555     int i;                   /* Loop counter */
  2436   2556     int rc;                  /* Result code from subfunctions */
  2437   2557   
         2558  +  const int nStat = p->nColumn+2;
         2559  +
  2438   2560     if( *pRC ) return;
  2439         -  a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) );
         2561  +  a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
  2440   2562     if( a==0 ){
  2441   2563       *pRC = SQLITE_NOMEM;
  2442   2564       return;
  2443   2565     }
  2444         -  pBlob = (char*)&a[p->nColumn+1];
         2566  +  pBlob = (char*)&a[nStat];
  2445   2567     rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
  2446   2568     if( rc ){
  2447   2569       sqlite3_free(a);
  2448   2570       *pRC = rc;
  2449   2571       return;
  2450   2572     }
  2451   2573     if( sqlite3_step(pStmt)==SQLITE_ROW ){
  2452         -    fts3DecodeIntArray(p->nColumn+1, a,
         2574  +    fts3DecodeIntArray(nStat, a,
  2453   2575            sqlite3_column_blob(pStmt, 0),
  2454   2576            sqlite3_column_bytes(pStmt, 0));
  2455   2577     }else{
  2456         -    memset(a, 0, sizeof(u32)*(p->nColumn+1) );
         2578  +    memset(a, 0, sizeof(u32)*(nStat) );
  2457   2579     }
  2458   2580     sqlite3_reset(pStmt);
  2459   2581     if( nChng<0 && a[0]<(u32)(-nChng) ){
  2460   2582       a[0] = 0;
  2461   2583     }else{
  2462   2584       a[0] += nChng;
  2463   2585     }
  2464         -  for(i=0; i<p->nColumn; i++){
         2586  +  for(i=0; i<p->nColumn+1; i++){
  2465   2587       u32 x = a[i+1];
  2466   2588       if( x+aSzIns[i] < aSzDel[i] ){
  2467   2589         x = 0;
  2468   2590       }else{
  2469   2591         x = x + aSzIns[i] - aSzDel[i];
  2470   2592       }
  2471   2593       a[i+1] = x;
  2472   2594     }
  2473         -  fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob);
         2595  +  fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
  2474   2596     rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
  2475   2597     if( rc ){
  2476   2598       sqlite3_free(a);
  2477   2599       *pRC = rc;
  2478   2600       return;
  2479   2601     }
  2480   2602     sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
................................................................................
  2512   2634     }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
  2513   2635       p->nMaxPendingData = atoi(&zVal[11]);
  2514   2636       rc = SQLITE_OK;
  2515   2637   #endif
  2516   2638     }else{
  2517   2639       rc = SQLITE_ERROR;
  2518   2640     }
         2641  +
         2642  +  sqlite3Fts3SegmentsClose(p);
         2643  +  return rc;
         2644  +}
         2645  +
         2646  +/*
         2647  +** Return the deferred doclist associated with deferred token pDeferred.
         2648  +** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
         2649  +** been called to allocate and populate the doclist.
         2650  +*/
         2651  +char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
         2652  +  if( pDeferred->pList ){
         2653  +    *pnByte = pDeferred->pList->nData;
         2654  +    return pDeferred->pList->aData;
         2655  +  }
         2656  +  *pnByte = 0;
         2657  +  return 0;
         2658  +}
         2659  +
         2660  +/*
         2661  +** Helper fucntion for FreeDeferredDoclists(). This function removes all
         2662  +** references to deferred doclists from within the tree of Fts3Expr 
         2663  +** structures headed by 
         2664  +*/
         2665  +static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
         2666  +  if( pExpr ){
         2667  +    fts3DeferredDoclistClear(pExpr->pLeft);
         2668  +    fts3DeferredDoclistClear(pExpr->pRight);
         2669  +    if( pExpr->isLoaded ){
         2670  +      sqlite3_free(pExpr->aDoclist);
         2671  +      pExpr->isLoaded = 0;
         2672  +      pExpr->aDoclist = 0;
         2673  +      pExpr->nDoclist = 0;
         2674  +      pExpr->pCurrent = 0;
         2675  +      pExpr->iCurrent = 0;
         2676  +    }
         2677  +  }
         2678  +}
         2679  +
         2680  +/*
         2681  +** Delete all cached deferred doclists. Deferred doclists are cached
         2682  +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
         2683  +*/
         2684  +void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
         2685  +  Fts3DeferredToken *pDef;
         2686  +  for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
         2687  +    sqlite3_free(pDef->pList);
         2688  +    pDef->pList = 0;
         2689  +  }
         2690  +  if( pCsr->pDeferred ){
         2691  +    fts3DeferredDoclistClear(pCsr->pExpr);
         2692  +  }
         2693  +}
         2694  +
         2695  +/*
         2696  +** Free all entries in the pCsr->pDeffered list. Entries are added to 
         2697  +** this list using sqlite3Fts3DeferToken().
         2698  +*/
         2699  +void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
         2700  +  Fts3DeferredToken *pDef;
         2701  +  Fts3DeferredToken *pNext;
         2702  +  for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
         2703  +    pNext = pDef->pNext;
         2704  +    sqlite3_free(pDef->pList);
         2705  +    sqlite3_free(pDef);
         2706  +  }
         2707  +  pCsr->pDeferred = 0;
         2708  +}
         2709  +
         2710  +/*
         2711  +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list
         2712  +** based on the row that pCsr currently points to.
         2713  +**
         2714  +** A deferred-doclist is like any other doclist with position information
         2715  +** included, except that it only contains entries for a single row of the
         2716  +** table, not for all rows.
         2717  +*/
         2718  +int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
         2719  +  int rc = SQLITE_OK;             /* Return code */
         2720  +  if( pCsr->pDeferred ){
         2721  +    int i;                        /* Used to iterate through table columns */
         2722  +    sqlite3_int64 iDocid;         /* Docid of the row pCsr points to */
         2723  +    Fts3DeferredToken *pDef;      /* Used to iterate through deferred tokens */
         2724  +  
         2725  +    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
         2726  +    sqlite3_tokenizer *pT = p->pTokenizer;
         2727  +    sqlite3_tokenizer_module const *pModule = pT->pModule;
         2728  +   
         2729  +    assert( pCsr->isRequireSeek==0 );
         2730  +    iDocid = sqlite3_column_int64(pCsr->pStmt, 0);
         2731  +  
         2732  +    for(i=0; i<p->nColumn && rc==SQLITE_OK; i++){
         2733  +      const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
         2734  +      sqlite3_tokenizer_cursor *pTC = 0;
         2735  +  
         2736  +      rc = pModule->xOpen(pT, zText, -1, &pTC);
         2737  +      while( rc==SQLITE_OK ){
         2738  +        char const *zToken;       /* Buffer containing token */
         2739  +        int nToken;               /* Number of bytes in token */
         2740  +        int iDum1, iDum2;         /* Dummy variables */
         2741  +        int iPos;                 /* Position of token in zText */
         2742  +  
         2743  +        pTC->pTokenizer = pT;
         2744  +        rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
         2745  +        for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
         2746  +          Fts3PhraseToken *pPT = pDef->pToken;
         2747  +          if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
         2748  +           && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
         2749  +           && (0==memcmp(zToken, pPT->z, pPT->n))
         2750  +          ){
         2751  +            fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc);
         2752  +          }
         2753  +        }
         2754  +      }
         2755  +      if( pTC ) pModule->xClose(pTC);
         2756  +      if( rc==SQLITE_DONE ) rc = SQLITE_OK;
         2757  +    }
         2758  +  
         2759  +    for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
         2760  +      if( pDef->pList ){
         2761  +        rc = fts3PendingListAppendVarint(&pDef->pList, 0);
         2762  +      }
         2763  +    }
         2764  +  }
  2519   2765   
  2520   2766     return rc;
  2521   2767   }
         2768  +
         2769  +/*
         2770  +** Add an entry for token pToken to the pCsr->pDeferred list.
         2771  +*/
         2772  +int sqlite3Fts3DeferToken(
         2773  +  Fts3Cursor *pCsr,               /* Fts3 table cursor */
         2774  +  Fts3PhraseToken *pToken,        /* Token to defer */
         2775  +  int iCol                        /* Column that token must appear in (or -1) */
         2776  +){
         2777  +  Fts3DeferredToken *pDeferred;
         2778  +  pDeferred = sqlite3_malloc(sizeof(*pDeferred));
         2779  +  if( !pDeferred ){
         2780  +    return SQLITE_NOMEM;
         2781  +  }
         2782  +  memset(pDeferred, 0, sizeof(*pDeferred));
         2783  +  pDeferred->pToken = pToken;
         2784  +  pDeferred->pNext = pCsr->pDeferred; 
         2785  +  pDeferred->iCol = iCol;
         2786  +  pCsr->pDeferred = pDeferred;
         2787  +
         2788  +  assert( pToken->pDeferred==0 );
         2789  +  pToken->pDeferred = pDeferred;
         2790  +
         2791  +  return SQLITE_OK;
         2792  +}
         2793  +
  2522   2794   
  2523   2795   /*
  2524   2796   ** This function does the work for the xUpdate method of FTS3 virtual
  2525   2797   ** tables.
  2526   2798   */
  2527   2799   int sqlite3Fts3UpdateMethod(
  2528   2800     sqlite3_vtab *pVtab,            /* FTS3 vtab object */
................................................................................
  2534   2806     int rc = SQLITE_OK;             /* Return Code */
  2535   2807     int isRemove = 0;               /* True for an UPDATE or DELETE */
  2536   2808     sqlite3_int64 iRemove = 0;      /* Rowid removed by UPDATE or DELETE */
  2537   2809     u32 *aSzIns;                    /* Sizes of inserted documents */
  2538   2810     u32 *aSzDel;                    /* Sizes of deleted documents */
  2539   2811     int nChng = 0;                  /* Net change in number of documents */
  2540   2812   
         2813  +  assert( p->pSegments==0 );
  2541   2814   
  2542   2815     /* Allocate space to hold the change in document sizes */
  2543         -  aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
         2816  +  aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
  2544   2817     if( aSzIns==0 ) return SQLITE_NOMEM;
  2545         -  aSzDel = &aSzIns[p->nColumn];
  2546         -  memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2);
         2818  +  aSzDel = &aSzIns[p->nColumn+1];
         2819  +  memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
  2547   2820   
  2548   2821     /* If this is a DELETE or UPDATE operation, remove the old record. */
  2549   2822     if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  2550         -    int isEmpty;
         2823  +    int isEmpty = 0;
  2551   2824       rc = fts3IsEmpty(p, apVal, &isEmpty);
  2552   2825       if( rc==SQLITE_OK ){
  2553   2826         if( isEmpty ){
  2554   2827           /* Deleting this row means the whole table is empty. In this case
  2555   2828           ** delete the contents of all three tables and throw away any
  2556   2829           ** data in the pendingTerms hash table.
  2557   2830           */
................................................................................
  2589   2862     }
  2590   2863   
  2591   2864     if( p->bHasDocsize ){
  2592   2865       fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
  2593   2866     }
  2594   2867   
  2595   2868     sqlite3_free(aSzIns);
         2869  +  sqlite3Fts3SegmentsClose(p);
  2596   2870     return rc;
  2597   2871   }
  2598   2872   
  2599   2873   /* 
  2600   2874   ** Flush any data in the pending-terms hash table to disk. If successful,
  2601   2875   ** merge all segments in the database (including the new segment, if 
  2602   2876   ** there was any data to flush) into a single segment. 
................................................................................
  2612   2886           sqlite3Fts3PendingTermsClear(p);
  2613   2887         }
  2614   2888       }else{
  2615   2889         sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
  2616   2890         sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
  2617   2891       }
  2618   2892     }
         2893  +  sqlite3Fts3SegmentsClose(p);
  2619   2894     return rc;
  2620   2895   }
  2621   2896   
  2622   2897   #endif

Added ext/fts3/fts3speed.tcl.

            1  +
            2  +
            3  +#--------------------------------------------------------------------------
            4  +# This script contains several sub-programs used to test FTS3/FTS4 
            5  +# performance. It does not run the queries directly, but generates SQL
            6  +# scripts that can be run using the shell tool.
            7  +#
            8  +# The following cases are tested:
            9  +#
           10  +#   1. Inserting documents into an FTS3 table.
           11  +#   2. Optimizing an FTS3 table (i.e. "INSERT INTO t1 VALUES('optimize')").
           12  +#   3. Deleting documents from an FTS3 table.
           13  +#   4. Querying FTS3 tables.
           14  +#
           15  +
           16  +# Number of tokens in vocabulary. And number of tokens in each document.
           17  +#
           18  +set VOCAB_SIZE  2000
           19  +set DOC_SIZE     100
           20  +
           21  +set NUM_INSERTS 100000
           22  +set NUM_SELECTS 1000
           23  +
           24  +# Force everything in this script to be deterministic.
           25  +#
           26  +expr {srand(0)}
           27  +
           28  +proc usage {} {
           29  +  puts stderr "Usage: $::argv0 <rows> <selects>"
           30  +  exit -1
           31  +}
           32  +
           33  +proc sql {sql} {
           34  +  puts $::fd $sql
           35  +}
           36  +
           37  +
           38  +# Return a list of $nWord randomly generated tokens each between 2 and 10
           39  +# characters in length.
           40  +#
           41  +proc build_vocab {nWord} {
           42  +  set ret [list]
           43  +  set chars [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
           44  +  for {set i 0} {$i<$nWord} {incr i} {
           45  +    set len [expr {int((rand()*9.0)+2)}]
           46  +    set term ""
           47  +    for {set j 0} {$j<$len} {incr j} {
           48  +      append term [lindex $chars [expr {int(rand()*[llength $chars])}]]
           49  +    }
           50  +    lappend ret $term
           51  +  }
           52  +  set ret
           53  +}
           54  +
           55  +proc select_term {} {
           56  +  set n [llength $::vocab]
           57  +  set t [expr int(rand()*$n*3)]
           58  +  if {$t>=2*$n} { set t [expr {($t-2*$n)/100}] }
           59  +  if {$t>=$n} { set t [expr {($t-$n)/10}] }
           60  +  lindex $::vocab $t
           61  +}
           62  +
           63  +proc select_doc {nTerm} {
           64  +  set ret [list]
           65  +  for {set i 0} {$i<$nTerm} {incr i} {
           66  +    lappend ret [select_term]
           67  +  }
           68  +  set ret
           69  +}
           70  +
           71  +proc test_1 {nInsert} {
           72  +  sql "PRAGMA synchronous = OFF;"
           73  +  sql "DROP TABLE IF EXISTS t1;"
           74  +  sql "CREATE VIRTUAL TABLE t1 USING fts4;"
           75  +  for {set i 0} {$i < $nInsert} {incr i} {
           76  +    set doc [select_doc $::DOC_SIZE]
           77  +    sql "INSERT INTO t1 VALUES('$doc');"
           78  +  }
           79  +}
           80  +
           81  +proc test_2 {} {
           82  +  sql "INSERT INTO t1(t1) VALUES('optimize');"
           83  +}
           84  +
           85  +proc test_3 {nSelect} {
           86  +  for {set i 0} {$i < $nSelect} {incr i} {
           87  +    sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term]';"
           88  +  }
           89  +}
           90  +
           91  +proc test_4 {nSelect} {
           92  +  for {set i 0} {$i < $nSelect} {incr i} {
           93  +    sql "SELECT count(*) FROM t1 WHERE t1 MATCH '[select_term] [select_term]';"
           94  +  }
           95  +}
           96  +
           97  +if {[llength $argv]!=0} usage
           98  +
           99  +set ::vocab [build_vocab $::VOCAB_SIZE]
          100  +
          101  +set ::fd [open fts3speed_insert.sql w]
          102  +test_1 $NUM_INSERTS
          103  +close $::fd
          104  +
          105  +set ::fd [open fts3speed_select.sql w]
          106  +test_3 $NUM_SELECTS
          107  +close $::fd
          108  +
          109  +set ::fd [open fts3speed_select2.sql w]
          110  +test_4 $NUM_SELECTS
          111  +close $::fd
          112  +
          113  +set ::fd [open fts3speed_optimize.sql w]
          114  +test_2
          115  +close $::fd
          116  +
          117  +puts "Success. Created files:"
          118  +puts "  fts3speed_insert.sql"
          119  +puts "  fts3speed_select.sql"
          120  +puts "  fts3speed_select2.sql"
          121  +puts "  fts3speed_optimize.sql"
          122  +

Changes to src/btree.c.

  8092   8092   ** (stored in BtCursor.aOverflow[]) is allocated and used by function
  8093   8093   ** accessPayload() (the worker function for sqlite3BtreeData() and
  8094   8094   ** sqlite3BtreePutData()).
  8095   8095   */
  8096   8096   void sqlite3BtreeCacheOverflow(BtCursor *pCur){
  8097   8097     assert( cursorHoldsMutex(pCur) );
  8098   8098     assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
  8099         -  assert(!pCur->isIncrblobHandle);
  8100         -  assert(!pCur->aOverflow);
         8099  +  invalidateOverflowCache(pCur);
  8101   8100     pCur->isIncrblobHandle = 1;
  8102   8101   }
  8103   8102   #endif
  8104   8103   
  8105   8104   /*
  8106   8105   ** Set both the "read version" (single byte at byte offset 18) and 
  8107   8106   ** "write version" (single byte at byte offset 19) fields in the database

Changes to src/sqlite.h.in.

  4786   4786     const char *zTable,
  4787   4787     const char *zColumn,
  4788   4788     sqlite3_int64 iRow,
  4789   4789     int flags,
  4790   4790     sqlite3_blob **ppBlob
  4791   4791   );
  4792   4792   
         4793  +/*
         4794  +** CAPI3REF: Move a BLOB Handle to a New Row
         4795  +**
         4796  +** This function is used to move an existing blob handle so that it points
         4797  +** to a different row of the same database table. The new row is identified
         4798  +** by the rowid value passed as the second argument. Only the row can be
         4799  +** changed, the database, table and column on which the blob handle is open
         4800  +** remain the same. Moving an existing blob handle to a new row can be
         4801  +** faster than closing the existing handle and opening a new one.
         4802  +**
         4803  +** The new row must meet the same criteria as for [sqlite3_blob_open()] -
         4804  +** it must exist and there must be either a blob or text value stored in
         4805  +** the nominated column. If the new row is not present in the table, or if
         4806  +** it does not contain a blob or text value, or if another error occurs, an
         4807  +** SQLite error code is returned and the blob handle is considered aborted.
         4808  +** All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or
         4809  +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return
         4810  +** SQLITE_ABORT.
         4811  +**
         4812  +** This function sets the database handle error code and message.
         4813  +*/
         4814  +SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
         4815  +
  4793   4816   /*
  4794   4817   ** CAPI3REF: Close A BLOB Handle
  4795   4818   **
  4796   4819   ** ^Closes an open [BLOB handle].
  4797   4820   **
  4798   4821   ** ^Closing a BLOB shall cause the current transaction to commit
  4799   4822   ** if there are no other BLOBs, no pending prepared statements, and the

Changes to src/test1.c.

  1585   1585     Tcl_SetObjResult(interp, pRet);
  1586   1586   
  1587   1587     return TCL_OK;
  1588   1588   }
  1589   1589   #endif
  1590   1590   
  1591   1591   #ifndef SQLITE_OMIT_INCRBLOB
         1592  +
         1593  +static int blobHandleFromObj(
         1594  +  Tcl_Interp *interp, 
         1595  +  Tcl_Obj *pObj,
         1596  +  sqlite3_blob **ppBlob
         1597  +){
         1598  +  char *z;
         1599  +  int n;
         1600  +
         1601  +  z = Tcl_GetStringFromObj(pObj, &n);
         1602  +  if( n==0 ){
         1603  +    *ppBlob = 0;
         1604  +  }else{
         1605  +    int notUsed;
         1606  +    Tcl_Channel channel;
         1607  +    ClientData instanceData;
         1608  +    
         1609  +    channel = Tcl_GetChannel(interp, z, &notUsed);
         1610  +    if( !channel ) return TCL_ERROR;
         1611  +
         1612  +    Tcl_Flush(channel);
         1613  +    Tcl_Seek(channel, 0, SEEK_SET);
         1614  +
         1615  +    instanceData = Tcl_GetChannelInstanceData(channel);
         1616  +    *ppBlob = *((sqlite3_blob **)instanceData);
         1617  +  }
         1618  +
         1619  +  return TCL_OK;
         1620  +}
         1621  +
         1622  +/*
         1623  +** sqlite3_blob_bytes  CHANNEL
         1624  +*/
         1625  +static int test_blob_bytes(
         1626  +  ClientData clientData, /* Not used */
         1627  +  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
         1628  +  int objc,              /* Number of arguments */
         1629  +  Tcl_Obj *CONST objv[]  /* Command arguments */
         1630  +){
         1631  +  sqlite3_blob *pBlob;
         1632  +  int nByte;
         1633  +  
         1634  +  if( objc!=2 ){
         1635  +    Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL");
         1636  +    return TCL_ERROR;
         1637  +  }
         1638  +
         1639  +  if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR;
         1640  +  nByte = sqlite3_blob_bytes(pBlob);
         1641  +  Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte));
         1642  +
         1643  +  return TCL_OK;
         1644  +}
         1645  +
         1646  +/*
         1647  +** sqlite3_blob_close  CHANNEL
         1648  +*/
         1649  +static int test_blob_close(
         1650  +  ClientData clientData, /* Not used */
         1651  +  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
         1652  +  int objc,              /* Number of arguments */
         1653  +  Tcl_Obj *CONST objv[]  /* Command arguments */
         1654  +){
         1655  +  sqlite3_blob *pBlob;
         1656  +  int nByte;
         1657  +  
         1658  +  if( objc!=2 ){
         1659  +    Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL");
         1660  +    return TCL_ERROR;
         1661  +  }
         1662  +
         1663  +  if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR;
         1664  +  sqlite3_blob_close(pBlob);
         1665  +
         1666  +  return TCL_OK;
         1667  +}
  1592   1668   
  1593   1669   /*
  1594   1670   ** sqlite3_blob_read  CHANNEL OFFSET N
  1595   1671   **
  1596   1672   **   This command is used to test the sqlite3_blob_read() in ways that
  1597   1673   **   the Tcl channel interface does not. The first argument should
  1598   1674   **   be the name of a valid channel created by the [incrblob] method
................................................................................
  1607   1683   */
  1608   1684   static int test_blob_read(
  1609   1685     ClientData clientData, /* Not used */
  1610   1686     Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
  1611   1687     int objc,              /* Number of arguments */
  1612   1688     Tcl_Obj *CONST objv[]  /* Command arguments */
  1613   1689   ){
  1614         -  Tcl_Channel channel;
  1615         -  ClientData instanceData;
  1616   1690     sqlite3_blob *pBlob;
  1617         -  int notUsed;
  1618   1691     int nByte;
  1619   1692     int iOffset;
  1620   1693     unsigned char *zBuf;
  1621   1694     int rc;
  1622   1695     
  1623   1696     if( objc!=4 ){
  1624   1697       Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N");
  1625   1698       return TCL_ERROR;
  1626   1699     }
  1627   1700   
  1628         -  channel = Tcl_GetChannel(interp, Tcl_GetString(objv[1]), &notUsed);
  1629         -  if( !channel
  1630         -   || TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset)
         1701  +  if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR;
         1702  +  if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset)
  1631   1703      || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte)
  1632         -   || nByte<0 || iOffset<0
  1633   1704     ){ 
  1634   1705       return TCL_ERROR;
  1635   1706     }
  1636   1707   
  1637         -  instanceData = Tcl_GetChannelInstanceData(channel);
  1638         -  pBlob = *((sqlite3_blob **)instanceData);
  1639         -
  1640   1708     zBuf = (unsigned char *)Tcl_Alloc(nByte);
  1641   1709     rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset);
  1642   1710     if( rc==SQLITE_OK ){
  1643   1711       Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte));
  1644   1712     }else{
  1645   1713       Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE);
  1646   1714     }
................................................................................
  1665   1733   */
  1666   1734   static int test_blob_write(
  1667   1735     ClientData clientData, /* Not used */
  1668   1736     Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
  1669   1737     int objc,              /* Number of arguments */
  1670   1738     Tcl_Obj *CONST objv[]  /* Command arguments */
  1671   1739   ){
  1672         -  Tcl_Channel channel;
  1673         -  ClientData instanceData;
  1674   1740     sqlite3_blob *pBlob;
  1675         -  int notUsed;
  1676   1741     int iOffset;
  1677   1742     int rc;
  1678   1743   
  1679   1744     unsigned char *zBuf;
  1680   1745     int nBuf;
  1681   1746     
  1682   1747     if( objc!=4 && objc!=5 ){
  1683   1748       Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET DATA ?NDATA?");
  1684   1749       return TCL_ERROR;
  1685   1750     }
  1686   1751   
  1687         -  channel = Tcl_GetChannel(interp, Tcl_GetString(objv[1]), &notUsed);
  1688         -  if( !channel || TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ 
         1752  +  if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR;
         1753  +  if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ 
  1689   1754       return TCL_ERROR;
  1690   1755     }
  1691   1756   
  1692         -  instanceData = Tcl_GetChannelInstanceData(channel);
  1693         -  pBlob = *((sqlite3_blob **)instanceData);
  1694         -
  1695   1757     zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf);
  1696   1758     if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){
  1697   1759       return TCL_ERROR;
  1698   1760     }
  1699   1761     rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset);
  1700   1762     if( rc!=SQLITE_OK ){
  1701   1763       Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE);
  1702   1764     }
  1703   1765   
  1704   1766     return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
  1705   1767   }
         1768  +
         1769  +static int test_blob_reopen(
         1770  +  ClientData clientData, /* Not used */
         1771  +  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
         1772  +  int objc,              /* Number of arguments */
         1773  +  Tcl_Obj *CONST objv[]  /* Command arguments */
         1774  +){
         1775  +  Tcl_WideInt iRowid;
         1776  +  sqlite3_blob *pBlob;
         1777  +  int rc;
         1778  +
         1779  +  unsigned char *zBuf;
         1780  +  int nBuf;
         1781  +  
         1782  +  if( objc!=3 ){
         1783  +    Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL ROWID");
         1784  +    return TCL_ERROR;
         1785  +  }
         1786  +
         1787  +  if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR;
         1788  +  if( Tcl_GetWideIntFromObj(interp, objv[2], &iRowid) ) return TCL_ERROR;
         1789  +
         1790  +  rc = sqlite3_blob_reopen(pBlob, iRowid);
         1791  +  if( rc!=SQLITE_OK ){
         1792  +    Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE);
         1793  +  }
         1794  +
         1795  +  return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
         1796  +}
         1797  +
  1706   1798   #endif
  1707   1799   
  1708   1800   /*
  1709   1801   ** Usage: sqlite3_create_collation_v2 DB-HANDLE NAME CMP-PROC DEL-PROC
  1710   1802   **
  1711   1803   **   This Tcl proc is used for testing the experimental
  1712   1804   **   sqlite3_create_collation_v2() interface.
................................................................................
  5322   5414        { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0},
  5323   5415   #endif
  5324   5416        { "sqlite3_libversion_number", test_libversion_number, 0  },
  5325   5417   #ifdef SQLITE_ENABLE_COLUMN_METADATA
  5326   5418        { "sqlite3_table_column_metadata", test_table_column_metadata, 0  },
  5327   5419   #endif
  5328   5420   #ifndef SQLITE_OMIT_INCRBLOB
  5329         -     { "sqlite3_blob_read",  test_blob_read, 0  },
  5330         -     { "sqlite3_blob_write", test_blob_write, 0  },
         5421  +     { "sqlite3_blob_read",   test_blob_read, 0  },
         5422  +     { "sqlite3_blob_write",  test_blob_write, 0  },
         5423  +     { "sqlite3_blob_reopen", test_blob_reopen, 0  },
         5424  +     { "sqlite3_blob_bytes",  test_blob_bytes, 0  },
         5425  +     { "sqlite3_blob_close",  test_blob_close, 0  },
  5331   5426   #endif
  5332   5427        { "pcache_stats",       test_pcache_stats, 0  },
  5333   5428   #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
  5334   5429        { "sqlite3_unlock_notify", test_unlock_notify, 0  },
  5335   5430   #endif
  5336   5431        { "sqlite3_wal_checkpoint", test_wal_checkpoint, 0  },
  5337   5432        { "test_sqlite3_log",     test_sqlite3_log, 0  },

Changes to src/vdbeblob.c.

    22     22   ** Valid sqlite3_blob* handles point to Incrblob structures.
    23     23   */
    24     24   typedef struct Incrblob Incrblob;
    25     25   struct Incrblob {
    26     26     int flags;              /* Copy of "flags" passed to sqlite3_blob_open() */
    27     27     int nByte;              /* Size of open blob, in bytes */
    28     28     int iOffset;            /* Byte offset of blob in cursor data */
           29  +  int iCol;               /* Table column this handle is open on */
    29     30     BtCursor *pCsr;         /* Cursor pointing at blob row */
    30     31     sqlite3_stmt *pStmt;    /* Statement holding cursor open */
    31     32     sqlite3 *db;            /* The associated database */
    32     33   };
    33     34   
           35  +
           36  +/*
           37  +** This function is used by both blob_open() and blob_reopen(). It seeks
           38  +** the b-tree cursor associated with blob handle p to point to row iRow.
           39  +** If successful, SQLITE_OK is returned and subsequent calls to
           40  +** sqlite3_blob_read() or sqlite3_blob_write() access the specified row.
           41  +**
           42  +** If an error occurs, or if the specified row does not exist or does not
           43  +** contain a value of type TEXT or BLOB in the column nominated when the
           44  +** blob handle was opened, then an error code is returned and *pzErr may
           45  +** be set to point to a buffer containing an error message. It is the
           46  +** responsibility of the caller to free the error message buffer using
           47  +** sqlite3DbFree().
           48  +**
           49  +** If an error does occur, then the b-tree cursor is closed. All subsequent
           50  +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will 
           51  +** immediately return SQLITE_ABORT.
           52  +*/
           53  +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
           54  +  int rc;                         /* Error code */
           55  +  char *zErr = 0;                 /* Error message */
           56  +  Vdbe *v = (Vdbe *)p->pStmt;
           57  +
           58  +  /* Set the value of the SQL statements only variable to integer iRow. 
           59  +  ** This is done directly instead of using sqlite3_bind_int64() to avoid 
           60  +  ** triggering asserts related to mutexes.
           61  +  */
           62  +  assert( v->aVar[0].flags&MEM_Int );
           63  +  v->aVar[0].u.i = iRow;
           64  +
           65  +  rc = sqlite3_step(p->pStmt);
           66  +  if( rc==SQLITE_ROW ){
           67  +    u32 type = v->apCsr[0]->aType[p->iCol];
           68  +    if( type<12 ){
           69  +      zErr = sqlite3MPrintf(p->db, "cannot open value of type %s",
           70  +          type==0?"null": type==7?"real": "integer"
           71  +      );
           72  +      rc = SQLITE_ERROR;
           73  +      sqlite3_finalize(p->pStmt);
           74  +      p->pStmt = 0;
           75  +    }else{
           76  +      p->iOffset = v->apCsr[0]->aOffset[p->iCol];
           77  +      p->nByte = sqlite3VdbeSerialTypeLen(type);
           78  +      p->pCsr =  v->apCsr[0]->pCursor;
           79  +      sqlite3BtreeEnterCursor(p->pCsr);
           80  +      sqlite3BtreeCacheOverflow(p->pCsr);
           81  +      sqlite3BtreeLeaveCursor(p->pCsr);
           82  +    }
           83  +  }
           84  +
           85  +  if( rc==SQLITE_ROW ){
           86  +    rc = SQLITE_OK;
           87  +  }else if( p->pStmt ){
           88  +    rc = sqlite3_finalize(p->pStmt);
           89  +    p->pStmt = 0;
           90  +    if( rc==SQLITE_OK ){
           91  +      zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow);
           92  +      rc = SQLITE_ERROR;
           93  +    }else{
           94  +      zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db));
           95  +    }
           96  +  }
           97  +
           98  +  assert( rc!=SQLITE_OK || zErr==0 );
           99  +  assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE );
          100  +
          101  +  *pzErr = zErr;
          102  +  return rc;
          103  +}
          104  +
    34    105   /*
    35    106   ** Open a blob handle.
    36    107   */
    37    108   int sqlite3_blob_open(
    38    109     sqlite3* db,            /* The database connection */
    39    110     const char *zDb,        /* The attached database containing the blob */
    40    111     const char *zTable,     /* The table containing the blob */
................................................................................
    67    138       {OP_TableLock, 0, 0, 0},       /* 2: Acquire a read or write lock */
    68    139   
    69    140       /* One of the following two instructions is replaced by an OP_Noop. */
    70    141       {OP_OpenRead, 0, 0, 0},        /* 3: Open cursor 0 for reading */
    71    142       {OP_OpenWrite, 0, 0, 0},       /* 4: Open cursor 0 for read/write */
    72    143   
    73    144       {OP_Variable, 1, 1, 1},        /* 5: Push the rowid to the stack */
    74         -    {OP_NotExists, 0, 9, 1},       /* 6: Seek the cursor */
          145  +    {OP_NotExists, 0, 10, 1},      /* 6: Seek the cursor */
    75    146       {OP_Column, 0, 0, 1},          /* 7  */
    76    147       {OP_ResultRow, 1, 0, 0},       /* 8  */
    77         -    {OP_Close, 0, 0, 0},           /* 9  */
    78         -    {OP_Halt, 0, 0, 0},            /* 10 */
          148  +    {OP_Goto, 0, 5, 0},            /* 9  */
          149  +    {OP_Close, 0, 0, 0},           /* 10 */
          150  +    {OP_Halt, 0, 0, 0},            /* 11 */
    79    151     };
    80    152   
    81         -  Vdbe *v = 0;
    82    153     int rc = SQLITE_OK;
    83    154     char *zErr = 0;
    84    155     Table *pTab;
    85         -  Parse *pParse;
          156  +  Parse *pParse = 0;
          157  +  Incrblob *pBlob = 0;
    86    158   
          159  +  flags = !!flags;                /* flags = (flags ? 1 : 0); */
    87    160     *ppBlob = 0;
          161  +
    88    162     sqlite3_mutex_enter(db->mutex);
          163  +
          164  +  pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
          165  +  if( !pBlob ) goto blob_open_out;
    89    166     pParse = sqlite3StackAllocRaw(db, sizeof(*pParse));
    90         -  if( pParse==0 ){
    91         -    rc = SQLITE_NOMEM;
    92         -    goto blob_open_out;
    93         -  }
          167  +  if( !pParse ) goto blob_open_out;
          168  +
    94    169     do {
    95    170       memset(pParse, 0, sizeof(Parse));
    96    171       pParse->db = db;
          172  +    sqlite3DbFree(db, zErr);
          173  +    zErr = 0;
    97    174   
    98    175       sqlite3BtreeEnterAll(db);
    99    176       pTab = sqlite3LocateTable(pParse, 0, zTable, zDb);
   100    177       if( pTab && IsVirtual(pTab) ){
   101    178         pTab = 0;
   102    179         sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable);
   103    180       }
................................................................................
   115    192         }
   116    193         rc = SQLITE_ERROR;
   117    194         sqlite3BtreeLeaveAll(db);
   118    195         goto blob_open_out;
   119    196       }
   120    197   
   121    198       /* Now search pTab for the exact column. */
   122         -    for(iCol=0; iCol < pTab->nCol; iCol++) {
          199  +    for(iCol=0; iCol<pTab->nCol; iCol++) {
   123    200         if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){
   124    201           break;
   125    202         }
   126    203       }
   127    204       if( iCol==pTab->nCol ){
   128    205         sqlite3DbFree(db, zErr);
   129    206         zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn);
................................................................................
   169    246           zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault);
   170    247           rc = SQLITE_ERROR;
   171    248           sqlite3BtreeLeaveAll(db);
   172    249           goto blob_open_out;
   173    250         }
   174    251       }
   175    252   
   176         -    v = sqlite3VdbeCreate(db);
   177         -    if( v ){
          253  +    pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db);
          254  +    assert( pBlob->pStmt || db->mallocFailed );
          255  +    if( pBlob->pStmt ){
          256  +      Vdbe *v = (Vdbe *)pBlob->pStmt;
   178    257         int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
          258  +
   179    259         sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob);
   180         -      flags = !!flags;                 /* flags = (flags ? 1 : 0); */
          260  +
   181    261   
   182    262         /* Configure the OP_Transaction */
   183    263         sqlite3VdbeChangeP1(v, 0, iDb);
   184    264         sqlite3VdbeChangeP2(v, 0, flags);
   185    265   
   186    266         /* Configure the OP_VerifyCookie */
   187    267         sqlite3VdbeChangeP1(v, 1, iDb);
................................................................................
   216    296         sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32);
   217    297         sqlite3VdbeChangeP2(v, 7, pTab->nCol);
   218    298         if( !db->mallocFailed ){
   219    299           sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0);
   220    300         }
   221    301       }
   222    302      
          303  +    pBlob->flags = flags;
          304  +    pBlob->iCol = iCol;
          305  +    pBlob->db = db;
   223    306       sqlite3BtreeLeaveAll(db);
   224    307       if( db->mallocFailed ){
   225    308         goto blob_open_out;
   226    309       }
   227         -
   228         -    sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow);
   229         -    rc = sqlite3_step((sqlite3_stmt *)v);
   230         -    if( rc!=SQLITE_ROW ){
   231         -      nAttempt++;
   232         -      rc = sqlite3_finalize((sqlite3_stmt *)v);
   233         -      sqlite3DbFree(db, zErr);
   234         -      zErr = sqlite3MPrintf(db, "%s", sqlite3_errmsg(db));
   235         -      v = 0;
   236         -    }
   237         -  } while( nAttempt<5 && rc==SQLITE_SCHEMA );
   238         -
   239         -  if( rc==SQLITE_ROW ){
   240         -    /* The row-record has been opened successfully. Check that the
   241         -    ** column in question contains text or a blob. If it contains
   242         -    ** text, it is up to the caller to get the encoding right.
   243         -    */
   244         -    Incrblob *pBlob;
   245         -    u32 type = v->apCsr[0]->aType[iCol];
   246         -
   247         -    if( type<12 ){
   248         -      sqlite3DbFree(db, zErr);
   249         -      zErr = sqlite3MPrintf(db, "cannot open value of type %s",
   250         -          type==0?"null": type==7?"real": "integer"
   251         -      );
   252         -      rc = SQLITE_ERROR;
   253         -      goto blob_open_out;
   254         -    }
   255         -    pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
   256         -    if( db->mallocFailed ){
   257         -      sqlite3DbFree(db, pBlob);
   258         -      goto blob_open_out;
   259         -    }
   260         -    pBlob->flags = flags;
   261         -    pBlob->pCsr =  v->apCsr[0]->pCursor;
   262         -    sqlite3BtreeEnterCursor(pBlob->pCsr);
   263         -    sqlite3BtreeCacheOverflow(pBlob->pCsr);
   264         -    sqlite3BtreeLeaveCursor(pBlob->pCsr);
   265         -    pBlob->pStmt = (sqlite3_stmt *)v;
   266         -    pBlob->iOffset = v->apCsr[0]->aOffset[iCol];
   267         -    pBlob->nByte = sqlite3VdbeSerialTypeLen(type);
   268         -    pBlob->db = db;
   269         -    *ppBlob = (sqlite3_blob *)pBlob;
   270         -    rc = SQLITE_OK;
   271         -  }else if( rc==SQLITE_OK ){
   272         -    sqlite3DbFree(db, zErr);
   273         -    zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow);
   274         -    rc = SQLITE_ERROR;
   275         -  }
          310  +    sqlite3_bind_int64(pBlob->pStmt, 1, iRow);
          311  +    rc = blobSeekToRow(pBlob, iRow, &zErr);
          312  +  } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA );
   276    313   
   277    314   blob_open_out:
   278         -  if( v && (rc!=SQLITE_OK || db->mallocFailed) ){
   279         -    sqlite3VdbeFinalize(v);
          315  +  if( rc==SQLITE_OK && db->mallocFailed==0 ){
          316  +    *ppBlob = (sqlite3_blob *)pBlob;
          317  +  }else{
          318  +    if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt);
          319  +    sqlite3DbFree(db, pBlob);
   280    320     }
   281    321     sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
   282    322     sqlite3DbFree(db, zErr);
   283    323     sqlite3StackFree(db, pParse);
   284    324     rc = sqlite3ApiExit(db, rc);
   285    325     sqlite3_mutex_leave(db->mutex);
   286    326     return rc;
................................................................................
   327    367     sqlite3_mutex_enter(db->mutex);
   328    368     v = (Vdbe*)p->pStmt;
   329    369   
   330    370     if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){
   331    371       /* Request is out of range. Return a transient error. */
   332    372       rc = SQLITE_ERROR;
   333    373       sqlite3Error(db, SQLITE_ERROR, 0);
   334         -  } else if( v==0 ){
          374  +  }else if( v==0 ){
   335    375       /* If there is no statement handle, then the blob-handle has
   336    376       ** already been invalidated. Return SQLITE_ABORT in this case.
   337    377       */
   338    378       rc = SQLITE_ABORT;
   339    379     }else{
   340    380       /* Call either BtreeData() or BtreePutData(). If SQLITE_ABORT is
   341    381       ** returned, clean-up the statement handle.
................................................................................
   377    417   ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob
   378    418   ** so no mutex is required for access.
   379    419   */
   380    420   int sqlite3_blob_bytes(sqlite3_blob *pBlob){
   381    421     Incrblob *p = (Incrblob *)pBlob;
   382    422     return p ? p->nByte : 0;
   383    423   }
          424  +
          425  +/*
          426  +** Move an existing blob handle to point to a different row of the same
          427  +** database table.
          428  +**
          429  +** If an error occurs, or if the specified row does not exist or does not
          430  +** contain a blob or text value, then an error code is returned and the
          431  +** database handle error code and message set. If this happens, then all 
          432  +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) 
          433  +** immediately return SQLITE_ABORT.
          434  +*/
          435  +int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){
          436  +  int rc;
          437  +  Incrblob *p = (Incrblob *)pBlob;
          438  +  sqlite3 *db;
          439  +
          440  +  if( p==0 ) return SQLITE_MISUSE_BKPT;
          441  +  db = p->db;
          442  +  sqlite3_mutex_enter(db->mutex);
          443  +
          444  +  if( p->pStmt==0 ){
          445  +    /* If there is no statement handle, then the blob-handle has
          446  +    ** already been invalidated. Return SQLITE_ABORT in this case.
          447  +    */
          448  +    rc = SQLITE_ABORT;
          449  +  }else{
          450  +    char *zErr;
          451  +    rc = blobSeekToRow(p, iRow, &zErr);
          452  +    if( rc!=SQLITE_OK ){
          453  +      sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
          454  +      sqlite3DbFree(db, zErr);
          455  +    }
          456  +    assert( rc!=SQLITE_SCHEMA );
          457  +  }
          458  +
          459  +  rc = sqlite3ApiExit(db, rc);
          460  +  sqlite3_mutex_leave(db->mutex);
          461  +  return rc;
          462  +}
   384    463   
   385    464   #endif /* #ifndef SQLITE_OMIT_INCRBLOB */

Changes to test/fts3ah.test.

     1         -# 2006 October 31 (scaaarey)
            1  +# 2006 October 31 
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
     2      5   #
     3         -# The author disclaims copyright to this source code.
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
     4      9   #
     5     10   #*************************************************************************
     6     11   # This file implements regression tests for SQLite library.  The focus
     7         -# here is testing correct handling of excessively long terms.
     8         -#
     9         -# $Id: fts3ah.test,v 1.1 2007/08/20 17:38:42 shess Exp $
           12  +# here is testing correct handling of very long terms.
    10     13   #
    11     14   
    12     15   set testdir [file dirname $argv0]
    13     16   source $testdir/tester.tcl
    14     17   
    15         -# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           18  +# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
    16     19   ifcapable !fts3 {
    17     20     finish_test
    18     21     return
    19     22   }
    20     23   
    21         -# Generate a term of len copies of char.
    22         -proc bigterm {char len} {
    23         -  for {set term ""} {$len>0} {incr len -1} {
    24         -    append term $char
    25         -  }
    26         -  return $term
    27         -}
    28         -
    29     24   # Generate a document of bigterms based on characters from the list
    30     25   # chars.
    31     26   proc bigtermdoc {chars len} {
    32     27     set doc ""
    33     28     foreach char $chars {
    34         -    append doc " " [bigterm $char $len]
           29  +    append doc " " [string repeat $char $len]
    35     30     }
    36     31     return $doc
    37     32   }
    38     33   
    39     34   set len 5000
    40     35   set doc1 [bigtermdoc {a b c d} $len]
    41     36   set doc2 [bigtermdoc {b d e f} $len]
    42     37   set doc3 [bigtermdoc {a c e} $len]
    43     38   
    44         -set aterm [bigterm a $len]
    45         -set bterm [bigterm b $len]
    46         -set xterm [bigterm x $len]
           39  +set aterm [string repeat a $len]
           40  +set bterm [string repeat b $len]
           41  +set xterm [string repeat x $len]
    47     42   
    48     43   db eval {
    49     44     CREATE VIRTUAL TABLE t1 USING fts3(content);
    50     45     INSERT INTO t1 (rowid, content) VALUES(1, $doc1);
    51     46     INSERT INTO t1 (rowid, content) VALUES(2, $doc2);
    52     47     INSERT INTO t1 (rowid, content) VALUES(3, $doc3);
    53     48   }
................................................................................
    57     52     execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something'}
    58     53   } {}
    59     54   
    60     55   do_test fts3ah-1.2 {
    61     56     execsql {SELECT rowid FROM t1 WHERE t1 MATCH $aterm}
    62     57   } {1 3}
    63     58   
    64         -do_test fts3ah-1.2 {
           59  +do_test fts3ah-1.3 {
    65     60     execsql {SELECT rowid FROM t1 WHERE t1 MATCH $xterm}
    66     61   } {}
    67     62   
    68         -do_test fts3ah-1.3 {
           63  +do_test fts3ah-1.4 {
    69     64     execsql "SELECT rowid FROM t1 WHERE t1 MATCH '$aterm -$xterm'"
    70     65   } {1 3}
    71     66   
    72         -do_test fts3ah-1.4 {
           67  +do_test fts3ah-1.5 {
    73     68     execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"$aterm $bterm\"'"
    74     69   } {1}
    75     70   
    76     71   finish_test

Changes to test/fts3ao.test.

    18     18   source $testdir/tester.tcl
    19     19   
    20     20   # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
    21     21   ifcapable !fts3 {
    22     22     finish_test
    23     23     return
    24     24   }
           25  +
           26  +set ::testprefix fts3ao
    25     27   
    26     28   #---------------------------------------------------------------------
    27     29   # These tests, fts3ao-1.*, test that ticket #2429 is fixed.
    28     30   #
    29     31   db eval {
    30     32     CREATE VIRTUAL TABLE t1 USING fts3(a, b, c);
    31     33     INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one four two');
................................................................................
   194    196         INSERT INTO t5 VALUES('Down came a jumbuck to drink at that billabong');
   195    197         ALTER TABLE t5 RENAME TO t6;
   196    198         INSERT INTO t6 VALUES('Down came the troopers, one, two, three');
   197    199       ROLLBACK;
   198    200       SELECT * FROM t5;
   199    201     }
   200    202   } {{the quick brown fox} {jumped over the} {lazy dog}}
          203  +
          204  +# Test that it is possible to rename an FTS4 table. Renaming an FTS4 table
          205  +# involves renaming the extra %_docsize and %_stat tables.
          206  +#
          207  +do_execsql_test 5.1 {
          208  +  CREATE VIRTUAL TABLE t7 USING FTS4;
          209  +  INSERT INTO t7 VALUES('coined by a German clinician');
          210  +  SELECT count(*) FROM sqlite_master WHERE name LIKE 't7%';
          211  +  SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%';
          212  +} {6 0}
          213  +do_execsql_test 5.2 {
          214  +  ALTER TABLE t7 RENAME TO t8;
          215  +  SELECT count(*) FROM sqlite_master WHERE name LIKE 't7%';
          216  +  SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%';
          217  +} {0 6}
   201    218   
   202    219   finish_test
   203    220   

Added test/fts3corrupt.test.

            1  +# 2010 October 27
            2  +#
            3  +#    May you do good and not evil.
            4  +#    May you find forgiveness for yourself and forgive others.
            5  +#    May you share freely, never taking more than you give.
            6  +#
            7  +#***********************************************************************
            8  +# Test that the FTS3 extension does not crash when it encounters a
            9  +# corrupt data structure on disk.
           10  +#
           11  +
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +
           16  +# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
           17  +ifcapable !fts3 { finish_test ; return }
           18  +
           19  +set ::testprefix fts3corrupt
           20  +
           21  +do_execsql_test 1.0 {
           22  +  CREATE VIRTUAL TABLE t1 USING fts3;
           23  +  INSERT INTO t1 VALUES('hello');
           24  +} {}
           25  +
           26  +do_test fts3corrupt-1.1 {
           27  +  set blob [db one {SELECT root from t1_segdir}]
           28  +  set blob [binary format a7ca* $blob 24 [string range $blob 8 end]]
           29  +  execsql { UPDATE t1_segdir SET root = $blob }
           30  +} {}
           31  +
           32  +do_test fts3corrupt-1.2 {
           33  +  foreach w {a b c d e f g h i j k l m n o} {
           34  +    execsql { INSERT INTO t1 VALUES($w) }
           35  +  }
           36  +} {}
           37  +
           38  +do_catchsql_test 1.3 {
           39  +  INSERT INTO t1 VALUES('world');
           40  +} {1 {database disk image is malformed}}
           41  +
           42  +finish_test
           43  +

Changes to test/fts3cov.test.

     2      2   #
     3      3   #    May you do good and not evil.
     4      4   #    May you find forgiveness for yourself and forgive others.
     5      5   #    May you share freely, never taking more than you give.
     6      6   #
     7      7   #***********************************************************************
     8      8   #
     9         -# The tests in this file are structural coverage tests. They are designed
    10         -# to complement the tests in fts3rnd.test and fts3doc.test. Between them,
    11         -# the three files should provide full coverage of the fts3 extension code.
            9  +# The tests in this file are structural coverage tests for FTS3.
    12     10   #
    13     11   
    14     12   set testdir [file dirname $argv0]
    15     13   source $testdir/tester.tcl
    16     14   
    17     15   # If this build does not include FTS3, skip the tests in this file.
    18     16   #
    19     17   ifcapable !fts3 { finish_test ; return }
    20     18   source $testdir/fts3_common.tcl
    21     19   source $testdir/malloc_common.tcl
    22     20   
    23     21   set DO_MALLOC_TEST 0
           22  +set testprefix fts3cov
    24     23   
    25     24   #--------------------------------------------------------------------------
    26     25   # When it first needs to read a block from the %_segments table, the FTS3 
    27     26   # module compiles an SQL statement for that purpose. The statement is 
    28     27   # stored and reused each subsequent time a block is read. This test case 
    29     28   # tests the effects of an OOM error occuring while compiling the statement.
    30     29   #
................................................................................
    78     77         INSERT INTO t1 VALUES('What makes her in the wood so late,');
    79     78         INSERT INTO t1 VALUES('A furlong from the castle gate?');
    80     79         INSERT INTO t1 VALUES('She had dreams all yesternight');
    81     80         INSERT INTO t1 VALUES('Of her own betrothed knight;');
    82     81         INSERT INTO t1 VALUES('And she in the midnight wood will pray');
    83     82         INSERT INTO t1 VALUES('For the weal of her lover that''s far away.');
    84     83       COMMIT;
    85         -
           84  +  }
           85  +  execsql {
    86     86       INSERT INTO t1(t1) VALUES('optimize');
    87     87       SELECT substr(hex(root), 1, 2) FROM t1_segdir;
    88     88     }
    89     89   } {03}
    90     90   
    91     91   # Test the "missing entry" case:
    92         -do_test fts3cov-2.1 {
           92  +do_test fts3cov-2.2 {
    93     93     set root [db one {SELECT root FROM t1_segdir}]
    94     94     read_fts3varint [string range $root 1 end] left_child
    95     95     execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
    96     96   } {}
    97         -do_error_test fts3cov-2.2 {
           97  +do_error_test fts3cov-2.3 {
    98     98     SELECT * FROM t1 WHERE t1 MATCH 'c*'
    99         -} {database disk image is malformed}
           99  +} {SQL logic error or missing database}
   100    100   
   101    101   # Test the "replaced with NULL" case:
   102         -do_test fts3cov-2.3 {
          102  +do_test fts3cov-2.4 {
   103    103     execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
   104    104   } {}
   105         -do_error_test fts3cov-2.4 {
          105  +do_error_test fts3cov-2.5 {
   106    106     SELECT * FROM t1 WHERE t1 MATCH 'cloud'
   107         -} {database disk image is malformed}
          107  +} {SQL logic error or missing database}
   108    108   
   109    109   #--------------------------------------------------------------------------
   110    110   # The following tests are to test the effects of OOM errors while storing
   111    111   # terms in the pending-hash table. Specifically, while creating doclist
   112    112   # blobs to store in the table. More specifically, to test OOM errors while
   113    113   # appending column numbers to doclists. For example, if a doclist consists
   114    114   # of:
................................................................................
   365    365     INSERT INTO t13 VALUES('scalar two functions');
   366    366     INSERT INTO t13 VALUES('functions scalar two');
   367    367   } -sqlbody {
   368    368     SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two';
   369    369     SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two';
   370    370     SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two';
   371    371   }
          372  +
          373  +do_execsql_test 14.0 {
          374  +  CREATE VIRTUAL TABLE t14 USING fts4(a, b);
          375  +  INSERT INTO t14 VALUES('one two three', 'one three four');
          376  +  INSERT INTO t14 VALUES('a b c', 'd e a');
          377  +}
          378  +do_execsql_test 14.1 {
          379  +  SELECT rowid FROM t14 WHERE t14 MATCH '"one two three"'
          380  +} {1}
          381  +do_execsql_test 14.2 {
          382  +  SELECT rowid FROM t14 WHERE t14 MATCH '"one four"'
          383  +} {}
          384  +do_execsql_test 14.3 {
          385  +  SELECT rowid FROM t14 WHERE t14 MATCH '"e a"'
          386  +} {2}
          387  +do_execsql_test 14.5 {
          388  +  SELECT rowid FROM t14 WHERE t14 MATCH '"e b"'
          389  +} {}
          390  +do_catchsql_test 14.6 {
          391  +  SELECT rowid FROM t14 WHERE rowid MATCH 'one'
          392  +} {1 {unable to use function MATCH in the requested context}}
          393  +do_catchsql_test 14.7 {
          394  +  SELECT rowid FROM t14 WHERE docid MATCH 'one'
          395  +} {1 {unable to use function MATCH in the requested context}}
          396  +
          397  +do_execsql_test 15.0 {
          398  +  CREATE VIRTUAL TABLE t15 USING fts4(a, b, c);
          399  +  INSERT INTO t15 VALUES('abc def ghi', 'abc2 def2 ghi2', 'abc3 def3 ghi3');
          400  +  INSERT INTO t15 VALUES('abc2 def2 ghi2', 'abc2 def2 ghi2', 'abc def3 ghi3');
          401  +}
          402  +do_execsql_test 15.1 {
          403  +  SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"'
          404  +} {1 2}
          405  +
          406  +# Test a corruption case.
          407  +#
          408  +do_execsql_test 16.1 {
          409  +  CREATE VIRTUAL TABLE t16 USING fts4;
          410  +  INSERT INTO t16 VALUES('theoretical work to examine the relationship');
          411  +  INSERT INTO t16 VALUES('solution of our problems on the invisible');
          412  +  DELETE FROM t16_content WHERE rowid = 2;
          413  +}
          414  +do_catchsql_test 16.2 {
          415  +  SELECT * FROM t16 WHERE t16 MATCH 'invisible'
          416  +} {1 {database disk image is malformed}}
          417  +
          418  +# And another corruption test case.
          419  +#
          420  +do_execsql_test 17.1 {
          421  +  CREATE VIRTUAL TABLE t17 USING fts4;
          422  +  INSERT INTO t17(content) VALUES('one one one');
          423  +  UPDATE t17_segdir SET root = X'00036F6E65FFFFFFFFFFFFFFFFFFFFFF02030300'
          424  +} {}
          425  +do_catchsql_test 17.2 {
          426  +  SELECT * FROM t17 WHERE t17 MATCH 'one'
          427  +} {1 {database disk image is malformed}}
          428  +
          429  +
          430  +
   372    431   
   373    432   finish_test

Added test/fts3defer.test.

            1  +# 2010 October 15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +
           12  +set testdir [file dirname $argv0]
           13  +source $testdir/tester.tcl
           14  +source $testdir/malloc_common.tcl
           15  +
           16  +ifcapable !fts3 {
           17  +  finish_test
           18  +  return
           19  +}
           20  +
           21  +set sqlite_fts3_enable_parentheses 1
           22  +
           23  +set ::testprefix fts3defer
           24  +
           25  +#--------------------------------------------------------------------------
           26  +# Test cases fts3defer-1.* are the "warm body" cases. The database contains
           27  +# one row with 15000 instances of the token "a". This makes the doclist for
           28  +# "a" so large that FTS3 will avoid loading it in most cases.
           29  +#
           30  +# To show this, test cases fts3defer-1.2.* execute a bunch of FTS3 queries
           31  +# involving token "a". Then, fts3defer-1.3.* replaces the doclist for token
           32  +# "a" with all zeroes and fts3defer-1.4.* repeats the tests from 1.2. If
           33  +# the tests still work, we can conclude that the doclist for "a" was not
           34  +# used.
           35  +# 
           36  +
           37  +set aaa [string repeat "a " 15000]
           38  +
           39  +do_execsql_test 1.1 {
           40  +  CREATE VIRTUAL TABLE t1 USING fts4;
           41  +  BEGIN;
           42  +    INSERT INTO t1 VALUES('this is a dog');
           43  +    INSERT INTO t1 VALUES('an instance of a phrase');
           44  +    INSERT INTO t1 VALUES('an instance of a longer phrase');
           45  +    INSERT INTO t1 VALUES($aaa);
           46  +  COMMIT;
           47  +} {}
           48  +
           49  +set tests {
           50  +  1  {SELECT rowid FROM t1 WHERE t1 MATCH '"a dog"'}                 {1}
           51  +  2  {SELECT rowid FROM t1 WHERE t1 MATCH '"is a dog"'}              {1}
           52  +  3  {SELECT rowid FROM t1 WHERE t1 MATCH '"a longer phrase"'}       {3}
           53  +  4  {SELECT snippet(t1) FROM t1 WHERE t1 MATCH '"a longer phrase"'}  
           54  +     {"an instance of <b>a</b> <b>longer</b> <b>phrase</b>"}
           55  +  5  {SELECT rowid FROM t1 WHERE t1 MATCH 'a dog'}                   {1}
           56  +}
           57  +
           58  +do_select_tests 1.2 $tests
           59  +
           60  +do_execsql_test 1.3 {
           61  +  SELECT count(*) FROM t1_segments WHERE length(block)>10000;
           62  +  UPDATE t1_segments 
           63  +    SET block = zeroblob(length(block)) 
           64  +    WHERE length(block)>10000;
           65  +} {1}
           66  +
           67  +do_select_tests 1.4 $tests
           68  +
           69  +# Drop the table. It is corrupt now anyhow, so not useful for subsequent tests.
           70  +#
           71  +do_execsql_test 1.5 { DROP TABLE t1 }
           72  +
           73  +#--------------------------------------------------------------------------
           74  +# These tests - fts3defer-2.* - are more rigorous. They test that for a
           75  +# variety of queries, FTS3 and FTS4 return the same results. And that 
           76  +# zeroing the very large doclists that FTS4 does not load does not change
           77  +# the results.
           78  +#
           79  +# They use the following pseudo-randomly generated document data. The
           80  +# tokens "zm" and "jk" are especially common in this dataset. Additionally,
           81  +# two documents are added to the pseudo-random data before it is loaded
           82  +# into FTS4 containing 100,000 instances of the "zm" and "jk" tokens. This
           83  +# makes the doclists for those tokens so large that FTS4 avoids loading them
           84  +# into memory if possible.
           85  +#
           86  +set data [list]
           87  +lappend data [string repeat "zm " 100000]
           88  +lappend data [string repeat "jk " 100000]
           89  +lappend data {*}{
           90  +  "zm zm agmckuiu uhzq nsab jk rrkx duszemmzl hyq jk"
           91  +  "jk uhzq zm zm rgpzmlnmd zm zk jk jk zm"
           92  +  "duszemmzl zm jk xldlpy zm jk sbptoa xh jk xldlpy"
           93  +  "zm xh zm xqf azavwm jk jk trqd rgpzmlnmd jk"
           94  +  "zm vwq urvysbnykk ubwrfqnbjf zk lsz jk doiwavhwwo jk jk"
           95  +  "jk xduvfhk orpfawpx zkhdvkw jk mjpavjuhw zm jk duszemmzl zm"
           96  +  "jk igju jk jk zm hmjf xh zm gwdfhwurx zk"
           97  +  "vgsld jk jk zm hrlipdm jn zm zsmhnf vgsld duszemmzl"
           98  +  "gtuiexzsu aayxpmve zm zm zm drir scpgna xh azavwm uhzq"
           99  +  "farlehdhq hkfoudzftq igju duszemmzl xnxhf ewle zm hrlipdm urvysbnykk kn"
          100  +  "xnxhf jk jk agmckuiu duszemmzl jk zm zm jk vgsld"
          101  +  "zm zm zm jk jk urvysbnykk ogttbykvt zm zm jk"
          102  +  "iasrqgqv zm azavwm zidhxhbtv jk jk mjpavjuhw zm zm ajmvcydy"
          103  +  "rgpzmlnmd tmt mjpavjuhw xh igju jk azavwm fibokdry vgsld ofm"
          104  +  "zm jk vgsld jk xh jk csjqxhgj drir jk pmrb"
          105  +  "xh jk jk zm rrkx duszemmzl mjpavjuhw xldlpy igju zm"
          106  +  "jk hkfoudzftq zf rrkx wdmy jupk jk zm urvysbnykk npywgdvgz"
          107  +  "zm jk zm zm zhbrzadb uenvbm aayxpmve urvysbnykk duszemmzl jk"
          108  +  "uenvbm jk zm fxw xh bdilwmjw mjpavjuhw uv jk zm"
          109  +  "nk jk bnhc pahlds jk igju dzadnqzprr jk jk jk"
          110  +  "uhzq uv zm duszemmzl tlqix jk jk xh jk zm"
          111  +  "jk zm agmckuiu urvysbnykk jk jk zm zm jk jk"
          112  +  "azavwm mjpavjuhw lsgshn trqd xldlpy ogyavjvv agmckuiu ryvwwhlbc jk jk"
          113  +  "tmt jk zk zm azavwm ofm acpgim bvgimjik iasrqgqv wuvajhwqz"
          114  +  "igju ogyavjvv xrbdak rrkx fibokdry zf ujfhmrllq jk zm hxgwvib"
          115  +  "zm pahlds jk uenvbm aayxpmve iaf hmjf xph vnlyvtkgx zm"
          116  +  "jk xnxhf igju jk xh jk nvfasfh zm js jk"
          117  +  "zm zm rwaj igju xr rrkx xnxhf nvfasfh skxbsqzvmt xatbxeqq"
          118  +  "vgsld zm ujfhmrllq uhzq ogyavjvv nsab azavwm zm vgsld jmfiqhwnjg"
          119  +  "ymjoym duszemmzl urvysbnykk azavwm jk jmfiqhwnjg bu qcdziqomqk vnlyvtkgx"
          120  +  "zm nbilqcnz dzadnqzprr xh bkfgzsxn urvysbnykk xrujfzxqf zm zf agmckuiu"
          121  +  "jk urvysbnykk nvfasfh zf xh zm zm qcdziqomqk qvxtclg wdmy"
          122  +  "fibokdry jk urvysbnykk jk xr osff zm cvnnsl zm vgsld"
          123  +  "jk mjpavjuhw hkfoudzftq jk zm xh xqf urvysbnykk jk iasrqgqv"
          124  +  "jk csjqxhgj duszemmzl iasrqgqv aayxpmve zm brsuoqww jk qpmhtvl wluvgsw"
          125  +  "jk mj azavwm jk zm jn dzadnqzprr zm jk uhzq"
          126  +  "zk xqf jupk fxw nbilqcnz zm jk jcpiwj tznlvbfcv nvfasfh"
          127  +  "jk jcpiwj zm xnxhf zm mjpavjuhw mj drir pa pvjrjlas"
          128  +  "duszemmzl dzadnqzprr jk swc duszemmzl tmt jk jk pahlds jk"
          129  +  "zk zm jk zm zm eczkjblu zm hi pmrb jk"
          130  +  "azavwm zm iz agmckuiu jk sntk jk duszemmzl duszemmzl zm"
          131  +  "jk zm jk eczkjblu urvysbnykk sk gnl jk ttvgf hmjf"
          132  +  "jk bnhc jjrxpjkb mjpavjuhw fibokdry igju jk zm zm xh"
          133  +  "wxe ogttbykvt uhzq xr iaf zf urvysbnykk aayxpmve oacaxgjoo mjpavjuhw"
          134  +  "gazrt jk ephknonq myjp uenvbm wuvajhwqz jk zm xnxhf nvfasfh"
          135  +  "zm aayxpmve csjqxhgj xnxhf xr jk aayxpmve xnxhf zm zm"
          136  +  "sokcyf zm ogyavjvv jk zm fibokdry zm jk igju igju"
          137  +  "vgsld bvgimjik xuprtlyle jk akmikrqyt jk aayxpmve hkfoudzftq ddjj ithtir"
          138  +  "zm uhzq ovkyevlgv zk uenvbm csjqxhgj jk vgsld pgybs jk"
          139  +  "zm agmckuiu zexh fibokdry jk uhzq bu tugflixoex xnxhf sk"
          140  +  "zm zf uenvbm jk azavwm zm zm agmckuiu zm jk"
          141  +  "rrkx jk zf jt zm oacaxgjoo fibokdry wdmy igju csjqxhgj"
          142  +  "hi igju zm jk zidhxhbtv dzadnqzprr jk jk trqd duszemmzl"
          143  +  "zm zm mjpavjuhw xrbdak qrvbjruc jk qzzqdxq guwq cvnnsl zm"
          144  +  "ithtir jk jk qcdziqomqk zm farlehdhq zm zm xrbdak jk"
          145  +  "ixfipk csjqxhgj azavwm sokcyf ttvgf vgsld jk sk xh zk"
          146  +  "nvfasfh azavwm zm zm zm fxw nvfasfh zk gnl trqd"
          147  +  "zm fibokdry csjqxhgj ofm dzadnqzprr jk akmikrqyt orpfawpx duszemmzl vwq"
          148  +  "csjqxhgj jk jk vgsld urvysbnykk jk nxum jk jk nxum"
          149  +  "zm hkfoudzftq jk ryvwwhlbc mjpavjuhw ephknonq jk zm ogyavjvv zm"
          150  +  "lwa hi xnxhf qdyerbws zk njtc jk uhzq zm jk"
          151  +  "trqd zm dzadnqzprr zm urvysbnykk jk lsz jk mjpavjuhw cmnnkna"
          152  +  "duszemmzl zk jk jk fibokdry jseuhjnzo zm aayxpmve zk jk"
          153  +  "fibokdry jk sviq qvxtclg wdmy jk doiwavhwwo zexh jk zm"
          154  +  "jupk zm xh jk mjpavjuhw zm jk nsab npywgdvgz duszemmzl"
          155  +  "zm igju zm zm nvfasfh eh hkfoudzftq fibokdry fxw xkblf"
          156  +  "jk zm jk jk zm xh zk abthnzcv zf csjqxhgj"
          157  +  "zm zm jk nkaotm urvysbnykk sbptoa bq jk ktxdty ubwrfqnbjf"
          158  +  "nvfasfh aayxpmve xdcuz zm tugflixoex jcpiwj zm mjpavjuhw fibokdry doiwavhwwo"
          159  +  "iaf jk mjpavjuhw zm duszemmzl jk jk uhzq pahlds fibokdry"
          160  +  "ddjj zk azavwm jk swc zm gjtexkv jk xh jk"
          161  +  "igju jk csjqxhgj zm jk dzadnqzprr duszemmzl ulvcbv jk jk"
          162  +  "jk fibokdry zm csjqxhgj jn zm zm zm zf uhzq"
          163  +  "duszemmzl jk xkblf zk hrlipdm aayxpmve uenvbm uhzq jk zf"
          164  +  "dzadnqzprr jk zm zdu nvfasfh zm jk urvysbnykk hmjf jk"
          165  +  "jk aayxpmve aserrdxm acpgim fibokdry jk drir wxe brsuoqww rrkx"
          166  +  "uhzq csjqxhgj nvfasfh jk rrkx qbamok trqd uenvbm sntk zm"
          167  +  "ps azavwm zkhdvkw jk zm jk jk zm csjqxhgj xedlrcfo"
          168  +  "jk jk ogyavjvv jk zm farlehdhq duszemmzl jk agitgxamxe jk"
          169  +  "qzzqdxq rwaj jk jk zm xqf jk uenvbm jk zk"
          170  +  "zm hxgwvib akmikrqyt zf agmckuiu uenvbm bq npywgdvgz azavwm jk"
          171  +  "zf jmfiqhwnjg js igju zm aayxpmve zm mbxnljomiv csjqxhgj nvfasfh"
          172  +  "zm jk jk gazrt jk jk lkc jk nvfasfh jk"
          173  +  "xldlpy orpfawpx zkhdvkw jk zm igju zm urvysbnykk dzadnqzprr mbxnljomiv"
          174  +  "urvysbnykk jk zk igju zm uenvbm jk zm ithtir jk"
          175  +  "zm zk zm zf ofm zm xdcuz dzadnqzprr zm vgsld"
          176  +  "sbptoa jk tugflixoex jk zm zm vgsld zm xh zm"
          177  +  "uhzq jk zk evvivo vgsld vniqnuynvf agmckuiu jk zm zm"
          178  +  "zm nvfasfh zm zm zm abthnzcv uenvbm jk zk dzadnqzprr"
          179  +  "zm azavwm igju qzzqdxq jk xnxhf abthnzcv jk nvfasfh zm"
          180  +  "qbamok fxw vgsld igju cmnnkna xnxhf vniqnuynvf zk xh zm"
          181  +  "nvfasfh zk zm mjpavjuhw dzadnqzprr jk jk duszemmzl xldlpy nvfasfh"
          182  +  "xnxhf sviq nsab npywgdvgz osff vgsld farlehdhq fibokdry wjbkhzsa hhac"
          183  +  "zm azavwm scpgna jk jk bq jk duszemmzl fibokdry ovkyevlgv"
          184  +  "csjqxhgj zm jk jk duszemmzl zk xh zm jk zf"
          185  +  "urvysbnykk dzadnqzprr csjqxhgj mjpavjuhw ubwrfqnbjf nkaotm jk jk zm drir"
          186  +  "nvfasfh xh igju zm wluvgsw jk zm srwwnezqk ewle ovnq"
          187  +  "jk nvfasfh eh ktxdty urvysbnykk vgsld zm jk eh uenvbm"
          188  +  "orpfawpx pahlds jk uhzq hi zm zm zf jk dzadnqzprr"
          189  +  "srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki"
          190  +}
          191  +
          192  +proc add_empty_records {n} {
          193  +  execsql BEGIN
          194  +  for {set i 0} {$i < $n} {incr i} {
          195  +    execsql { INSERT INTO t1 VALUES('') }
          196  +  }
          197  +  execsql COMMIT
          198  +}
          199  +
          200  +
          201  +#set e [list]
          202  +#foreach d $data {set e [concat $e $d]}
          203  +#puts [lsort -unique $e]
          204  +#exit
          205  +
          206  +set zero_long_doclists {
          207  +  UPDATE t1_segments SET block=zeroblob(length(block)) WHERE length(block)>10000
          208  +}
          209  +
          210  +foreach {tn setup} {
          211  +  1 {
          212  +    set dmt_modes 0
          213  +    execsql { CREATE VIRTUAL TABLE t1 USING FTS3 }
          214  +    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
          215  +  }
          216  +  2 {
          217  +    set dmt_modes 0
          218  +    execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
          219  +    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
          220  +  }
          221  +  3 {
          222  +    set dmt_modes {0 1 2}
          223  +    execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
          224  +    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
          225  +    add_empty_records 1000
          226  +    execsql $zero_long_doclists
          227  +  }
          228  +  4 {
          229  +    set dmt_modes 0
          230  +    execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
          231  +    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
          232  +    add_empty_records 1000
          233  +    execsql "INSERT INTO t1(t1) VALUES('optimize')"
          234  +    execsql $zero_long_doclists
          235  +  }
          236  +} {
          237  +
          238  +  execsql { DROP TABLE IF EXISTS t1 }
          239  +  eval $setup
          240  +  set ::testprefix fts3defer-2.$tn
          241  +  set DO_MALLOC_TEST 0
          242  +
          243  +  do_execsql_test 0 { 
          244  +    SELECT count(*) FROM t1_segments WHERE length(block)>10000 
          245  +  } {2}
          246  +
          247  +  do_select_test 1.1 {
          248  +    SELECT rowid FROM t1 WHERE t1 MATCH 'jk xnxhf'
          249  +  } {13 29 40 47 48 52 63 92}
          250  +  do_select_test 1.2 {
          251  +    SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
          252  +  } {100}
          253  +if {$tn==3} breakpoint
          254  +  do_select_test 1.3 {
          255  +    SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
          256  +  } {7 70 98}
          257  +  do_select_test 1.4 {
          258  +    SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl jk'
          259  +  } {3 5 8 10 13 18 20 23 32 37 41 43 55 60 65 67 72 74 76 81 94 96 97}
          260  +  do_select_test 1.5 {
          261  +    SELECT rowid FROM t1 WHERE t1 MATCH 'ubwrfqnbjf jk'
          262  +  } {7 70 98}
          263  +  do_select_test 1.6 {
          264  +    SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf jk jk jk jk'
          265  +  } {7 70 98}
          266  +  do_select_test 1.7 {
          267  +    SELECT rowid FROM t1 WHERE t1 MATCH 'zm xnxhf'
          268  +  } {12 13 29 30 40 47 48 52 63 92 93}
          269  +  do_select_test 1.8 {
          270  +    SELECT rowid FROM t1 WHERE t1 MATCH 'zm eh'
          271  +  } {68 100}
          272  +  do_select_test 1.9 {
          273  +    SELECT rowid FROM t1 WHERE t1 MATCH 'zm ubwrfqnbjf'
          274  +  } {7 70 98}
          275  +  do_select_test 1.10 {
          276  +    SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
          277  +  } {10 13 17 31 35 51 58 88 89 90 93 100}
          278  +  do_select_test 1.11 {
          279  +    SELECT rowid FROM t1 
          280  +    WHERE t1 MATCH '(
          281  +      zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR 
          282  +      zk OR zkhdvkw OR zm OR zsmhnf
          283  +    ) vgsld'
          284  +  } {10 13 17 31 35 51 58 88 89 90 93 100}
          285  +
          286  +  do_select_test 2.1 {
          287  +    SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
          288  +  } {3 24 52 53}
          289  +  do_select_test 2.2 {
          290  +    SELECT rowid FROM t1 WHERE t1 MATCH '"zm zf"'
          291  +  } {33 53 75 88 101}
          292  +  do_select_test 2.3 {
          293  +    SELECT rowid FROM t1 WHERE t1 MATCH '"zm aayxpmve"'
          294  +  } {48 65 84}
          295  +  do_select_test 2.4 {
          296  +    SELECT rowid FROM t1 WHERE t1 MATCH '"aayxpmve zm"'
          297  +  } {11 37 84}
          298  +  do_select_test 2.5 {
          299  +    SELECT rowid FROM t1 WHERE t1 MATCH '"jk azavwm"'
          300  +  } {16 53}
          301  +  do_select_test 2.6 {
          302  +    SELECT rowid FROM t1 WHERE t1 MATCH '"xh jk jk"'
          303  +  } {18}
          304  +  do_select_test 2.7 {
          305  +    SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld"'
          306  +  } {13 17}
          307  +  do_select_test 2.8 {
          308  +    SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld lkjlkjlkj"'
          309  +  } {}
          310  +
          311  +  do_select_test 3.1 {
          312  +    SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
          313  +  } {
          314  +    {zm [zm] [agmckuiu] uhzq nsab jk rrkx duszemmzl hyq jk} 
          315  +    {jk [zm] [agmckuiu] urvysbnykk jk jk zm zm jk jk} 
          316  +    {[zm] [agmckuiu] zexh fibokdry jk uhzq bu tugflixoex xnxhf sk} 
          317  +    {zm zf uenvbm jk azavwm zm [zm] [agmckuiu] zm jk}
          318  +  }
          319  +
          320  +  do_select_test 3.2 {
          321  +    SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'xnxhf jk'
          322  +  } {
          323  +    {[xnxhf] [jk] [jk] agmckuiu duszemmzl [jk] zm zm [jk] vgsld} 
          324  +    {[jk] [xnxhf] igju [jk] xh [jk] nvfasfh zm js [jk]} 
          325  +    {[jk] jcpiwj zm [xnxhf] zm mjpavjuhw mj drir pa pvjrjlas} 
          326  +    {gazrt [jk] ephknonq myjp uenvbm wuvajhwqz [jk] zm [xnxhf] nvfasfh} 
          327  +    {zm aayxpmve csjqxhgj [xnxhf] xr [jk] aayxpmve [xnxhf] zm zm} 
          328  +    {zm agmckuiu zexh fibokdry [jk] uhzq bu tugflixoex [xnxhf] sk} 
          329  +    {lwa hi [xnxhf] qdyerbws zk njtc [jk] uhzq zm [jk]} 
          330  +    {zm azavwm igju qzzqdxq [jk] [xnxhf] abthnzcv [jk] nvfasfh zm}
          331  +  }
          332  +
          333  +  do_select_test 4.1 {
          334  +    SELECT offsets(t1) FROM t1 WHERE t1 MATCH '"jk uenvbm"'
          335  +  } {
          336  +    {0 0 10 2 0 1 13 6} {0 0 26 2 0 1 29 6}
          337  +  }
          338  +
          339  +  do_select_test 4.2 {
          340  +    SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'duszemmzl jk fibokdry'
          341  +  } {
          342  +    {0 2 3 8 0 1 36 2 0 0 58 9} 
          343  +    {0 0 0 9 0 1 13 2 0 1 16 2 0 2 19 8 0 1 53 2} 
          344  +    {0 1 4 2 0 0 20 9 0 1 30 2 0 1 33 2 0 2 48 8} 
          345  +    {0 1 17 2 0 1 20 2 0 1 26 2 0 0 29 9 0 2 39 8}
          346  +  }
          347  +
          348  +  do_select_test 4.3 {
          349  +    SELECT offsets(t1) FROM t1 
          350  +    WHERE t1 MATCH 'vgsld (hrlipdm OR (aapmve NEAR duszemmzl))'
          351  +  } {{0 0 0 5 0 1 15 7 0 0 36 5}}
          352  +
          353  +  # The following block of tests runs normally with FTS3 or FTS4 without the
          354  +  # long doclists zeroed. And with OOM-injection for FTS4 with long doclists
          355  +  # zeroed. Change this by messing with the [set dmt_modes] commands above.
          356  +  #
          357  +  foreach DO_MALLOC_TEST $dmt_modes {
          358  +    
          359  +    # Phrase search.
          360  +    do_select_test 5.$DO_MALLOC_TEST.1 {
          361  +      SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"'
          362  +    } {8 15 36 64 67 72}
          363  +
          364  +    # Multiple tokens search.
          365  +    do_select_test 5.$DO_MALLOC_TEST.2 {
          366  +      SELECT rowid FROM t1 WHERE t1 MATCH 'duszemmzl zm'
          367  +    } {3 5 8 10 12 13 18 20 23 37 43 55 60 65 67 72 74 81 94 96 97}
          368  +
          369  +    # snippet() function with phrase.
          370  +    do_select_test 5.$DO_MALLOC_TEST.3 {
          371  +      SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm aayxpmve"'
          372  +    } {
          373  +      {[zm] [aayxpmve] csjqxhgj xnxhf xr jk aayxpmve xnxhf zm zm} 
          374  +      {duszemmzl zk jk jk fibokdry jseuhjnzo [zm] [aayxpmve] zk jk} 
          375  +      {zf jmfiqhwnjg js igju [zm] [aayxpmve] zm mbxnljomiv csjqxhgj nvfasfh}
          376  +    }
          377  +    
          378  +    # snippet() function with multiple tokens.
          379  +    do_select_test 5.$DO_MALLOC_TEST.4 {
          380  +      SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'zm zhbrzadb'
          381  +    } {
          382  +      {[zm] jk [zm] [zm] [zhbrzadb] uenvbm aayxpmve urvysbnykk duszemmzl jk}
          383  +    }
          384  +    
          385  +    # snippet() function with phrase.
          386  +    do_select_test 5.$DO_MALLOC_TEST.5 {
          387  +      SELECT offsets(t1) FROM t1 WHERE t1 MATCH '"zm aayxpmve"'
          388  +    } {
          389  +      {0 0 0 2 0 1 3 8} {0 0 38 2 0 1 41 8} {0 0 22 2 0 1 25 8}
          390  +    }
          391  +    
          392  +    # snippet() function with multiple tokens.
          393  +    do_select_test 5.$DO_MALLOC_TEST.6 {
          394  +      SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'zm zhbrzadb'
          395  +    } {
          396  +      {0 0 0 2 0 0 6 2 0 0 9 2 0 1 12 8}
          397  +    }
          398  +
          399  +    set DO_MALLOC_TEST 0
          400  +  }
          401  +
          402  +  do_select_test 6.1 {
          403  +    SELECT rowid FROM t1 
          404  +    WHERE t1 MATCH 'vgsld (hrlipdm OR (aayxpmve duszemmzl))'
          405  +  } {10}
          406  +  do_select_test 6.2.1 {
          407  +    SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk"'
          408  +  } {8}
          409  +  do_select_test 6.2.2 {
          410  +    SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"'
          411  +  } {15 26 92 96}
          412  +  do_select_test 6.2.3 {
          413  +    SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
          414  +  } {8 15 26 92 96}
          415  +}
          416  +
          417  +set testprefix fts3defer
          418  +
          419  +do_execsql_test 3.1 {
          420  +  CREATE VIRTUAL TABLE x1 USING fts4(a, b);
          421  +  INSERT INTO x1 VALUES('a b c', 'd e f');
          422  +  INSERT INTO x1 SELECT * FROM x1;
          423  +  INSERT INTO x1 SELECT * FROM x1;
          424  +  INSERT INTO x1 SELECT * FROM x1;
          425  +  INSERT INTO x1 SELECT * FROM x1;
          426  +}
          427  +do_execsql_test 3.2 "
          428  +  INSERT INTO x1 VALUES(
          429  +    '[string repeat {d } 3000]', '[string repeat {f } 30000]'
          430  +  );
          431  +  INSERT INTO x1(x1) VALUES('optimize');
          432  +"
          433  +
          434  +do_execsql_test 3.3 {
          435  +  SELECT count(*) FROM x1 WHERE x1 MATCH '"d e f"'
          436  +} {16}
          437  +
          438  +
          439  +finish_test

Added test/fts3defer2.test.

            1  +# 2010 October 23
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +source $testdir/malloc_common.tcl
           16  +ifcapable !fts3 { finish_test ; return }
           17  +
           18  +set testprefix fts3defer2
           19  +
           20  +proc mit {blob} {
           21  +  set scan(littleEndian) i*
           22  +  set scan(bigEndian) I*
           23  +  binary scan $blob $scan($::tcl_platform(byteOrder)) r
           24  +  return $r
           25  +}
           26  +db func mit mit
           27  +
           28  +#-----------------------------------------------------------------------------
           29  +# This block of tests - fts3defer2-1.* - test the interaction of deferred NEAR
           30  +# expressions and the snippet, offsets and matchinfo functions.
           31  +# 
           32  +do_execsql_test 1.1.1 {
           33  +  CREATE VIRTUAL TABLE t1 USING fts4;
           34  +}
           35  +do_execsql_test 1.1.2 "INSERT INTO t1 VALUES('[string repeat {a } 20000]')"
           36  +do_execsql_test 1.1.3 "INSERT INTO t1 VALUES('[string repeat {z } 20000]')"
           37  +do_execsql_test 1.1.4 {
           38  +  INSERT INTO t1 VALUES('a b c d e f a x y');
           39  +  INSERT INTO t1 VALUES('');
           40  +  INSERT INTO t1 VALUES('');
           41  +  INSERT INTO t1 VALUES('');
           42  +  INSERT INTO t1 VALUES('');
           43  +  INSERT INTO t1 VALUES('');
           44  +  INSERT INTO t1(t1) VALUES('optimize');
           45  +}
           46  +do_execsql_test 1.1.4 {
           47  +  SELECT count(*) FROM t1_segments WHERE length(block)>10000;
           48  +  UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000;
           49  +} {2}
           50  +
           51  +do_execsql_test 1.2.1 {
           52  +  SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
           53  +} {{a b c d e f a x y}}
           54  +
           55  +do_execsql_test 1.2.2 {
           56  +  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
           57  +  FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
           58  +} [list                              \
           59  +   {a b c d [e] [f] [a] x y}         \
           60  +   {0 1 8 1 0 0 10 1 0 2 12 1}       \
           61  +   [list 3 1   1 1 1   1 8 8   1 8 8   8 5001 9]
           62  +]
           63  +
           64  +do_execsql_test 1.2.3 {
           65  +  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
           66  +  FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
           67  +} [list                                 \
           68  +   {[a] b c d [e] [f] [a] x y}          \
           69  +   {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
           70  +   [list 3 1   1 1 1   1 8 8   2 8 8   8 5001 9]
           71  +]
           72  +
           73  +do_execsql_test 1.3.1 { DROP TABLE t1 }
           74  +
           75  +#-----------------------------------------------------------------------------
           76  +# Test cases fts3defer2-2.* focus specifically on the matchinfo function.
           77  +# 
           78  +do_execsql_test 2.1.1 "CREATE VIRTUAL TABLE t2 USING fts4"
           79  +do_execsql_test 2.1.2 "INSERT INTO t2 VALUES('[string repeat {a } 10000]')"
           80  +do_execsql_test 2.1.3 "INSERT INTO t2 VALUES('b [string repeat {z } 10000]')"
           81  +do_execsql_test 2.1.4 [string repeat "INSERT INTO t2 VALUES('x');" 50]
           82  +do_execsql_test 2.1.5 {
           83  +  INSERT INTO t2 VALUES('a b c d e f g');
           84  +  INSERT INTO t2 VALUES('a b c d e f g');
           85  +}
           86  +foreach {tn sql} {
           87  +  1 {}
           88  +  2 { INSERT INTO t2(t2) VALUES('optimize') }
           89  +  3 { UPDATE t2_segments SET block = zeroblob(length(block)) 
           90  +      WHERE length(block)>10000;
           91  +  }
           92  +} {
           93  +  execsql $sql
           94  +  do_execsql_test 2.2.$tn {
           95  +    SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'a b';
           96  +  } [list                                          \
           97  +    [list 2 1  1 54 54  1 3 3  54 372 7]        \
           98  +    [list 2 1  1 54 54  1 3 3  54 372 7]        \
           99  +  ]
          100  +}
          101  +
          102  +do_execsql_test 2.3.1 {
          103  +  CREATE VIRTUAL TABLE t3 USING fts4;
          104  +  INSERT INTO t3 VALUES('a b c d e f');
          105  +  INSERT INTO t3 VALUES('x b c d e f');
          106  +  INSERT INTO t3 VALUES('d e f a b c');
          107  +  INSERT INTO t3 VALUES('b c d e f');
          108  +  INSERT INTO t3 VALUES('');
          109  +  INSERT INTO t3 VALUES('');
          110  +  INSERT INTO t3 VALUES('');
          111  +  INSERT INTO t3 VALUES('');
          112  +  INSERT INTO t3 VALUES('');
          113  +  INSERT INTO t3 VALUES('');
          114  +}
          115  +do_execsql_test 2.3.2 "
          116  +  INSERT INTO t3 VALUES('f e d c b [string repeat {a } 10000]')
          117  +"
          118  +foreach {tn sql} {
          119  +  1 {}
          120  +  2 { INSERT INTO t3(t3) VALUES('optimize') }
          121  +  3 { UPDATE t3_segments SET block = zeroblob(length(block)) 
          122  +      WHERE length(block)>10000;
          123  +  }
          124  +} {
          125  +  execsql $sql
          126  +  do_execsql_test 2.4.$tn {
          127  +    SELECT docid, mit(matchinfo(t3)) FROM t3 WHERE t3 MATCH '"a b c"';
          128  +  } {1 {1 1 1 4 4 11 912 6} 3 {1 1 1 4 4 11 912 6}}
          129  +}
          130  +
          131  +
          132  +finish_test
          133  +

Added test/fts3fault.test.

            1  +# 2010 June 15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +
           13  +set testdir [file dirname $argv0]
           14  +source $testdir/tester.tcl
           15  +
           16  +set ::testprefix fts3fault
           17  +
           18  +# Test error handling in the sqlite3Fts3Init() function. This is the 
           19  +# function that registers the FTS3 module and various support functions
           20  +# with SQLite.
           21  +#
           22  +do_faultsim_test 1 -body { 
           23  +  sqlite3 db test.db 
           24  +  expr 0
           25  +} -test {
           26  +  catch { db close }
           27  +}
           28  +
           29  +# Test error handling in an "ALTER TABLE ... RENAME TO" statement on an
           30  +# FTS3 table. Specifically, test renaming the table within a transaction
           31  +# after it has been written to.
           32  +#
           33  +faultsim_delete_and_reopen
           34  +do_execsql_test 2.0 {
           35  +  CREATE VIRTUAL TABLE t1 USING fts3;
           36  +  INSERT INTO t1 VALUES('test renaming the table');
           37  +  INSERT INTO t1 VALUES(' after it has been written');
           38  +}
           39  +do_faultsim_test 2 -prep { 
           40  +  sqlite3 db test.db
           41  +  execsql {
           42  +    BEGIN;
           43  +      INSERT INTO t1 VALUES('registers the FTS3 module');
           44  +      INSERT INTO t1 VALUES('various support functions');
           45  +  }
           46  +} -body {
           47  +  execsql { ALTER TABLE t1 RENAME TO t2 }
           48  +} -test {
           49  +  faultsim_test_result {0 {}} 
           50  +}
           51  +
           52  +# Test error handling in the special case where a single prefix query 
           53  +# matches terms that reside on a large range of leaf nodes.
           54  +#
           55  +do_test fts3fault-3.0 {
           56  +  sqlite3 db test.db
           57  +  execsql { CREATE VIRTUAL TABLE t3 USING fts4; }
           58  +  execsql { INSERT INTO t3(t3) VALUES('nodesize=50') }
           59  +  execsql { BEGIN }
           60  +  for {set i 0} {$i < 1000} {incr i} {
           61  +    execsql { INSERT INTO t3 VALUES('aaa' || $i) }
           62  +  }
           63  +  execsql { COMMIT }
           64  +} {}
           65  +
           66  +do_faultsim_test 3 -faults oom-transient -prep { 
           67  +  sqlite3 db test.db
           68  +  execsql { SELECT * FROM t3 WHERE t3 MATCH 'x' }
           69  +} -body {
           70  +  execsql { SELECT count(rowid) FROM t3 WHERE t3 MATCH 'aa*' }
           71  +} -test {
           72  +  faultsim_test_result {0 1000} 
           73  +}
           74  +
           75  +do_test fts3fault-4.0 {
           76  +  faultsim_delete_and_reopen
           77  +  execsql { 
           78  +    CREATE VIRTUAL TABLE t4 USING fts4; 
           79  +    INSERT INTO t4 VALUES('The British Government called on');
           80  +    INSERT INTO t4 VALUES('as pesetas then became much');
           81  +  }
           82  +} {}
           83  +faultsim_save_and_close
           84  +do_faultsim_test 4 -prep { 
           85  +  faultsim_restore_and_reopen
           86  +  execsql { SELECT content FROM t4 }
           87  +} -body {
           88  +  execsql { SELECT optimize(t4) FROM t4 LIMIT 1 }
           89  +} -test {
           90  +  faultsim_test_result {0 {{Index optimized}}}
           91  +}
           92  +
           93  +do_test fts3fault-5.0 {
           94  +  faultsim_delete_and_reopen
           95  +  execsql { 
           96  +    CREATE VIRTUAL TABLE t5 USING fts4; 
           97  +    INSERT INTO t5 VALUES('The British Government called on');
           98  +    INSERT INTO t5 VALUES('as pesetas then became much');
           99  +  }
          100  +} {}
          101  +faultsim_save_and_close
          102  +do_faultsim_test 5 -prep { 
          103  +  faultsim_restore_and_reopen
          104  +  execsql { 
          105  +    BEGIN;
          106  +      INSERT INTO t5 VALUES('influential in shaping his future outlook');
          107  +      INSERT INTO t5 VALUES('might be acceptable to the British electorate');
          108  +  }
          109  +} -body {
          110  +  execsql { SELECT rowid FROM t5 WHERE t5 MATCH 'british' }
          111  +} -test {
          112  +  faultsim_test_result {0 {1 4}}
          113  +}
          114  +
          115  +do_test fts3fault-6.0 {
          116  +  faultsim_delete_and_reopen
          117  +  execsql { CREATE VIRTUAL TABLE t6 USING fts4 }
          118  +} {}
          119  +faultsim_save_and_close
          120  +do_faultsim_test 6 -prep { 
          121  +  faultsim_restore_and_reopen
          122  +  execsql { SELECT rowid FROM t6 }
          123  +} -body {
          124  +  execsql { DROP TABLE t6 }
          125  +} -test {
          126  +  faultsim_test_result {0 {}}
          127  +}
          128  +
          129  +finish_test

Changes to test/fts3malloc.test.

    39     39   
    40     40   
    41     41   proc normal_list {l} {
    42     42     set ret [list]
    43     43     foreach elem $l {lappend ret $elem}
    44     44     set ret
    45     45   }
    46         -
    47     46   
    48     47   do_write_test fts3_malloc-1.1 sqlite_master {
    49     48     CREATE VIRTUAL TABLE ft1 USING fts3(a, b)
    50     49   }
    51     50   do_write_test fts3_malloc-1.2 sqlite_master {
    52     51     CREATE VIRTUAL TABLE ft2 USING fts3([a], [b]);
    53     52   }

Changes to test/fts3query.test.

    20     20   # If this build does not include FTS3, skip the tests in this file.
    21     21   #
    22     22   ifcapable !fts3 { finish_test ; return }
    23     23   source $testdir/malloc_common.tcl
    24     24   source $testdir/fts3_common.tcl
    25     25   set DO_MALLOC_TEST 0
    26     26   
           27  +set testprefix fts3query
           28  +
    27     29   do_test fts3query-1.1 {
    28     30     execsql {
    29     31       CREATE VIRTUAL TABLE t1 USING fts3(x);
    30     32       BEGIN;
    31     33         INSERT INTO t1 VALUES('The source code for SQLite is in the public');
    32     34     }
    33     35   } {}
................................................................................
   125    127   do_test fts3query-4.4 {
   126    128     eqp "SELECT t1.number FROM t1, bt WHERE t1.number=bt.rowid ORDER BY t1.date"
   127    129   } {0 0 {TABLE t1 WITH INDEX i1 ORDER BY} 1 1 {TABLE bt USING PRIMARY KEY}}
   128    130   do_test fts3query-4.5 {
   129    131     eqp "SELECT t1.number FROM bt, t1 WHERE t1.number=bt.rowid ORDER BY t1.date"
   130    132   } {0 1 {TABLE t1 WITH INDEX i1 ORDER BY} 1 0 {TABLE bt USING PRIMARY KEY}}
   131    133   
          134  +
          135  +# Test that calling matchinfo() with the wrong number of arguments, or with
          136  +# an invalid argument returns an error.
          137  +#
          138  +do_execsql_test 5.1 {
          139  +  CREATE VIRTUAL TABLE t2 USING FTS4;
          140  +  INSERT INTO t2 VALUES('it was the first time in history');
          141  +}
          142  +do_select_tests 5.2 -errorformat {
          143  +  wrong number of arguments to function %s()
          144  +} {
          145  +  1 "SELECT matchinfo() FROM t2 WHERE t2 MATCH 'history'"       matchinfo
          146  +  2 "SELECT matchinfo(t2, t2) FROM t2 WHERE t2 MATCH 'history'" matchinfo
          147  +
          148  +  3 "SELECT snippet(t2, 1, 2, 3, 4, 5, 6) FROM t2 WHERE t2 MATCH 'history'" 
          149  +    snippet
          150  +}
          151  +do_select_tests 5.3 -errorformat {
          152  +  illegal first argument to %s
          153  +} {
          154  +  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
          155  +  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
          156  +  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
          157  +  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
          158  +}
          159  +do_execsql_test 5.4.0 { UPDATE t2_content SET c0content = X'1234' }
          160  +do_select_tests 5.4 -errorformat {
          161  +  illegal first argument to %s
          162  +} {
          163  +  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
          164  +  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
          165  +  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
          166  +  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
          167  +}
          168  +do_execsql_test 5.5 { DROP TABLE t2 }
          169  +
          170  +# Test the snippet() function with 1 to 6 arguments.
          171  +# 
          172  +do_execsql_test 6.1 {
          173  +  CREATE VIRTUAL TABLE t3 USING FTS4(a, b);
          174  +  INSERT INTO t3 VALUES('no gestures', 'another intriguing discovery by observing the hand gestures (called beats) people make while speaking. Research has shown that such gestures do more than add visual emphasis to our words (many people gesture while they''re on the telephone, for example); it seems they actually help our brains find words');
          175  +}
          176  +do_select_tests 6.2 {
          177  +  1 "SELECT snippet(t3) FROM t3 WHERE t3 MATCH 'gestures'"
          178  +  {{<b>...</b>hand <b>gestures</b> (called beats) people make while speaking. Research has shown that such <b>gestures</b> do<b>...</b>}}
          179  +
          180  +  2 "SELECT snippet(t3, 'XXX') FROM t3 WHERE t3 MATCH 'gestures'" 
          181  +  {{<b>...</b>hand XXXgestures</b> (called beats) people make while speaking. Research has shown that such XXXgestures</b> do<b>...</b>}}
          182  +
          183  +  3 "SELECT snippet(t3, 'XXX', 'YYY') FROM t3 WHERE t3 MATCH 'gestures'" 
          184  +  {{<b>...</b>hand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY do<b>...</b>}}
          185  +
          186  +  4 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ') FROM t3 WHERE t3 MATCH 'gestures'" 
          187  +  {{ZZZhand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY doZZZ}}
          188  +
          189  +  5 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 1) FROM t3 WHERE t3 MATCH 'gestures'" 
          190  +  {{ZZZhand XXXgesturesYYY (called beats) people make while speaking. Research has shown that such XXXgesturesYYY doZZZ}}
          191  +
          192  +  6 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 0) FROM t3 WHERE t3 MATCH 'gestures'" 
          193  +  {{no XXXgesturesYYY}}
          194  +
          195  +  7 "SELECT snippet(t3, 'XXX', 'YYY', 'ZZZ', 1, 5) FROM t3 WHERE t3 MATCH 'gestures'" 
          196  +  {{ZZZthe hand XXXgesturesYYY (called beatsZZZ}}
          197  +}
          198  +
   132    199   
   133    200   finish_test
   134    201   

Changes to test/fts3snippet.test.

    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   
    20     20   # If SQLITE_ENABLE_FTS3 is not defined, omit this file.
    21     21   ifcapable !fts3 { finish_test ; return }
    22     22   source $testdir/fts3_common.tcl
    23         -source $testdir/malloc_common.tcl
    24     23   
    25     24   set sqlite_fts3_enable_parentheses 1
    26     25   set DO_MALLOC_TEST 0
    27     26   
    28     27   # Transform the list $L to its "normal" form. So that it can be compared to
    29     28   # another list with the same set of elements using [string compare].
    30     29   #
................................................................................
   455    454     do_select_test $T.10.5 {
   456    455       SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft;
   457    456     } {0 blob 0 blob 0 blob}
   458    457     do_select_test $T.10.6 {
   459    458       SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r
   460    459     } {0 blob}
   461    460   }
   462         -
   463    461   
   464    462   set sqlite_fts3_enable_parentheses 0
   465    463   finish_test

Added test/incrblob3.test.

            1  +# 2010 October 20
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +#
           13  +
           14  +set testdir [file dirname $argv0]
           15  +source $testdir/tester.tcl
           16  +
           17  +sqlite3 db test.db
           18  +sqlite3_db_config_lookaside db 0 0 0
           19  +
           20  +do_execsql_test incrblob3-1.1 {
           21  +  CREATE TABLE blobs(k INTEGER PRIMARY KEY, v BLOB);
           22  +  INSERT INTO blobs VALUES(1, zeroblob(100));
           23  +  INSERT INTO blobs VALUES(2, zeroblob(100));
           24  +} {}
           25  +
           26  +# Test the sqlite3_blob_reopen()/read()/write() functions.
           27  +#
           28  +do_test incrblob3-1.2 {
           29  +  set ::blob [db incrblob blobs v 1]
           30  +  puts $::blob "hello world"
           31  +} {}
           32  +
           33  +do_test incrblob3-1.3 {
           34  +  sqlite3_blob_reopen $::blob 2
           35  +  puts $::blob "world hello"
           36  +} {}
           37  +
           38  +do_test incrblob3-1.4 {
           39  +  sqlite3_blob_reopen $::blob 1
           40  +  gets $::blob
           41  +} {hello world}
           42  +
           43  +do_test incrblob3-1.5 {
           44  +  sqlite3_blob_reopen $::blob 2
           45  +  gets $::blob
           46  +} {world hello}
           47  +
           48  +do_test incrblob3-1.6 { close $::blob } {}
           49  +
           50  +# Test some error conditions.
           51  +#
           52  +#   incrblob3-2.1: Attempting to reopen a row that does not exist.
           53  +#   incrblob3-2.2: Attempting to reopen a row that does not contain a blob
           54  +#                  or text value.
           55  +#
           56  +do_test incrblob3-2.1.1 {
           57  +  set ::blob [db incrblob blobs v 1]
           58  +  list [catch {sqlite3_blob_reopen $::blob 3} msg] $msg
           59  +} {1 SQLITE_ERROR}
           60  +do_test incrblob3-2.1.2 {
           61  +  list [sqlite3_errcode db] [sqlite3_errmsg db]
           62  +} {SQLITE_ERROR {no such rowid: 3}}
           63  +do_test incrblob3-2.1.3 {
           64  +  list [catch {sqlite3_blob_reopen $::blob 1} msg] $msg
           65  +} {1 SQLITE_ABORT}
           66  +do_test incrblob3-2.1.4 { close $::blob } {}
           67  +
           68  +do_execsql_test incrblob3-2.2.1 {
           69  +  INSERT INTO blobs VALUES(3, 42);
           70  +  INSERT INTO blobs VALUES(4, 54.4);
           71  +  INSERT INTO blobs VALUES(5, NULL);
           72  +}
           73  +foreach {tn rowid type} {
           74  +  1 3 integer
           75  +  2 4 real
           76  +  3 5 null
           77  +} {
           78  +  do_test incrblob3-2.2.$tn.1 {
           79  +    set ::blob [db incrblob blobs v 1]
           80  +    list [catch {sqlite3_blob_reopen $::blob $rowid} msg] $msg
           81  +  } {1 SQLITE_ERROR}
           82  +  do_test incrblob3-2.2.$tn.2 {
           83  +    list [sqlite3_errcode db] [sqlite3_errmsg db]
           84  +  } "SQLITE_ERROR {cannot open value of type $type}"
           85  +
           86  +  do_test incrblob3-2.2.$tn.3 {
           87  +    list [catch {sqlite3_blob_reopen $::blob 1} msg] $msg
           88  +  } {1 SQLITE_ABORT}
           89  +  do_test incrblob3-2.2.$tn.4 {
           90  +    list [catch {sqlite3_blob_read $::blob 0 10} msg] $msg
           91  +  } {1 SQLITE_ABORT}
           92  +  do_test incrblob3-2.2.$tn.5 {
           93  +    list [catch {sqlite3_blob_write $::blob 0 "abcd"} msg] $msg
           94  +  } {1 SQLITE_ABORT}
           95  +
           96  +  do_test incrblob3-2.2.$tn.4 { close $::blob } {}
           97  +}
           98  +
           99  +# Test that passing NULL to sqlite3_blob_XXX() APIs returns SQLITE_MISUSE.
          100  +#
          101  +#   incrblob3-3.1: sqlite3_blob_reopen()
          102  +#   incrblob3-3.2: sqlite3_blob_read()
          103  +#   incrblob3-3.3: sqlite3_blob_write()
          104  +#   incrblob3-3.4: sqlite3_blob_bytes()
          105  +#
          106  +do_test incrblob3-3.1 {
          107  +  list [catch {sqlite3_blob_reopen {} 3} msg] $msg
          108  +} {1 SQLITE_MISUSE}
          109  +
          110  +do_test incrblob3-3.2 {
          111  +  list [catch {sqlite3_blob_read {} 0 10} msg] $msg
          112  +} {1 SQLITE_MISUSE}
          113  +
          114  +do_test incrblob3-3.3 {
          115  +  list [catch {sqlite3_blob_write {} 0 "abcd"} msg] $msg
          116  +} {1 SQLITE_MISUSE}
          117  +
          118  +do_test incrblob3-3.4 { sqlite3_blob_bytes {} } {0}
          119  +
          120  +do_test incrblob3-3.5 { sqlite3_blob_close {} } {}
          121  +
          122  +# Test out-of-range reading and writing
          123  +#
          124  +do_test incrblob3-4.1 {
          125  +  set ::blob [db incrblob blobs v 1]
          126  +  sqlite3_blob_bytes $::blob
          127  +} {100}
          128  +do_test incrblob3-4.2 {
          129  +  list [catch { sqlite3_blob_read $::blob -1 10 } msg] $msg
          130  +} {1 SQLITE_ERROR}
          131  +do_test incrblob3-4.3 {
          132  +  list [catch { sqlite3_blob_read $::blob 0 -10 } msg] $msg
          133  +} {1 SQLITE_ERROR}
          134  +do_test incrblob3-4.4 {
          135  +  list [catch { sqlite3_blob_read $::blob 95 10 } msg] $msg
          136  +} {1 SQLITE_ERROR}
          137  +do_test incrblob3-4.5 {
          138  +  list [catch { sqlite3_blob_write $::blob -1 "abcdefghij" 10 } msg] $msg
          139  +} {1 SQLITE_ERROR}
          140  +do_test incrblob3-4.6 {
          141  +  list [catch { sqlite3_blob_write $::blob 0 "abcdefghij" -10 } msg] $msg
          142  +} {1 SQLITE_ERROR}
          143  +do_test incrblob3-4.7 {
          144  +  list [catch { sqlite3_blob_write $::blob 95 "abcdefghij" } msg] $msg
          145  +} {1 SQLITE_ERROR}
          146  +
          147  +do_test incrblob3-4.8 { close $::blob } {}
          148  +
          149  +# Test that modifying the row a blob handle points to aborts the blob.
          150  +#
          151  +do_test incrblob3-5.1 {
          152  +  set ::blob [db incrblob blobs v 1]
          153  +  sqlite3_blob_bytes $::blob
          154  +} {100}
          155  +do_test incrblob3-5.2 {
          156  +  execsql { UPDATE blobs SET v = '123456789012345678901234567890' WHERE k = 1 }
          157  +  list [catch { sqlite3_blob_read $::blob 0 10 } msg] $msg
          158  +} {1 SQLITE_ABORT}
          159  +
          160  +# Test various errors that can occur in sqlite3_blob_open():
          161  +#
          162  +#   1. Trying to open a virtual table column.
          163  +#   2. Trying to open a view column.
          164  +#   3. Trying to open a column that does not exist.
          165  +#   4. Trying to open a read/write handle on an indexed column.
          166  +#   5. Trying to open a read/write handle on the child key of an FK constraint.
          167  +#
          168  +ifcapable fts3 {
          169  +  do_test incrblob3-6.1 {
          170  +    execsql {
          171  +      CREATE VIRTUAL TABLE ft USING fts3;
          172  +      INSERT INTO ft VALUES('rules to open a column to which');
          173  +    }
          174  +
          175  +    list [catch { db incrblob ft content 1 } msg] $msg
          176  +  } {1 {cannot open virtual table: ft}}
          177  +}
          178  +ifcapable view {
          179  +  do_test incrblob3-6.2 {
          180  +    execsql { CREATE VIEW v1 AS SELECT * FROM blobs }
          181  +    list [catch { db incrblob v1 content 1 } msg] $msg
          182  +  } {1 {cannot open view: v1}}
          183  +}
          184  +
          185  +do_test incrblob3-6.3 {
          186  +  list [catch { db incrblob blobs content 1 } msg] $msg
          187  +} {1 {no such column: "content"}}
          188  +
          189  +do_test incrblob3-6.4.1 {
          190  +  execsql { 
          191  +    CREATE TABLE t1(a, b);
          192  +    CREATE INDEX i1 ON t1(b);
          193  +    INSERT INTO t1 VALUES(zeroblob(100), zeroblob(100));
          194  +  }
          195  +  list [catch { db incrblob t1 b 1 } msg] $msg
          196  +} {1 {cannot open indexed column for writing}}
          197  +do_test incrblob3-6.4.2 {
          198  +  set ::blob [db incrblob t1 a 1]
          199  +  close $::blob
          200  +} {}
          201  +do_test incrblob3-6.4.3 {
          202  +  set ::blob [db incrblob -readonly t1 b 1]
          203  +  close $::blob
          204  +} {}
          205  +
          206  +do_test incrblob3-6.5.1 {
          207  +  execsql { 
          208  +    CREATE TABLE p1(a PRIMARY KEY);
          209  +    CREATE TABLE c1(a, b REFERENCES p1);
          210  +    PRAGMA foreign_keys = 1;
          211  +    INSERT INTO p1 VALUES(zeroblob(100));
          212  +    INSERT INTO c1 VALUES(zeroblob(100), zeroblob(100));
          213  +  }
          214  +  list [catch { db incrblob c1 b 1 } msg] $msg
          215  +} {1 {cannot open foreign key column for writing}}
          216  +
          217  +do_test incrblob3-6.5.2 {
          218  +  set ::blob [db incrblob c1 a 1]
          219  +  close $::blob
          220  +} {}
          221  +do_test incrblob3-6.5.3 {
          222  +  set ::blob [db incrblob -readonly c1 b 1]
          223  +  close $::blob
          224  +} {}
          225  +do_test incrblob3-6.5.4 {
          226  +  execsql { PRAGMA foreign_keys = 0 }
          227  +  set ::blob [db incrblob c1 b 1]
          228  +  close $::blob
          229  +} {}
          230  +
          231  +
          232  +# Test that sqlite3_blob_open() handles transient and persistent schema 
          233  +# errors correctly.
          234  +#
          235  +do_test incrblob3-7.1 {
          236  +  sqlite3 db2 test.db
          237  +  sqlite3_db_config_lookaside db2 0 0 0
          238  +  execsql { CREATE TABLE t2(x) } db2
          239  +  set ::blob [db incrblob blobs v 1]
          240  +  close $::blob
          241  +} {}
          242  +db2 close
          243  +
          244  +testvfs tvfs -default 1
          245  +tvfs filter xAccess
          246  +tvfs script access_method
          247  +
          248  +proc access_method {args} {
          249  +  set schemacookie [hexio_get_int [hexio_read test.db 40 4]]
          250  +  incr schemacookie
          251  +  hexio_write test.db 40 [hexio_render_int32 $schemacookie]
          252  +
          253  +  set dbversion [hexio_get_int [hexio_read test.db 24 4]]
          254  +  incr dbversion
          255  +  hexio_write test.db 24 [hexio_render_int32 $dbversion]
          256  +
          257  +  return ""
          258  +}
          259  +
          260  +do_test incrblob3-7.2 {
          261  +  sqlite3 db test.db 
          262  +  sqlite3_db_config_lookaside db 0 0 0
          263  +  list [catch {db incrblob blobs v 1} msg] $msg
          264  +} {1 {database schema has changed}}
          265  +db close
          266  +tvfs delete
          267  +
          268  +finish_test
          269  +

Added test/incrblobfault.test.

            1  +# 2010 October 26
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +#
           13  +
           14  +set testdir [file dirname $argv0]
           15  +source $testdir/tester.tcl
           16  +
           17  +set testprefix incrblobfault
           18  +
           19  +do_execsql_test 1.0 {
           20  +  CREATE TABLE blob(x INTEGER PRIMARY KEY, v BLOB);
           21  +  INSERT INTO blob VALUES(1, 'hello world');
           22  +  INSERT INTO blob VALUES(2, 'world hello');
           23  +  INSERT INTO blob SELECT NULL, v FROM blob;
           24  +  INSERT INTO blob SELECT NULL, v FROM blob;
           25  +  INSERT INTO blob SELECT NULL, v FROM blob;
           26  +  INSERT INTO blob SELECT NULL, v FROM blob;
           27  +  INSERT INTO blob SELECT NULL, v FROM blob;
           28  +  INSERT INTO blob SELECT NULL, v FROM blob;
           29  +  INSERT INTO blob SELECT NULL, v FROM blob;
           30  +  INSERT INTO blob SELECT NULL, v FROM blob;
           31  +  INSERT INTO blob SELECT NULL, v FROM blob;
           32  +  INSERT INTO blob SELECT NULL, v FROM blob;
           33  +}
           34  +
           35  +do_faultsim_test 1 -prep {
           36  +  sqlite3 db test.db
           37  +  set ::blob [db incrblob blob v 1]
           38  +} -body {
           39  +  if {[catch {sqlite3_blob_reopen $::blob 1000}]} {
           40  +    error [sqlite3_errmsg db]
           41  +  }
           42  +} -test {
           43  +  faultsim_test_result {0 {}}
           44  +  close $::blob
           45  +}
           46  +
           47  +do_faultsim_test 2 -prep {
           48  +  sqlite3 db test.db
           49  +  set ::blob [db incrblob blob v 1]
           50  +} -body {
           51  +  if {[catch {sqlite3_blob_reopen $::blob -1}]} {
           52  +    error [sqlite3_errmsg db]
           53  +  }
           54  +} -test {
           55  +  faultsim_test_result {1 {no such rowid: -1}}
           56  +  close $::blob
           57  +}
           58  +
           59  +do_faultsim_test 3 -prep {
           60  +  sqlite3 db test.db
           61  +} -body {
           62  +  set ::blob [db incrblob blob v 1]
           63  +  gets $::blob
           64  +} -test {
           65  +  faultsim_test_result {0 {hello world}}
           66  +  catch { close $::blob }
           67  +}
           68  +
           69  +finish_test
           70  +

Changes to test/malloc_common.tcl.

   109    109   proc do_faultsim_test {name args} {
   110    110     global FAULTSIM
   111    111     
   112    112     set DEFAULT(-faults)        [array names FAULTSIM]
   113    113     set DEFAULT(-prep)          ""
   114    114     set DEFAULT(-body)          ""
   115    115     set DEFAULT(-test)          ""
          116  +
          117  +  fix_testname name
   116    118   
   117    119     array set O [array get DEFAULT]
   118    120     array set O $args
   119    121     foreach o [array names O] {
   120    122       if {[info exists DEFAULT($o)]==0} { error "unknown option: $o" }
   121    123     }
   122    124   
................................................................................
   522    524   # by parameter $result, or (b) TCL throws an "out of memory" error.
   523    525   #
   524    526   # If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
   525    527   # is executed just once. In this case the test case passes if the results
   526    528   # match the expected results passed via parameter $result.
   527    529   #
   528    530   proc do_select_test {name sql result} {
   529         -  uplevel [list doPassiveTest 0 $name $sql [list 0 $result]]
          531  +  uplevel [list doPassiveTest 0 $name $sql [list 0 [list {*}$result]]]
   530    532   }
   531    533   
   532    534   proc do_restart_select_test {name sql result} {
   533    535     uplevel [list doPassiveTest 1 $name $sql [list 0 $result]]
   534    536   }
   535    537   
   536    538   proc do_error_test {name sql error} {
   537    539     uplevel [list doPassiveTest 0 $name $sql [list 1 $error]]
   538    540   }
   539    541   
   540    542   proc doPassiveTest {isRestart name sql catchres} {
   541    543     if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }
          544  +
          545  +  if {[info exists ::testprefix] 
          546  +   && [string is integer [string range $name 0 0]]
          547  +  } {
          548  +    set name $::testprefix.$name
          549  +  }
   542    550   
   543    551     switch $::DO_MALLOC_TEST {
   544    552       0 { # No malloc failures.
   545    553         do_test $name [list set {} [uplevel [list catchsql $sql]]] $catchres
   546    554         return
   547    555       }
   548    556       1 { # Simulate transient failures.

Changes to test/permutations.test.

   155    155     All multi-threaded tests.
   156    156   } -files {
   157    157     notify2.test   thread001.test thread002.test thread003.test 
   158    158     thread004.test thread005.test walthread.test
   159    159   }
   160    160   
   161    161   test_suite "fts3" -prefix "" -description {
   162         -  All FTS3 tests except fts3malloc.test and fts3rnd.test.
          162  +  All FTS3 tests except fts3rnd.test.
   163    163   } -files {
   164    164     fts3aa.test fts3ab.test fts3ac.test fts3ad.test fts3ae.test
   165    165     fts3af.test fts3ag.test fts3ah.test fts3ai.test fts3aj.test
   166    166     fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test
   167    167     fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
   168         -  fts3e.test fts3expr.test fts3expr2.test fts3near.test 
   169         -  fts3query.test fts3snippet.test
          168  +  fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
          169  +  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
          170  +
          171  +  fts3fault.test fts3malloc.test
   170    172   }
   171    173   
   172    174   
   173    175   lappend ::testsuitelist xxx
   174    176   #-------------------------------------------------------------------------
   175    177   # Define the coverage related test suites:
   176    178   #

Changes to test/tester.tcl.

  1371   1371   }
  1372   1372   
  1373   1373   # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set
  1374   1374   # to non-zero, then set the global variable $AUTOVACUUM to 1.
  1375   1375   set AUTOVACUUM $sqlite_options(default_autovacuum)
  1376   1376   
  1377   1377   source $testdir/thread_common.tcl
         1378  +source $testdir/malloc_common.tcl