/ Check-in [35b1b8d4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the 3.8.12 beta changes from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | sessions
Files: files | file ages | folders
SHA1: 35b1b8d4b97715030700e37b292bb4f1bb3f44d6
User & Date: drh 2015-10-08 14:55:30
Context
2015-10-09
15:29
Merge the version 3.9.0 changes and the incorporation of JSON1 and FTS5 into the amalgamation from trunk. check-in: c1d96fb6 user: drh tags: sessions
2015-10-08
14:55
Merge the 3.8.12 beta changes from trunk. check-in: 35b1b8d4 user: drh tags: sessions
02:44
Remove two unused lines of code - discovered by scan-build. check-in: 77b707b7 user: drh tags: trunk
2015-10-01
16:35
Changes to the sesssions module ensure that tables appear within changesets and patchsets in the same order that they were attached to the sessions object. check-in: 76958902 user: dan tags: sessions
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_expr.c.

   789    789     Fts3Expr *pFree = 0;            /* List of free nodes. Linked by pParent. */
   790    790     int eType = pRoot->eType;       /* Type of node in this tree */
   791    791   
   792    792     if( nMaxDepth==0 ){
   793    793       rc = SQLITE_ERROR;
   794    794     }
   795    795   
   796         -  if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
   797         -    Fts3Expr **apLeaf;
   798         -    apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
   799         -    if( 0==apLeaf ){
   800         -      rc = SQLITE_NOMEM;
   801         -    }else{
   802         -      memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
   803         -    }
   804         -
   805         -    if( rc==SQLITE_OK ){
   806         -      int i;
   807         -      Fts3Expr *p;
   808         -
   809         -      /* Set $p to point to the left-most leaf in the tree of eType nodes. */
   810         -      for(p=pRoot; p->eType==eType; p=p->pLeft){
   811         -        assert( p->pParent==0 || p->pParent->pLeft==p );
   812         -        assert( p->pLeft && p->pRight );
   813         -      }
   814         -
   815         -      /* This loop runs once for each leaf in the tree of eType nodes. */
   816         -      while( 1 ){
   817         -        int iLvl;
   818         -        Fts3Expr *pParent = p->pParent;     /* Current parent of p */
   819         -
   820         -        assert( pParent==0 || pParent->pLeft==p );
   821         -        p->pParent = 0;
   822         -        if( pParent ){
   823         -          pParent->pLeft = 0;
   824         -        }else{
   825         -          pRoot = 0;
   826         -        }
   827         -        rc = fts3ExprBalance(&p, nMaxDepth-1);
   828         -        if( rc!=SQLITE_OK ) break;
   829         -
   830         -        for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
   831         -          if( apLeaf[iLvl]==0 ){
   832         -            apLeaf[iLvl] = p;
   833         -            p = 0;
   834         -          }else{
   835         -            assert( pFree );
   836         -            pFree->pLeft = apLeaf[iLvl];
   837         -            pFree->pRight = p;
   838         -            pFree->pLeft->pParent = pFree;
   839         -            pFree->pRight->pParent = pFree;
   840         -
   841         -            p = pFree;
   842         -            pFree = pFree->pParent;
   843         -            p->pParent = 0;
   844         -            apLeaf[iLvl] = 0;
   845         -          }
   846         -        }
   847         -        if( p ){
   848         -          sqlite3Fts3ExprFree(p);
   849         -          rc = SQLITE_TOOBIG;
   850         -          break;
   851         -        }
   852         -
   853         -        /* If that was the last leaf node, break out of the loop */
   854         -        if( pParent==0 ) break;
   855         -
   856         -        /* Set $p to point to the next leaf in the tree of eType nodes */
   857         -        for(p=pParent->pRight; p->eType==eType; p=p->pLeft);
   858         -
   859         -        /* Remove pParent from the original tree. */
   860         -        assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
   861         -        pParent->pRight->pParent = pParent->pParent;
   862         -        if( pParent->pParent ){
   863         -          pParent->pParent->pLeft = pParent->pRight;
   864         -        }else{
   865         -          assert( pParent==pRoot );
   866         -          pRoot = pParent->pRight;
   867         -        }
   868         -
   869         -        /* Link pParent into the free node list. It will be used as an
   870         -        ** internal node of the new tree.  */
   871         -        pParent->pParent = pFree;
   872         -        pFree = pParent;
          796  +  if( rc==SQLITE_OK ){
          797  +    if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
          798  +      Fts3Expr **apLeaf;
          799  +      apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
          800  +      if( 0==apLeaf ){
          801  +        rc = SQLITE_NOMEM;
          802  +      }else{
          803  +        memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
   873    804         }
   874    805   
   875    806         if( rc==SQLITE_OK ){
   876         -        p = 0;
   877         -        for(i=0; i<nMaxDepth; i++){
   878         -          if( apLeaf[i] ){
   879         -            if( p==0 ){
   880         -              p = apLeaf[i];
   881         -              p->pParent = 0;
          807  +        int i;
          808  +        Fts3Expr *p;
          809  +
          810  +        /* Set $p to point to the left-most leaf in the tree of eType nodes. */
          811  +        for(p=pRoot; p->eType==eType; p=p->pLeft){
          812  +          assert( p->pParent==0 || p->pParent->pLeft==p );
          813  +          assert( p->pLeft && p->pRight );
          814  +        }
          815  +
          816  +        /* This loop runs once for each leaf in the tree of eType nodes. */
          817  +        while( 1 ){
          818  +          int iLvl;
          819  +          Fts3Expr *pParent = p->pParent;     /* Current parent of p */
          820  +
          821  +          assert( pParent==0 || pParent->pLeft==p );
          822  +          p->pParent = 0;
          823  +          if( pParent ){
          824  +            pParent->pLeft = 0;
          825  +          }else{
          826  +            pRoot = 0;
          827  +          }
          828  +          rc = fts3ExprBalance(&p, nMaxDepth-1);
          829  +          if( rc!=SQLITE_OK ) break;
          830  +
          831  +          for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
          832  +            if( apLeaf[iLvl]==0 ){
          833  +              apLeaf[iLvl] = p;
          834  +              p = 0;
   882    835               }else{
   883         -              assert( pFree!=0 );
          836  +              assert( pFree );
          837  +              pFree->pLeft = apLeaf[iLvl];
   884    838                 pFree->pRight = p;
   885         -              pFree->pLeft = apLeaf[i];
   886    839                 pFree->pLeft->pParent = pFree;
   887    840                 pFree->pRight->pParent = pFree;
   888    841   
   889    842                 p = pFree;
   890    843                 pFree = pFree->pParent;
   891    844                 p->pParent = 0;
          845  +              apLeaf[iLvl] = 0;
          846  +            }
          847  +          }
          848  +          if( p ){
          849  +            sqlite3Fts3ExprFree(p);
          850  +            rc = SQLITE_TOOBIG;
          851  +            break;
          852  +          }
          853  +
          854  +          /* If that was the last leaf node, break out of the loop */
          855  +          if( pParent==0 ) break;
          856  +
          857  +          /* Set $p to point to the next leaf in the tree of eType nodes */
          858  +          for(p=pParent->pRight; p->eType==eType; p=p->pLeft);
          859  +
          860  +          /* Remove pParent from the original tree. */
          861  +          assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
          862  +          pParent->pRight->pParent = pParent->pParent;
          863  +          if( pParent->pParent ){
          864  +            pParent->pParent->pLeft = pParent->pRight;
          865  +          }else{
          866  +            assert( pParent==pRoot );
          867  +            pRoot = pParent->pRight;
          868  +          }
          869  +
          870  +          /* Link pParent into the free node list. It will be used as an
          871  +          ** internal node of the new tree.  */
          872  +          pParent->pParent = pFree;
          873  +          pFree = pParent;
          874  +        }
          875  +
          876  +        if( rc==SQLITE_OK ){
          877  +          p = 0;
          878  +          for(i=0; i<nMaxDepth; i++){
          879  +            if( apLeaf[i] ){
          880  +              if( p==0 ){
          881  +                p = apLeaf[i];
          882  +                p->pParent = 0;
          883  +              }else{
          884  +                assert( pFree!=0 );
          885  +                pFree->pRight = p;
          886  +                pFree->pLeft = apLeaf[i];
          887  +                pFree->pLeft->pParent = pFree;
          888  +                pFree->pRight->pParent = pFree;
          889  +
          890  +                p = pFree;
          891  +                pFree = pFree->pParent;
          892  +                p->pParent = 0;
          893  +              }
   892    894               }
   893    895             }
   894         -        }
   895         -        pRoot = p;
   896         -      }else{
   897         -        /* An error occurred. Delete the contents of the apLeaf[] array 
   898         -        ** and pFree list. Everything else is cleaned up by the call to
   899         -        ** sqlite3Fts3ExprFree(pRoot) below.  */
   900         -        Fts3Expr *pDel;
   901         -        for(i=0; i<nMaxDepth; i++){
   902         -          sqlite3Fts3ExprFree(apLeaf[i]);
          896  +          pRoot = p;
          897  +        }else{
          898  +          /* An error occurred. Delete the contents of the apLeaf[] array 
          899  +          ** and pFree list. Everything else is cleaned up by the call to
          900  +          ** sqlite3Fts3ExprFree(pRoot) below.  */
          901  +          Fts3Expr *pDel;
          902  +          for(i=0; i<nMaxDepth; i++){
          903  +            sqlite3Fts3ExprFree(apLeaf[i]);
          904  +          }
          905  +          while( (pDel=pFree)!=0 ){
          906  +            pFree = pDel->pParent;
          907  +            sqlite3_free(pDel);
          908  +          }
   903    909           }
   904         -        while( (pDel=pFree)!=0 ){
   905         -          pFree = pDel->pParent;
   906         -          sqlite3_free(pDel);
   907         -        }
          910  +
          911  +        assert( pFree==0 );
          912  +        sqlite3_free( apLeaf );
          913  +      }
          914  +    }else if( eType==FTSQUERY_NOT ){
          915  +      Fts3Expr *pLeft = pRoot->pLeft;
          916  +      Fts3Expr *pRight = pRoot->pRight;
          917  +
          918  +      pRoot->pLeft = 0;
          919  +      pRoot->pRight = 0;
          920  +      pLeft->pParent = 0;
          921  +      pRight->pParent = 0;
          922  +
          923  +      rc = fts3ExprBalance(&pLeft, nMaxDepth-1);
          924  +      if( rc==SQLITE_OK ){
          925  +        rc = fts3ExprBalance(&pRight, nMaxDepth-1);
   908    926         }
   909    927   
   910         -      assert( pFree==0 );
   911         -      sqlite3_free( apLeaf );
          928  +      if( rc!=SQLITE_OK ){
          929  +        sqlite3Fts3ExprFree(pRight);
          930  +        sqlite3Fts3ExprFree(pLeft);
          931  +      }else{
          932  +        assert( pLeft && pRight );
          933  +        pRoot->pLeft = pLeft;
          934  +        pLeft->pParent = pRoot;
          935  +        pRoot->pRight = pRight;
          936  +        pRight->pParent = pRoot;
          937  +      }
   912    938       }
   913    939     }
   914         -
          940  +  
   915    941     if( rc!=SQLITE_OK ){
   916    942       sqlite3Fts3ExprFree(pRoot);
   917    943       pRoot = 0;
   918    944     }
   919    945     *pp = pRoot;
   920    946     return rc;
   921    947   }

Changes to ext/fts3/fts3_write.c.

  1318   1318       pNext = &pReader->aDoclist[pReader->nDoclist];
  1319   1319     }
  1320   1320   
  1321   1321     if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
  1322   1322   
  1323   1323       if( fts3SegReaderIsPending(pReader) ){
  1324   1324         Fts3HashElem *pElem = *(pReader->ppNextElem);
  1325         -      if( pElem==0 ){
  1326         -        pReader->aNode = 0;
  1327         -      }else{
         1325  +      sqlite3_free(pReader->aNode);
         1326  +      pReader->aNode = 0;
         1327  +      if( pElem ){
         1328  +        char *aCopy;
  1328   1329           PendingList *pList = (PendingList *)fts3HashData(pElem);
         1330  +        int nCopy = pList->nData+1;
  1329   1331           pReader->zTerm = (char *)fts3HashKey(pElem);
  1330   1332           pReader->nTerm = fts3HashKeysize(pElem);
  1331         -        pReader->nNode = pReader->nDoclist = pList->nData + 1;
  1332         -        pReader->aNode = pReader->aDoclist = pList->aData;
         1333  +        aCopy = (char*)sqlite3_malloc(nCopy);
         1334  +        if( !aCopy ) return SQLITE_NOMEM;
         1335  +        memcpy(aCopy, pList->aData, nCopy);
         1336  +        pReader->nNode = pReader->nDoclist = nCopy;
         1337  +        pReader->aNode = pReader->aDoclist = aCopy;
  1333   1338           pReader->ppNextElem++;
  1334   1339           assert( pReader->aNode );
  1335   1340         }
  1336   1341         return SQLITE_OK;
  1337   1342       }
  1338   1343   
  1339   1344       fts3SegReaderSetEof(pReader);
................................................................................
  1565   1570   }
  1566   1571   
  1567   1572   /*
  1568   1573   ** Free all allocations associated with the iterator passed as the 
  1569   1574   ** second argument.
  1570   1575   */
  1571   1576   void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
  1572         -  if( pReader && !fts3SegReaderIsPending(pReader) ){
  1573         -    sqlite3_free(pReader->zTerm);
         1577  +  if( pReader ){
         1578  +    if( !fts3SegReaderIsPending(pReader) ){
         1579  +      sqlite3_free(pReader->zTerm);
         1580  +    }
  1574   1581       if( !fts3SegReaderIsRootOnly(pReader) ){
  1575   1582         sqlite3_free(pReader->aNode);
  1576         -      sqlite3_blob_close(pReader->pBlob);
  1577   1583       }
         1584  +    sqlite3_blob_close(pReader->pBlob);
  1578   1585     }
  1579   1586     sqlite3_free(pReader);
  1580   1587   }
  1581   1588   
  1582   1589   /*
  1583   1590   ** Allocate a new SegReader object.
  1584   1591   */

Changes to ext/fts5/fts5Int.h.

    77     77   extern int sqlite3_fts5_may_be_corrupt;
    78     78   # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
    79     79   #else
    80     80   # define assert_nc(x) assert(x)
    81     81   #endif
    82     82   
    83     83   typedef struct Fts5Global Fts5Global;
           84  +typedef struct Fts5Colset Fts5Colset;
           85  +
           86  +/* If a NEAR() clump or phrase may only match a specific set of columns, 
           87  +** then an object of the following type is used to record the set of columns.
           88  +** Each entry in the aiCol[] array is a column that may be matched.
           89  +**
           90  +** This object is used by fts5_expr.c and fts5_index.c.
           91  +*/
           92  +struct Fts5Colset {
           93  +  int nCol;
           94  +  int aiCol[1];
           95  +};
           96  +
           97  +
    84     98   
    85     99   /**************************************************************************
    86    100   ** Interface to code in fts5_config.c. fts5_config.c contains contains code
    87    101   ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
    88    102   */
    89    103   
    90    104   typedef struct Fts5Config Fts5Config;
................................................................................
   301    315   ** Create/destroy an Fts5Index object.
   302    316   */
   303    317   int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
   304    318   int sqlite3Fts5IndexClose(Fts5Index *p);
   305    319   
   306    320   /*
   307    321   ** for(
   308         -**   pIter = sqlite3Fts5IndexQuery(p, "token", 5, 0);
          322  +**   sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
   309    323   **   0==sqlite3Fts5IterEof(pIter);
   310    324   **   sqlite3Fts5IterNext(pIter)
   311    325   ** ){
   312    326   **   i64 iRowid = sqlite3Fts5IterRowid(pIter);
   313    327   ** }
   314    328   */
   315    329   
................................................................................
   317    331   ** Open a new iterator to iterate though all rowids that match the 
   318    332   ** specified token or token prefix.
   319    333   */
   320    334   int sqlite3Fts5IndexQuery(
   321    335     Fts5Index *p,                   /* FTS index to query */
   322    336     const char *pToken, int nToken, /* Token (or prefix) to query for */
   323    337     int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
   324         -  Fts5IndexIter **ppIter
          338  +  Fts5Colset *pColset,            /* Match these columns only */
          339  +  Fts5IndexIter **ppIter          /* OUT: New iterator object */
   325    340   );
   326    341   
   327    342   /*
   328    343   ** The various operations on open token or token prefix iterators opened
   329    344   ** using sqlite3Fts5IndexQuery().
   330    345   */
   331    346   int sqlite3Fts5IterEof(Fts5IndexIter*);
................................................................................
   366    381   
   367    382   /*
   368    383   ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
   369    384   ** document iDocid.
   370    385   */
   371    386   int sqlite3Fts5IndexBeginWrite(
   372    387     Fts5Index *p,                   /* Index to write to */
          388  +  int bDelete,                    /* True if current operation is a delete */
   373    389     i64 iDocid                      /* Docid to add or remove data from */
   374    390   );
   375    391   
   376    392   /*
   377    393   ** Flush any data stored in the in-memory hash tables to the database.
   378    394   ** If the bCommit flag is true, also close any open blob handles.
   379    395   */
................................................................................
   429    445   int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
   430    446   int sqlite3Fts5GetVarintLen(u32 iVal);
   431    447   u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
   432    448   int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
   433    449   
   434    450   #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
   435    451   #define fts5GetVarint    sqlite3Fts5GetVarint
          452  +
          453  +#define fts5FastGetVarint32(a, iOff, nVal) {      \
          454  +  nVal = (a)[iOff++];                             \
          455  +  if( nVal & 0x80 ){                              \
          456  +    iOff--;                                       \
          457  +    iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
          458  +  }                                               \
          459  +}
          460  +
   436    461   
   437    462   /*
   438    463   ** End of interface to code in fts5_varint.c.
   439    464   **************************************************************************/
   440    465   
   441    466   
   442    467   /**************************************************************************
................................................................................
   522    547   int sqlite3Fts5StorageClose(Fts5Storage *p);
   523    548   int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
   524    549   
   525    550   int sqlite3Fts5DropAll(Fts5Config*);
   526    551   int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
   527    552   
   528    553   int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
   529         -int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
          554  +int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
          555  +int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
   530    556   
   531    557   int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
   532    558   
   533    559   int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
   534    560   void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
   535    561   
   536    562   int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
................................................................................
   561    587   */
   562    588   typedef struct Fts5Expr Fts5Expr;
   563    589   typedef struct Fts5ExprNode Fts5ExprNode;
   564    590   typedef struct Fts5Parse Fts5Parse;
   565    591   typedef struct Fts5Token Fts5Token;
   566    592   typedef struct Fts5ExprPhrase Fts5ExprPhrase;
   567    593   typedef struct Fts5ExprNearset Fts5ExprNearset;
   568         -typedef struct Fts5ExprColset Fts5ExprColset;
   569    594   
   570    595   struct Fts5Token {
   571    596     const char *p;                  /* Token text (not NULL terminated) */
   572    597     int n;                          /* Size of buffer p in bytes */
   573    598   };
   574    599   
   575    600   /* Parse a MATCH expression. */
................................................................................
   629    654   
   630    655   Fts5ExprNearset *sqlite3Fts5ParseNearset(
   631    656     Fts5Parse*, 
   632    657     Fts5ExprNearset*,
   633    658     Fts5ExprPhrase* 
   634    659   );
   635    660   
   636         -Fts5ExprColset *sqlite3Fts5ParseColset(
          661  +Fts5Colset *sqlite3Fts5ParseColset(
   637    662     Fts5Parse*, 
   638         -  Fts5ExprColset*, 
          663  +  Fts5Colset*, 
   639    664     Fts5Token *
   640    665   );
   641    666   
   642    667   void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
   643    668   void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
   644    669   void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
   645    670   
   646    671   void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
   647         -void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5ExprColset*);
          672  +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
   648    673   void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
   649    674   void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
   650    675   
   651    676   /*
   652    677   ** End of interface to code in fts5_expr.c.
   653    678   **************************************************************************/
   654    679   

Changes to ext/fts5/fts5_buffer.c.

   181    181     if( i>=n ){
   182    182       /* EOF */
   183    183       *piOff = -1;
   184    184       return 1;  
   185    185     }else{
   186    186       i64 iOff = *piOff;
   187    187       int iVal;
   188         -    i += fts5GetVarint32(&a[i], iVal);
          188  +    fts5FastGetVarint32(a, i, iVal);
   189    189       if( iVal==1 ){
   190         -      i += fts5GetVarint32(&a[i], iVal);
          190  +      fts5FastGetVarint32(a, i, iVal);
   191    191         iOff = ((i64)iVal) << 32;
   192         -      i += fts5GetVarint32(&a[i], iVal);
          192  +      fts5FastGetVarint32(a, i, iVal);
   193    193       }
   194    194       *piOff = iOff + (iVal-2);
   195    195       *pi = i;
   196    196       return 0;
   197    197     }
   198    198   }
   199    199   
................................................................................
   229    229   int sqlite3Fts5PoslistWriterAppend(
   230    230     Fts5Buffer *pBuf, 
   231    231     Fts5PoslistWriter *pWriter,
   232    232     i64 iPos
   233    233   ){
   234    234     static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
   235    235     int rc = SQLITE_OK;
   236         -  if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
   237         -    fts5BufferAppendVarint(&rc, pBuf, 1);
   238         -    fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32));
   239         -    pWriter->iPrev = (iPos & colmask);
          236  +  if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){
          237  +    if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
          238  +      pBuf->p[pBuf->n++] = 1;
          239  +      pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
          240  +      pWriter->iPrev = (iPos & colmask);
          241  +    }
          242  +    pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
          243  +    pWriter->iPrev = iPos;
   240    244     }
   241         -  fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2);
   242         -  pWriter->iPrev = iPos;
   243    245     return rc;
   244    246   }
   245    247   
   246    248   void *sqlite3Fts5MallocZero(int *pRc, int nByte){
   247    249     void *pRet = 0;
   248    250     if( *pRc==SQLITE_OK ){
   249    251       pRet = sqlite3_malloc(nByte);
................................................................................
   286    288   ** Return true if character 't' may be part of an FTS5 bareword, or false
   287    289   ** otherwise. Characters that may be part of barewords:
   288    290   **
   289    291   **   * All non-ASCII characters,
   290    292   **   * The 52 upper and lower case ASCII characters, and
   291    293   **   * The 10 integer ASCII characters.
   292    294   **   * The underscore character "_" (0x5F).
          295  +**   * The unicode "subsitute" character (0x1A).
   293    296   */
   294    297   int sqlite3Fts5IsBareword(char t){
   295    298     u8 aBareword[128] = {
   296    299       0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x00 .. 0x0F */
   297         -    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x10 .. 0x1F */
          300  +    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 1, 0, 0, 0, 0, 0,   /* 0x10 .. 0x1F */
   298    301       0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x20 .. 0x2F */
   299    302       1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 0, 0, 0, 0, 0, 0,   /* 0x30 .. 0x3F */
   300    303       0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x40 .. 0x4F */
   301    304       1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 1,   /* 0x50 .. 0x5F */
   302    305       0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60 .. 0x6F */
   303    306       1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
   304    307     };
   305    308   
   306    309     return (t & 0x80) || aBareword[(int)t];
   307    310   }
   308    311   
   309    312   

Changes to ext/fts5/fts5_expr.c.

    85     85   struct Fts5ExprPhrase {
    86     86     Fts5ExprNode *pNode;            /* FTS5_STRING node this phrase is part of */
    87     87     Fts5Buffer poslist;             /* Current position list */
    88     88     int nTerm;                      /* Number of entries in aTerm[] */
    89     89     Fts5ExprTerm aTerm[1];          /* Terms that make up this phrase */
    90     90   };
    91     91   
    92         -/*
    93         -** If a NEAR() clump may only match a specific set of columns, then
    94         -** Fts5ExprNearset.pColset points to an object of the following type.
    95         -** Each entry in the aiCol[] array
    96         -*/
    97         -struct Fts5ExprColset {
    98         -  int nCol;
    99         -  int aiCol[1];
   100         -};
   101         -
   102     92   /*
   103     93   ** One or more phrases that must appear within a certain token distance of
   104     94   ** each other within each matching document.
   105     95   */
   106     96   struct Fts5ExprNearset {
   107     97     int nNear;                      /* NEAR parameter */
   108         -  Fts5ExprColset *pColset;        /* Columns to search (NULL -> all columns) */
           98  +  Fts5Colset *pColset;            /* Columns to search (NULL -> all columns) */
   109     99     int nPhrase;                    /* Number of entries in aPhrase[] array */
   110    100     Fts5ExprPhrase *apPhrase[1];    /* Array of phrase pointers */
   111    101   };
   112    102   
   113    103   
   114    104   /*
   115    105   ** Parse context.
................................................................................
   272    262     if( p ){
   273    263       sqlite3Fts5ParseNodeFree(p->pRoot);
   274    264       sqlite3_free(p->apExprPhrase);
   275    265       sqlite3_free(p);
   276    266     }
   277    267   }
   278    268   
   279         -static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
          269  +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
   280    270     int i;
   281    271     for(i=0; i<pColset->nCol; i++){
   282    272       if( pColset->aiCol[i]==iCol ) return 1;
   283    273     }
   284    274     return 0;
   285    275   }
   286    276   
................................................................................
   401    391   **
   402    392   ** SQLITE_OK is returned if an error occurs, or an SQLite error code 
   403    393   ** otherwise. It is not considered an error code if the current rowid is 
   404    394   ** not a match.
   405    395   */
   406    396   static int fts5ExprPhraseIsMatch(
   407    397     Fts5ExprNode *pNode,            /* Node pPhrase belongs to */
   408         -  Fts5ExprColset *pColset,        /* Restrict matches to these columns */
          398  +  Fts5Colset *pColset,            /* Restrict matches to these columns */
   409    399     Fts5ExprPhrase *pPhrase,        /* Phrase object to initialize */
   410    400     int *pbMatch                    /* OUT: Set to true if really a match */
   411    401   ){
   412    402     Fts5PoslistWriter writer = {0};
   413    403     Fts5PoslistReader aStatic[4];
   414    404     Fts5PoslistReader *aIter = aStatic;
   415    405     int i;
................................................................................
   801    791     while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
   802    792       prev = *p++;
   803    793     }
   804    794     return p - (*pa);
   805    795   }
   806    796   
   807    797   static int fts5ExprExtractColset (
   808         -  Fts5ExprColset *pColset,        /* Colset to filter on */
          798  +  Fts5Colset *pColset,            /* Colset to filter on */
   809    799     const u8 *pPos, int nPos,       /* Position list */
   810    800     Fts5Buffer *pBuf                /* Output buffer */
   811    801   ){
   812    802     int rc = SQLITE_OK;
   813    803     int i;
   814    804   
   815    805     fts5BufferZero(pBuf);
................................................................................
   864    854     ** of a single term only, grab pointers into the poslist managed by the
   865    855     ** fts5_index.c iterator object. This is much faster than synthesizing 
   866    856     ** a new poslist the way we have to for more complicated phrase or NEAR
   867    857     ** expressions.  */
   868    858     Fts5ExprNearset *pNear = pNode->pNear;
   869    859     Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
   870    860     Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
   871         -  Fts5ExprColset *pColset = pNear->pColset;
          861  +  Fts5Colset *pColset = pNear->pColset;
   872    862     const u8 *pPos;
   873    863     int nPos;
   874    864     int rc;
   875    865   
   876    866     assert( pNode->eType==FTS5_TERM );
   877    867     assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
   878    868     assert( pPhrase->aTerm[0].pSynonym==0 );
................................................................................
   998    988             sqlite3Fts5IterClose(p->pIter);
   999    989             p->pIter = 0;
  1000    990           }
  1001    991           rc = sqlite3Fts5IndexQuery(
  1002    992               pExpr->pIndex, p->zTerm, strlen(p->zTerm),
  1003    993               (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
  1004    994               (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
          995  +            pNear->pColset,
  1005    996               &p->pIter
  1006    997           );
  1007    998           assert( rc==SQLITE_OK || p->pIter==0 );
  1008    999           if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
  1009   1000             bEof = 0;
  1010   1001           }
  1011   1002         }
................................................................................
  1750   1741       nNear = FTS5_DEFAULT_NEARDIST;
  1751   1742     }
  1752   1743     pNear->nNear = nNear;
  1753   1744   }
  1754   1745   
  1755   1746   /*
  1756   1747   ** The second argument passed to this function may be NULL, or it may be
  1757         -** an existing Fts5ExprColset object. This function returns a pointer to
         1748  +** an existing Fts5Colset object. This function returns a pointer to
  1758   1749   ** a new colset object containing the contents of (p) with new value column
  1759   1750   ** number iCol appended. 
  1760   1751   **
  1761   1752   ** If an OOM error occurs, store an error code in pParse and return NULL.
  1762   1753   ** The old colset object (if any) is not freed in this case.
  1763   1754   */
  1764         -static Fts5ExprColset *fts5ParseColset(
         1755  +static Fts5Colset *fts5ParseColset(
  1765   1756     Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  1766         -  Fts5ExprColset *p,              /* Existing colset object */
         1757  +  Fts5Colset *p,                  /* Existing colset object */
  1767   1758     int iCol                        /* New column to add to colset object */
  1768   1759   ){
  1769   1760     int nCol = p ? p->nCol : 0;     /* Num. columns already in colset object */
  1770         -  Fts5ExprColset *pNew;           /* New colset object to return */
         1761  +  Fts5Colset *pNew;               /* New colset object to return */
  1771   1762   
  1772   1763     assert( pParse->rc==SQLITE_OK );
  1773   1764     assert( iCol>=0 && iCol<pParse->pConfig->nCol );
  1774   1765   
  1775         -  pNew = sqlite3_realloc(p, sizeof(Fts5ExprColset) + sizeof(int)*nCol);
         1766  +  pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol);
  1776   1767     if( pNew==0 ){
  1777   1768       pParse->rc = SQLITE_NOMEM;
  1778   1769     }else{
  1779   1770       int *aiCol = pNew->aiCol;
  1780   1771       int i, j;
  1781   1772       for(i=0; i<nCol; i++){
  1782   1773         if( aiCol[i]==iCol ) return pNew;
................................................................................
  1793   1784       for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] );
  1794   1785   #endif
  1795   1786     }
  1796   1787   
  1797   1788     return pNew;
  1798   1789   }
  1799   1790   
  1800         -Fts5ExprColset *sqlite3Fts5ParseColset(
         1791  +Fts5Colset *sqlite3Fts5ParseColset(
  1801   1792     Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  1802         -  Fts5ExprColset *pColset,        /* Existing colset object */
         1793  +  Fts5Colset *pColset,            /* Existing colset object */
  1803   1794     Fts5Token *p
  1804   1795   ){
  1805         -  Fts5ExprColset *pRet = 0;
         1796  +  Fts5Colset *pRet = 0;
  1806   1797     int iCol;
  1807   1798     char *z;                        /* Dequoted copy of token p */
  1808   1799   
  1809   1800     z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n);
  1810   1801     if( pParse->rc==SQLITE_OK ){
  1811   1802       Fts5Config *pConfig = pParse->pConfig;
  1812   1803       sqlite3Fts5Dequote(z);
................................................................................
  1828   1819   
  1829   1820     return pRet;
  1830   1821   }
  1831   1822   
  1832   1823   void sqlite3Fts5ParseSetColset(
  1833   1824     Fts5Parse *pParse, 
  1834   1825     Fts5ExprNearset *pNear, 
  1835         -  Fts5ExprColset *pColset 
         1826  +  Fts5Colset *pColset 
  1836   1827   ){
  1837   1828     if( pNear ){
  1838   1829       pNear->pColset = pColset;
  1839   1830     }else{
  1840   1831       sqlite3_free(pColset);
  1841   1832     }
  1842   1833   }

Changes to ext/fts5/fts5_index.c.

   287    287     ** Variables related to the accumulation of tokens and doclists within the
   288    288     ** in-memory hash tables before they are flushed to disk.
   289    289     */
   290    290     Fts5Hash *pHash;                /* Hash table for in-memory data */
   291    291     int nMaxPendingData;            /* Max pending data before flush to disk */
   292    292     int nPendingData;               /* Current bytes of pending data */
   293    293     i64 iWriteRowid;                /* Rowid for current doc being written */
   294         -  Fts5Buffer scratch;
          294  +  int bDelete;                    /* Current write is a delete */
   295    295   
   296    296     /* Error state. */
   297    297     int rc;                         /* Current error code */
   298    298   
   299    299     /* State used by the fts5DataXXX() functions. */
   300    300     sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
   301    301     sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
................................................................................
   303    303     sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
   304    304     sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
   305    305     sqlite3_stmt *pIdxSelect;
   306    306     int nRead;                      /* Total number of blocks read */
   307    307   };
   308    308   
   309    309   struct Fts5DoclistIter {
   310         -  u8 *a;
   311         -  int n;
   312         -  int i;
          310  +  u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
   313    311   
   314    312     /* Output variables. aPoslist==0 at EOF */
   315    313     i64 iRowid;
   316    314     u8 *aPoslist;
   317    315     int nPoslist;
          316  +  int nSize;
   318    317   };
   319    318   
   320    319   /*
   321    320   ** The contents of the "structure" record for each index are represented
   322    321   ** using an Fts5Structure record in memory. Which uses instances of the 
   323    322   ** other Fts5StructureXXX types as components.
   324    323   */
................................................................................
  1776   1775           }else{
  1777   1776             pIter->pLeaf->p = (u8*)pList;
  1778   1777             pIter->pLeaf->nn = nList;
  1779   1778             pIter->pLeaf->szLeaf = nList;
  1780   1779             pIter->iEndofDoclist = nList+1;
  1781   1780             sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm);
  1782   1781             pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
         1782  +          if( pbNewTerm ) *pbNewTerm = 1;
  1783   1783           }
  1784   1784         }else{
  1785   1785           iOff = 0;
  1786   1786           /* Next entry is not on the current page */
  1787   1787           while( iOff==0 ){
  1788   1788             fts5SegIterNextPage(p, pIter);
  1789   1789             pLeaf = pIter->pLeaf;
................................................................................
  1936   1936     ){
  1937   1937       return;
  1938   1938     }
  1939   1939   
  1940   1940     pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
  1941   1941   }
  1942   1942   
  1943         -#define fts5IndexGetVarint32(a, iOff, nVal) {     \
  1944         -  nVal = (a)[iOff++];                             \
  1945         -  if( nVal & 0x80 ){                              \
  1946         -    iOff--;                                       \
  1947         -    iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
  1948         -  }                                               \
  1949         -}
  1950         -
  1951   1943   #define fts5IndexSkipVarint(a, iOff) {            \
  1952   1944     int iEnd = iOff+9;                              \
  1953   1945     while( (a[iOff++] & 0x80) && iOff<iEnd );       \
  1954   1946   }
  1955   1947   
  1956   1948   /*
  1957   1949   ** The iterator object passed as the second argument currently contains
................................................................................
  1990   1982     iPgidx = szLeaf;
  1991   1983     iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
  1992   1984     iOff = iTermOff;
  1993   1985   
  1994   1986     while( 1 ){
  1995   1987   
  1996   1988       /* Figure out how many new bytes are in this term */
  1997         -    fts5IndexGetVarint32(a, iOff, nNew);
         1989  +    fts5FastGetVarint32(a, iOff, nNew);
  1998   1990       if( nKeep<nMatch ){
  1999   1991         goto search_failed;
  2000   1992       }
  2001   1993   
  2002   1994       assert( nKeep>=nMatch );
  2003   1995       if( nKeep==nMatch ){
  2004   1996         int nCmp;
................................................................................
  2026   2018       }
  2027   2019   
  2028   2020       iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
  2029   2021       iTermOff += nKeep;
  2030   2022       iOff = iTermOff;
  2031   2023   
  2032   2024       /* Read the nKeep field of the next term. */
  2033         -    fts5IndexGetVarint32(a, iOff, nKeep);
         2025  +    fts5FastGetVarint32(a, iOff, nKeep);
  2034   2026     }
  2035   2027   
  2036   2028    search_failed:
  2037   2029     if( bGe==0 ){
  2038   2030       fts5DataRelease(pIter->pLeaf);
  2039   2031       pIter->pLeaf = 0;
  2040   2032       return;
................................................................................
  3701   3693         if( (ret + i) > nMax ) break;
  3702   3694         ret += i;
  3703   3695       }
  3704   3696     }
  3705   3697     return ret;
  3706   3698   }
  3707   3699   
  3708         -#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
  3709         -  assert( pBuf->nSpace>=(pBuf->n+nBlob) );             \
  3710         -  memcpy(&pBuf->p[pBuf->n], pBlob, nBlob);             \
  3711         -  pBuf->n += nBlob;                                    \
         3700  +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) {     \
         3701  +  assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) );             \
         3702  +  memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob);             \
         3703  +  (pBuf)->n += nBlob;                                      \
         3704  +}
         3705  +
         3706  +#define fts5BufferSafeAppendVarint(pBuf, iVal) {                \
         3707  +  (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal));  \
         3708  +  assert( (pBuf)->nSpace>=(pBuf)->n );                          \
  3712   3709   }
  3713   3710   
  3714   3711   /*
  3715   3712   ** Flush the contents of in-memory hash table iHash to a new level-0 
  3716   3713   ** segment on disk. Also update the corresponding structure record.
  3717   3714   **
  3718   3715   ** If an error occurs, set the Fts5Index.rc error code. If an error has 
................................................................................
  3934   3931     fts5StructureRelease(pStruct);
  3935   3932   
  3936   3933     return fts5IndexReturn(p);
  3937   3934   }
  3938   3935   
  3939   3936   static void fts5PoslistCallback(
  3940   3937     Fts5Index *p, 
  3941         -  void *pCtx, 
         3938  +  void *pContext, 
         3939  +  const u8 *pChunk, int nChunk
         3940  +){
         3941  +  assert_nc( nChunk>=0 );
         3942  +  if( nChunk>0 ){
         3943  +    fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pContext, nChunk, pChunk);
         3944  +  }
         3945  +}
         3946  +
         3947  +typedef struct PoslistCallbackCtx PoslistCallbackCtx;
         3948  +struct PoslistCallbackCtx {
         3949  +  Fts5Buffer *pBuf;               /* Append to this buffer */
         3950  +  Fts5Colset *pColset;            /* Restrict matches to this column */
         3951  +  int eState;                     /* See above */
         3952  +};
         3953  +
         3954  +/*
         3955  +** TODO: Make this more efficient!
         3956  +*/
         3957  +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
         3958  +  int i;
         3959  +  for(i=0; i<pColset->nCol; i++){
         3960  +    if( pColset->aiCol[i]==iCol ) return 1;
         3961  +  }
         3962  +  return 0;
         3963  +}
         3964  +
         3965  +static void fts5PoslistFilterCallback(
         3966  +  Fts5Index *p, 
         3967  +  void *pContext, 
  3942   3968     const u8 *pChunk, int nChunk
  3943   3969   ){
         3970  +  PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
  3944   3971     assert_nc( nChunk>=0 );
  3945   3972     if( nChunk>0 ){
  3946         -    fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pCtx, nChunk, pChunk);
         3973  +    /* Search through to find the first varint with value 1. This is the
         3974  +    ** start of the next columns hits. */
         3975  +    int i = 0;
         3976  +    int iStart = 0;
         3977  +
         3978  +    if( pCtx->eState==2 ){
         3979  +      int iCol;
         3980  +      fts5FastGetVarint32(pChunk, i, iCol);
         3981  +      if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
         3982  +        pCtx->eState = 1;
         3983  +        fts5BufferAppendVarint(&p->rc, pCtx->pBuf, 1);
         3984  +      }else{
         3985  +        pCtx->eState = 0;
         3986  +      }
         3987  +    }
         3988  +
         3989  +    do {
         3990  +      while( i<nChunk && pChunk[i]!=0x01 ){
         3991  +        while( pChunk[i] & 0x80 ) i++;
         3992  +        i++;
         3993  +      }
         3994  +      if( pCtx->eState ){
         3995  +        fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]);
         3996  +      }
         3997  +      if( i<nChunk ){
         3998  +        int iCol;
         3999  +        iStart = i;
         4000  +        i++;
         4001  +        if( i>=nChunk ){
         4002  +          pCtx->eState = 2;
         4003  +        }else{
         4004  +          fts5FastGetVarint32(pChunk, i, iCol);
         4005  +          pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
         4006  +          if( pCtx->eState ){
         4007  +            fts5BufferAppendBlob(&p->rc, pCtx->pBuf, i-iStart, &pChunk[iStart]);
         4008  +            iStart = i;
         4009  +          }
         4010  +        }
         4011  +      }
         4012  +    }while( i<nChunk );
  3947   4013     }
  3948   4014   }
  3949   4015   
  3950   4016   /*
  3951   4017   ** Iterator pIter currently points to a valid entry (not EOF). This
  3952   4018   ** function appends the position list data for the current entry to
  3953   4019   ** buffer pBuf. It does not make a copy of the position-list size
  3954   4020   ** field.
  3955   4021   */
  3956   4022   static void fts5SegiterPoslist(
  3957   4023     Fts5Index *p,
  3958   4024     Fts5SegIter *pSeg,
         4025  +  Fts5Colset *pColset,
  3959   4026     Fts5Buffer *pBuf
  3960   4027   ){
  3961         -  fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
         4028  +  if( pColset==0 ){
         4029  +    fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
         4030  +  }else{
         4031  +    PoslistCallbackCtx sCtx;
         4032  +    sCtx.pBuf = pBuf;
         4033  +    sCtx.pColset = pColset;
         4034  +    sCtx.eState = pColset ? fts5IndexColsetTest(pColset, 0) : 1;
         4035  +    assert( sCtx.eState==0 || sCtx.eState==1 );
         4036  +    fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
         4037  +  }
  3962   4038   }
  3963   4039   
  3964   4040   /*
  3965   4041   ** Iterator pMulti currently points to a valid entry (not EOF). This
  3966   4042   ** function appends a copy of the position-list of the entry pMulti 
  3967   4043   ** currently points to to buffer pBuf.
  3968   4044   **
  3969   4045   ** If an error occurs, an error code is left in p->rc. It is assumed
  3970   4046   ** no error has already occurred when this function is called.
  3971   4047   */
  3972         -static void fts5MultiIterPoslist(
         4048  +static int fts5MultiIterPoslist(
  3973   4049     Fts5Index *p,
  3974   4050     Fts5IndexIter *pMulti,
  3975         -  int bSz,                        /* Append a size field before the data */
         4051  +  Fts5Colset *pColset,
  3976   4052     Fts5Buffer *pBuf
  3977   4053   ){
  3978   4054     if( p->rc==SQLITE_OK ){
         4055  +    int iSz;
         4056  +    int iData;
         4057  +
  3979   4058       Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  3980   4059       assert( fts5MultiIterEof(p, pMulti)==0 );
  3981   4060   
  3982         -    if( bSz ){
  3983         -      /* WRITEPOSLISTSIZE */
  3984         -      fts5BufferAppendVarint(&p->rc, pBuf, pSeg->nPos*2);
         4061  +    /* WRITEPOSLISTSIZE */
         4062  +    iSz = pBuf->n;
         4063  +    fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
         4064  +    iData = pBuf->n;
         4065  +
         4066  +    fts5SegiterPoslist(p, pSeg, pColset, pBuf);
         4067  +
         4068  +    if( pColset ){
         4069  +      int nActual = pBuf->n - iData;
         4070  +      if( nActual!=pSeg->nPos ){
         4071  +        /* WRITEPOSLISTSIZE */
         4072  +        if( nActual==0 ){
         4073  +          return 1;
         4074  +        }else{
         4075  +          int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
         4076  +          while( iSz<(iData-nReq) ){ pBuf->p[iSz++] = 0x80; }
         4077  +          sqlite3Fts5PutVarint(&pBuf->p[iSz], nActual*2);
         4078  +        }
         4079  +      }
  3985   4080       }
  3986         -    fts5SegiterPoslist(p, pSeg, pBuf);
  3987   4081     }
         4082  +
         4083  +  return 0;
  3988   4084   }
  3989   4085   
  3990   4086   static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  3991         -  if( pIter->i<pIter->n ){
  3992         -    int bDummy;
  3993         -    if( pIter->i ){
  3994         -      i64 iDelta;
  3995         -      pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta);
  3996         -      pIter->iRowid += iDelta;
         4087  +  u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
         4088  +
         4089  +  assert( pIter->aPoslist );
         4090  +  if( p>=pIter->aEof ){
         4091  +    pIter->aPoslist = 0;
         4092  +  }else{
         4093  +    i64 iDelta;
         4094  +
         4095  +    p += fts5GetVarint(p, (u64*)&iDelta);
         4096  +    pIter->iRowid += iDelta;
         4097  +
         4098  +    /* Read position list size */
         4099  +    if( p[0] & 0x80 ){
         4100  +      int nPos;
         4101  +      pIter->nSize = fts5GetVarint32(p, nPos);
         4102  +      pIter->nPoslist = (nPos>>1);
  3997   4103       }else{
  3998         -      pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
         4104  +      pIter->nPoslist = ((int)(p[0])) >> 1;
         4105  +      pIter->nSize = 1;
  3999   4106       }
  4000         -    pIter->i += fts5GetPoslistSize(
  4001         -        &pIter->a[pIter->i], &pIter->nPoslist, &bDummy
  4002         -    );
  4003         -    pIter->aPoslist = &pIter->a[pIter->i];
  4004         -    pIter->i += pIter->nPoslist;
  4005         -  }else{
  4006         -    pIter->aPoslist = 0;
         4107  +
         4108  +    pIter->aPoslist = p;
  4007   4109     }
  4008   4110   }
  4009   4111   
  4010   4112   static void fts5DoclistIterInit(
  4011   4113     Fts5Buffer *pBuf, 
  4012   4114     Fts5DoclistIter *pIter
  4013   4115   ){
  4014   4116     memset(pIter, 0, sizeof(*pIter));
  4015         -  pIter->a = pBuf->p;
  4016         -  pIter->n = pBuf->n;
         4117  +  pIter->aPoslist = pBuf->p;
         4118  +  pIter->aEof = &pBuf->p[pBuf->n];
  4017   4119     fts5DoclistIterNext(pIter);
  4018   4120   }
  4019   4121   
         4122  +#if 0
  4020   4123   /*
  4021   4124   ** Append a doclist to buffer pBuf.
         4125  +**
         4126  +** This function assumes that space within the buffer has already been
         4127  +** allocated.
  4022   4128   */
  4023   4129   static void fts5MergeAppendDocid(
  4024         -  int *pRc,                       /* IN/OUT: Error code */
  4025   4130     Fts5Buffer *pBuf,               /* Buffer to write to */
  4026   4131     i64 *piLastRowid,               /* IN/OUT: Previous rowid written (if any) */
  4027   4132     i64 iRowid                      /* Rowid to append */
  4028   4133   ){
  4029         -  if( pBuf->n==0 ){
  4030         -    fts5BufferAppendVarint(pRc, pBuf, iRowid);
  4031         -  }else{
  4032         -    fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
  4033         -  }
         4134  +  assert( pBuf->n!=0 || (*piLastRowid)==0 );
         4135  +  fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid);
  4034   4136     *piLastRowid = iRowid;
  4035   4137   }
         4138  +#endif
         4139  +
         4140  +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) {       \
         4141  +  assert( (pBuf)->n!=0 || (iLastRowid)==0 );                   \
         4142  +  fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
         4143  +  (iLastRowid) = (iRowid);                                     \
         4144  +}
  4036   4145   
  4037   4146   /*
  4038   4147   ** Buffers p1 and p2 contain doclists. This function merges the content
  4039   4148   ** of the two doclists together and sets buffer p1 to the result before
  4040   4149   ** returning.
  4041   4150   **
  4042   4151   ** If an error occurs, an error code is left in p->rc. If an error has
................................................................................
  4052   4161       Fts5DoclistIter i1;
  4053   4162       Fts5DoclistIter i2;
  4054   4163       Fts5Buffer out;
  4055   4164       Fts5Buffer tmp;
  4056   4165       memset(&out, 0, sizeof(out));
  4057   4166       memset(&tmp, 0, sizeof(tmp));
  4058   4167   
         4168  +    sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n);
  4059   4169       fts5DoclistIterInit(p1, &i1);
  4060   4170       fts5DoclistIterInit(p2, &i2);
  4061   4171       while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
  4062   4172         if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
  4063   4173           /* Copy entry from i1 */
  4064         -        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i1.iRowid);
  4065         -        /* WRITEPOSLISTSIZE */
  4066         -        fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2);
  4067         -        fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
         4174  +        fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
         4175  +        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
  4068   4176           fts5DoclistIterNext(&i1);
  4069   4177         }
  4070   4178         else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
  4071   4179           /* Copy entry from i2 */
  4072         -        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
  4073         -        /* WRITEPOSLISTSIZE */
  4074         -        fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2);
  4075         -        fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
         4180  +        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4181  +        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
  4076   4182           fts5DoclistIterNext(&i2);
  4077   4183         }
  4078   4184         else{
  4079         -        Fts5PoslistReader r1;
  4080         -        Fts5PoslistReader r2;
         4185  +        i64 iPos1 = 0;
         4186  +        i64 iPos2 = 0;
         4187  +        int iOff1 = 0;
         4188  +        int iOff2 = 0;
         4189  +        u8 *a1 = &i1.aPoslist[i1.nSize];
         4190  +        u8 *a2 = &i2.aPoslist[i2.nSize];
         4191  +
  4081   4192           Fts5PoslistWriter writer;
  4082         -
  4083   4193           memset(&writer, 0, sizeof(writer));
  4084   4194   
  4085   4195           /* Merge the two position lists. */ 
  4086         -        fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
         4196  +        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4087   4197           fts5BufferZero(&tmp);
  4088         -        sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1);
  4089         -        sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2);
  4090         -        while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){
         4198  +
         4199  +        sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4200  +        sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4201  +
         4202  +        while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){
  4091   4203             i64 iNew;
  4092         -          if( r2.bEof || (r1.bEof==0 && r1.iPos<r2.iPos) ){
  4093         -            iNew = r1.iPos;
  4094         -            sqlite3Fts5PoslistReaderNext(&r1);
         4204  +          if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
         4205  +            iNew = iPos1;
         4206  +            sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
  4095   4207             }else{
  4096         -            iNew = r2.iPos;
  4097         -            sqlite3Fts5PoslistReaderNext(&r2);
  4098         -            if( r1.iPos==r2.iPos ) sqlite3Fts5PoslistReaderNext(&r1);
         4208  +            iNew = iPos2;
         4209  +            sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4210  +            if( iPos1==iPos2 ){
         4211  +              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
         4212  +            }
  4099   4213             }
  4100   4214             p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
  4101   4215           }
  4102   4216   
  4103   4217           /* WRITEPOSLISTSIZE */
  4104         -        fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2);
  4105         -        fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p);
         4218  +        fts5BufferSafeAppendVarint(&out, tmp.n * 2);
         4219  +        fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
  4106   4220           fts5DoclistIterNext(&i1);
  4107   4221           fts5DoclistIterNext(&i2);
  4108   4222         }
  4109   4223       }
  4110   4224   
  4111   4225       fts5BufferSet(&p->rc, p1, out.n, out.p);
  4112   4226       fts5BufferFree(&tmp);
................................................................................
  4121   4235   }
  4122   4236   
  4123   4237   static void fts5SetupPrefixIter(
  4124   4238     Fts5Index *p,                   /* Index to read from */
  4125   4239     int bDesc,                      /* True for "ORDER BY rowid DESC" */
  4126   4240     const u8 *pToken,               /* Buffer containing prefix to match */
  4127   4241     int nToken,                     /* Size of buffer pToken in bytes */
  4128         -  Fts5IndexIter **ppIter       /* OUT: New iterator */
         4242  +  Fts5Colset *pColset,            /* Restrict matches to these columns */
         4243  +  Fts5IndexIter **ppIter          /* OUT: New iterator */
  4129   4244   ){
  4130   4245     Fts5Structure *pStruct;
  4131   4246     Fts5Buffer *aBuf;
  4132   4247     const int nBuf = 32;
  4133   4248   
  4134   4249     aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  4135   4250     pStruct = fts5StructureRead(p);
................................................................................
  4160   4275               fts5BufferSwap(&doclist, &aBuf[i]);
  4161   4276               fts5BufferZero(&doclist);
  4162   4277             }else{
  4163   4278               fts5MergePrefixLists(p, &doclist, &aBuf[i]);
  4164   4279               fts5BufferZero(&aBuf[i]);
  4165   4280             }
  4166   4281           }
         4282  +        iLastRowid = 0;
  4167   4283         }
  4168   4284   
  4169         -      fts5MergeAppendDocid(&p->rc, &doclist, &iLastRowid, iRowid);
  4170         -      fts5MultiIterPoslist(p, p1, 1, &doclist);
         4285  +      if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){
         4286  +        int iSave = doclist.n;
         4287  +        assert( doclist.n!=0 || iLastRowid==0 );
         4288  +        fts5BufferSafeAppendVarint(&doclist, iRowid - iLastRowid);
         4289  +        if( fts5MultiIterPoslist(p, p1, pColset, &doclist) ){
         4290  +          doclist.n = iSave;
         4291  +        }else{
         4292  +          iLastRowid = iRowid;
         4293  +        }
         4294  +      }
  4171   4295       }
  4172   4296   
  4173   4297       for(i=0; i<nBuf; i++){
  4174         -      fts5MergePrefixLists(p, &doclist, &aBuf[i]);
         4298  +      if( p->rc==SQLITE_OK ){
         4299  +        fts5MergePrefixLists(p, &doclist, &aBuf[i]);
         4300  +      }
  4175   4301         fts5BufferFree(&aBuf[i]);
  4176   4302       }
  4177   4303       fts5MultiIterFree(p, p1);
  4178   4304   
  4179   4305       pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
  4180   4306       if( pData ){
  4181   4307         pData->p = (u8*)&pData[1];
................................................................................
  4191   4317   }
  4192   4318   
  4193   4319   
  4194   4320   /*
  4195   4321   ** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
  4196   4322   ** to the document with rowid iRowid.
  4197   4323   */
  4198         -int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
         4324  +int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  4199   4325     assert( p->rc==SQLITE_OK );
  4200   4326   
  4201   4327     /* Allocate the hash table if it has not already been allocated */
  4202   4328     if( p->pHash==0 ){
  4203   4329       p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
  4204   4330     }
  4205   4331   
  4206   4332     /* Flush the hash table to disk if required */
  4207         -  if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
         4333  +  if( iRowid<p->iWriteRowid 
         4334  +   || (iRowid==p->iWriteRowid && p->bDelete==0)
         4335  +   || (p->nPendingData > p->nMaxPendingData) 
         4336  +  ){
  4208   4337       fts5IndexFlush(p);
  4209   4338     }
         4339  +
  4210   4340     p->iWriteRowid = iRowid;
         4341  +  p->bDelete = bDelete;
  4211   4342     return fts5IndexReturn(p);
  4212   4343   }
  4213   4344   
  4214   4345   /*
  4215   4346   ** Commit data to disk.
  4216   4347   */
  4217   4348   int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
................................................................................
  4302   4433       assert( p->pReader==0 );
  4303   4434       sqlite3_finalize(p->pWriter);
  4304   4435       sqlite3_finalize(p->pDeleter);
  4305   4436       sqlite3_finalize(p->pIdxWriter);
  4306   4437       sqlite3_finalize(p->pIdxDeleter);
  4307   4438       sqlite3_finalize(p->pIdxSelect);
  4308   4439       sqlite3Fts5HashFree(p->pHash);
  4309         -    sqlite3Fts5BufferFree(&p->scratch);
  4310   4440       sqlite3_free(p->zDataTbl);
  4311   4441       sqlite3_free(p);
  4312   4442     }
  4313   4443     return rc;
  4314   4444   }
  4315   4445   
  4316   4446   /*
................................................................................
  4363   4493     const char *pToken, int nToken  /* Token to add or remove to or from index */
  4364   4494   ){
  4365   4495     int i;                          /* Used to iterate through indexes */
  4366   4496     int rc = SQLITE_OK;             /* Return code */
  4367   4497     Fts5Config *pConfig = p->pConfig;
  4368   4498   
  4369   4499     assert( p->rc==SQLITE_OK );
         4500  +  assert( (iCol<0)==p->bDelete );
  4370   4501   
  4371   4502     /* Add the entry to the main terms index. */
  4372   4503     rc = sqlite3Fts5HashWrite(
  4373   4504         p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken
  4374   4505     );
  4375   4506   
  4376   4507     for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){
................................................................................
  4389   4520   ** Open a new iterator to iterate though all rowid that match the 
  4390   4521   ** specified token or token prefix.
  4391   4522   */
  4392   4523   int sqlite3Fts5IndexQuery(
  4393   4524     Fts5Index *p,                   /* FTS index to query */
  4394   4525     const char *pToken, int nToken, /* Token (or prefix) to query for */
  4395   4526     int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
         4527  +  Fts5Colset *pColset,            /* Match these columns only */
  4396   4528     Fts5IndexIter **ppIter          /* OUT: New iterator object */
  4397   4529   ){
  4398   4530     Fts5Config *pConfig = p->pConfig;
  4399   4531     Fts5IndexIter *pRet = 0;
  4400   4532     int iIdx = 0;
  4401   4533     Fts5Buffer buf = {0, 0, 0};
  4402   4534   
................................................................................
  4432   4564         if( pStruct ){
  4433   4565           fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet);
  4434   4566           fts5StructureRelease(pStruct);
  4435   4567         }
  4436   4568       }else{
  4437   4569         int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
  4438   4570         buf.p[0] = FTS5_MAIN_PREFIX;
  4439         -      fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, &pRet);
         4571  +      fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
  4440   4572       }
  4441   4573   
  4442   4574       if( p->rc ){
  4443   4575         sqlite3Fts5IterClose(pRet);
  4444   4576         pRet = 0;
  4445   4577         fts5CloseReader(p);
  4446   4578       }
................................................................................
  4534   4666     assert( pIter->pIndex->rc==SQLITE_OK );
  4535   4667     *piRowid = pSeg->iRowid;
  4536   4668     *pn = pSeg->nPos;
  4537   4669     if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->szLeaf ){
  4538   4670       *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4539   4671     }else{
  4540   4672       fts5BufferZero(&pIter->poslist);
  4541         -    fts5SegiterPoslist(pIter->pIndex, pSeg, &pIter->poslist);
         4673  +    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
  4542   4674       *pp = pIter->poslist.p;
  4543   4675     }
  4544   4676     return fts5IndexReturn(pIter->pIndex);
  4545   4677   }
  4546   4678   
  4547   4679   /*
  4548   4680   ** This function is similar to sqlite3Fts5IterPoslist(), except that it
  4549   4681   ** copies the position list into the buffer supplied as the second 
  4550   4682   ** argument.
  4551   4683   */
  4552   4684   int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
  4553   4685     Fts5Index *p = pIter->pIndex;
  4554         -
         4686  +  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  4555   4687     assert( p->rc==SQLITE_OK );
  4556   4688     fts5BufferZero(pBuf);
  4557         -  fts5MultiIterPoslist(p, pIter, 0, pBuf);
         4689  +  fts5SegiterPoslist(p, pSeg, 0, pBuf);
  4558   4690     return fts5IndexReturn(p);
  4559   4691   }
  4560   4692   
  4561   4693   /*
  4562   4694   ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
  4563   4695   */
  4564   4696   void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
................................................................................
  4725   4857     const char *z,                  /* Index key to query for */
  4726   4858     int n,                          /* Size of index key in bytes */
  4727   4859     int flags,                      /* Flags for Fts5IndexQuery */
  4728   4860     u64 *pCksum                     /* IN/OUT: Checksum value */
  4729   4861   ){
  4730   4862     u64 cksum = *pCksum;
  4731   4863     Fts5IndexIter *pIdxIter = 0;
  4732         -  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
         4864  +  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
  4733   4865   
  4734   4866     while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  4735   4867       i64 dummy;
  4736   4868       const u8 *pPos;
  4737   4869       int nPos;
  4738   4870       i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
  4739   4871       rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos, &dummy);
................................................................................
  5099   5231       i64 iRowid = fts5MultiIterRowid(pIter);
  5100   5232       char *z = (char*)fts5MultiIterTerm(pIter, &n);
  5101   5233   
  5102   5234       /* If this is a new term, query for it. Update cksum3 with the results. */
  5103   5235       fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
  5104   5236   
  5105   5237       poslist.n = 0;
  5106         -    fts5MultiIterPoslist(p, pIter, 0, &poslist);
         5238  +    fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
  5107   5239       while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
  5108   5240         int iCol = FTS5_POS2COLUMN(iPos);
  5109   5241         int iTokOff = FTS5_POS2OFFSET(iPos);
  5110   5242         cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
  5111   5243       }
  5112   5244     }
  5113   5245     fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
................................................................................
  5238   5370       *pRc = rc;
  5239   5371       return;
  5240   5372     }
  5241   5373   
  5242   5374     fts5DebugStructure(pRc, pBuf, p);
  5243   5375     fts5StructureRelease(p);
  5244   5376   }
         5377  +
         5378  +/*
         5379  +** This is part of the fts5_decode() debugging aid.
         5380  +**
         5381  +** Arguments pBlob/nBlob contain an "averages" record. This function 
         5382  +** appends a human-readable representation of record to the buffer passed 
         5383  +** as the second argument. 
         5384  +*/
         5385  +static void fts5DecodeAverages(
         5386  +  int *pRc,                       /* IN/OUT: error code */
         5387  +  Fts5Buffer *pBuf,
         5388  +  const u8 *pBlob, int nBlob
         5389  +){
         5390  +  int i = 0;
         5391  +  const char *zSpace = "";
         5392  +
         5393  +  while( i<nBlob ){
         5394  +    u64 iVal;
         5395  +    i += sqlite3Fts5GetVarint(&pBlob[i], &iVal);
         5396  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal);
         5397  +    zSpace = " ";
         5398  +  }
         5399  +}
  5245   5400   
  5246   5401   /*
  5247   5402   ** Buffer (a/n) is assumed to contain a list of serialized varints. Read
  5248   5403   ** each varint and append its string representation to buffer pBuf. Return
  5249   5404   ** after either the input buffer is exhausted or a 0 value is read.
  5250   5405   **
  5251   5406   ** The return value is the number of bytes read from the input buffer.
................................................................................
  5340   5495       for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){
  5341   5496         sqlite3Fts5BufferAppendPrintf(&rc, &s, 
  5342   5497             " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
  5343   5498         );
  5344   5499       }
  5345   5500     }else if( iSegid==0 ){
  5346   5501       if( iRowid==FTS5_AVERAGES_ROWID ){
  5347         -      /* todo */
         5502  +      fts5DecodeAverages(&rc, &s, a, n);
  5348   5503       }else{
  5349   5504         fts5DecodeStructure(&rc, &s, a, n);
  5350   5505       }
  5351   5506     }else{
  5352   5507       Fts5Buffer term;              /* Current term read from page */
  5353   5508       int szLeaf;                   /* Offset of pgidx in a[] */
  5354   5509       int iPgidxOff;

Changes to ext/fts5/fts5_main.c.

   435    435   */
   436    436   #define FTS5_PLAN_MATCH          1       /* (<tbl> MATCH ?) */
   437    437   #define FTS5_PLAN_SOURCE         2       /* A source cursor for SORTED_MATCH */
   438    438   #define FTS5_PLAN_SPECIAL        3       /* An internal query */
   439    439   #define FTS5_PLAN_SORTED_MATCH   4       /* (<tbl> MATCH ? ORDER BY rank) */
   440    440   #define FTS5_PLAN_SCAN           5       /* No usable constraint */
   441    441   #define FTS5_PLAN_ROWID          6       /* (rowid = ?) */
          442  +
          443  +/*
          444  +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
          445  +** extension is currently being used by a version of SQLite too old to
          446  +** support index-info flags. In that case this function is a no-op.
          447  +*/
          448  +static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
          449  +#if SQLITE_VERSION_NUMBER>=3008012
          450  +  if( sqlite3_libversion_number()>=3008012 ){
          451  +    pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
          452  +  }
          453  +#endif
          454  +}
   442    455   
   443    456   /*
   444    457   ** Implementation of the xBestIndex method for FTS5 tables. Within the 
   445    458   ** WHERE constraint, it searches for the following:
   446    459   **
   447    460   **   1. A MATCH constraint against the special column.
   448    461   **   2. A MATCH constraint against the "rank" column.
................................................................................
   488    501     struct Constraint {
   489    502       int op;                       /* Mask against sqlite3_index_constraint.op */
   490    503       int fts5op;                   /* FTS5 mask for idxFlags */
   491    504       int iCol;                     /* 0==rowid, 1==tbl, 2==rank */
   492    505       int omit;                     /* True to omit this if found */
   493    506       int iConsIndex;               /* Index in pInfo->aConstraint[] */
   494    507     } aConstraint[] = {
   495         -    {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_MATCH,    1, 1, -1},
   496         -    {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_RANK,     2, 1, -1},
          508  +    {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 
          509  +                                    FTS5_BI_MATCH,    1, 1, -1},
          510  +    {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 
          511  +                                    FTS5_BI_RANK,     2, 1, -1},
   497    512       {SQLITE_INDEX_CONSTRAINT_EQ,    FTS5_BI_ROWID_EQ, 0, 0, -1},
   498    513       {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, 
   499    514                                       FTS5_BI_ROWID_LE, 0, 0, -1},
   500    515       {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, 
   501    516                                       FTS5_BI_ROWID_GE, 0, 0, -1},
   502    517     };
   503    518   
................................................................................
   542    557       }
   543    558     }
   544    559   
   545    560     /* Calculate the estimated cost based on the flags set in idxFlags. */
   546    561     bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH);
   547    562     if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){
   548    563       pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0;
          564  +    if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo);
   549    565     }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
   550    566       pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0;
   551    567     }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
   552    568       pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0;
   553    569     }else{
   554    570       pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0;
   555    571     }
................................................................................
  1280   1296   **
  1281   1297   ** The commands implemented by this function are documented in the "Special
  1282   1298   ** INSERT Directives" section of the documentation. It should be updated if
  1283   1299   ** more commands are added to this function.
  1284   1300   */
  1285   1301   static int fts5SpecialInsert(
  1286   1302     Fts5Table *pTab,                /* Fts5 table object */
  1287         -  sqlite3_value *pCmd,            /* Value inserted into special column */
         1303  +  const char *zCmd,               /* Text inserted into table-name column */
  1288   1304     sqlite3_value *pVal             /* Value inserted into rank column */
  1289   1305   ){
  1290   1306     Fts5Config *pConfig = pTab->pConfig;
  1291         -  const char *z = (const char*)sqlite3_value_text(pCmd);
  1292   1307     int rc = SQLITE_OK;
  1293   1308     int bError = 0;
  1294   1309   
  1295         -  if( 0==sqlite3_stricmp("delete-all", z) ){
         1310  +  if( 0==sqlite3_stricmp("delete-all", zCmd) ){
  1296   1311       if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
  1297   1312         fts5SetVtabError(pTab, 
  1298   1313             "'delete-all' may only be used with a "
  1299   1314             "contentless or external content fts5 table"
  1300   1315         );
  1301   1316         rc = SQLITE_ERROR;
  1302   1317       }else{
  1303   1318         rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
  1304   1319       }
  1305         -  }else if( 0==sqlite3_stricmp("rebuild", z) ){
         1320  +  }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
  1306   1321       if( pConfig->eContent==FTS5_CONTENT_NONE ){
  1307   1322         fts5SetVtabError(pTab, 
  1308   1323             "'rebuild' may not be used with a contentless fts5 table"
  1309   1324         );
  1310   1325         rc = SQLITE_ERROR;
  1311   1326       }else{
  1312   1327         rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
  1313   1328       }
  1314         -  }else if( 0==sqlite3_stricmp("optimize", z) ){
         1329  +  }else if( 0==sqlite3_stricmp("optimize", zCmd) ){
  1315   1330       rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
  1316         -  }else if( 0==sqlite3_stricmp("merge", z) ){
         1331  +  }else if( 0==sqlite3_stricmp("merge", zCmd) ){
  1317   1332       int nMerge = sqlite3_value_int(pVal);
  1318   1333       rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
  1319         -  }else if( 0==sqlite3_stricmp("integrity-check", z) ){
         1334  +  }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){
  1320   1335       rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
  1321   1336   #ifdef SQLITE_DEBUG
  1322         -  }else if( 0==sqlite3_stricmp("prefix-index", z) ){
         1337  +  }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
  1323   1338       pConfig->bPrefixIndex = sqlite3_value_int(pVal);
  1324   1339   #endif
  1325   1340     }else{
  1326   1341       rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
  1327   1342       if( rc==SQLITE_OK ){
  1328         -      rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
         1343  +      rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, zCmd, pVal, &bError);
  1329   1344       }
  1330   1345       if( rc==SQLITE_OK ){
  1331   1346         if( bError ){
  1332   1347           rc = SQLITE_ERROR;
  1333   1348         }else{
  1334         -        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal, 0);
         1349  +        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0);
  1335   1350         }
  1336   1351       }
  1337   1352     }
  1338   1353     return rc;
  1339   1354   }
  1340   1355   
  1341   1356   static int fts5SpecialDelete(
................................................................................
  1347   1362     int eType1 = sqlite3_value_type(apVal[1]);
  1348   1363     if( eType1==SQLITE_INTEGER ){
  1349   1364       sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
  1350   1365       rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
  1351   1366     }
  1352   1367     return rc;
  1353   1368   }
         1369  +
         1370  +static void fts5StorageInsert(
         1371  +  int *pRc, 
         1372  +  Fts5Table *pTab, 
         1373  +  sqlite3_value **apVal, 
         1374  +  i64 *piRowid
         1375  +){
         1376  +  int rc = *pRc;
         1377  +  if( rc==SQLITE_OK ){
         1378  +    rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid);
         1379  +  }
         1380  +  if( rc==SQLITE_OK ){
         1381  +    rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid);
         1382  +  }
         1383  +  *pRc = rc;
         1384  +}
  1354   1385   
  1355   1386   /* 
  1356   1387   ** This function is the implementation of the xUpdate callback used by 
  1357   1388   ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
  1358   1389   ** inserted, updated or deleted.
         1390  +**
         1391  +** A delete specifies a single argument - the rowid of the row to remove.
         1392  +** 
         1393  +** Update and insert operations pass:
         1394  +**
         1395  +**   1. The "old" rowid, or NULL.
         1396  +**   2. The "new" rowid.
         1397  +**   3. Values for each of the nCol matchable columns.
         1398  +**   4. Values for the two hidden columns (<tablename> and "rank").
  1359   1399   */
  1360   1400   static int fts5UpdateMethod(
  1361   1401     sqlite3_vtab *pVtab,            /* Virtual table handle */
  1362   1402     int nArg,                       /* Size of argument array */
  1363   1403     sqlite3_value **apVal,          /* Array of arguments */
  1364   1404     sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
  1365   1405   ){
  1366   1406     Fts5Table *pTab = (Fts5Table*)pVtab;
  1367   1407     Fts5Config *pConfig = pTab->pConfig;
  1368   1408     int eType0;                     /* value_type() of apVal[0] */
  1369         -  int eConflict;                  /* ON CONFLICT for this DML */
  1370   1409     int rc = SQLITE_OK;             /* Return code */
  1371   1410   
  1372   1411     /* A transaction must be open when this is called. */
  1373   1412     assert( pTab->ts.eState==1 );
  1374   1413   
         1414  +  assert( pVtab->zErrMsg==0 );
         1415  +  assert( nArg==1 || nArg==(2+pConfig->nCol+2) );
         1416  +  assert( nArg==1 
         1417  +      || sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
         1418  +      || sqlite3_value_type(apVal[1])==SQLITE_NULL 
         1419  +  );
  1375   1420     assert( pTab->pConfig->pzErrmsg==0 );
  1376   1421     pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
  1377   1422   
  1378         -  /* A delete specifies a single argument - the rowid of the row to remove.
  1379         -  ** Update and insert operations pass:
  1380         -  **
  1381         -  **   1. The "old" rowid, or NULL.
  1382         -  **   2. The "new" rowid.
  1383         -  **   3. Values for each of the nCol matchable columns.
  1384         -  **   4. Values for the two hidden columns (<tablename> and "rank").
  1385         -  */
         1423  +  /* Put any active cursors into REQUIRE_SEEK state. */
         1424  +  fts5TripCursors(pTab);
  1386   1425   
  1387   1426     eType0 = sqlite3_value_type(apVal[0]);
  1388         -  eConflict = sqlite3_vtab_on_conflict(pConfig->db);
         1427  +  if( eType0==SQLITE_NULL 
         1428  +   && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL 
         1429  +  ){
         1430  +    /* A "special" INSERT op. These are handled separately. */
         1431  +    const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]);
         1432  +    if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
         1433  +      && 0==sqlite3_stricmp("delete", z) 
         1434  +    ){
         1435  +      rc = fts5SpecialDelete(pTab, apVal, pRowid);
         1436  +    }else{
         1437  +      rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
         1438  +    }
         1439  +  }else{
         1440  +    /* A regular INSERT, UPDATE or DELETE statement. The trick here is that
         1441  +    ** any conflict on the rowid value must be detected before any 
         1442  +    ** modifications are made to the database file. There are 4 cases:
         1443  +    **
         1444  +    **   1) DELETE
         1445  +    **   2) UPDATE (rowid not modified)
         1446  +    **   3) UPDATE (rowid modified)
         1447  +    **   4) INSERT
         1448  +    **
         1449  +    ** Cases 3 and 4 may violate the rowid constraint.
         1450  +    */
         1451  +    int eConflict = sqlite3_vtab_on_conflict(pConfig->db);
  1389   1452   
  1390         -  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  1391         -  assert( pVtab->zErrMsg==0 );
  1392         -  assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) );
         1453  +    assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
         1454  +    assert( nArg!=1 || eType0==SQLITE_INTEGER );
  1393   1455   
  1394         -  fts5TripCursors(pTab);
  1395         -  if( eType0==SQLITE_INTEGER ){
  1396         -    if( fts5IsContentless(pTab) ){
         1456  +    /* Filter out attempts to run UPDATE or DELETE on contentless tables.
         1457  +    ** This is not suported.  */
         1458  +    if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){
  1397   1459         pTab->base.zErrMsg = sqlite3_mprintf(
  1398   1460             "cannot %s contentless fts5 table: %s", 
  1399   1461             (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
  1400   1462         );
  1401   1463         rc = SQLITE_ERROR;
  1402         -    }else{
         1464  +    }
         1465  +
         1466  +    /* Case 1: DELETE */
         1467  +    else if( nArg==1 ){
  1403   1468         i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
  1404   1469         rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
  1405   1470       }
  1406         -  }else{
  1407         -    sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
  1408         -    assert( nArg>1 );
  1409         -    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
  1410         -      const char *z = (const char*)sqlite3_value_text(pCmd);
  1411         -      if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
  1412         -       && 0==sqlite3_stricmp("delete", z) 
         1471  +
         1472  +    /* Case 2: INSERT */
         1473  +    else if( eType0!=SQLITE_INTEGER ){     
         1474  +      /* If this is a REPLACE, first remove the current entry (if any) */
         1475  +      if( eConflict==SQLITE_REPLACE 
         1476  +       && sqlite3_value_type(apVal[1])==SQLITE_INTEGER 
  1413   1477         ){
  1414         -        rc = fts5SpecialDelete(pTab, apVal, pRowid);
         1478  +        i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
         1479  +        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
         1480  +      }
         1481  +      fts5StorageInsert(&rc, pTab, apVal, pRowid);
         1482  +    }
         1483  +
         1484  +    /* Case 2: UPDATE */
         1485  +    else{
         1486  +      i64 iOld = sqlite3_value_int64(apVal[0]);  /* Old rowid */
         1487  +      i64 iNew = sqlite3_value_int64(apVal[1]);  /* New rowid */
         1488  +      if( iOld!=iNew ){
         1489  +        if( eConflict==SQLITE_REPLACE ){
         1490  +          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
         1491  +          if( rc==SQLITE_OK ){
         1492  +            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew);
         1493  +          }
         1494  +          fts5StorageInsert(&rc, pTab, apVal, pRowid);
         1495  +        }else{
         1496  +          rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid);
         1497  +          if( rc==SQLITE_OK ){
         1498  +            rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
         1499  +          }
         1500  +          if( rc==SQLITE_OK ){
         1501  +            rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid);
         1502  +          }
         1503  +        }
  1415   1504         }else{
  1416         -        rc = fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]);
         1505  +        rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld);
         1506  +        fts5StorageInsert(&rc, pTab, apVal, pRowid);
  1417   1507         }
  1418         -      goto update_method_out;
  1419   1508       }
  1420   1509     }
  1421   1510   
  1422         -
  1423         -  if( rc==SQLITE_OK && nArg>1 ){
  1424         -    rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid);
  1425         -  }
  1426         -
  1427         - update_method_out:
  1428   1511     pTab->pConfig->pzErrmsg = 0;
  1429   1512     return rc;
  1430   1513   }
  1431   1514   
  1432   1515   /*
  1433   1516   ** Implementation of xSync() method. 
  1434   1517   */

Changes to ext/fts5/fts5_storage.c.

   388    388       int rc2;
   389    389       sqlite3_bind_int64(pSeek, 1, iDel);
   390    390       if( sqlite3_step(pSeek)==SQLITE_ROW ){
   391    391         int iCol;
   392    392         Fts5InsertCtx ctx;
   393    393         ctx.pStorage = p;
   394    394         ctx.iCol = -1;
   395         -      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
          395  +      rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   396    396         for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
   397    397           if( pConfig->abUnindexed[iCol-1] ) continue;
   398    398           ctx.szCol = 0;
   399    399           rc = sqlite3Fts5Tokenize(pConfig, 
   400    400               FTS5_TOKENIZE_DOCUMENT,
   401    401               (const char*)sqlite3_column_text(pSeek, iCol),
   402    402               sqlite3_column_bytes(pSeek, iCol),
................................................................................
   545    545     /* Delete the index records */
   546    546     if( rc==SQLITE_OK ){
   547    547       int iCol;
   548    548       Fts5InsertCtx ctx;
   549    549       ctx.pStorage = p;
   550    550       ctx.iCol = -1;
   551    551   
   552         -    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
          552  +    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   553    553       for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
   554    554         if( pConfig->abUnindexed[iCol] ) continue;
   555    555         ctx.szCol = 0;
   556    556         rc = sqlite3Fts5Tokenize(pConfig, 
   557    557           FTS5_TOKENIZE_DOCUMENT,
   558    558           (const char*)sqlite3_value_text(apVal[iCol]),
   559    559           sqlite3_value_bytes(apVal[iCol]),
................................................................................
   635    635       rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
   636    636     }
   637    637   
   638    638     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
   639    639       i64 iRowid = sqlite3_column_int64(pScan, 0);
   640    640   
   641    641       sqlite3Fts5BufferZero(&buf);
   642         -    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid);
          642  +    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
   643    643       for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
   644    644         ctx.szCol = 0;
   645    645         if( pConfig->abUnindexed[ctx.iCol]==0 ){
   646    646           rc = sqlite3Fts5Tokenize(pConfig, 
   647    647               FTS5_TOKENIZE_DOCUMENT,
   648    648               (const char*)sqlite3_column_text(pScan, ctx.iCol+1),
   649    649               sqlite3_column_bytes(pScan, ctx.iCol+1),
................................................................................
   701    701         *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
   702    702       }
   703    703     }
   704    704     return rc;
   705    705   }
   706    706   
   707    707   /*
   708         -** Insert a new row into the FTS table.
          708  +** Insert a new row into the FTS content table.
          709  +*/
          710  +int sqlite3Fts5StorageContentInsert(
          711  +  Fts5Storage *p, 
          712  +  sqlite3_value **apVal, 
          713  +  i64 *piRowid
          714  +){
          715  +  Fts5Config *pConfig = p->pConfig;
          716  +  int rc = SQLITE_OK;
          717  +
          718  +  /* Insert the new row into the %_content table. */
          719  +  if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
          720  +    if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
          721  +      *piRowid = sqlite3_value_int64(apVal[1]);
          722  +    }else{
          723  +      rc = fts5StorageNewRowid(p, piRowid);
          724  +    }
          725  +  }else{
          726  +    sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
          727  +    int i;                        /* Counter variable */
          728  +#if 0
          729  +    if( eConflict==SQLITE_REPLACE ){
          730  +      eStmt = FTS5_STMT_REPLACE_CONTENT;
          731  +      rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
          732  +    }else{
          733  +      eStmt = FTS5_STMT_INSERT_CONTENT;
          734  +    }
          735  +#endif
          736  +    if( rc==SQLITE_OK ){
          737  +      rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
          738  +    }
          739  +    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
          740  +      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
          741  +    }
          742  +    if( rc==SQLITE_OK ){
          743  +      sqlite3_step(pInsert);
          744  +      rc = sqlite3_reset(pInsert);
          745  +    }
          746  +    *piRowid = sqlite3_last_insert_rowid(pConfig->db);
          747  +  }
          748  +
          749  +  return rc;
          750  +}
          751  +
          752  +/*
          753  +** Insert new entries into the FTS index and %_docsize table.
   709    754   */
   710         -int sqlite3Fts5StorageInsert(
   711         -  Fts5Storage *p,                 /* Storage module to write to */
   712         -  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
   713         -  int eConflict,                  /* on conflict clause */
   714         -  i64 *piRowid                    /* OUT: rowid of new record */
          755  +int sqlite3Fts5StorageIndexInsert(
          756  +  Fts5Storage *p, 
          757  +  sqlite3_value **apVal, 
          758  +  i64 iRowid
   715    759   ){
   716    760     Fts5Config *pConfig = p->pConfig;
   717    761     int rc = SQLITE_OK;             /* Return code */
   718         -  sqlite3_stmt *pInsert = 0;      /* Statement used to write %_content table */
   719         -  int eStmt = 0;                  /* Type of statement used on %_content */
   720         -  int i;                          /* Counter variable */
   721    762     Fts5InsertCtx ctx;              /* Tokenization callback context object */
   722    763     Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
   723    764   
   724    765     memset(&buf, 0, sizeof(Fts5Buffer));
          766  +  ctx.pStorage = p;
   725    767     rc = fts5StorageLoadTotals(p, 1);
   726    768   
   727         -  /* Insert the new row into the %_content table. */
   728    769     if( rc==SQLITE_OK ){
   729         -    if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
   730         -      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
   731         -        *piRowid = sqlite3_value_int64(apVal[1]);
   732         -      }else{
   733         -        rc = fts5StorageNewRowid(p, piRowid);
   734         -      }
   735         -    }else{
   736         -      if( eConflict==SQLITE_REPLACE ){
   737         -        eStmt = FTS5_STMT_REPLACE_CONTENT;
   738         -        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
   739         -      }else{
   740         -        eStmt = FTS5_STMT_INSERT_CONTENT;
   741         -      }
   742         -      if( rc==SQLITE_OK ){
   743         -        rc = fts5StorageGetStmt(p, eStmt, &pInsert, 0);
   744         -      }
   745         -      for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
   746         -        rc = sqlite3_bind_value(pInsert, i, apVal[i]);
   747         -      }
   748         -      if( rc==SQLITE_OK ){
   749         -        sqlite3_step(pInsert);
   750         -        rc = sqlite3_reset(pInsert);
   751         -      }
   752         -      *piRowid = sqlite3_last_insert_rowid(pConfig->db);
   753         -    }
   754         -  }
   755         -
   756         -  /* Add new entries to the FTS index */
   757         -  if( rc==SQLITE_OK ){
   758         -    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
   759         -    ctx.pStorage = p;
          770  +    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
   760    771     }
   761    772     for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
   762    773       ctx.szCol = 0;
   763    774       if( pConfig->abUnindexed[ctx.iCol]==0 ){
   764    775         rc = sqlite3Fts5Tokenize(pConfig, 
   765    776             FTS5_TOKENIZE_DOCUMENT,
   766    777             (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
................................................................................
   772    783       sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
   773    784       p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
   774    785     }
   775    786     p->nTotalRow++;
   776    787   
   777    788     /* Write the %_docsize record */
   778    789     if( rc==SQLITE_OK ){
   779         -    rc = fts5StorageInsertDocsize(p, *piRowid, &buf);
          790  +    rc = fts5StorageInsertDocsize(p, iRowid, &buf);
   780    791     }
   781    792     sqlite3_free(buf.p);
   782    793   
   783    794     /* Write the averages record */
   784    795     if( rc==SQLITE_OK ){
   785    796       rc = fts5StorageSaveTotals(p);
   786    797     }

Changes to ext/fts5/fts5_vocab.c.

   398    398     sqlite3_value **apVal           /* Arguments for the indexing scheme */
   399    399   ){
   400    400     Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
   401    401     int rc;
   402    402     const int flags = FTS5INDEX_QUERY_SCAN;
   403    403   
   404    404     fts5VocabResetCursor(pCsr);
   405         -  rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, &pCsr->pIter);
          405  +  rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, 0, &pCsr->pIter);
   406    406     if( rc==SQLITE_OK ){
   407    407       rc = fts5VocabNextMethod(pCursor);
   408    408     }
   409    409   
   410    410     return rc;
   411    411   }
   412    412   

Changes to ext/fts5/fts5parse.y.

    97     97     A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
    98     98   }
    99     99   cnearset(A) ::= colset(X) COLON nearset(Y). { 
   100    100     sqlite3Fts5ParseSetColset(pParse, Y, X);
   101    101     A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
   102    102   }
   103    103   
   104         -%type colset {Fts5ExprColset*}
          104  +%type colset {Fts5Colset*}
   105    105   %destructor colset { sqlite3_free($$); }
   106         -%type colsetlist {Fts5ExprColset*}
          106  +%type colsetlist {Fts5Colset*}
   107    107   %destructor colsetlist { sqlite3_free($$); }
   108    108   
   109    109   colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
   110    110   colset(A) ::= STRING(X). {
   111    111     A = sqlite3Fts5ParseColset(pParse, 0, &X);
   112    112   }
   113    113   

Changes to ext/fts5/test/fts5al.test.

   247    247     INSERT INTO t3 VALUES('a one');
   248    248     INSERT INTO t3 VALUES('a two');
   249    249     INSERT INTO t3 VALUES('a three');
   250    250     INSERT INTO t3 VALUES('a four');
   251    251     INSERT INTO t3 VALUES('a five');
   252    252     INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()');
   253    253   }
   254         -breakpoint
   255    254   
   256    255   do_execsql_test 4.3.2 {
   257    256     SELECT * FROM t3
   258    257     WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(4)' 
   259    258     ORDER BY rank ASC
   260    259   } {
   261    260     {a four} {a one} {a five} {a two} {a three}
   262    261   }
          262  +
   263    263   do_execsql_test 4.3.3 {
   264    264     SELECT *, rank FROM t3
   265    265     WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' 
   266    266     ORDER BY rank ASC
   267    267   } {
   268    268     {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
   269    269   }
          270  +
          271  +do_execsql_test 4.3.4 {
          272  +  SELECT * FROM t3('a', 'rowidmod(4)') ORDER BY rank ASC;
          273  +} {
          274  +  {a four} {a one} {a five} {a two} {a three}
          275  +}
          276  +
          277  +do_execsql_test 4.3.5 {
          278  +  SELECT *, rank FROM t3('a', 'rowidmod(3)') ORDER BY rank ASC
          279  +} {
          280  +  {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
          281  +}
   270    282   
   271    283   do_catchsql_test 4.4.3 {
   272    284     SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 
   273    285   } {1 {no such function: xyz}}
   274    286   do_catchsql_test 4.4.4 {
   275    287     SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
   276    288   } {1 {parse error in rank function: }}
   277    289   
   278    290   
   279    291   
   280    292   finish_test
   281    293   

Added ext/fts5/test/fts5onepass.test.

            1  +# 2015 Sep 27
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +set testprefix fts5onepass
           15  +
           16  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           17  +ifcapable !fts5 {
           18  +  finish_test
           19  +  return
           20  +}
           21  +
           22  +do_execsql_test 1.0 {
           23  +  CREATE VIRTUAL TABLE ft USING fts5(content);
           24  +  INSERT INTO ft(rowid, content) VALUES(1, '1 2 3');
           25  +  INSERT INTO ft(rowid, content) VALUES(2, '4 5 6');
           26  +  INSERT INTO ft(rowid, content) VALUES(3, '7 8 9');
           27  +}
           28  +
           29  +#-------------------------------------------------------------------------
           30  +# Check that UPDATE and DELETE statements that feature "WHERE rowid=?" or 
           31  +# or "WHERE rowid=?" clauses do not use statement journals. But that other
           32  +# DELETE and UPDATE statements do.
           33  +#
           34  +# Note: "MATCH ? AND rowid=?" does use a statement journal.
           35  +#
           36  +foreach {tn sql uses} {
           37  +  1.1 { DELETE FROM ft } 1
           38  +  1.2 { DELETE FROM ft WHERE rowid=? } 0
           39  +  1.3 { DELETE FROM ft WHERE rowid=? } 0
           40  +  1.4 { DELETE FROM ft WHERE ft MATCH '1' } 1
           41  +  1.5 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1
           42  +  1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1
           43  +
           44  +  2.1 { UPDATE ft SET content='a b c' } 1
           45  +  2.2 { UPDATE ft SET content='a b c' WHERE rowid=? } 0
           46  +  2.3 { UPDATE ft SET content='a b c' WHERE rowid=? } 0
           47  +  2.4 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' } 1
           48  +  2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1
           49  +  2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1
           50  +} {
           51  +  do_test 1.$tn { sql_uses_stmt db $sql } $uses
           52  +}
           53  +
           54  +#-------------------------------------------------------------------------
           55  +# Check that putting a "DELETE/UPDATE ... WHERE rowid=?" statement in a
           56  +# trigger program does not prevent the VM from using a statement 
           57  +# transaction. Even if the calling statement cannot hit a constraint.
           58  +#
           59  +do_execsql_test 2.0 {
           60  +  CREATE TABLE t1(x);
           61  +
           62  +  CREATE TRIGGER t1_ai AFTER INSERT ON t1 BEGIN
           63  +    DELETE FROM ft WHERE rowid=new.x;
           64  +  END;
           65  +
           66  +  CREATE TRIGGER t1_ad AFTER DELETE ON t1 BEGIN
           67  +    UPDATE ft SET content = 'a b c' WHERE rowid=old.x;
           68  +  END;
           69  +
           70  +  CREATE TRIGGER t1_bu BEFORE UPDATE ON t1 BEGIN
           71  +    DELETE FROM ft WHERE rowid=old.x;
           72  +  END;
           73  +}
           74  +
           75  +foreach {tn sql uses} {
           76  +  1 { INSERT INTO t1 VALUES(1)      } 1
           77  +  2 { DELETE FROM t1 WHERE x=4      } 1
           78  +  3 { UPDATE t1 SET x=10 WHERE x=11 } 1
           79  +} {
           80  +  do_test 2.$tn { sql_uses_stmt db $sql } $uses
           81  +}
           82  +
           83  +#-------------------------------------------------------------------------
           84  +# Test that an "UPDATE ... WHERE rowid=?" works and does not corrupt the
           85  +# index when it strikes a constraint. Both inside and outside a 
           86  +# transaction.
           87  +#
           88  +foreach {tn tcl1 tcl2}  {
           89  +  1 {} {}
           90  +
           91  +  2 {
           92  +    execsql BEGIN
           93  +  } {
           94  +    if {[sqlite3_get_autocommit db]==1} { error "transaction rolled back!" }
           95  +    execsql COMMIT
           96  +  }
           97  +} {
           98  +
           99  +  do_execsql_test 3.$tn.0 {
          100  +    DROP TABLE IF EXISTS ft2;
          101  +    CREATE VIRTUAL TABLE ft2 USING fts5(content);
          102  +    INSERT INTO ft2(rowid, content) VALUES(1, 'a b c');
          103  +    INSERT INTO ft2(rowid, content) VALUES(2, 'a b d');
          104  +    INSERT INTO ft2(rowid, content) VALUES(3, 'a b e');
          105  +  }
          106  +
          107  +  eval $tcl1
          108  +  foreach {tn2 sql content} {
          109  +    1 { UPDATE ft2 SET rowid=2 WHERE rowid=1 }
          110  +      { 1 {a b c} 2 {a b d} 3 {a b e} }
          111  +
          112  +    2 { 
          113  +      INSERT INTO ft2(rowid, content) VALUES(4, 'a b f');
          114  +      UPDATE ft2 SET rowid=5 WHERE rowid=4;
          115  +      UPDATE ft2 SET rowid=3 WHERE rowid=5;
          116  +    } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }
          117  +
          118  +    3 {
          119  +      UPDATE ft2 SET rowid=3 WHERE rowid=4;           -- matches 0 rows
          120  +      UPDATE ft2 SET rowid=2 WHERE rowid=3;
          121  +    } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }
          122  +
          123  +    4 {
          124  +      INSERT INTO ft2(rowid, content) VALUES(4, 'a b g');
          125  +      UPDATE ft2 SET rowid=-1 WHERE rowid=4;
          126  +      UPDATE ft2 SET rowid=3 WHERE rowid=-1;
          127  +    } {-1 {a b g} 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }
          128  +
          129  +    5 {
          130  +      DELETE FROM ft2 WHERE rowid=451;
          131  +      DELETE FROM ft2 WHERE rowid=-1;
          132  +      UPDATE ft2 SET rowid = 2 WHERE rowid = 1;
          133  +    } {1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} }
          134  +  } {
          135  +    do_catchsql_test 3.$tn.$tn2.a $sql {1 {constraint failed}}
          136  +    do_execsql_test  3.$tn.$tn2.b { SELECT rowid, content FROM ft2 } $content
          137  +
          138  +    do_execsql_test  3.$tn.$tn2.c { 
          139  +      INSERT INTO ft2(ft2) VALUES('integrity-check');
          140  +    }
          141  +  }
          142  +  eval $tcl2
          143  +}
          144  +
          145  +#-------------------------------------------------------------------------
          146  +# Check that DELETE and UPDATE operations can be done without flushing
          147  +# the in-memory hash table to disk.
          148  +#
          149  +reset_db
          150  +do_execsql_test 4.1.1 {
          151  +  CREATE VIRTUAL TABLE ttt USING fts5(x);
          152  +  BEGIN;
          153  +    INSERT INTO ttt(rowid, x) VALUES(1, 'a b c');
          154  +    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
          155  +    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
          156  +  COMMIT
          157  +}
          158  +do_test 4.1.2 { fts5_level_segs ttt } {1}
          159  +
          160  +do_execsql_test 4.2.1 {
          161  +  BEGIN;
          162  +    DELETE FROM ttt WHERE rowid=1;
          163  +    DELETE FROM ttt WHERE rowid=3;
          164  +    INSERT INTO ttt(rowid, x) VALUES(4, 'd e f');
          165  +    INSERT INTO ttt(rowid, x) VALUES(5, 'd e f');
          166  +  COMMIT;
          167  +} {}
          168  +do_test 4.2.2 { fts5_level_segs ttt } {2}
          169  +
          170  +
          171  +do_execsql_test 4.3.1 {
          172  +  BEGIN;
          173  +    UPDATE ttt SET x = 'd e f' WHERE rowid = 2;
          174  +    UPDATE ttt SET x = 'A B C' WHERE rowid = 4;
          175  +    INSERT INTO ttt(rowid, x) VALUES(6, 'd e f');
          176  +  COMMIT;
          177  +} {}
          178  +do_test 4.2.2 { fts5_level_segs ttt } {3}
          179  +
          180  +finish_test
          181  +

Added ext/fts5/test/fts5phrase.test.

            1  +# 2014 Jan 08
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Tests focused on phrase queries.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5phrase
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t3 USING fts5(a, b, c);
           26  +  INSERT INTO t3 VALUES('d e a', 'd i j j f', 'i j i e b f h'); -- 1
           27  +  INSERT INTO t3 VALUES('g a e', 'f g i g a', 'h d g i g h c'); -- 2
           28  +  INSERT INTO t3 VALUES('e a d', 'e i h a f', 'c e h i f b i'); -- 3
           29  +  INSERT INTO t3 VALUES('a g c', 'd j d j c', 'c d f j i g j'); -- 4
           30  +  INSERT INTO t3 VALUES('b c b', 'j g c d f', 'j c j d g f b'); -- 5
           31  +  INSERT INTO t3 VALUES('j a d', 'e b i h h', 'c c f g d i d'); -- 6
           32  +  INSERT INTO t3 VALUES('a d f', 'h g i i i', 'e a g c i f b'); -- 7
           33  +  INSERT INTO t3 VALUES('g f d', 'f c g b j', 'b b h h h j j'); -- 8
           34  +  INSERT INTO t3 VALUES('f h g', 'c j f g j', 'd h d f e b h'); -- 9
           35  +  INSERT INTO t3 VALUES('f h d', 'c i a d b', 'g b j b a d e'); -- 10
           36  +  INSERT INTO t3 VALUES('j h h', 'j i h a g', 'd e i e a g j'); -- 11
           37  +  INSERT INTO t3 VALUES('a b e', 'h g a g c', 'h c a a d e g'); -- 12
           38  +  INSERT INTO t3 VALUES('a j g', 'i h i f i', 'a g h j g i b'); -- 13
           39  +  INSERT INTO t3 VALUES('j h e', 'f e d i e', 'i d c f e d c'); -- 14
           40  +  INSERT INTO t3 VALUES('d j d', 'd b i a c', 'g d h i d b e'); -- 15
           41  +  INSERT INTO t3 VALUES('h j e', 'e b b c f', 'j a f g h d j'); -- 16
           42  +  INSERT INTO t3 VALUES('c b j', 'c a b a i', 'h f i d a d c'); -- 17
           43  +  INSERT INTO t3 VALUES('e e d', 'i d f c c', 'g i d a f e a'); -- 18
           44  +  INSERT INTO t3 VALUES('e i g', 'e a b i h', 'i f d d a d f'); -- 19
           45  +  INSERT INTO t3 VALUES('h g f', 'b h h j d', 'i f d e g j a'); -- 20
           46  +  INSERT INTO t3 VALUES('e h f', 'j c b c f', 'j a j g h a c'); -- 21
           47  +  INSERT INTO t3 VALUES('d c h', 'b g i c e', 'i i c d e h i'); -- 22
           48  +  INSERT INTO t3 VALUES('a h i', 'a g d f f', 'e f i i b b h'); -- 23
           49  +  INSERT INTO t3 VALUES('d d g', 'c c b c g', 'g c h e b c e'); -- 24
           50  +  INSERT INTO t3 VALUES('a b b', 'b f a d i', 'd a h a b c i'); -- 25
           51  +  INSERT INTO t3 VALUES('a f d', 'a j e a h', 'j i h j a i f'); -- 26
           52  +  INSERT INTO t3 VALUES('d j d', 'h a d i a', 'h h f j h g a'); -- 27
           53  +  INSERT INTO t3 VALUES('g a e', 'd g f a g', 'i d b c g g j'); -- 28
           54  +  INSERT INTO t3 VALUES('j e h', 'g h j h g', 'd a e j a a h'); -- 29
           55  +  INSERT INTO t3 VALUES('e j e', 'g e j g c', 'f c e b e e a'); -- 30
           56  +  INSERT INTO t3 VALUES('h f f', 'i j g e c', 'j j f c a i j'); -- 31
           57  +  INSERT INTO t3 VALUES('a g c', 'c g d b i', 'g h c b a a f'); -- 32
           58  +  INSERT INTO t3 VALUES('c h i', 'j d h e e', 'a h i d c c j'); -- 33
           59  +  INSERT INTO t3 VALUES('d a c', 'e d d b j', 'c e b b h i h'); -- 34
           60  +  INSERT INTO t3 VALUES('d f h', 'c a f c c', 'j b b c c j f'); -- 35
           61  +  INSERT INTO t3 VALUES('b g h', 'g c c c f', 'c g c f h e e'); -- 36
           62  +  INSERT INTO t3 VALUES('f e a', 'b h f j h', 'j g h f d g f'); -- 37
           63  +  INSERT INTO t3 VALUES('h f a', 'a e i j g', 'f d a f d f c'); -- 38
           64  +  INSERT INTO t3 VALUES('f i c', 'f i i i i', 'e c f d h j f'); -- 39
           65  +  INSERT INTO t3 VALUES('h h d', 'd i e d i', 'd f e i a h a'); -- 40
           66  +  INSERT INTO t3 VALUES('f g c', 'd a f c h', 'b b g j c e g'); -- 41
           67  +  INSERT INTO t3 VALUES('h i h', 'h d j d e', 'e d b b i e g'); -- 42
           68  +  INSERT INTO t3 VALUES('b h i', 'j e i d a', 'j j h e e c a'); -- 43
           69  +  INSERT INTO t3 VALUES('g i g', 'f c c f d', 'a c i c a d a'); -- 44
           70  +  INSERT INTO t3 VALUES('c c f', 'a b j d b', 'c a e g f e c'); -- 45
           71  +  INSERT INTO t3 VALUES('d h j', 'g c b j d', 'e a h f h j g'); -- 46
           72  +  INSERT INTO t3 VALUES('a a d', 'j e j a i', 'i d c f f f b'); -- 47
           73  +  INSERT INTO t3 VALUES('b g j', 'e c i h f', 'd d h b g a d'); -- 48
           74  +  INSERT INTO t3 VALUES('c i a', 'a c c c c', 'e h i e h i e'); -- 49
           75  +  INSERT INTO t3 VALUES('f f c', 'f f b i i', 'f f a j e c i'); -- 50
           76  +}
           77  +
           78  +proc pmatch {col expr} {
           79  +  return [expr {[string first $expr $col]>=0}]
           80  +}
           81  +db func pmatch pmatch
           82  +
           83  +foreach {tn cols tokens} {
           84  +  1 a         "c c"
           85  +  2 b         "c c"
           86  +  3 c         "c c"
           87  +  4 {a b c}   "c c"
           88  +  5 {a b c}   "b h"
           89  +  6 {a b}     "b h"
           90  +  7 {a c}     "b h"
           91  +  8 {c a}     "b h"
           92  +  9 {c}       "i e"
           93  +  10 {b}      "i e"
           94  +  11 {a}      "i e"
           95  +} {
           96  +  set fts   "{$cols}:[join $tokens +]"
           97  +  set where [list]
           98  +  foreach c $cols { lappend where "pmatch($c, '$tokens')" }
           99  +  set where [join $where " OR "]
          100  +
          101  +  set res [db eval "SELECT rowid FROM t3 WHERE $where"]
          102  +  do_execsql_test "1.$tn.$fts->([llength $res] rows)" { 
          103  +    SELECT rowid FROM t3($fts) 
          104  +  } $res
          105  +}
          106  +
          107  +do_execsql_test 2.0 {
          108  +  SELECT rowid,
          109  +    highlight(t3, 0, '*', '*'),
          110  +    highlight(t3, 1, '*', '*'),
          111  +    highlight(t3, 2, '*', '*')
          112  +  FROM t3('a:f+f')
          113  +} {
          114  +  31 {h *f f*} {i j g e c} {j j f c a i j} 
          115  +  50 {*f f* c} {f f b i i} {f f a j e c i}
          116  +}
          117  +
          118  +finish_test
          119  +

Changes to ext/fts5/test/fts5prefix.test.

    58     58   foreach {tn q res} {
    59     59     1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1
    60     60     2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2
    61     61   } {
    62     62     do_execsql_test 2.3.$tn $q $res
    63     63   }
    64     64   
           65  +#-------------------------------------------------------------------------
           66  +# Check that prefix queries with:
           67  +#
           68  +#   * a column filter, and
           69  +#   * no prefix index.
           70  +#
           71  +# work Ok.
           72  +#
           73  +do_execsql_test 3.0 {
           74  +  CREATE VIRTUAL TABLE t3 USING fts5(a, b, c);
           75  +  INSERT INTO t3(t3, rank) VALUES('pgsz', 32);
           76  +  BEGIN;
           77  +    INSERT INTO t3 VALUES('acb ccc bba', 'cca bba bca', 'bbc ccc bca'); -- 1
           78  +    INSERT INTO t3 VALUES('cbb cac cab', 'abb aac bba', 'aab ccc cac'); -- 2
           79  +    INSERT INTO t3 VALUES('aac bcb aac', 'acb bcb caa', 'aca bab bca'); -- 3
           80  +    INSERT INTO t3 VALUES('aab ccb ccc', 'aca cba cca', 'aca aac cbb'); -- 4
           81  +    INSERT INTO t3 VALUES('bac aab bab', 'ccb bac cba', 'acb aba abb'); -- 5
           82  +    INSERT INTO t3 VALUES('bab abc ccb', 'acb cba abb', 'cbb aaa cab'); -- 6
           83  +    INSERT INTO t3 VALUES('cbb bbc baa', 'aab aca baa', 'bcc cca aca'); -- 7
           84  +    INSERT INTO t3 VALUES('abc bba abb', 'cac abc cba', 'acc aac cac'); -- 8
           85  +    INSERT INTO t3 VALUES('bbc bbc cab', 'bcb ccb cba', 'bcc cac acb'); -- 9
           86  +  COMMIT;
           87  +}
           88  +
           89  +foreach {tn match res} {
           90  +  1 "a : c*" {1 2 4 6 7 9}
           91  +  2 "b : c*" {1 3 4 5 6 8 9}
           92  +  3 "c : c*" {1 2 4 6 7 8 9}
           93  +  4 "a : b*" {1 3 5 6 7 8 9}
           94  +  5 "b : b*" {1 2 3 5 7 9}
           95  +  6 "c : b*" {1 3 7 9}
           96  +  7 "a : a*" {1 3 4 5 6 8}
           97  +  8 "b : a*" {2 3 4 6 7 8}
           98  +  9 "c : a*" {2 3 4 5 6 7 8 9}
           99  +} {
          100  +  do_execsql_test 3.1.$tn {
          101  +    SELECT rowid FROM t3($match)
          102  +  } $res
          103  +}
          104  +
          105  +do_test 3.2 {
          106  +  expr srand(0)
          107  +  execsql { DELETE FROM t3 }
          108  +  for {set i 0} {$i < 1000} {incr i} {
          109  +    set a [fts5_rnddoc 3]
          110  +    set b [fts5_rnddoc 8]
          111  +    set c [fts5_rnddoc 20]
          112  +    execsql { INSERT INTO t3 VALUES($a, $b, $c) }
          113  +  }
          114  +  execsql { INSERT INTO t3(t3) VALUES('integrity-check') }
          115  +} {}
          116  +
          117  +proc gmatch {col pattern} {
          118  +  expr {[lsearch -glob $col $pattern]>=0}
          119  +}
          120  +db func gmatch gmatch
          121  +
          122  +proc ghl {col pattern} {
          123  +  foreach t $col {
          124  +    if {[string match $pattern $t]} {
          125  +      lappend res "*$t*"
          126  +    } else {
          127  +      lappend res $t
          128  +    }
          129  +  }
          130  +  set res
          131  +}
          132  +db func ghl ghl
          133  +
          134  +set COLS(a) 0
          135  +set COLS(b) 1
          136  +set COLS(c) 2
          137  +
          138  +for {set x 0} {$x<2} {incr x} {
          139  +  foreach {tn pattern} {
          140  +    1  {xa*}
          141  +    2  {xb*}
          142  +    3  {xc*}
          143  +    4  {xd*}
          144  +    5  {xe*}
          145  +    6  {xf*}
          146  +    7  {xg*}
          147  +    8  {xh*}
          148  +    9  {xi*}
          149  +    10 {xj*}
          150  +  } {
          151  +    foreach col {a b c} {
          152  +
          153  +      # Check that the list of returned rowids is correct.
          154  +      #
          155  +      set res [db eval "SELECT rowid FROM t3 WHERE gmatch($col, '$pattern')"]
          156  +      set query "$col : $pattern"
          157  +      do_execsql_test 3.3.$x.$tn.$col.rowid {
          158  +        SELECT rowid FROM t3($query);
          159  +      } $res
          160  +
          161  +      # Check that the highlight() function works.
          162  +      #
          163  +      set res [db eval \
          164  +        "SELECT ghl($col, '$pattern') FROM t3 WHERE gmatch($col, '$pattern')"
          165  +      ]
          166  +      set idx $COLS($col)
          167  +      do_execsql_test 3.3.$x.$tn.$col.highlight {
          168  +        SELECT highlight(t3, $idx, '*', '*') FROM t3($query);
          169  +      } $res
          170  +    }
          171  +
          172  +    foreach colset {{a b} {b c} {c a} {a c} {b a}} {
          173  +      # Check that the list of returned rowids is correct.
          174  +      #
          175  +      foreach {col1 col2} $colset {}
          176  +      set expr "gmatch($col1, '$pattern') OR gmatch($col2, '$pattern')"
          177  +      set res [db eval "SELECT rowid FROM t3 WHERE $expr"]
          178  +      set query "{$colset} : $pattern"
          179  +      do_execsql_test 3.3.$x.$tn.{$colset}.rowid {
          180  +        SELECT rowid FROM t3($query);
          181  +      } $res
          182  +
          183  +      set resq    "SELECT ghl($col1, '$pattern'), ghl($col2, '$pattern')"
          184  +      append resq " FROM t3 WHERE $expr"
          185  +      set res [db eval $resq]
          186  +      set idx1 $COLS($col1)
          187  +      set idx2 $COLS($col2)
          188  +      do_execsql_test 3.3.$x.$tn.{$colset}.highlight {
          189  +        SELECT highlight(t3, $idx1, '*', '*'), highlight(t3, $idx2, '*', '*')
          190  +          FROM t3($query)
          191  +      } $res
          192  +    }
          193  +  }
          194  +  execsql { INSERT INTO t3(t3) VALUES('optimize') }
          195  +  execsql { INSERT INTO t3(t3) VALUES('integrity-check') }
          196  +}
          197  +
    65    198   
    66    199   finish_test
          200  +
    67    201   

Changes to ext/fts5/test/fts5simple.test.

    15     15   
    16     16   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    17     17   ifcapable !fts5 {
    18     18     finish_test
    19     19     return
    20     20   }
    21     21   
    22         -if 1 {
    23     22   #-------------------------------------------------------------------------
    24     23   #
    25     24   set doc "x x [string repeat {y } 50]z z"
    26     25   do_execsql_test 1.0 {
    27     26     CREATE VIRTUAL TABLE t1 USING fts5(x);
    28     27     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    29     28     BEGIN;
................................................................................
   133    132     COMMIT;
   134    133   } {}
   135    134   
   136    135   do_execsql_test 5.4 {
   137    136     SELECT rowid FROM tt WHERE tt MATCH 'a*';
   138    137   } {1 2}
   139    138   
   140         -}
   141         -
   142    139   do_execsql_test 5.5 {
   143    140     DELETE FROM tt;
   144    141     BEGIN;
   145    142       INSERT INTO tt VALUES('aa');
   146    143       INSERT INTO tt VALUES('ab');
   147    144       INSERT INTO tt VALUES('aa');
   148    145       INSERT INTO tt VALUES('ab');
................................................................................
   179    176   
   180    177   do_catchsql_test 6.2 { 
   181    178     SELECT * FROM xyz WHERE xyz MATCH '' 
   182    179   } {1 {fts5: syntax error near ""}}
   183    180   do_catchsql_test 6.3 { 
   184    181     SELECT * FROM xyz WHERE xyz MATCH NULL 
   185    182   } {1 {fts5: syntax error near ""}}
          183  +
          184  +#-------------------------------------------------------------------------
          185  +
          186  +do_execsql_test 7.1 {
          187  +  CREATE VIRTUAL TABLE ft2 USING fts5(content);
          188  +  INSERT INTO ft2(rowid, content) VALUES(1, 'a b c');
          189  +  INSERT INTO ft2(rowid, content) VALUES(2, 'a b d');
          190  +} 
          191  +
          192  +do_catchsql_test 7.2 {
          193  +  BEGIN;
          194  +    UPDATE ft2 SET rowid=2 WHERE rowid=1;
          195  +} {1 {constraint failed}} 
          196  +
          197  +do_execsql_test 7.3 {
          198  +  COMMIT;
          199  +  INSERT INTO ft2(ft2) VALUES('integrity-check');
          200  +} {}
          201  +
          202  +do_execsql_test 7.4 {
          203  +  SELECT * FROM ft2;
          204  +} {{a b c} {a b d}}
          205  +
          206  +#-------------------------------------------------------------------------
          207  +#
          208  +reset_db
          209  +do_execsql_test 8.1 {
          210  +  CREATE VIRTUAL TABLE ft2 USING fts5(content);
          211  +  INSERT INTO ft2(rowid, content) VALUES(1, 'a b');
          212  +}
          213  +
          214  +do_execsql_test 8.2 {
          215  +  BEGIN;
          216  +    INSERT INTO ft2(rowid, content) VALUES(4, 'a x');
          217  +}
          218  +
          219  +do_execsql_test 8.3 {
          220  +  INSERT INTO ft2(ft2) VALUES('integrity-check');
          221  +}
          222  +
          223  +#-------------------------------------------------------------------------
          224  +# Check that the "table function" syntax works.
          225  +#
          226  +reset_db
          227  +do_execsql_test 9.1 {
          228  +  CREATE VIRTUAL TABLE ft2 USING fts5(content);
          229  +  INSERT INTO ft2(rowid, content) VALUES(1, 'a b');
          230  +  INSERT INTO ft2(rowid, content) VALUES(2, 'a b c d');
          231  +  INSERT INTO ft2(rowid, content) VALUES(3, 'c d e f');
          232  +}
          233  +
          234  +do_execsql_test 9.2 {
          235  +  SELECT rowid FROM ft2('a');
          236  +} {1 2}
          237  +
          238  +do_execsql_test 9.3 {
          239  +  SELECT rowid FROM ft2('b AND c');
          240  +} {2}
          241  +
          242  +#-------------------------------------------------------------------------
          243  +#
          244  +do_execsql_test 10.0 {
          245  +  CREATE VIRTUAL TABLE t3 USING fts5(a, b, c);
          246  +  INSERT INTO t3 VALUES('bac aab bab', 'c bac c', 'acb aba abb'); -- 1
          247  +  INSERT INTO t3 VALUES('bab abc c', 'acb c abb', 'c aaa c');     -- 2
          248  +}
          249  +
          250  +do_execsql_test 10.1 {
          251  +  SELECT rowid FROM t3('c: c*');
          252  +} {2}
          253  +
          254  +#-------------------------------------------------------------------------
          255  +# Test that character 0x1A is allowed in fts5 barewords.
          256  +#
          257  +do_test 11.0 {
          258  +  execsql "CREATE VIRTUAL TABLE t4 USING fts5(x, tokenize=\"ascii tokenchars '\x1A'\")"
          259  +  execsql "
          260  +    INSERT INTO t4 VALUES('a b c \x1A');
          261  +    INSERT INTO t4 VALUES('a b c d\x1A');
          262  +    INSERT INTO t4 VALUES('a b c \x1Ad');
          263  +    INSERT INTO t4 VALUES('a b c d');
          264  +  "
          265  +} {}
          266  +
          267  +do_test 11.1 {
          268  +  execsql "SELECT rowid FROM t4('\x1A')"
          269  +} {1}
          270  +do_test 11.2 {
          271  +  execsql "SELECT rowid FROM t4('\x1A*')"
          272  +} {1 3}
          273  +do_test 11.3 {
          274  +  execsql "SELECT rowid FROM t4('d\x1A')"
          275  +} {2}
          276  +
          277  +do_test 11.4 {
          278  +  catchsql "SELECT rowid FROM t4('d\x1B')"
          279  +} {/fts5: syntax error/}
          280  +do_test 11.5 {
          281  +  catchsql "SELECT rowid FROM t4('d\x19')"
          282  +} {/fts5: syntax error/}
   186    283   
   187    284   
   188    285   finish_test
   189    286   

Added ext/fts5/tool/fts5txt2db.tcl.

            1  +
            2  +
            3  +proc usage {} {
            4  +  puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..."
            5  +  puts stderr ""
            6  +  puts stderr "Options are"
            7  +  puts stderr "  -fts5"
            8  +  puts stderr "  -fts4"
            9  +  puts stderr "  -colsize <list of column sizes>"
           10  +  puts stderr {
           11  +This script is designed to create fts4/5 tables with more than one column.
           12  +The -colsize option should be set to a Tcl list of integer values, one for
           13  +each column in the table. Each value is the number of tokens that will be
           14  +inserted into the column value for each row. For example, setting the -colsize
           15  +option to "5 10" creates an FTS table with 2 columns, with roughly 5 and 10
           16  +tokens per row in each, respectively.
           17  +
           18  +Each "FILE" argument should be a text file. The contents of these text files is
           19  +split on whitespace characters to form a list of tokens. The first N1 tokens
           20  +are used for the first column of the first row, where N1 is the first element
           21  +of the -colsize list. The next N2 are used for the second column of the first
           22  +row, and so on. Rows are added to the table until the entire list of tokens
           23  +is exhausted.
           24  +}
           25  +  exit -1
           26  +}
           27  +
           28  +set O(aColsize)       [list 10 10 10]
           29  +set O(tblname)        t1
           30  +set O(fts)            fts5
           31  +
           32  +
           33  +set options_with_values {-colsize}
           34  +
           35  +for {set i 0} {$i < [llength $argv]} {incr i} {
           36  +  set opt [lindex $argv $i]
           37  +  if {[string range $opt 0 0]!="-"} break
           38  +
           39  +  if {[lsearch $options_with_values $opt]>=0} {
           40  +    incr i
           41  +    if {$i==[llength $argv]} usage
           42  +    set val [lindex $argv $i]
           43  +  }
           44  +
           45  +  switch -- $opt {
           46  +    -colsize {
           47  +      set O(aColSize) $val
           48  +    }
           49  +
           50  +    -fts4 {
           51  +      set O(fts) fts4
           52  +    }
           53  +
           54  +    -fts5 {
           55  +      set O(fts) fts5
           56  +    }
           57  +  }
           58  +}
           59  +
           60  +if {$i > [llength $argv]-2} usage
           61  +set O(db) [lindex $argv $i]
           62  +set O(files) [lrange $argv [expr $i+1] end]
           63  +
           64  +foreach {k v} [lrange $argv 0 end-2] {
           65  +  switch -- $k {
           66  +    -colsize {
           67  +      set O(aColSize) $v
           68  +    }
           69  +
           70  +    -colsize {
           71  +      set O(aColSize) $v
           72  +    }
           73  +  }
           74  +
           75  +}
           76  +
           77  +sqlite3 db $O(db)
           78  +load_static_extension db fts5
           79  +
           80  +
           81  +# Create the FTS table in the db. Return a list of the table columns.
           82  +#
           83  +proc create_table {} {
           84  +  global O
           85  +  set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
           86  +
           87  +  set nCol [llength $O(aColsize)]
           88  +  set cols [lrange $cols 0 [expr $nCol-1]]
           89  +
           90  +  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) ("
           91  +  append sql [join $cols ,]
           92  +  append sql ");"
           93  +
           94  +  db eval $sql
           95  +  return $cols
           96  +}
           97  +
           98  +# Return a list of tokens from the named file.
           99  +#
          100  +proc readfile {file} {
          101  +  set fd [open $file]
          102  +  set data [read $fd]
          103  +  close $fd
          104  +  split $data
          105  +}
          106  +
          107  +
          108  +# Load all the data into a big list of tokens.
          109  +#
          110  +set tokens [list]
          111  +foreach f $O(files) {
          112  +  set tokens [concat $tokens [readfile $f]]
          113  +}
          114  +
          115  +set N [llength $tokens]
          116  +set i 0
          117  +set cols [create_table]
          118  +set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]"
          119  +foreach c [lrange $cols 1 end] {
          120  +  append sql ", \$$c"
          121  +}
          122  +append sql ")"
          123  +
          124  +db eval BEGIN
          125  +  while {$i < $N} {
          126  +    foreach c $cols s $O(aColsize) {
          127  +      set $c [lrange $tokens $i [expr $i+$s-1]]
          128  +      incr i $s
          129  +    }
          130  +    db eval $sql
          131  +  }
          132  +db eval COMMIT
          133  +
          134  +
          135  +

Changes to ext/rtree/rtree1.test.

    30     30   #   rtree-5.*: Test DELETE
    31     31   #   rtree-6.*: Test UPDATE
    32     32   #   rtree-7.*: Test renaming an r-tree table.
    33     33   #   rtree-8.*: Test constrained scans of r-tree data.
    34     34   #
    35     35   #   rtree-12.*: Test that on-conflict clauses are supported.
    36     36   #   rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed.
           37  +#   rtree-14.*: Test if a non-integer is inserted into the PK column of an
           38  +#               r-tree table, it is converted to an integer before being
           39  +#               inserted. Also that if a non-numeric is inserted into one
           40  +#               of the min/max dimension columns, it is converted to the
           41  +#               required type before being inserted.
    37     42   #
    38     43   
    39     44   ifcapable !rtree {
    40     45     finish_test
    41     46     return
    42     47   }
    43     48   
................................................................................
   530    535     WITH r(x) AS (
   531    536       SELECT 1 UNION ALL
   532    537       SELECT 2 UNION ALL
   533    538       SELECT 3
   534    539     )
   535    540     SELECT * FROM r CROSS JOIN t9 WHERE id=x;
   536    541   } {1 1 0.0 0.0 2 2 0.0 0.0}
          542  +
          543  +#-------------------------------------------------------------------------
          544  +# Test if a non-integer is inserted into the PK column of an r-tree
          545  +# table, it is converted to an integer before being inserted. Also
          546  +# that if a non-numeric is inserted into one of the min/max dimension
          547  +# columns, it is converted to the required type before being inserted.
          548  +#
          549  +do_execsql_test 14.1 {
          550  +  CREATE VIRTUAL TABLE t10 USING rtree(ii, x1, x2);
          551  +}
          552  +
          553  +do_execsql_test 14.2 {
          554  +  INSERT INTO t10 VALUES(NULL,   1, 2);
          555  +  INSERT INTO t10 VALUES(NULL,   2, 3);
          556  +  INSERT INTO t10 VALUES('4xxx', 3, 4);
          557  +  INSERT INTO t10 VALUES(5.0,    4, 5);
          558  +  INSERT INTO t10 VALUES(6.4,    5, 6);
          559  +}
          560  +do_execsql_test 14.3 {
          561  +  SELECT * FROM t10;
          562  +} {
          563  +  1 1.0 2.0   2 2.0 3.0   4 3.0 4.0   5 4.0 5.0   6 5.0 6.0
          564  +}
          565  +
          566  +do_execsql_test 14.4 {
          567  +  DELETE FROM t10;
          568  +  INSERT INTO t10 VALUES(1, 'one', 'two');
          569  +  INSERT INTO t10 VALUES(2, '52xyz', '81...');
          570  +}
          571  +do_execsql_test 14.5 {
          572  +  SELECT * FROM t10;
          573  +} {
          574  +  1 0.0 0.0
          575  +  2 52.0 81.0
          576  +}
          577  +
          578  +do_execsql_test 14.4 {
          579  +  DROP TABLE t10;
          580  +  CREATE VIRTUAL TABLE t10 USING rtree_i32(ii, x1, x2);
          581  +  INSERT INTO t10 VALUES(1, 'one', 'two');
          582  +  INSERT INTO t10 VALUES(2, '52xyz', '81...');
          583  +  INSERT INTO t10 VALUES(3, 42.3, 49.9);
          584  +}
          585  +do_execsql_test 14.5 {
          586  +  SELECT * FROM t10;
          587  +} {
          588  +  1 0 0
          589  +  2 52 81
          590  +  3 42 49
          591  +}
   537    592   
   538    593   finish_test

Changes to src/mutex_w32.c.

    83     83   ** compiled without mutexes (SQLITE_THREADSAFE=0).
    84     84   */
    85     85   void sqlite3MemoryBarrier(void){
    86     86   #if defined(SQLITE_MEMORY_BARRIER)
    87     87     SQLITE_MEMORY_BARRIER;
    88     88   #elif defined(__GNUC__)
    89     89     __sync_synchronize();
    90         -#else
           90  +#elif !defined(SQLITE_DISABLE_INTRINSIC) && \
           91  +      defined(_MSC_VER) && _MSC_VER>=1300
           92  +  _ReadWriteBarrier();
           93  +#elif defined(MemoryBarrier)
    91     94     MemoryBarrier();
    92     95   #endif
    93     96   }
    94     97   
    95     98   /*
    96     99   ** Initialize and deinitialize the mutex subsystem.
    97    100   */

Changes to src/pragma.c.

  1357   1357   
  1358   1358       /* Code that appears at the end of the integrity check.  If no error
  1359   1359       ** messages have been generated, output OK.  Otherwise output the
  1360   1360       ** error message
  1361   1361       */
  1362   1362       static const int iLn = VDBE_OFFSET_LINENO(2);
  1363   1363       static const VdbeOpList endCode[] = {
  1364         -      { OP_IfNeg,       1, 0,        0},    /* 0 */
  1365         -      { OP_String8,     0, 3,        0},    /* 1 */
         1364  +      { OP_AddImm,      1, 0,        0},    /* 0 */
         1365  +      { OP_If,          1, 0,        0},    /* 1 */
         1366  +      { OP_String8,     0, 3,        0},    /* 2 */
  1366   1367         { OP_ResultRow,   3, 1,        0},
  1367   1368       };
  1368   1369   
  1369   1370       int isQuick = (sqlite3Tolower(zLeft[0])=='q');
  1370   1371   
  1371   1372       /* If the PRAGMA command was of the form "PRAGMA <db>.integrity_check",
  1372   1373       ** then iDb is set to the index of the database identified by <db>.
................................................................................
  1559   1560             sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7);
  1560   1561             sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1);
  1561   1562           }
  1562   1563   #endif /* SQLITE_OMIT_BTREECOUNT */
  1563   1564         } 
  1564   1565       }
  1565   1566       addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn);
  1566         -    sqlite3VdbeChangeP3(v, addr, -mxErr);
  1567         -    sqlite3VdbeJumpHere(v, addr);
  1568         -    sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC);
         1567  +    sqlite3VdbeChangeP2(v, addr, -mxErr);
         1568  +    sqlite3VdbeJumpHere(v, addr+1);
         1569  +    sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC);
  1569   1570     }
  1570   1571     break;
  1571   1572   #endif /* SQLITE_OMIT_INTEGRITY_CHECK */
  1572   1573   
  1573   1574   #ifndef SQLITE_OMIT_UTF16
  1574   1575     /*
  1575   1576     **   PRAGMA encoding

Changes to src/select.c.

   575    575       int addr;
   576    576       int iLimit;
   577    577       if( pSelect->iOffset ){
   578    578         iLimit = pSelect->iOffset+1;
   579    579       }else{
   580    580         iLimit = pSelect->iLimit;
   581    581       }
   582         -    addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, -1); VdbeCoverage(v);
          582  +    addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v);
   583    583       sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
   584    584       sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
   585    585       sqlite3VdbeJumpHere(v, addr);
   586    586     }
   587    587   }
   588    588   
   589    589   /*
................................................................................
   591    591   */
   592    592   static void codeOffset(
   593    593     Vdbe *v,          /* Generate code into this VM */
   594    594     int iOffset,      /* Register holding the offset counter */
   595    595     int iContinue     /* Jump here to skip the current record */
   596    596   ){
   597    597     if( iOffset>0 ){
   598         -    int addr;
   599         -    addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v);
   600         -    sqlite3VdbeGoto(v, iContinue);
   601         -    VdbeComment((v, "skip OFFSET records"));
   602         -    sqlite3VdbeJumpHere(v, addr);
          598  +    sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v);
          599  +    VdbeComment((v, "OFFSET"));
   603    600     }
   604    601   }
   605    602   
   606    603   /*
   607    604   ** Add code that will check to make sure the N registers starting at iMem
   608    605   ** form a distinct entry.  iTab is a sorting index that holds previously
   609    606   ** seen combinations of the N values.  A new entry is made in iTab
................................................................................
  1811   1808   ** the reuse of the same limit and offset registers across multiple
  1812   1809   ** SELECT statements.
  1813   1810   */
  1814   1811   static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
  1815   1812     Vdbe *v = 0;
  1816   1813     int iLimit = 0;
  1817   1814     int iOffset;
  1818         -  int addr1, n;
         1815  +  int n;
  1819   1816     if( p->iLimit ) return;
  1820   1817   
  1821   1818     /* 
  1822   1819     ** "LIMIT -1" always shows all rows.  There is some
  1823   1820     ** controversy about what the correct behavior should be.
  1824   1821     ** The current implementation interprets "LIMIT 0" to mean
  1825   1822     ** no rows.
................................................................................
  1846   1843       }
  1847   1844       if( p->pOffset ){
  1848   1845         p->iOffset = iOffset = ++pParse->nMem;
  1849   1846         pParse->nMem++;   /* Allocate an extra register for limit+offset */
  1850   1847         sqlite3ExprCode(pParse, p->pOffset, iOffset);
  1851   1848         sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v);
  1852   1849         VdbeComment((v, "OFFSET counter"));
  1853         -      addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v);
  1854         -      sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset);
  1855         -      sqlite3VdbeJumpHere(v, addr1);
         1850  +      sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iOffset, iOffset, 0);
  1856   1851         sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1);
  1857   1852         VdbeComment((v, "LIMIT+OFFSET"));
  1858         -      addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v);
  1859         -      sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1);
  1860         -      sqlite3VdbeJumpHere(v, addr1);
         1853  +      sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iLimit, iOffset+1, -1);
  1861   1854       }
  1862   1855     }
  1863   1856   }
  1864   1857   
  1865   1858   #ifndef SQLITE_OMIT_COMPOUND_SELECT
  1866   1859   /*
  1867   1860   ** Return the appropriate collating sequence for the iCol-th column of
................................................................................
  2269   2262         }
  2270   2263         p->pPrior = 0;
  2271   2264         p->iLimit = pPrior->iLimit;
  2272   2265         p->iOffset = pPrior->iOffset;
  2273   2266         if( p->iLimit ){
  2274   2267           addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v);
  2275   2268           VdbeComment((v, "Jump ahead if LIMIT reached"));
         2269  +        if( p->iOffset ){
         2270  +          sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iOffset, p->iOffset, 0);
         2271  +          sqlite3VdbeAddOp3(v, OP_Add, p->iLimit, p->iOffset, p->iOffset+1);
         2272  +          sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iLimit, p->iOffset+1, -1);
         2273  +        }
  2276   2274         }
  2277   2275         explainSetInteger(iSub2, pParse->iNextSelectId);
  2278   2276         rc = sqlite3Select(pParse, p, &dest);
  2279   2277         testcase( rc!=SQLITE_OK );
  2280   2278         pDelete = p->pPrior;
  2281   2279         p->pPrior = pPrior;
  2282   2280         p->nSelectRow += pPrior->nSelectRow;

Changes to src/sqliteInt.h.

   192    192   */
   193    193   #if !defined(SQLITE_DISABLE_INTRINSIC)
   194    194   #  if defined(_MSC_VER) && _MSC_VER>=1300
   195    195   #    if !defined(_WIN32_WCE)
   196    196   #      include <intrin.h>
   197    197   #      pragma intrinsic(_byteswap_ushort)
   198    198   #      pragma intrinsic(_byteswap_ulong)
          199  +#      pragma intrinsic(_ReadWriteBarrier)
   199    200   #    else
   200    201   #      include <cmnintrin.h>
   201    202   #    endif
   202    203   #  endif
   203    204   #endif
   204    205   
   205    206   /*

Changes to src/vdbe.c.

  5728   5728     if( pIn1->u.i<pIn2->u.i){
  5729   5729       pIn1->u.i = pIn2->u.i;
  5730   5730     }
  5731   5731     break;
  5732   5732   }
  5733   5733   #endif /* SQLITE_OMIT_AUTOINCREMENT */
  5734   5734   
  5735         -/* Opcode: IfPos P1 P2 * * *
  5736         -** Synopsis: if r[P1]>0 goto P2
         5735  +/* Opcode: IfPos P1 P2 P3 * *
         5736  +** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2
  5737   5737   **
  5738   5738   ** Register P1 must contain an integer.
  5739         -** If the value of register P1 is 1 or greater, jump to P2 and
  5740         -** add the literal value P3 to register P1.
         5739  +** If the value of register P1 is 1 or greater, subtract P3 from the
         5740  +** value in P1 and jump to P2.
  5741   5741   **
  5742   5742   ** If the initial value of register P1 is less than 1, then the
  5743   5743   ** value is unchanged and control passes through to the next instruction.
  5744   5744   */
  5745   5745   case OP_IfPos: {        /* jump, in1 */
  5746   5746     pIn1 = &aMem[pOp->p1];
  5747   5747     assert( pIn1->flags&MEM_Int );
  5748   5748     VdbeBranchTaken( pIn1->u.i>0, 2);
  5749         -  if( pIn1->u.i>0 ) goto jump_to_p2;
         5749  +  if( pIn1->u.i>0 ){
         5750  +    pIn1->u.i -= pOp->p3;
         5751  +    goto jump_to_p2;
         5752  +  }
  5750   5753     break;
  5751   5754   }
  5752   5755   
  5753         -/* Opcode: IfNeg P1 P2 P3 * *
  5754         -** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2
         5756  +/* Opcode: SetIfNotPos P1 P2 P3 * *
         5757  +** Synopsis: if r[P1]<=0 then r[P2]=P3
  5755   5758   **
  5756         -** Register P1 must contain an integer.  Add literal P3 to the value in
  5757         -** register P1 then if the value of register P1 is less than zero, jump to P2. 
         5759  +** Register P1 must contain an integer.
         5760  +** If the value of register P1 is not positive (if it is less than 1) then
         5761  +** set the value of register P2 to be the integer P3.
  5758   5762   */
  5759         -case OP_IfNeg: {        /* jump, in1 */
         5763  +case OP_SetIfNotPos: {        /* in1, in2 */
  5760   5764     pIn1 = &aMem[pOp->p1];
  5761   5765     assert( pIn1->flags&MEM_Int );
  5762         -  pIn1->u.i += pOp->p3;
  5763         -  VdbeBranchTaken(pIn1->u.i<0, 2);
  5764         -  if( pIn1->u.i<0 ) goto jump_to_p2;
         5766  +  if( pIn1->u.i<=0 ){
         5767  +    pOut = out2Prerelease(p, pOp);
         5768  +    pOut->u.i = pOp->p3;
         5769  +  }
  5765   5770     break;
  5766   5771   }
  5767   5772   
  5768   5773   /* Opcode: IfNotZero P1 P2 P3 * *
  5769         -** Synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2
         5774  +** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2
  5770   5775   **
  5771   5776   ** Register P1 must contain an integer.  If the content of register P1 is
  5772         -** initially nonzero, then add P3 to P1 and jump to P2.  If register P1 is
  5773         -** initially zero, leave it unchanged and fall through.
         5777  +** initially nonzero, then subtract P3 from the value in register P1 and
         5778  +** jump to P2.  If register P1 is initially zero, leave it unchanged
         5779  +** and fall through.
  5774   5780   */
  5775   5781   case OP_IfNotZero: {        /* jump, in1 */
  5776   5782     pIn1 = &aMem[pOp->p1];
  5777   5783     assert( pIn1->flags&MEM_Int );
  5778   5784     VdbeBranchTaken(pIn1->u.i<0, 2);
  5779   5785     if( pIn1->u.i ){
  5780         -     pIn1->u.i += pOp->p3;
         5786  +     pIn1->u.i -= pOp->p3;
  5781   5787        goto jump_to_p2;
  5782   5788     }
  5783   5789     break;
  5784   5790   }
  5785   5791   
  5786   5792   /* Opcode: DecrJumpZero P1 P2 * * *
  5787   5793   ** Synopsis: if (--r[P1])==0 goto P2

Changes to src/vtab.c.

  1141   1141   
  1142   1142   /*
  1143   1143   ** Erase the eponymous virtual table instance associated with
  1144   1144   ** virtual table module pMod, if it exists.
  1145   1145   */
  1146   1146   void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
  1147   1147     Table *pTab = pMod->pEpoTab;
  1148         -  if( (pTab = pMod->pEpoTab)!=0 ){
         1148  +  if( pTab!=0 ){
  1149   1149       sqlite3DeleteColumnNames(db, pTab);
  1150   1150       sqlite3VtabClear(db, pTab);
  1151   1151       sqlite3DbFree(db, pTab);
  1152   1152       pMod->pEpoTab = 0;
  1153   1153     }
  1154   1154   }
  1155   1155   

Changes to src/whereexpr.c.

   946    946             eExtraOp = WO_EQUIV;
   947    947           }
   948    948         }else{
   949    949           pDup = pExpr;
   950    950           pNew = pTerm;
   951    951         }
   952    952         exprCommute(pParse, pDup);
   953         -      pLeft = sqlite3ExprSkipCollate(pDup->pLeft);
   954    953         pNew->leftCursor = iCur;
   955    954         pNew->u.leftColumn = iColumn;
   956    955         testcase( (prereqLeft | extraRight) != prereqLeft );
   957    956         pNew->prereqRight = prereqLeft | extraRight;
   958    957         pNew->prereqAll = prereqAll;
   959    958         pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask;
   960    959       }

Changes to test/fts3expr3.test.

   117    117   proc balanced_andor_tree {nEntry} {
   118    118     set tree [balanced_exprtree_structure $nEntry]
   119    119     set node "{[balanced_and_tree $nEntry]}"
   120    120     regsub -all AND $node OR node
   121    121     regsub -all xxx $tree $node tree
   122    122     return $tree
   123    123   }
          124  +
          125  +if 1 {
   124    126   
   125    127   # Test that queries like "1 AND 2 AND 3 AND 4..." are transformed to 
   126    128   # balanced trees by FTS.
   127    129   #
   128    130   for {set i 1} {$i < 100} {incr i} {
   129    131     do_test 1.$i {
   130    132       test_fts3expr2 [random_and_query $i]
................................................................................
   197    199   set query [random_andor_query 12]
   198    200   set result [balanced_andor_tree 12]
   199    201   do_faultsim_test fts3expr3-fault-1 -faults oom-* -body {
   200    202     test_fts3expr2 $::query
   201    203   } -test {
   202    204     faultsim_test_result [list 0 $::result]
   203    205   }
          206  +
          207  +}
          208  +
          209  +#-------------------------------------------------------------------
          210  +
          211  +foreach {tn expr res} {
          212  +  1 {1 OR 2 OR 3 OR 4}           {OR {OR {P 1} {P 2}} {OR {P 3} {P 4}}} 
          213  +  2 {1 OR (2 AND 3 AND 4 AND 5)} 
          214  +    {OR {P 1} {AND {AND {P 2} {P 3}} {AND {P 4} {P 5}}}}
          215  +  3 {(2 AND 3 AND 4 AND 5) OR 1} 
          216  +    {OR {AND {AND {P 2} {P 3}} {AND {P 4} {P 5}}} {P 1}}
          217  +
          218  +  4 {1 AND (2 OR 3 OR 4 OR 5)} 
          219  +    {AND {P 1} {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}}}
          220  +  5 {(2 OR 3 OR 4 OR 5) AND 1} 
          221  +    {AND {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}} {P 1}}
          222  +
          223  +  6 {(2 OR 3 OR 4 OR 5) NOT 1} 
          224  +    {NOT {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}} {P 1}}
          225  +
          226  +  7 {1 NOT (2 OR 3 OR 4 OR 5)} 
          227  +    {NOT {P 1} {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}}}
          228  +
          229  +  8 {(1 OR 2 OR 3 OR 4) NOT (5 AND 6 AND 7 AND 8)}
          230  +    {NOT {OR {OR {P 1} {P 2}} {OR {P 3} {P 4}}} {AND {AND {P 5} {P 6}} {AND {P 7} {P 8}}}}
          231  +} {
          232  +  do_test 5.1.$tn {
          233  +    test_fts3expr2 $expr
          234  +  } $res
          235  +}
   204    236   
   205    237   set sqlite_fts3_enable_parentheses 0
   206    238   finish_test

Added test/offset1.test.

            1  +# 2015-10-06
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# 
           12  +# This file implements test cases for the [b65cb2c8d91f6685841d7d1e13b6]
           13  +# bug:  Correct handling of LIMIT and OFFSET on a UNION ALL query where
           14  +# the right-hand SELECT contains an ORDER BY in a subquery.
           15  +#
           16  +
           17  +set testdir [file dirname $argv0]
           18  +source $testdir/tester.tcl
           19  +
           20  +ifcapable !compound {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +do_execsql_test offset1-1.1 {
           26  +  CREATE TABLE t1(a,b);
           27  +  INSERT INTO t1 VALUES(1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e');
           28  +  CREATE TABLE t2(x,y);
           29  +  INSERT INTO t2 VALUES(8,'y'),(9,'z'),(6,'w'),(7,'x');
           30  +  SELECT count(*) FROM t1, t2;
           31  +} {20}
           32  +
           33  +do_execsql_test offset1-1.2.0 {
           34  +  SELECT a, b FROM t1
           35  +  UNION ALL
           36  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           37  +  LIMIT 3 OFFSET 0;
           38  +} {1 a 2 b 3 c}
           39  +do_execsql_test offset1-1.2.1 {
           40  +  SELECT a, b FROM t1
           41  +  UNION ALL
           42  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           43  +  LIMIT 3 OFFSET 1;
           44  +} {2 b 3 c 4 d}
           45  +do_execsql_test offset1-1.2.2 {
           46  +  SELECT a, b FROM t1
           47  +  UNION ALL
           48  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           49  +  LIMIT 3 OFFSET 2;
           50  +} {3 c 4 d 5 e}
           51  +do_execsql_test offset1-1.2.3 {
           52  +  SELECT a, b FROM t1
           53  +  UNION ALL
           54  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           55  +  LIMIT 3 OFFSET 3;
           56  +} {4 d 5 e 6 w}
           57  +do_execsql_test offset1-1.2.4 {
           58  +  SELECT a, b FROM t1
           59  +  UNION ALL
           60  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           61  +  LIMIT 3 OFFSET 4;
           62  +} {5 e 6 w 7 x}
           63  +do_execsql_test offset1-1.2.5 {
           64  +  SELECT a, b FROM t1
           65  +  UNION ALL
           66  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           67  +  LIMIT 3 OFFSET 5;
           68  +} {6 w 7 x 8 y}
           69  +do_execsql_test offset1-1.2.6 {
           70  +  SELECT a, b FROM t1
           71  +  UNION ALL
           72  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           73  +  LIMIT 3 OFFSET 6;
           74  +} {7 x 8 y 9 z}
           75  +do_execsql_test offset1-1.2.7 {
           76  +  SELECT a, b FROM t1
           77  +  UNION ALL
           78  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           79  +  LIMIT 3 OFFSET 7;
           80  +} {8 y 9 z}
           81  +do_execsql_test offset1-1.2.8 {
           82  +  SELECT a, b FROM t1
           83  +  UNION ALL
           84  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           85  +  LIMIT 3 OFFSET 8;
           86  +} {9 z}
           87  +do_execsql_test offset1-1.2.9 {
           88  +  SELECT a, b FROM t1
           89  +  UNION ALL
           90  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
           91  +  LIMIT 3 OFFSET 9;
           92  +} {}
           93  +
           94  +do_execsql_test offset1-1.3.0 {
           95  +  SELECT * FROM t1 LIMIT 0;
           96  +} {}
           97  +
           98  +do_execsql_test offset1-1.4.0 {
           99  +  SELECT a, b FROM t1
          100  +  UNION ALL
          101  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          102  +  LIMIT 0 OFFSET 1;
          103  +} {}
          104  +do_execsql_test offset1-1.4.1 {
          105  +  SELECT a, b FROM t1
          106  +  UNION ALL
          107  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          108  +  LIMIT 1 OFFSET 1;
          109  +} {2 b}
          110  +do_execsql_test offset1-1.4.2 {
          111  +  SELECT a, b FROM t1
          112  +  UNION ALL
          113  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          114  +  LIMIT 2 OFFSET 1;
          115  +} {2 b 3 c}
          116  +do_execsql_test offset1-1.4.3 {
          117  +  SELECT a, b FROM t1
          118  +  UNION ALL
          119  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          120  +  LIMIT 3 OFFSET 1;
          121  +} {2 b 3 c 4 d}
          122  +do_execsql_test offset1-1.4.4 {
          123  +  SELECT a, b FROM t1
          124  +  UNION ALL
          125  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          126  +  LIMIT 4 OFFSET 1;
          127  +} {2 b 3 c 4 d 5 e}
          128  +do_execsql_test offset1-1.4.5 {
          129  +  SELECT a, b FROM t1
          130  +  UNION ALL
          131  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          132  +  LIMIT 5 OFFSET 1;
          133  +} {2 b 3 c 4 d 5 e 6 w}
          134  +do_execsql_test offset1-1.4.6 {
          135  +  SELECT a, b FROM t1
          136  +  UNION ALL
          137  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          138  +  LIMIT 6 OFFSET 1;
          139  +} {2 b 3 c 4 d 5 e 6 w 7 x}
          140  +do_execsql_test offset1-1.4.7 {
          141  +  SELECT a, b FROM t1
          142  +  UNION ALL
          143  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          144  +  LIMIT 7 OFFSET 1;
          145  +} {2 b 3 c 4 d 5 e 6 w 7 x 8 y}
          146  +do_execsql_test offset1-1.4.8 {
          147  +  SELECT a, b FROM t1
          148  +  UNION ALL
          149  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          150  +  LIMIT 8 OFFSET 1;
          151  +} {2 b 3 c 4 d 5 e 6 w 7 x 8 y 9 z}
          152  +do_execsql_test offset1-1.4.9 {
          153  +  SELECT a, b FROM t1
          154  +  UNION ALL
          155  +  SELECT * FROM (SELECT x, y FROM t2 ORDER BY y)
          156  +  LIMIT 9 OFFSET 1;
          157  +} {2 b 3 c 4 d 5 e 6 w 7 x 8 y 9 z}
          158  +
          159  +
          160  +
          161  +finish_test

Changes to test/spellfix2.test.

    72     72     WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20;
    73     73   } {
    74     74     32 20
    75     75   }
    76     76   
    77     77   do_execsql_test 1.6 {
    78     78     SELECT word, distance, matchlen FROM demo 
    79         -  WHERE word MATCH 'amstedam*' AND distance <= 100;
           79  +  WHERE word MATCH 'amstedam*' AND distance <= 100
           80  +  ORDER BY distance, word;
    80     81   } {
    81         -  amsterdam         100 9        amsterdamh        100 9
    82         -  amsterdamm        100 9        amsterdamn        100 9
    83         -  amsterdama        100 9        amsterdame        100 9
    84         -  amsterdami        100 9        amsterdamo        100 9
    85         -  amsterdamu        100 9        amsterdamy        100 9
    86         -  amsterdammetje    100 9        amsterdamania     100 9
    87         -  amsterdamb        100 9        amsterdamf        100 9
    88         -  amsterdamp        100 9        amsterdamv        100 9
    89         -  amsterdamw        100 9        amsterdamweg      100 9
    90         -  amsterdamc        100 9        amsterdamg        100 9
    91         -  amsterdamj        100 9        amsterdamk        100 9
    92         -  amsterdamq        100 9        amsterdams        100 9
    93         -  amsterdamx        100 9        amsterdamz        100 9
    94         -  amsterdamsestraat 100 9        amsterdamd        100 9
    95         -  amsterdamt        100 9        amsterdaml        100 9
    96         -  amsterdamlaan     100 9        amsterdamr        100 9
           82  +  amsterdam         100 9        amsterdama        100 9
           83  +  amsterdamania     100 9        amsterdamb        100 9
           84  +  amsterdamc        100 9        amsterdamd        100 9
           85  +  amsterdame        100 9        amsterdamf        100 9
           86  +  amsterdamg        100 9        amsterdamh        100 9
           87  +  amsterdami        100 9        amsterdamj        100 9
           88  +  amsterdamk        100 9        amsterdaml        100 9
           89  +  amsterdamlaan     100 9        amsterdamm        100 9
           90  +  amsterdammetje    100 9        amsterdamn        100 9
           91  +  amsterdamo        100 9        amsterdamp        100 9
           92  +  amsterdamq        100 9        amsterdamr        100 9
           93  +  amsterdams        100 9        amsterdamsestraat 100 9
           94  +  amsterdamt        100 9        amsterdamu        100 9
           95  +  amsterdamv        100 9        amsterdamw        100 9
           96  +  amsterdamweg      100 9        amsterdamx        100 9
           97  +  amsterdamy        100 9        amsterdamz        100 9
    97     98   }
    98     99   
    99    100   do_execsql_test 1.7 {
   100    101     SELECT word, distance, matchlen FROM demo 
   101         -  WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20;
          102  +  WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20
          103  +  ORDER BY distance, word;
   102    104   } {
   103         -  amsterdam         100 9        amsterdamh        100 9
   104         -  amsterdamm        100 9        amsterdamn        100 9
   105         -  amsterdama        100 9        amsterdame        100 9
   106         -  amsterdami        100 9        amsterdamo        100 9
   107         -  amsterdamu        100 9        amsterdamy        100 9
   108         -  amsterdammetje    100 9        amsterdamania     100 9
   109         -  amsterdamb        100 9        amsterdamf        100 9
   110         -  amsterdamp        100 9        amsterdamv        100 9
   111         -  amsterdamw        100 9        amsterdamweg      100 9
   112         -  amsterdamc        100 9        amsterdamg        100 9
          105  +  amsterdam         100 9        amsterdama        100 9
          106  +  amsterdamania     100 9        amsterdamb        100 9
          107  +  amsterdamc        100 9        amsterdame        100 9
          108  +  amsterdamf        100 9        amsterdamg        100 9
          109  +  amsterdamh        100 9        amsterdami        100 9
          110  +  amsterdamm        100 9        amsterdammetje    100 9
          111  +  amsterdamn        100 9        amsterdamo        100 9
          112  +  amsterdamp        100 9        amsterdamu        100 9
          113  +  amsterdamv        100 9        amsterdamw        100 9
          114  +  amsterdamweg      100 9        amsterdamy        100 9
   113    115   }
   114    116   
   115    117   
   116    118   finish_test

Changes to test/uri.test.

    59     59       #
    60     60       # NOTE: Due to limits on legal characters for file names imposed by
    61     61       #       Windows, we must skip the final two tests here (i.e. the
    62     62       #       question mark is illegal in a file name on Windows).
    63     63       #
    64     64       if {$tn>14} break
    65     65   
           66  +    #
           67  +    # NOTE: When running on Tcl 8.6 (or higher?) on Windows, a colon within
           68  +    #       the file name no longer tries to access an alternate data stream
           69  +    #       (ADS) named "test.db" for the "http" file, causing some spurious
           70  +    #       failures of this test.
           71  +    #
           72  +    if {$tn==12 && $::tcl_version>=8.6} continue
           73  +
    66     74       #
    67     75       # NOTE: On Windows, we need to account for the fact that the current
    68     76       #       directory does not start with a forward slash.
    69     77       #
    70     78       set uri  [string map [list PWD/ /[test_pwd /]] $uri]
    71     79     } else {
    72     80       set uri  [string map [list PWD/ [test_pwd /]] $uri]

Deleted tool/diffdb.c.

     1         -/*
     2         -** A utility for printing the differences between two SQLite database files.
     3         -*/
     4         -#include <stdio.h>
     5         -#include <ctype.h>
     6         -#include <sys/types.h>
     7         -#include <sys/stat.h>
     8         -#include <fcntl.h>
     9         -#include <unistd.h>
    10         -#include <stdlib.h>
    11         -
    12         -
    13         -#define PAGESIZE 1024
    14         -static int db1 = -1;
    15         -static int db2 = -1;
    16         -
    17         -int main(int argc, char **argv){
    18         -  int iPg;
    19         -  unsigned char a1[PAGESIZE], a2[PAGESIZE];
    20         -  if( argc!=3 ){
    21         -    fprintf(stderr,"Usage: %s FILENAME FILENAME\n", argv[0]);
    22         -    exit(1);
    23         -  }
    24         -  db1 = open(argv[1], O_RDONLY);
    25         -  if( db1<0 ){
    26         -    fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]);
    27         -    exit(1);
    28         -  }
    29         -  db2 = open(argv[2], O_RDONLY);
    30         -  if( db2<0 ){
    31         -    fprintf(stderr,"%s: can't open %s\n", argv[0], argv[2]);
    32         -    exit(1);
    33         -  }
    34         -  iPg = 1;
    35         -  while( read(db1, a1, PAGESIZE)==PAGESIZE && read(db2,a2,PAGESIZE)==PAGESIZE ){
    36         -    if( memcmp(a1,a2,PAGESIZE) ){
    37         -      printf("Page %d\n", iPg);
    38         -    }
    39         -    iPg++;
    40         -  }
    41         -  printf("%d pages checked\n", iPg-1);
    42         -  close(db1);
    43         -  close(db2);
    44         -}

Deleted tool/opcodeDoc.awk.

     1         -#
     2         -# Extract opcode documentation for sqliteVdbe.c and generate HTML
     3         -#
     4         -BEGIN {
     5         -  print "<html><body bgcolor=white>"
     6         -  print "<h1>SQLite Virtual Database Engine Opcodes</h1>"
     7         -  print "<table>"
     8         -}
     9         -/ Opcode: /,/\*\// {
    10         -  if( $2=="Opcode:" ){
    11         -    printf "<tr><td>%s&nbsp;%s&nbsp;%s&nbsp;%s</td>\n<td>\n", $3, $4, $5, $6
    12         -  }else if( $1=="*/" ){
    13         -    printf "</td></tr>\n"
    14         -  }else if( NF>1 ){
    15         -    sub(/^ *\*\* /,"")
    16         -    gsub(/</,"&lt;")
    17         -    gsub(/&/,"&amp;")
    18         -    print
    19         -  }
    20         -}
    21         -END {
    22         -  print "</table></body></html>"
    23         -}

Deleted tool/space_used.tcl.

     1         -# Run this TCL script using "testfixture" in order get a report that shows
     2         -# how much disk space is used by a particular data to actually store data
     3         -# versus how much space is unused.
     4         -#
     5         -
     6         -# Get the name of the database to analyze
     7         -#
     8         -if {[llength $argv]!=1} {
     9         -  puts stderr "Usage: $argv0 database-name"
    10         -  exit 1
    11         -}
    12         -set file_to_analyze [lindex $argv 0]
    13         -
    14         -# Open the database
    15         -#
    16         -sqlite db [lindex $argv 0]
    17         -set DB [btree_open [lindex $argv 0]]
    18         -
    19         -# Output the schema for the generated report
    20         -#
    21         -puts \
    22         -{BEGIN;
    23         -CREATE TABLE space_used(
    24         -   name clob,        -- Name of a table or index in the database file
    25         -   is_index boolean, -- TRUE if it is an index, false for a table
    26         -   payload int,      -- Total amount of data stored in this table or index
    27         -   pri_pages int,    -- Number of primary pages used
    28         -   ovfl_pages int,   -- Number of overflow pages used
    29         -   pri_unused int,   -- Number of unused bytes on primary pages
    30         -   ovfl_unused int   -- Number of unused bytes on overflow pages
    31         -);}
    32         -
    33         -# This query will be used to find the root page number for every index and
    34         -# table in the database.
    35         -#
    36         -set sql {
    37         -  SELECT name, type, rootpage FROM sqlite_master
    38         -  UNION ALL
    39         -  SELECT 'sqlite_master', 'table', 2
    40         -  ORDER BY 1
    41         -}
    42         -
    43         -# Initialize variables used for summary statistics.
    44         -#
    45         -set total_size 0
    46         -set total_primary 0
    47         -set total_overflow 0
    48         -set total_unused_primary 0
    49         -set total_unused_ovfl 0
    50         -
    51         -# Analyze every table in the database, one at a time.
    52         -#
    53         -foreach {name type rootpage} [db eval $sql] {
    54         -  set cursor [btree_cursor $DB $rootpage 0]
    55         -  set go [btree_first $cursor]
    56         -  set size 0
    57         -  catch {unset pg_used}
    58         -  set unused_ovfl 0
    59         -  set n_overflow 0
    60         -  while {$go==0} {
    61         -    set payload [btree_payload_size $cursor]
    62         -    incr size $payload
    63         -    set stat [btree_cursor_dump $cursor]
    64         -    set pgno [lindex $stat 0]
    65         -    set freebytes [lindex $stat 4]
    66         -    set pg_used($pgno) $freebytes
    67         -    if {$payload>238} {
    68         -      set n [expr {($payload-238+1019)/1020}]
    69         -      incr n_overflow $n
    70         -      incr unused_ovfl [expr {$n*1020+238-$payload}]
    71         -    }
    72         -    set go [btree_next $cursor]
    73         -  }
    74         -  btree_close_cursor $cursor
    75         -  set n_primary [llength [array names pg_used]]
    76         -  set unused_primary 0
    77         -  foreach x [array names pg_used] {incr unused_primary $pg_used($x)}
    78         -  regsub -all ' $name '' name
    79         -  puts -nonewline "INSERT INTO space_used VALUES('$name'"
    80         -  puts -nonewline ",[expr {$type=="index"}]"
    81         -  puts ",$size,$n_primary,$n_overflow,$unused_primary,$unused_ovfl);"
    82         -  incr total_size $size
    83         -  incr total_primary $n_primary
    84         -  incr total_overflow $n_overflow
    85         -  incr total_unused_primary $unused_primary
    86         -  incr total_unused_ovfl $unused_ovfl
    87         -}
    88         -
    89         -# Output summary statistics:
    90         -#
    91         -puts "-- Total payload size: $total_size"
    92         -puts "-- Total pages used: $total_primary primary and $total_overflow overflow"
    93         -set file_pgcnt [expr {[file size [lindex $argv 0]]/1024}]
    94         -puts -nonewline "-- Total unused bytes on primary pages: $total_unused_primary"
    95         -if {$total_primary>0} {
    96         -  set upp [expr {$total_unused_primary/$total_primary}]
    97         -  puts " (avg $upp bytes/page)"
    98         -} else {
    99         -  puts ""
   100         -}
   101         -puts -nonewline "-- Total unused bytes on overflow pages: $total_unused_ovfl"
   102         -if {$total_overflow>0} {
   103         -  set upp [expr {$total_unused_ovfl/$total_overflow}]
   104         -  puts " (avg $upp bytes/page)"
   105         -} else {
   106         -  puts ""
   107         -}
   108         -set n_free [expr {$file_pgcnt-$total_primary-$total_overflow}]
   109         -if {$n_free>0} {incr n_free -1}
   110         -puts "-- Total pages on freelist: $n_free"
   111         -puts "COMMIT;"