/ Check-in [d85774e0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest enhancements and fixes from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA1: d85774e0dcb6cb1958ecd85891ef785e4692f67b
User & Date: drh 2016-01-14 15:03:12
Context
2016-01-20
11:40
Merge all recent enhancements from trunk. check-in: 3ed49691 user: drh tags: apple-osx
2016-01-14
15:03
Merge the latest enhancements and fixes from trunk. check-in: d85774e0 user: drh tags: apple-osx
14:33
Yet another change to FTS5 trying to get it to merge successfully into sessions. check-in: 8dedff3b user: drh tags: trunk
2016-01-11
13:03
Import the version number change to 3.11.0. check-in: 132772d1 user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

   175    175            func.lo global.lo hash.lo \
   176    176            icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \
   177    177            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   178    178            memjournal.lo \
   179    179            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   180    180            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   181    181            pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   182         -         random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
          182  +         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
   183    183            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   184    184            update.lo util.lo vacuum.lo \
   185    185            vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   186    186            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   187    187            utf.lo vtab.lo
   188    188   
   189    189   # Object files for the amalgamation.
................................................................................
  1036   1036   fts5.c: $(FTS5_SRC)
  1037   1037   	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
  1038   1038   	cp $(TOP)/ext/fts5/fts5.h .
  1039   1039   
  1040   1040   fts5.lo:	fts5.c $(HDR) $(EXTHDR)
  1041   1041   	$(LTCOMPILE) -DSQLITE_CORE -c fts5.c
  1042   1042   
         1043  +sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
         1044  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
         1045  +
  1043   1046   
  1044   1047   # Rules to build the 'testfixture' application.
  1045   1048   #
  1046   1049   # If using the amalgamation, use sqlite3.c directly to build the test
  1047   1050   # fixture.  Otherwise link against libsqlite3.la.  (This distinction is
  1048   1051   # necessary because the test fixture requires non-API symbols which are
  1049   1052   # hidden when the library is built via the amalgamation).
................................................................................
  1138   1141   	$(LTLINK) -I. -o $@ $(TOP)/tool/logest.c
  1139   1142   
  1140   1143   wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.c
  1141   1144   	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS)
  1142   1145   
  1143   1146   speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.lo
  1144   1147   	$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
         1148  +
         1149  +rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
         1150  +	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
         1151  +
         1152  +loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
         1153  +	$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
  1145   1154   
  1146   1155   # This target will fail if the SQLite amalgamation contains any exported
  1147   1156   # symbols that do not begin with "sqlite3_". It is run as part of the
  1148   1157   # releasetest.tcl script.
  1149   1158   #
  1150   1159   checksymbols: sqlite3.lo
  1151   1160   	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0

Changes to Makefile.msc.

   835    835            func.lo global.lo hash.lo \
   836    836            icu.lo insert.lo journal.lo legacy.lo loadext.lo \
   837    837            main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
   838    838            memjournal.lo \
   839    839            mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
   840    840            notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
   841    841            pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
   842         -         random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
          842  +         random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
   843    843            table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
   844    844            update.lo util.lo vacuum.lo \
   845    845            vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
   846    846            vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
   847    847            utf.lo vtab.lo
   848    848   
   849    849   # Object files for the amalgamation.
................................................................................
  1710   1710   
  1711   1711   fts5_ext.lo:	fts5.c $(HDR) $(EXTHDR)
  1712   1712   	$(LTCOMPILE) $(NO_WARN) -c fts5.c
  1713   1713   
  1714   1714   fts5.dll:	fts5_ext.lo
  1715   1715   	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo
  1716   1716   
         1717  +sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
         1718  +	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
         1719  +
  1717   1720   # Rules to build the 'testfixture' application.
  1718   1721   #
  1719   1722   # If using the amalgamation, use sqlite3.c directly to build the test
  1720   1723   # fixture.  Otherwise link against libsqlite3.lib.  (This distinction is
  1721   1724   # necessary because the test fixture requires non-API symbols which are
  1722   1725   # hidden when the library is built via the amalgamation).
  1723   1726   #
................................................................................
  1831   1834   wordcount.exe:	$(TOP)\test\wordcount.c $(SQLITE3C)
  1832   1835   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1833   1836   		$(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  1834   1837   
  1835   1838   speedtest1.exe:	$(TOP)\test\speedtest1.c $(SQLITE3C)
  1836   1839   	$(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \
  1837   1840   		$(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         1841  +
         1842  +rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C)
         1843  +	$(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \
         1844  +		$(LDFLAGS) $(LTLINKOPTS)
  1838   1845   
  1839   1846   clean:
  1840   1847   	del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
  1841   1848   	del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL
  1842   1849   	del /Q sqlite3.h opcodes.c opcodes.h 2>NUL
  1843   1850   	del /Q lemon.* lempar.c parse.* 2>NUL
  1844   1851   	del /Q mkkeywordhash.* keywordhash.h 2>NUL

Changes to autoconf/configure.ac.

    69     69   #
    70     70   AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING(
    71     71     [--enable-threadsafe], [build a thread-safe library [default=yes]])], 
    72     72     [], [enable_threadsafe=yes])
    73     73   THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
    74     74   if test x"$enable_threadsafe" != "xno"; then
    75     75     THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
    76         -  AC_SEARCH_LIBS(pthread_create, pthread)
           76  +  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
    77     77   fi
    78     78   AC_SUBST(THREADSAFE_FLAGS)
    79     79   #-----------------------------------------------------------------------
    80     80   
    81     81   #-----------------------------------------------------------------------
    82     82   #   --enable-dynamic-extensions
    83     83   #

Changes to configure.

 10460  10460     SQLITE_THREADSAFE=1
 10461  10461     { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 10462  10462   $as_echo "yes" >&6; }
 10463  10463   fi
 10464  10464   
 10465  10465   
 10466  10466   if test "$SQLITE_THREADSAFE" = "1"; then
 10467         -  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5
 10468         -$as_echo_n "checking for library containing pthread_create... " >&6; }
 10469         -if ${ac_cv_search_pthread_create+:} false; then :
        10467  +  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5
        10468  +$as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; }
        10469  +if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
 10470  10470     $as_echo_n "(cached) " >&6
 10471  10471   else
 10472  10472     ac_func_search_save_LIBS=$LIBS
 10473  10473   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 10474  10474   /* end confdefs.h.  */
 10475  10475   
 10476  10476   /* Override any GCC internal prototype to avoid an error.
 10477  10477      Use char because int might match the return type of a GCC
 10478  10478      builtin and then its argument prototype would still apply.  */
 10479  10479   #ifdef __cplusplus
 10480  10480   extern "C"
 10481  10481   #endif
 10482         -char pthread_create ();
        10482  +char pthread_mutexattr_init ();
 10483  10483   int
 10484  10484   main ()
 10485  10485   {
 10486         -return pthread_create ();
        10486  +return pthread_mutexattr_init ();
 10487  10487     ;
 10488  10488     return 0;
 10489  10489   }
 10490  10490   _ACEOF
 10491  10491   for ac_lib in '' pthread; do
 10492  10492     if test -z "$ac_lib"; then
 10493  10493       ac_res="none required"
 10494  10494     else
 10495  10495       ac_res=-l$ac_lib
 10496  10496       LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
 10497  10497     fi
 10498  10498     if ac_fn_c_try_link "$LINENO"; then :
 10499         -  ac_cv_search_pthread_create=$ac_res
        10499  +  ac_cv_search_pthread_mutexattr_init=$ac_res
 10500  10500   fi
 10501  10501   rm -f core conftest.err conftest.$ac_objext \
 10502  10502       conftest$ac_exeext
 10503         -  if ${ac_cv_search_pthread_create+:} false; then :
        10503  +  if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
 10504  10504     break
 10505  10505   fi
 10506  10506   done
 10507         -if ${ac_cv_search_pthread_create+:} false; then :
        10507  +if ${ac_cv_search_pthread_mutexattr_init+:} false; then :
 10508  10508   
 10509  10509   else
 10510         -  ac_cv_search_pthread_create=no
        10510  +  ac_cv_search_pthread_mutexattr_init=no
 10511  10511   fi
 10512  10512   rm conftest.$ac_ext
 10513  10513   LIBS=$ac_func_search_save_LIBS
 10514  10514   fi
 10515         -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5
 10516         -$as_echo "$ac_cv_search_pthread_create" >&6; }
 10517         -ac_res=$ac_cv_search_pthread_create
        10515  +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5
        10516  +$as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; }
        10517  +ac_res=$ac_cv_search_pthread_mutexattr_init
 10518  10518   if test "$ac_res" != no; then :
 10519  10519     test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
 10520  10520   
 10521  10521   fi
 10522  10522   
 10523  10523   fi
 10524  10524   

Changes to configure.ac.

   190    190   else
   191    191     SQLITE_THREADSAFE=1
   192    192     AC_MSG_RESULT([yes])
   193    193   fi
   194    194   AC_SUBST(SQLITE_THREADSAFE)
   195    195   
   196    196   if test "$SQLITE_THREADSAFE" = "1"; then
   197         -  AC_SEARCH_LIBS(pthread_create, pthread)
          197  +  AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
   198    198   fi
   199    199   
   200    200   ##########
   201    201   # Do we want to support release
   202    202   #
   203    203   AC_ARG_ENABLE(releasemode, 
   204    204   AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)

Changes to ext/fts5/fts5.h.

    80     80   **   *pnToken to the number of tokens in column iCol of the current row.
    81     81   **
    82     82   **   If parameter iCol is greater than or equal to the number of columns
    83     83   **   in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
    84     84   **   an OOM condition or IO error), an appropriate SQLite error code is 
    85     85   **   returned.
    86     86   **
           87  +**   This function may be quite inefficient if used with an FTS5 table
           88  +**   created with the "columnsize=0" option.
           89  +**
    87     90   ** xColumnText:
    88     91   **   This function attempts to retrieve the text of column iCol of the
    89     92   **   current document. If successful, (*pz) is set to point to a buffer
    90     93   **   containing the text in utf-8 encoding, (*pn) is set to the size in bytes
    91     94   **   (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
    92     95   **   if an error occurs, an SQLite error code is returned and the final values
    93     96   **   of (*pz) and (*pn) are undefined.
................................................................................
   100    103   **   are numbered starting from zero.
   101    104   **
   102    105   ** xInstCount:
   103    106   **   Set *pnInst to the total number of occurrences of all phrases within
   104    107   **   the query within the current row. Return SQLITE_OK if successful, or
   105    108   **   an error code (i.e. SQLITE_NOMEM) if an error occurs.
   106    109   **
          110  +**   This API can be quite slow if used with an FTS5 table created with the
          111  +**   "detail=none" or "detail=column" option. If the FTS5 table is created 
          112  +**   with either "detail=none" or "detail=column" and "content=" option 
          113  +**   (i.e. if it is a contentless table), then this API always returns 0.
          114  +**
   107    115   ** xInst:
   108    116   **   Query for the details of phrase match iIdx within the current row.
   109    117   **   Phrase matches are numbered starting from zero, so the iIdx argument
   110    118   **   should be greater than or equal to zero and smaller than the value
   111    119   **   output by xInstCount().
          120  +**
          121  +**   Usually, output parameter *piPhrase is set to the phrase number, *piCol
          122  +**   to the column in which it occurs and *piOff the token offset of the
          123  +**   first token of the phrase. The exception is if the table was created
          124  +**   with the offsets=0 option specified. In this case *piOff is always
          125  +**   set to -1.
   112    126   **
   113    127   **   Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 
   114    128   **   if an error occurs.
          129  +**
          130  +**   This API can be quite slow if used with an FTS5 table created with the
          131  +**   "detail=none" or "detail=column" option. 
   115    132   **
   116    133   ** xRowid:
   117    134   **   Returns the rowid of the current row.
   118    135   **
   119    136   ** xTokenize:
   120    137   **   Tokenize text using the tokenizer belonging to the FTS5 table.
   121    138   **
................................................................................
   192    209   **   xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
   193    210   **   to use, this API may be faster under some circumstances. To iterate 
   194    211   **   through instances of phrase iPhrase, use the following code:
   195    212   **
   196    213   **       Fts5PhraseIter iter;
   197    214   **       int iCol, iOff;
   198    215   **       for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
   199         -**           iOff>=0;
          216  +**           iCol>=0;
   200    217   **           pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
   201    218   **       ){
   202    219   **         // An instance of phrase iPhrase at offset iOff of column iCol
   203    220   **       }
   204    221   **
   205    222   **   The Fts5PhraseIter structure is defined above. Applications should not
   206    223   **   modify this structure directly - it should only be used as shown above
   207         -**   with the xPhraseFirst() and xPhraseNext() API methods.
          224  +**   with the xPhraseFirst() and xPhraseNext() API methods (and by
          225  +**   xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
          226  +**
          227  +**   This API can be quite slow if used with an FTS5 table created with the
          228  +**   "detail=none" or "detail=column" option. If the FTS5 table is created 
          229  +**   with either "detail=none" or "detail=column" and "content=" option 
          230  +**   (i.e. if it is a contentless table), then this API always iterates
          231  +**   through an empty set (all calls to xPhraseFirst() set iCol to -1).
   208    232   **
   209    233   ** xPhraseNext()
   210    234   **   See xPhraseFirst above.
          235  +**
          236  +** xPhraseFirstColumn()
          237  +**   This function and xPhraseNextColumn() are similar to the xPhraseFirst()
          238  +**   and xPhraseNext() APIs described above. The difference is that instead
          239  +**   of iterating through all instances of a phrase in the current row, these
          240  +**   APIs are used to iterate through the set of columns in the current row
          241  +**   that contain one or more instances of a specified phrase. For example:
          242  +**
          243  +**       Fts5PhraseIter iter;
          244  +**       int iCol;
          245  +**       for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
          246  +**           iCol>=0;
          247  +**           pApi->xPhraseNextColumn(pFts, &iter, &iCol)
          248  +**       ){
          249  +**         // Column iCol contains at least one instance of phrase iPhrase
          250  +**       }
          251  +**
          252  +**   This API can be quite slow if used with an FTS5 table created with the
          253  +**   "detail=none" option. If the FTS5 table is created with either 
          254  +**   "detail=none" "content=" option (i.e. if it is a contentless table), 
          255  +**   then this API always iterates through an empty set (all calls to 
          256  +**   xPhraseFirstColumn() set iCol to -1).
          257  +**
          258  +**   The information accessed using this API and its companion
          259  +**   xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
          260  +**   (or xInst/xInstCount). The chief advantage of this API is that it is
          261  +**   significantly more efficient than those alternatives when used with
          262  +**   "detail=column" tables.  
          263  +**
          264  +** xPhraseNextColumn()
          265  +**   See xPhraseFirstColumn above.
   211    266   */
   212    267   struct Fts5ExtensionApi {
   213         -  int iVersion;                   /* Currently always set to 1 */
          268  +  int iVersion;                   /* Currently always set to 3 */
   214    269   
   215    270     void *(*xUserData)(Fts5Context*);
   216    271   
   217    272     int (*xColumnCount)(Fts5Context*);
   218    273     int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
   219    274     int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
   220    275   
................................................................................
   236    291   
   237    292     int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
   238    293       int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
   239    294     );
   240    295     int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
   241    296     void *(*xGetAuxdata)(Fts5Context*, int bClear);
   242    297   
   243         -  void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
          298  +  int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
   244    299     void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
          300  +
          301  +  int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
          302  +  void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
   245    303   };
   246    304   
   247    305   /* 
   248    306   ** CUSTOM AUXILIARY FUNCTIONS
   249    307   *************************************************************************/
   250    308   
   251    309   /*************************************************************************

Changes to ext/fts5/fts5Int.h.

   147    147     u8 *abUnindexed;                /* True for unindexed columns */
   148    148     int nPrefix;                    /* Number of prefix indexes */
   149    149     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   150    150     int eContent;                   /* An FTS5_CONTENT value */
   151    151     char *zContent;                 /* content table */ 
   152    152     char *zContentRowid;            /* "content_rowid=" option value */ 
   153    153     int bColumnsize;                /* "columnsize=" option value (dflt==1) */
          154  +  int eDetail;                    /* FTS5_DETAIL_XXX value */
   154    155     char *zContentExprlist;
   155    156     Fts5Tokenizer *pTok;
   156    157     fts5_tokenizer *pTokApi;
   157    158   
   158    159     /* Values loaded from the %_config table */
   159    160     int iCookie;                    /* Incremented when %_config is modified */
   160    161     int pgsz;                       /* Approximate page size used in %_data */
................................................................................
   175    176   /* Current expected value of %_config table 'version' field */
   176    177   #define FTS5_CURRENT_VERSION 4
   177    178   
   178    179   #define FTS5_CONTENT_NORMAL   0
   179    180   #define FTS5_CONTENT_NONE     1
   180    181   #define FTS5_CONTENT_EXTERNAL 2
   181    182   
          183  +#define FTS5_DETAIL_FULL    0
          184  +#define FTS5_DETAIL_NONE    1
          185  +#define FTS5_DETAIL_COLUMNS 2
   182    186   
   183    187   
   184    188   
   185    189   int sqlite3Fts5ConfigParse(
   186    190       Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
   187    191   );
   188    192   void sqlite3Fts5ConfigFree(Fts5Config*);
................................................................................
   288    292   /* Malloc utility */
   289    293   void *sqlite3Fts5MallocZero(int *pRc, int nByte);
   290    294   char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
   291    295   
   292    296   /* Character set tests (like isspace(), isalpha() etc.) */
   293    297   int sqlite3Fts5IsBareword(char t);
   294    298   
          299  +
          300  +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
          301  +typedef struct Fts5Termset Fts5Termset;
          302  +int sqlite3Fts5TermsetNew(Fts5Termset**);
          303  +int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
          304  +void sqlite3Fts5TermsetFree(Fts5Termset*);
          305  +
   295    306   /*
   296    307   ** End of interface to code in fts5_buffer.c.
   297    308   **************************************************************************/
   298    309   
   299    310   /**************************************************************************
   300    311   ** Interface to code in fts5_index.c. fts5_index.c contains contains code
   301    312   ** to access the data stored in the %_data table.
................................................................................
   323    334   **   sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter);
   324    335   **   0==sqlite3Fts5IterEof(pIter);
   325    336   **   sqlite3Fts5IterNext(pIter)
   326    337   ** ){
   327    338   **   i64 iRowid = sqlite3Fts5IterRowid(pIter);
   328    339   ** }
   329    340   */
          341  +
          342  +/*
          343  +** Return a simple checksum value based on the arguments.
          344  +*/
          345  +u64 sqlite3Fts5IndexEntryCksum(
          346  +  i64 iRowid, 
          347  +  int iCol, 
          348  +  int iPos, 
          349  +  int iIdx,
          350  +  const char *pTerm,
          351  +  int nTerm
          352  +);
          353  +
          354  +/*
          355  +** Argument p points to a buffer containing utf-8 text that is n bytes in 
          356  +** size. Return the number of bytes in the nChar character prefix of the
          357  +** buffer, or 0 if there are less than nChar characters in total.
          358  +*/
          359  +int sqlite3Fts5IndexCharlenToBytelen(
          360  +  const char *p, 
          361  +  int nByte, 
          362  +  int nChar
          363  +);
   330    364   
   331    365   /*
   332    366   ** Open a new iterator to iterate though all rowids that match the 
   333    367   ** specified token or token prefix.
   334    368   */
   335    369   int sqlite3Fts5IndexQuery(
   336    370     Fts5Index *p,                   /* FTS index to query */
................................................................................
   409    443   */
   410    444   int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
   411    445   int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
   412    446   
   413    447   /*
   414    448   ** Functions called by the storage module as part of integrity-check.
   415    449   */
   416         -u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int);
   417    450   int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
   418    451   
   419    452   /* 
   420    453   ** Called during virtual module initialization to register UDF 
   421    454   ** fts5_decode() with SQLite 
   422    455   */
   423    456   int sqlite3Fts5IndexInit(sqlite3*);
................................................................................
   431    464   int sqlite3Fts5IndexReads(Fts5Index *p);
   432    465   
   433    466   int sqlite3Fts5IndexReinit(Fts5Index *p);
   434    467   int sqlite3Fts5IndexOptimize(Fts5Index *p);
   435    468   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
   436    469   
   437    470   int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
          471  +
          472  +int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);
   438    473   
   439    474   /*
   440    475   ** End of interface to code in fts5_index.c.
   441    476   **************************************************************************/
   442    477   
   443    478   /**************************************************************************
   444    479   ** Interface to code in fts5_varint.c. 
................................................................................
   488    523   ** Interface to code in fts5_hash.c. 
   489    524   */
   490    525   typedef struct Fts5Hash Fts5Hash;
   491    526   
   492    527   /*
   493    528   ** Create a hash table, free a hash table.
   494    529   */
   495         -int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize);
          530  +int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
   496    531   void sqlite3Fts5HashFree(Fts5Hash*);
   497    532   
   498    533   int sqlite3Fts5HashWrite(
   499    534     Fts5Hash*,
   500    535     i64 iRowid,                     /* Rowid for this entry */
   501    536     int iCol,                       /* Column token appears in (-ve -> delete) */
   502    537     int iPos,                       /* Position of token within column */
................................................................................
   624    659   
   625    660   /* Called during startup to register a UDF with SQLite */
   626    661   int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
   627    662   
   628    663   int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
   629    664   int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
   630    665   int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
          666  +
          667  +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
          668  +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
          669  +int sqlite3Fts5ExprPopulatePoslists(
          670  +    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
          671  +);
          672  +void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
          673  +void sqlite3Fts5ExprClearEof(Fts5Expr*);
   631    674   
   632    675   int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
          676  +
          677  +int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
   633    678   
   634    679   /*******************************************
   635    680   ** The fts5_expr.c API above this point is used by the other hand-written
   636    681   ** C code in this module. The interfaces below this point are called by
   637    682   ** the parser code in fts5parse.y.  */
   638    683   
   639    684   void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);

Changes to ext/fts5/fts5_buffer.c.

   286    286       1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 1,   /* 0x50 .. 0x5F */
   287    287       0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60 .. 0x6F */
   288    288       1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
   289    289     };
   290    290   
   291    291     return (t & 0x80) || aBareword[(int)t];
   292    292   }
          293  +
          294  +
          295  +/*************************************************************************
          296  +*/
          297  +typedef struct Fts5TermsetEntry Fts5TermsetEntry;
          298  +struct Fts5TermsetEntry {
          299  +  char *pTerm;
          300  +  int nTerm;
          301  +  int iIdx;                       /* Index (main or aPrefix[] entry) */
          302  +  Fts5TermsetEntry *pNext;
          303  +};
          304  +
          305  +struct Fts5Termset {
          306  +  Fts5TermsetEntry *apHash[512];
          307  +};
          308  +
          309  +int sqlite3Fts5TermsetNew(Fts5Termset **pp){
          310  +  int rc = SQLITE_OK;
          311  +  *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
          312  +  return rc;
          313  +}
          314  +
          315  +int sqlite3Fts5TermsetAdd(
          316  +  Fts5Termset *p, 
          317  +  int iIdx,
          318  +  const char *pTerm, int nTerm, 
          319  +  int *pbPresent
          320  +){
          321  +  int rc = SQLITE_OK;
          322  +  *pbPresent = 0;
          323  +  if( p ){
          324  +    int i;
          325  +    int hash;
          326  +    Fts5TermsetEntry *pEntry;
          327  +
          328  +    /* Calculate a hash value for this term */
          329  +    hash = 104 + iIdx;
          330  +    for(i=0; i<nTerm; i++){
          331  +      hash += (hash << 3) + (int)pTerm[i];
          332  +    }
          333  +    hash = hash % ArraySize(p->apHash);
          334  +
          335  +    for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
          336  +      if( pEntry->iIdx==iIdx 
          337  +          && pEntry->nTerm==nTerm 
          338  +          && memcmp(pEntry->pTerm, pTerm, nTerm)==0 
          339  +        ){
          340  +        *pbPresent = 1;
          341  +        break;
          342  +      }
          343  +    }
          344  +
          345  +    if( pEntry==0 ){
          346  +      pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
          347  +      if( pEntry ){
          348  +        pEntry->pTerm = (char*)&pEntry[1];
          349  +        pEntry->nTerm = nTerm;
          350  +        pEntry->iIdx = iIdx;
          351  +        memcpy(pEntry->pTerm, pTerm, nTerm);
          352  +        pEntry->pNext = p->apHash[hash];
          353  +        p->apHash[hash] = pEntry;
          354  +      }
          355  +    }
          356  +  }
          357  +
          358  +  return rc;
          359  +}
          360  +
          361  +void sqlite3Fts5TermsetFree(Fts5Termset *p){
          362  +  if( p ){
          363  +    int i;
          364  +    for(i=0; i<ArraySize(p->apHash); i++){
          365  +      Fts5TermsetEntry *pEntry = p->apHash[i];
          366  +      while( pEntry ){
          367  +        Fts5TermsetEntry *pDel = pEntry;
          368  +        pEntry = pEntry->pNext;
          369  +        sqlite3_free(pDel);
          370  +      }
          371  +    }
          372  +    sqlite3_free(p);
          373  +  }
          374  +}
          375  +
   293    376   
   294    377   

Changes to ext/fts5/fts5_config.c.

    10     10   **
    11     11   ******************************************************************************
    12     12   **
    13     13   ** This is an SQLite module implementing full-text search.
    14     14   */
    15     15   
    16     16   
    17         -
    18     17   #include "fts5Int.h"
    19     18   
    20     19   #define FTS5_DEFAULT_PAGE_SIZE   4050
    21     20   #define FTS5_DEFAULT_AUTOMERGE      4
    22     21   #define FTS5_DEFAULT_CRISISMERGE   16
    23     22   #define FTS5_DEFAULT_HASHSIZE    (1024*1024)
    24     23   
................................................................................
   191    190     assert( 0==fts5_iswhitespace(z[0]) );
   192    191     quote = z[0];
   193    192     if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
   194    193       fts5Dequote(z);
   195    194     }
   196    195   }
   197    196   
          197  +
          198  +struct Fts5Enum {
          199  +  const char *zName;
          200  +  int eVal;
          201  +};
          202  +typedef struct Fts5Enum Fts5Enum;
          203  +
          204  +static int fts5ConfigSetEnum(
          205  +  const Fts5Enum *aEnum, 
          206  +  const char *zEnum, 
          207  +  int *peVal
          208  +){
          209  +  int nEnum = strlen(zEnum);
          210  +  int i;
          211  +  int iVal = -1;
          212  +
          213  +  for(i=0; aEnum[i].zName; i++){
          214  +    if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
          215  +      if( iVal>=0 ) return SQLITE_ERROR;
          216  +      iVal = aEnum[i].eVal;
          217  +    }
          218  +  }
          219  +
          220  +  *peVal = iVal;
          221  +  return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
          222  +}
          223  +
   198    224   /*
   199    225   ** Parse a "special" CREATE VIRTUAL TABLE directive and update
   200    226   ** configuration object pConfig as appropriate.
   201    227   **
   202    228   ** If successful, object pConfig is updated and SQLITE_OK returned. If
   203    229   ** an error occurs, an SQLite error code is returned and an error message
   204    230   ** may be left in *pzErr. It is the responsibility of the caller to
................................................................................
   340    366         *pzErr = sqlite3_mprintf("malformed columnsize=... directive");
   341    367         rc = SQLITE_ERROR;
   342    368       }else{
   343    369         pConfig->bColumnsize = (zArg[0]=='1');
   344    370       }
   345    371       return rc;
   346    372     }
          373  +
          374  +  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
          375  +    const Fts5Enum aDetail[] = {
          376  +      { "none", FTS5_DETAIL_NONE },
          377  +      { "full", FTS5_DETAIL_FULL },
          378  +      { "columns", FTS5_DETAIL_COLUMNS },
          379  +      { 0, 0 }
          380  +    };
          381  +
          382  +    if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
          383  +      *pzErr = sqlite3_mprintf("malformed detail=... directive");
          384  +    }
          385  +    return rc;
          386  +  }
   347    387   
   348    388     *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
   349    389     return SQLITE_ERROR;
   350    390   }
   351    391   
   352    392   /*
   353    393   ** Allocate an instance of the default tokenizer ("simple") at 
................................................................................
   496    536   
   497    537     nByte = nArg * (sizeof(char*) + sizeof(u8));
   498    538     pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
   499    539     pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
   500    540     pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
   501    541     pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
   502    542     pRet->bColumnsize = 1;
          543  +  pRet->eDetail = FTS5_DETAIL_FULL;
   503    544   #ifdef SQLITE_DEBUG
   504    545     pRet->bPrefixIndex = 1;
   505    546   #endif
   506    547     if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
   507    548       *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
   508    549       rc = SQLITE_ERROR;
   509    550     }

Changes to ext/fts5/fts5_expr.c.

    36     36   #include <stdio.h>
    37     37   void sqlite3Fts5ParserTrace(FILE*, char*);
    38     38   #endif
    39     39   
    40     40   
    41     41   struct Fts5Expr {
    42     42     Fts5Index *pIndex;
           43  +  Fts5Config *pConfig;
    43     44     Fts5ExprNode *pRoot;
    44     45     int bDesc;                      /* Iterate in descending rowid order */
    45     46     int nPhrase;                    /* Number of phrases in expression */
    46     47     Fts5ExprPhrase **apExprPhrase;  /* Pointers to phrase objects */
    47     48   };
    48     49   
    49     50   /*
................................................................................
   231    232       *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
   232    233       if( pNew==0 ){
   233    234         sParse.rc = SQLITE_NOMEM;
   234    235         sqlite3Fts5ParseNodeFree(sParse.pExpr);
   235    236       }else{
   236    237         pNew->pRoot = sParse.pExpr;
   237    238         pNew->pIndex = 0;
          239  +      pNew->pConfig = pConfig;
   238    240         pNew->apExprPhrase = sParse.apPhrase;
   239    241         pNew->nPhrase = sParse.nPhrase;
   240    242         sParse.apPhrase = 0;
   241    243       }
   242    244     }
   243    245   
   244    246     sqlite3_free(sParse.apPhrase);
................................................................................
   295    297     if( pbEof && bRetValid==0 ) *pbEof = 1;
   296    298     return iRet;
   297    299   }
   298    300   
   299    301   /*
   300    302   ** Argument pTerm must be a synonym iterator.
   301    303   */
   302         -static int fts5ExprSynonymPoslist(
          304  +static int fts5ExprSynonymList(
   303    305     Fts5ExprTerm *pTerm, 
          306  +  int bCollist, 
   304    307     Fts5Colset *pColset,
   305    308     i64 iRowid,
   306    309     int *pbDel,                     /* OUT: Caller should sqlite3_free(*pa) */
   307    310     u8 **pa, int *pn
   308    311   ){
   309    312     Fts5PoslistReader aStatic[4];
   310    313     Fts5PoslistReader *aIter = aStatic;
................................................................................
   315    318   
   316    319     assert( pTerm->pSynonym );
   317    320     for(p=pTerm; p; p=p->pSynonym){
   318    321       Fts5IndexIter *pIter = p->pIter;
   319    322       if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
   320    323         const u8 *a;
   321    324         int n;
   322         -      i64 dummy;
   323         -      rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
          325  +
          326  +      if( bCollist ){
          327  +        rc = sqlite3Fts5IterCollist(pIter, &a, &n);
          328  +      }else{
          329  +        i64 dummy;
          330  +        rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
          331  +      }
          332  +
   324    333         if( rc!=SQLITE_OK ) goto synonym_poslist_out;
          334  +      if( n==0 ) continue;
   325    335         if( nIter==nAlloc ){
   326    336           int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
   327    337           Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   328    338           if( aNew==0 ){
   329    339             rc = SQLITE_NOMEM;
   330    340             goto synonym_poslist_out;
   331    341           }
................................................................................
   418    428     for(i=0; i<pPhrase->nTerm; i++){
   419    429       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
   420    430       i64 dummy;
   421    431       int n = 0;
   422    432       int bFlag = 0;
   423    433       const u8 *a = 0;
   424    434       if( pTerm->pSynonym ){
   425         -      rc = fts5ExprSynonymPoslist(
   426         -          pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
          435  +      rc = fts5ExprSynonymList(
          436  +          pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
   427    437         );
   428    438       }else{
   429    439         rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
   430    440       }
   431    441       if( rc!=SQLITE_OK ) goto ismatch_out;
   432    442       sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
   433    443       aIter[i].bFlag = (u8)bFlag;
................................................................................
   753    763   static int fts5ExprNearTest(
   754    764     int *pRc,
   755    765     Fts5Expr *pExpr,                /* Expression that pNear is a part of */
   756    766     Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_STRING) */
   757    767   ){
   758    768     Fts5ExprNearset *pNear = pNode->pNear;
   759    769     int rc = *pRc;
   760         -  int i;
   761         -
   762         -  /* Check that each phrase in the nearset matches the current row.
   763         -  ** Populate the pPhrase->poslist buffers at the same time. If any
   764         -  ** phrase is not a match, break out of the loop early.  */
   765         -  for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   766         -    Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   767         -    if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
   768         -      int bMatch = 0;
   769         -      rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
   770         -      if( bMatch==0 ) break;
   771         -    }else{
   772         -      rc = sqlite3Fts5IterPoslistBuffer(
   773         -          pPhrase->aTerm[0].pIter, &pPhrase->poslist
   774         -      );
   775         -    }
   776         -  }
   777         -
   778         -  *pRc = rc;
   779         -  if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
   780         -    return 1;
   781         -  }
   782         -
   783         -  return 0;
          770  +
          771  +  if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){
          772  +    Fts5ExprTerm *pTerm;
          773  +    Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
          774  +    pPhrase->poslist.n = 0;
          775  +    for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
          776  +      Fts5IndexIter *pIter = pTerm->pIter;
          777  +      if( sqlite3Fts5IterEof(pIter)==0 ){
          778  +        int n;
          779  +        i64 iRowid;
          780  +        rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
          781  +        if( rc!=SQLITE_OK ){
          782  +          *pRc = rc;
          783  +          return 0;
          784  +        }else if( iRowid==pNode->iRowid && n>0 ){
          785  +          pPhrase->poslist.n = 1;
          786  +        }
          787  +      }
          788  +    }
          789  +    return pPhrase->poslist.n;
          790  +  }else{
          791  +    int i;
          792  +
          793  +    /* Check that each phrase in the nearset matches the current row.
          794  +    ** Populate the pPhrase->poslist buffers at the same time. If any
          795  +    ** phrase is not a match, break out of the loop early.  */
          796  +    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
          797  +      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
          798  +      if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
          799  +        int bMatch = 0;
          800  +        rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
          801  +        if( bMatch==0 ) break;
          802  +      }else{
          803  +        rc = sqlite3Fts5IterPoslistBuffer(
          804  +            pPhrase->aTerm[0].pIter, &pPhrase->poslist
          805  +        );
          806  +      }
          807  +    }
          808  +
          809  +    *pRc = rc;
          810  +    if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
          811  +      return 1;
          812  +    }
          813  +    return 0;
          814  +  }
   784    815   }
   785    816   
   786    817   static int fts5ExprTokenTest(
   787    818     Fts5Expr *pExpr,                /* Expression that pNear is a part of */
   788    819     Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_TERM) */
   789    820   ){
   790    821     /* As this "NEAR" object is actually a single phrase that consists 
................................................................................
  1214   1245             }
  1215   1246             assert( rc!=SQLITE_OK || cmp<=0 );
  1216   1247             if( cmp || p2->bNomatch ) break;
  1217   1248             rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
  1218   1249           }
  1219   1250           pNode->bEof = p1->bEof;
  1220   1251           pNode->iRowid = p1->iRowid;
         1252  +        if( p1->bEof ){
         1253  +          fts5ExprNodeZeroPoslist(p2);
         1254  +        }
  1221   1255           break;
  1222   1256         }
  1223   1257       }
  1224   1258     }
  1225   1259     return rc;
  1226   1260   }
  1227   1261   
................................................................................
  1599   1633         sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  1600   1634       }
  1601   1635     }
  1602   1636   
  1603   1637     if( rc==SQLITE_OK ){
  1604   1638       /* All the allocations succeeded. Put the expression object together. */
  1605   1639       pNew->pIndex = pExpr->pIndex;
         1640  +    pNew->pConfig = pExpr->pConfig;
  1606   1641       pNew->nPhrase = 1;
  1607   1642       pNew->apExprPhrase[0] = sCtx.pPhrase;
  1608   1643       pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
  1609   1644       pNew->pRoot->pNear->nPhrase = 1;
  1610   1645       sCtx.pPhrase->pNode = pNew->pRoot;
  1611   1646   
  1612   1647       if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
................................................................................
  1740   1775   }
  1741   1776   
  1742   1777   void sqlite3Fts5ParseSetColset(
  1743   1778     Fts5Parse *pParse, 
  1744   1779     Fts5ExprNearset *pNear, 
  1745   1780     Fts5Colset *pColset 
  1746   1781   ){
         1782  +  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1783  +    pParse->rc = SQLITE_ERROR;
         1784  +    pParse->zErr = sqlite3_mprintf(
         1785  +      "fts5: column queries are not supported (detail=none)"
         1786  +    );
         1787  +    sqlite3_free(pColset);
         1788  +    return;
         1789  +  }
         1790  +
  1747   1791     if( pNear ){
  1748   1792       pNear->pColset = pColset;
  1749   1793     }else{
  1750   1794       sqlite3_free(pColset);
  1751   1795     }
  1752   1796   }
  1753   1797   
................................................................................
  1801   1845         pRet->eType = eType;
  1802   1846         pRet->pNear = pNear;
  1803   1847         if( eType==FTS5_STRING ){
  1804   1848           int iPhrase;
  1805   1849           for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
  1806   1850             pNear->apPhrase[iPhrase]->pNode = pRet;
  1807   1851           }
  1808         -        if( pNear->nPhrase==1 
  1809         -         && pNear->apPhrase[0]->nTerm==1 
  1810         -         && pNear->apPhrase[0]->aTerm[0].pSynonym==0
  1811         -        ){
  1812         -          pRet->eType = FTS5_TERM;
         1852  +        if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
         1853  +          if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){
         1854  +            pRet->eType = FTS5_TERM;
         1855  +          }
         1856  +        }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
         1857  +          assert( pParse->rc==SQLITE_OK );
         1858  +          pParse->rc = SQLITE_ERROR;
         1859  +          assert( pParse->zErr==0 );
         1860  +          pParse->zErr = sqlite3_mprintf(
         1861  +              "fts5: %s queries are not supported (detail!=full)", 
         1862  +              pNear->nPhrase==1 ? "phrase": "NEAR"
         1863  +          );
         1864  +          sqlite3_free(pRet);
         1865  +          pRet = 0;
  1813   1866           }
  1814   1867         }else{
  1815   1868           fts5ExprAddChildren(pRet, pLeft);
  1816   1869           fts5ExprAddChildren(pRet, pRight);
  1817   1870         }
  1818   1871       }
  1819   1872     }
................................................................................
  1919   1972       for(i=0; i<pNear->nPhrase; i++){
  1920   1973         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
  1921   1974   
  1922   1975         zRet = fts5PrintfAppend(zRet, " {");
  1923   1976         for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
  1924   1977           char *zTerm = pPhrase->aTerm[iTerm].zTerm;
  1925   1978           zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
         1979  +        if( pPhrase->aTerm[iTerm].bPrefix ){
         1980  +          zRet = fts5PrintfAppend(zRet, "*");
         1981  +        }
  1926   1982         }
  1927   1983   
  1928   1984         if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
  1929   1985         if( zRet==0 ) return 0;
  1930   1986       }
  1931   1987   
  1932   1988     }else{
................................................................................
  2231   2287       nRet = pPhrase->poslist.n;
  2232   2288     }else{
  2233   2289       *pa = 0;
  2234   2290       nRet = 0;
  2235   2291     }
  2236   2292     return nRet;
  2237   2293   }
         2294  +
         2295  +struct Fts5PoslistPopulator {
         2296  +  Fts5PoslistWriter writer;
         2297  +  int bOk;                        /* True if ok to populate */
         2298  +  int bMiss;
         2299  +};
         2300  +
         2301  +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
         2302  +  Fts5PoslistPopulator *pRet;
         2303  +  pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
         2304  +  if( pRet ){
         2305  +    int i;
         2306  +    memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
         2307  +    for(i=0; i<pExpr->nPhrase; i++){
         2308  +      Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
         2309  +      Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
         2310  +      assert( pExpr->apExprPhrase[i]->nTerm==1 );
         2311  +      if( bLive && 
         2312  +          (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
         2313  +      ){
         2314  +        pRet[i].bMiss = 1;
         2315  +      }else{
         2316  +        pBuf->n = 0;
         2317  +      }
         2318  +    }
         2319  +  }
         2320  +  return pRet;
         2321  +}
         2322  +
         2323  +struct Fts5ExprCtx {
         2324  +  Fts5Expr *pExpr;
         2325  +  Fts5PoslistPopulator *aPopulator;
         2326  +  i64 iOff;
         2327  +};
         2328  +typedef struct Fts5ExprCtx Fts5ExprCtx;
         2329  +
         2330  +/*
         2331  +** TODO: Make this more efficient!
         2332  +*/
         2333  +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
         2334  +  int i;
         2335  +  for(i=0; i<pColset->nCol; i++){
         2336  +    if( pColset->aiCol[i]==iCol ) return 1;
         2337  +  }
         2338  +  return 0;
         2339  +}
         2340  +
         2341  +static int fts5ExprPopulatePoslistsCb(
         2342  +  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
         2343  +  int tflags,                /* Mask of FTS5_TOKEN_* flags */
         2344  +  const char *pToken,        /* Pointer to buffer containing token */
         2345  +  int nToken,                /* Size of token in bytes */
         2346  +  int iStart,                /* Byte offset of token within input text */
         2347  +  int iEnd                   /* Byte offset of end of token within input text */
         2348  +){
         2349  +  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
         2350  +  Fts5Expr *pExpr = p->pExpr;
         2351  +  int i;
         2352  +
         2353  +  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
         2354  +  for(i=0; i<pExpr->nPhrase; i++){
         2355  +    Fts5ExprTerm *pTerm;
         2356  +    if( p->aPopulator[i].bOk==0 ) continue;
         2357  +    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
         2358  +      int nTerm = strlen(pTerm->zTerm);
         2359  +      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
         2360  +       && memcmp(pTerm->zTerm, pToken, nTerm)==0
         2361  +      ){
         2362  +        int rc = sqlite3Fts5PoslistWriterAppend(
         2363  +            &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
         2364  +        );
         2365  +        if( rc ) return rc;
         2366  +        break;
         2367  +      }
         2368  +    }
         2369  +  }
         2370  +  return SQLITE_OK;
         2371  +}
         2372  +
         2373  +int sqlite3Fts5ExprPopulatePoslists(
         2374  +  Fts5Config *pConfig,
         2375  +  Fts5Expr *pExpr, 
         2376  +  Fts5PoslistPopulator *aPopulator,
         2377  +  int iCol, 
         2378  +  const char *z, int n
         2379  +){
         2380  +  int i;
         2381  +  Fts5ExprCtx sCtx;
         2382  +  sCtx.pExpr = pExpr;
         2383  +  sCtx.aPopulator = aPopulator;
         2384  +  sCtx.iOff = (((i64)iCol) << 32) - 1;
         2385  +
         2386  +  for(i=0; i<pExpr->nPhrase; i++){
         2387  +    Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
         2388  +    Fts5Colset *pColset = pNode->pNear->pColset;
         2389  +    if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) 
         2390  +     || aPopulator[i].bMiss
         2391  +    ){
         2392  +      aPopulator[i].bOk = 0;
         2393  +    }else{
         2394  +      aPopulator[i].bOk = 1;
         2395  +    }
         2396  +  }
         2397  +
         2398  +  return sqlite3Fts5Tokenize(pConfig, 
         2399  +      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
         2400  +  );
         2401  +}
         2402  +
         2403  +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
         2404  +  if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){
         2405  +    pNode->pNear->apPhrase[0]->poslist.n = 0;
         2406  +  }else{
         2407  +    int i;
         2408  +    for(i=0; i<pNode->nChild; i++){
         2409  +      fts5ExprClearPoslists(pNode->apChild[i]);
         2410  +    }
         2411  +  }
         2412  +}
         2413  +
         2414  +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
         2415  +  if( pNode ){
         2416  +    pNode->iRowid = iRowid;
         2417  +    pNode->bEof = 0;
         2418  +    switch( pNode->eType ){
         2419  +      case FTS5_TERM:
         2420  +      case FTS5_STRING:
         2421  +        return (pNode->pNear->apPhrase[0]->poslist.n>0);
         2422  +
         2423  +      case FTS5_AND: {
         2424  +        int i;
         2425  +        for(i=0; i<pNode->nChild; i++){
         2426  +          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
         2427  +            fts5ExprClearPoslists(pNode);
         2428  +            return 0;
         2429  +          }
         2430  +        }
         2431  +        break;
         2432  +      }
         2433  +
         2434  +      case FTS5_OR: {
         2435  +        int i;
         2436  +        int bRet = 0;
         2437  +        for(i=0; i<pNode->nChild; i++){
         2438  +          if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
         2439  +            bRet = 1;
         2440  +          }
         2441  +        }
         2442  +        if( bRet==0 ){
         2443  +          fts5ExprClearPoslists(pNode);
         2444  +        }
         2445  +        return bRet;
         2446  +      }
         2447  +
         2448  +      default: {
         2449  +        assert( pNode->eType==FTS5_NOT );
         2450  +        if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
         2451  +         || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
         2452  +        ){
         2453  +          fts5ExprClearPoslists(pNode);
         2454  +          return 0;
         2455  +        }
         2456  +        break;
         2457  +      }
         2458  +    }
         2459  +  }
         2460  +  return 1;
         2461  +}
         2462  +
         2463  +void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
         2464  +  fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
         2465  +}
         2466  +
         2467  +static void fts5ExprClearEof(Fts5ExprNode *pNode){
         2468  +  int i;
         2469  +  for(i=0; i<pNode->nChild; i++){
         2470  +    fts5ExprClearEof(pNode->apChild[i]);
         2471  +  }
         2472  +  pNode->bEof = 0;
         2473  +}
         2474  +void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
         2475  +  fts5ExprClearEof(pExpr->pRoot);
         2476  +}
         2477  +
         2478  +/*
         2479  +** This function is only called for detail=columns tables. 
         2480  +*/
         2481  +int sqlite3Fts5ExprPhraseCollist(
         2482  +  Fts5Expr *pExpr, 
         2483  +  int iPhrase, 
         2484  +  const u8 **ppCollist, 
         2485  +  int *pnCollist
         2486  +){
         2487  +  Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
         2488  +  Fts5ExprNode *pNode = pPhrase->pNode;
         2489  +  int rc = SQLITE_OK;
         2490  +
         2491  +  assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
         2492  +  if( pNode->bEof==0 
         2493  +   && pNode->iRowid==pExpr->pRoot->iRowid 
         2494  +   && pPhrase->poslist.n>0
         2495  +  ){
         2496  +    Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
         2497  +    if( pTerm->pSynonym ){
         2498  +      int bDel = 0;
         2499  +      u8 *a;
         2500  +      rc = fts5ExprSynonymList(
         2501  +          pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist
         2502  +      );
         2503  +      if( bDel ){
         2504  +        sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a);
         2505  +        *ppCollist = pPhrase->poslist.p;
         2506  +        sqlite3_free(a);
         2507  +      }else{
         2508  +        *ppCollist = a;
         2509  +      }
         2510  +    }else{
         2511  +      sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
         2512  +    }
         2513  +  }else{
         2514  +    *ppCollist = 0;
         2515  +    *pnCollist = 0;
         2516  +  }
         2517  +
         2518  +  return rc;
         2519  +}
         2520  +

Changes to ext/fts5/fts5_hash.c.

    22     22   ** This file contains the implementation of an in-memory hash table used
    23     23   ** to accumuluate "term -> doclist" content before it is flused to a level-0
    24     24   ** segment.
    25     25   */
    26     26   
    27     27   
    28     28   struct Fts5Hash {
           29  +  int eDetail;                    /* Copy of Fts5Config.eDetail */
    29     30     int *pnByte;                    /* Pointer to bytes counter */
    30     31     int nEntry;                     /* Number of entries currently in hash */
    31     32     int nSlot;                      /* Size of aSlot[] array */
    32     33     Fts5HashEntry *pScan;           /* Current ordered scan item */
    33     34     Fts5HashEntry **aSlot;          /* Array of hash slots */
    34     35   };
    35     36   
................................................................................
    58     59     Fts5HashEntry *pHashNext;       /* Next hash entry with same hash-key */
    59     60     Fts5HashEntry *pScanNext;       /* Next entry in sorted order */
    60     61     
    61     62     int nAlloc;                     /* Total size of allocation */
    62     63     int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
    63     64     int nData;                      /* Total bytes of data (incl. structure) */
    64     65     u8 bDel;                        /* Set delete-flag @ iSzPoslist */
           66  +  u8 bContent;                    /* Set content-flag (detail=none mode) */
    65     67   
    66     68     int iCol;                       /* Column of last value written */
    67     69     int iPos;                       /* Position of last value written */
    68     70     i64 iRowid;                     /* Rowid of last value written */
    69     71     char zKey[8];                   /* Nul-terminated entry key */
    70     72   };
    71     73   
................................................................................
    75     77   #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
    76     78   
    77     79   
    78     80   
    79     81   /*
    80     82   ** Allocate a new hash table.
    81     83   */
    82         -int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
           84  +int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
    83     85     int rc = SQLITE_OK;
    84     86     Fts5Hash *pNew;
    85     87   
    86     88     *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash));
    87     89     if( pNew==0 ){
    88     90       rc = SQLITE_NOMEM;
    89     91     }else{
    90     92       int nByte;
    91     93       memset(pNew, 0, sizeof(Fts5Hash));
    92     94       pNew->pnByte = pnByte;
           95  +    pNew->eDetail = pConfig->eDetail;
    93     96   
    94     97       pNew->nSlot = 1024;
    95     98       nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
    96     99       pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
    97    100       if( pNew->aSlot==0 ){
    98    101         sqlite3_free(pNew);
    99    102         *ppNew = 0;
................................................................................
   178    181   
   179    182     sqlite3_free(apOld);
   180    183     pHash->nSlot = nNew;
   181    184     pHash->aSlot = apNew;
   182    185     return SQLITE_OK;
   183    186   }
   184    187   
   185         -static void fts5HashAddPoslistSize(Fts5HashEntry *p){
          188  +static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
   186    189     if( p->iSzPoslist ){
   187    190       u8 *pPtr = (u8*)p;
   188         -    int nSz = (p->nData - p->iSzPoslist - 1);         /* Size in bytes */
   189         -    int nPos = nSz*2 + p->bDel;                       /* Value of nPos field */
   190         -
   191         -    assert( p->bDel==0 || p->bDel==1 );
   192         -    if( nPos<=127 ){
   193         -      pPtr[p->iSzPoslist] = (u8)nPos;
          191  +    if( pHash->eDetail==FTS5_DETAIL_NONE ){
          192  +      assert( p->nData==p->iSzPoslist );
          193  +      if( p->bDel ){
          194  +        pPtr[p->nData++] = 0x00;
          195  +        if( p->bContent ){
          196  +          pPtr[p->nData++] = 0x00;
          197  +        }
          198  +      }
   194    199       }else{
   195         -      int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
   196         -      memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
   197         -      sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
   198         -      p->nData += (nByte-1);
          200  +      int nSz = (p->nData - p->iSzPoslist - 1);       /* Size in bytes */
          201  +      int nPos = nSz*2 + p->bDel;                     /* Value of nPos field */
          202  +
          203  +      assert( p->bDel==0 || p->bDel==1 );
          204  +      if( nPos<=127 ){
          205  +        pPtr[p->iSzPoslist] = (u8)nPos;
          206  +      }else{
          207  +        int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
          208  +        memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
          209  +        sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
          210  +        p->nData += (nByte-1);
          211  +      }
   199    212       }
   200         -    p->bDel = 0;
          213  +
   201    214       p->iSzPoslist = 0;
          215  +    p->bDel = 0;
          216  +    p->bContent = 0;
   202    217     }
   203    218   }
   204    219   
          220  +/*
          221  +** Add an entry to the in-memory hash table. The key is the concatenation
          222  +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
          223  +**
          224  +**     (bByte || pToken) -> (iRowid,iCol,iPos)
          225  +**
          226  +** Or, if iCol is negative, then the value is a delete marker.
          227  +*/
   205    228   int sqlite3Fts5HashWrite(
   206    229     Fts5Hash *pHash,
   207    230     i64 iRowid,                     /* Rowid for this entry */
   208    231     int iCol,                       /* Column token appears in (-ve -> delete) */
   209    232     int iPos,                       /* Position of token within column */
   210    233     char bByte,                     /* First byte of token */
   211    234     const char *pToken, int nToken  /* Token to add or remove to or from index */
   212    235   ){
   213    236     unsigned int iHash;
   214    237     Fts5HashEntry *p;
   215    238     u8 *pPtr;
   216    239     int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
          240  +  int bNew;                       /* If non-delete entry should be written */
          241  +  
          242  +  bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
   217    243   
   218    244     /* Attempt to locate an existing hash entry */
   219    245     iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   220    246     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   221    247       if( p->zKey[0]==bByte 
   222    248        && memcmp(&p->zKey[1], pToken, nToken)==0 
   223    249        && p->zKey[nToken+1]==0 
................................................................................
   224    250       ){
   225    251         break;
   226    252       }
   227    253     }
   228    254   
   229    255     /* If an existing hash entry cannot be found, create a new one. */
   230    256     if( p==0 ){
          257  +    /* Figure out how much space to allocate */
   231    258       int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
   232    259       if( nByte<128 ) nByte = 128;
   233    260   
          261  +    /* Grow the Fts5Hash.aSlot[] array if necessary. */
   234    262       if( (pHash->nEntry*2)>=pHash->nSlot ){
   235    263         int rc = fts5HashResize(pHash);
   236    264         if( rc!=SQLITE_OK ) return rc;
   237    265         iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   238    266       }
   239    267   
          268  +    /* Allocate new Fts5HashEntry and add it to the hash table. */
   240    269       p = (Fts5HashEntry*)sqlite3_malloc(nByte);
   241    270       if( !p ) return SQLITE_NOMEM;
   242    271       memset(p, 0, FTS5_HASHENTRYSIZE);
   243    272       p->nAlloc = nByte;
   244    273       p->zKey[0] = bByte;
   245    274       memcpy(&p->zKey[1], pToken, nToken);
   246    275       assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
   247    276       p->zKey[nToken+1] = '\0';
   248    277       p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
   249         -    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
   250         -    p->iSzPoslist = p->nData;
   251         -    p->nData += 1;
   252         -    p->iRowid = iRowid;
   253    278       p->pHashNext = pHash->aSlot[iHash];
   254    279       pHash->aSlot[iHash] = p;
   255    280       pHash->nEntry++;
          281  +
          282  +    /* Add the first rowid field to the hash-entry */
          283  +    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
          284  +    p->iRowid = iRowid;
          285  +
          286  +    p->iSzPoslist = p->nData;
          287  +    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
          288  +      p->nData += 1;
          289  +      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
          290  +    }
          291  +
   256    292       nIncr += p->nData;
   257         -  }
   258         -
   259         -  /* Check there is enough space to append a new entry. Worst case scenario
   260         -  ** is:
   261         -  **
   262         -  **     + 9 bytes for a new rowid,
   263         -  **     + 4 byte reserved for the "poslist size" varint.
   264         -  **     + 1 byte for a "new column" byte,
   265         -  **     + 3 bytes for a new column number (16-bit max) as a varint,
   266         -  **     + 5 bytes for the new position offset (32-bit max).
   267         -  */
   268         -  if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
   269         -    int nNew = p->nAlloc * 2;
   270         -    Fts5HashEntry *pNew;
   271         -    Fts5HashEntry **pp;
   272         -    pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
   273         -    if( pNew==0 ) return SQLITE_NOMEM;
   274         -    pNew->nAlloc = nNew;
   275         -    for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
   276         -    *pp = pNew;
   277         -    p = pNew;
          293  +  }else{
          294  +
          295  +    /* Appending to an existing hash-entry. Check that there is enough 
          296  +    ** space to append the largest possible new entry. Worst case scenario 
          297  +    ** is:
          298  +    **
          299  +    **     + 9 bytes for a new rowid,
          300  +    **     + 4 byte reserved for the "poslist size" varint.
          301  +    **     + 1 byte for a "new column" byte,
          302  +    **     + 3 bytes for a new column number (16-bit max) as a varint,
          303  +    **     + 5 bytes for the new position offset (32-bit max).
          304  +    */
          305  +    if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
          306  +      int nNew = p->nAlloc * 2;
          307  +      Fts5HashEntry *pNew;
          308  +      Fts5HashEntry **pp;
          309  +      pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
          310  +      if( pNew==0 ) return SQLITE_NOMEM;
          311  +      pNew->nAlloc = nNew;
          312  +      for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
          313  +      *pp = pNew;
          314  +      p = pNew;
          315  +    }
          316  +    nIncr -= p->nData;
   278    317     }
          318  +  assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
          319  +
   279    320     pPtr = (u8*)p;
   280         -  nIncr -= p->nData;
   281    321   
   282    322     /* If this is a new rowid, append the 4-byte size field for the previous
   283    323     ** entry, and the new rowid for this entry.  */
   284    324     if( iRowid!=p->iRowid ){
   285         -    fts5HashAddPoslistSize(p);
          325  +    fts5HashAddPoslistSize(pHash, p);
   286    326       p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
          327  +    p->iRowid = iRowid;
          328  +    bNew = 1;
   287    329       p->iSzPoslist = p->nData;
   288         -    p->nData += 1;
   289         -    p->iCol = 0;
   290         -    p->iPos = 0;
   291         -    p->iRowid = iRowid;
          330  +    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
          331  +      p->nData += 1;
          332  +      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
          333  +      p->iPos = 0;
          334  +    }
   292    335     }
   293    336   
   294    337     if( iCol>=0 ){
   295         -    /* Append a new column value, if necessary */
   296         -    assert( iCol>=p->iCol );
   297         -    if( iCol!=p->iCol ){
   298         -      pPtr[p->nData++] = 0x01;
   299         -      p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
   300         -      p->iCol = iCol;
   301         -      p->iPos = 0;
          338  +    if( pHash->eDetail==FTS5_DETAIL_NONE ){
          339  +      p->bContent = 1;
          340  +    }else{
          341  +      /* Append a new column value, if necessary */
          342  +      assert( iCol>=p->iCol );
          343  +      if( iCol!=p->iCol ){
          344  +        if( pHash->eDetail==FTS5_DETAIL_FULL ){
          345  +          pPtr[p->nData++] = 0x01;
          346  +          p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
          347  +          p->iCol = iCol;
          348  +          p->iPos = 0;
          349  +        }else{
          350  +          bNew = 1;
          351  +          p->iCol = iPos = iCol;
          352  +        }
          353  +      }
          354  +
          355  +      /* Append the new position offset, if necessary */
          356  +      if( bNew ){
          357  +        p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
          358  +        p->iPos = iPos;
          359  +      }
   302    360       }
   303         -
   304         -    /* Append the new position offset */
   305         -    p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
   306         -    p->iPos = iPos;
   307    361     }else{
   308    362       /* This is a delete. Set the delete flag. */
   309    363       p->bDel = 1;
   310    364     }
          365  +
   311    366     nIncr += p->nData;
   312         -
   313    367     *pHash->pnByte += nIncr;
   314    368     return SQLITE_OK;
   315    369   }
   316    370   
   317    371   
   318    372   /*
   319    373   ** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
................................................................................
   419    473     Fts5HashEntry *p;
   420    474   
   421    475     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   422    476       if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
   423    477     }
   424    478   
   425    479     if( p ){
   426         -    fts5HashAddPoslistSize(p);
          480  +    fts5HashAddPoslistSize(pHash, p);
   427    481       *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   428    482       *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
   429    483     }else{
   430    484       *ppDoclist = 0;
   431    485       *pnDoclist = 0;
   432    486     }
   433    487   
................................................................................
   455    509     const char **pzTerm,            /* OUT: term (nul-terminated) */
   456    510     const u8 **ppDoclist,           /* OUT: pointer to doclist */
   457    511     int *pnDoclist                  /* OUT: size of doclist in bytes */
   458    512   ){
   459    513     Fts5HashEntry *p;
   460    514     if( (p = pHash->pScan) ){
   461    515       int nTerm = (int)strlen(p->zKey);
   462         -    fts5HashAddPoslistSize(p);
          516  +    fts5HashAddPoslistSize(pHash, p);
   463    517       *pzTerm = p->zKey;
   464    518       *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   465    519       *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
   466    520     }else{
   467    521       *pzTerm = 0;
   468    522       *ppDoclist = 0;
   469    523       *pnDoclist = 0;
   470    524     }
   471    525   }
   472    526   

Changes to ext/fts5/fts5_index.c.

   429    429     Fts5StructureSegment *pSeg;     /* Segment to iterate through */
   430    430     int flags;                      /* Mask of configuration flags */
   431    431     int iLeafPgno;                  /* Current leaf page number */
   432    432     Fts5Data *pLeaf;                /* Current leaf data */
   433    433     Fts5Data *pNextLeaf;            /* Leaf page (iLeafPgno+1) */
   434    434     int iLeafOffset;                /* Byte offset within current leaf */
   435    435   
          436  +  /* Next method */
          437  +  void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
          438  +
   436    439     /* The page and offset from which the current term was read. The offset 
   437    440     ** is the offset of the first rowid in the current doclist.  */
   438    441     int iTermLeafPgno;
   439    442     int iTermLeafOffset;
   440    443   
   441    444     int iPgidxOff;                  /* Next offset in pgidx */
   442    445     int iEndofDoclist;
................................................................................
   448    451   
   449    452     Fts5DlidxIter *pDlidx;          /* If there is a doclist-index */
   450    453   
   451    454     /* Variables populated based on current entry. */
   452    455     Fts5Buffer term;                /* Current term */
   453    456     i64 iRowid;                     /* Current rowid */
   454    457     int nPos;                       /* Number of bytes in current position list */
   455         -  int bDel;                       /* True if the delete flag is set */
          458  +  u8 bDel;                        /* True if the delete flag is set */
   456    459   };
   457    460   
   458    461   /*
   459    462   ** Argument is a pointer to an Fts5Data structure that contains a 
   460    463   ** leaf page.
   461    464   */
   462    465   #define ASSERT_SZLEAF_OK(x) assert( \
   463    466       (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \
   464    467   )
   465    468   
   466    469   #define FTS5_SEGITER_ONETERM 0x01
   467    470   #define FTS5_SEGITER_REVERSE 0x02
   468         -
   469    471   
   470    472   /* 
   471    473   ** Argument is a pointer to an Fts5Data structure that contains a leaf
   472    474   ** page. This macro evaluates to true if the leaf contains no terms, or
   473    475   ** false if it contains at least one term.
   474    476   */
   475    477   #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn)
................................................................................
  1488   1490   **
  1489   1491   ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the 
  1490   1492   ** position list content (if any).
  1491   1493   */
  1492   1494   static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
  1493   1495     if( p->rc==SQLITE_OK ){
  1494   1496       int iOff = pIter->iLeafOffset;  /* Offset to read at */
  1495         -    int nSz;
  1496   1497       ASSERT_SZLEAF_OK(pIter->pLeaf);
  1497         -    fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
  1498         -    pIter->bDel = (nSz & 0x0001);
  1499         -    pIter->nPos = nSz>>1;
         1498  +    if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1499  +      int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf);
         1500  +      pIter->bDel = 0;
         1501  +      pIter->nPos = 1;
         1502  +      if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
         1503  +        pIter->bDel = 1;
         1504  +        iOff++;
         1505  +        if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
         1506  +          pIter->nPos = 1;
         1507  +          iOff++;
         1508  +        }else{
         1509  +          pIter->nPos = 0;
         1510  +        }
         1511  +      }
         1512  +    }else{
         1513  +      int nSz;
         1514  +      fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
         1515  +      pIter->bDel = (nSz & 0x0001);
         1516  +      pIter->nPos = nSz>>1;
         1517  +      assert_nc( pIter->nPos>=0 );
         1518  +    }
  1500   1519       pIter->iLeafOffset = iOff;
  1501         -    assert_nc( pIter->nPos>=0 );
  1502   1520     }
  1503   1521   }
  1504   1522   
  1505   1523   static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
  1506   1524     u8 *a = pIter->pLeaf->p;        /* Buffer to read data from */
  1507   1525     int iOff = pIter->iLeafOffset;
  1508   1526   
................................................................................
  1554   1572       int nExtra;
  1555   1573       pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra);
  1556   1574       pIter->iEndofDoclist += nExtra;
  1557   1575     }
  1558   1576   
  1559   1577     fts5SegIterLoadRowid(p, pIter);
  1560   1578   }
         1579  +
         1580  +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*);
         1581  +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*);
         1582  +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*);
         1583  +
         1584  +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
         1585  +  if( pIter->flags & FTS5_SEGITER_REVERSE ){
         1586  +    pIter->xNext = fts5SegIterNext_Reverse;
         1587  +  }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1588  +    pIter->xNext = fts5SegIterNext_None;
         1589  +  }else{
         1590  +    pIter->xNext = fts5SegIterNext;
         1591  +  }
         1592  +}
  1561   1593   
  1562   1594   /*
  1563   1595   ** Initialize the iterator object pIter to iterate through the entries in
  1564   1596   ** segment pSeg. The iterator is left pointing to the first entry when 
  1565   1597   ** this function returns.
  1566   1598   **
  1567   1599   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
................................................................................
  1580   1612       ** at EOF already. */
  1581   1613       assert( pIter->pLeaf==0 );
  1582   1614       return;
  1583   1615     }
  1584   1616   
  1585   1617     if( p->rc==SQLITE_OK ){
  1586   1618       memset(pIter, 0, sizeof(*pIter));
         1619  +    fts5SegIterSetNext(p, pIter);
  1587   1620       pIter->pSeg = pSeg;
  1588   1621       pIter->iLeafPgno = pSeg->pgnoFirst-1;
  1589   1622       fts5SegIterNextPage(p, pIter);
  1590   1623     }
  1591   1624   
  1592   1625     if( p->rc==SQLITE_OK ){
  1593   1626       pIter->iLeafOffset = 4;
................................................................................
  1611   1644   ** This function advances the iterator so that it points to the last 
  1612   1645   ** relevant rowid on the page and, if necessary, initializes the 
  1613   1646   ** aRowidOffset[] and iRowidOffset variables. At this point the iterator
  1614   1647   ** is in its regular state - Fts5SegIter.iLeafOffset points to the first
  1615   1648   ** byte of the position list content associated with said rowid.
  1616   1649   */
  1617   1650   static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
         1651  +  int eDetail = p->pConfig->eDetail;
  1618   1652     int n = pIter->pLeaf->szLeaf;
  1619   1653     int i = pIter->iLeafOffset;
  1620   1654     u8 *a = pIter->pLeaf->p;
  1621   1655     int iRowidOffset = 0;
  1622   1656   
  1623   1657     if( n>pIter->iEndofDoclist ){
  1624   1658       n = pIter->iEndofDoclist;
  1625   1659     }
  1626   1660   
  1627   1661     ASSERT_SZLEAF_OK(pIter->pLeaf);
  1628   1662     while( 1 ){
  1629   1663       i64 iDelta = 0;
  1630         -    int nPos;
  1631         -    int bDummy;
  1632   1664   
  1633         -    i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
  1634         -    i += nPos;
         1665  +    if( eDetail==FTS5_DETAIL_NONE ){
         1666  +      /* todo */
         1667  +      if( i<n && a[i]==0 ){
         1668  +        i++;
         1669  +        if( i<n && a[i]==0 ) i++;
         1670  +      }
         1671  +    }else{
         1672  +      int nPos;
         1673  +      int bDummy;
         1674  +      i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
         1675  +      i += nPos;
         1676  +    }
  1635   1677       if( i>=n ) break;
  1636   1678       i += fts5GetVarint(&a[i], (u64*)&iDelta);
  1637   1679       pIter->iRowid += iDelta;
  1638   1680   
         1681  +    /* If necessary, grow the pIter->aRowidOffset[] array. */
  1639   1682       if( iRowidOffset>=pIter->nRowidOffset ){
  1640   1683         int nNew = pIter->nRowidOffset + 8;
  1641   1684         int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int));
  1642   1685         if( aNew==0 ){
  1643   1686           p->rc = SQLITE_NOMEM;
  1644   1687           break;
  1645   1688         }
................................................................................
  1709   1752   ** points to a delete marker. A delete marker is an entry with a 0 byte
  1710   1753   ** position-list.
  1711   1754   */
  1712   1755   static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){
  1713   1756     Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
  1714   1757     return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
  1715   1758   }
         1759  +
         1760  +/*
         1761  +** Advance iterator pIter to the next entry.
         1762  +**
         1763  +** This version of fts5SegIterNext() is only used by reverse iterators.
         1764  +*/
         1765  +static void fts5SegIterNext_Reverse(
         1766  +  Fts5Index *p,                   /* FTS5 backend object */
         1767  +  Fts5SegIter *pIter,             /* Iterator to advance */
         1768  +  int *pbNewTerm                  /* OUT: Set for new term */
         1769  +){
         1770  +  assert( pIter->flags & FTS5_SEGITER_REVERSE );
         1771  +  assert( pIter->pNextLeaf==0 );
         1772  +  if( pIter->iRowidOffset>0 ){
         1773  +    u8 *a = pIter->pLeaf->p;
         1774  +    int iOff;
         1775  +    i64 iDelta;
         1776  +
         1777  +    pIter->iRowidOffset--;
         1778  +    pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset];
         1779  +    fts5SegIterLoadNPos(p, pIter);
         1780  +    iOff = pIter->iLeafOffset;
         1781  +    if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){
         1782  +      iOff += pIter->nPos;
         1783  +    }
         1784  +    fts5GetVarint(&a[iOff], (u64*)&iDelta);
         1785  +    pIter->iRowid -= iDelta;
         1786  +  }else{
         1787  +    fts5SegIterReverseNewPage(p, pIter);
         1788  +  }
         1789  +}
         1790  +
         1791  +/*
         1792  +** Advance iterator pIter to the next entry.
         1793  +**
         1794  +** This version of fts5SegIterNext() is only used if detail=none and the
         1795  +** iterator is not a reverse direction iterator.
         1796  +*/
         1797  +static void fts5SegIterNext_None(
         1798  +  Fts5Index *p,                   /* FTS5 backend object */
         1799  +  Fts5SegIter *pIter,             /* Iterator to advance */
         1800  +  int *pbNewTerm                  /* OUT: Set for new term */
         1801  +){
         1802  +  int iOff;
         1803  +
         1804  +  assert( p->rc==SQLITE_OK );
         1805  +  assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 );
         1806  +  assert( p->pConfig->eDetail==FTS5_DETAIL_NONE );
         1807  +
         1808  +  ASSERT_SZLEAF_OK(pIter->pLeaf);
         1809  +  iOff = pIter->iLeafOffset;
         1810  +
         1811  +  /* Next entry is on the next page */
         1812  +  if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
         1813  +    fts5SegIterNextPage(p, pIter);
         1814  +    if( p->rc || pIter->pLeaf==0 ) return;
         1815  +    pIter->iRowid = 0;
         1816  +    iOff = 4;
         1817  +  }
         1818  +
         1819  +  if( iOff<pIter->iEndofDoclist ){
         1820  +    /* Next entry is on the current page */
         1821  +    i64 iDelta;
         1822  +    iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta);
         1823  +    pIter->iLeafOffset = iOff;
         1824  +    pIter->iRowid += iDelta;
         1825  +  }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){
         1826  +    if( pIter->pSeg ){
         1827  +      int nKeep = 0;
         1828  +      if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){
         1829  +        iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep);
         1830  +      }
         1831  +      pIter->iLeafOffset = iOff;
         1832  +      fts5SegIterLoadTerm(p, pIter, nKeep);
         1833  +    }else{
         1834  +      const u8 *pList = 0;
         1835  +      const char *zTerm = 0;
         1836  +      int nList;
         1837  +      sqlite3Fts5HashScanNext(p->pHash);
         1838  +      sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
         1839  +      if( pList==0 ) goto next_none_eof;
         1840  +      pIter->pLeaf->p = (u8*)pList;
         1841  +      pIter->pLeaf->nn = nList;
         1842  +      pIter->pLeaf->szLeaf = nList;
         1843  +      pIter->iEndofDoclist = nList;
         1844  +      sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm);
         1845  +      pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
         1846  +    }
         1847  +
         1848  +    if( pbNewTerm ) *pbNewTerm = 1;
         1849  +  }else{
         1850  +    goto next_none_eof;
         1851  +  }
         1852  +
         1853  +  fts5SegIterLoadNPos(p, pIter);
         1854  +
         1855  +  return;
         1856  + next_none_eof:
         1857  +  fts5DataRelease(pIter->pLeaf);
         1858  +  pIter->pLeaf = 0;
         1859  +}
         1860  +
  1716   1861   
  1717   1862   /*
  1718   1863   ** Advance iterator pIter to the next entry. 
  1719   1864   **
  1720   1865   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It 
  1721   1866   ** is not considered an error if the iterator reaches EOF. If an error has 
  1722   1867   ** already occurred when this function is called, it is a no-op.
  1723   1868   */
  1724   1869   static void fts5SegIterNext(
  1725   1870     Fts5Index *p,                   /* FTS5 backend object */
  1726   1871     Fts5SegIter *pIter,             /* Iterator to advance */
  1727   1872     int *pbNewTerm                  /* OUT: Set for new term */
  1728   1873   ){
         1874  +  Fts5Data *pLeaf = pIter->pLeaf;
         1875  +  int iOff;
         1876  +  int bNewTerm = 0;
         1877  +  int nKeep = 0;
         1878  +
  1729   1879     assert( pbNewTerm==0 || *pbNewTerm==0 );
  1730         -  if( p->rc==SQLITE_OK ){
  1731         -    if( pIter->flags & FTS5_SEGITER_REVERSE ){
  1732         -      assert( pIter->pNextLeaf==0 );
  1733         -      if( pIter->iRowidOffset>0 ){
  1734         -        u8 *a = pIter->pLeaf->p;
  1735         -        int iOff;
  1736         -        int nPos;
  1737         -        int bDummy;
  1738         -        i64 iDelta;
  1739         -
  1740         -        pIter->iRowidOffset--;
  1741         -        pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
  1742         -        iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy);
  1743         -        iOff += nPos;
  1744         -        fts5GetVarint(&a[iOff], (u64*)&iDelta);
  1745         -        pIter->iRowid -= iDelta;
  1746         -        fts5SegIterLoadNPos(p, pIter);
  1747         -      }else{
  1748         -        fts5SegIterReverseNewPage(p, pIter);
  1749         -      }
  1750         -    }else{
  1751         -      Fts5Data *pLeaf = pIter->pLeaf;
  1752         -      int iOff;
  1753         -      int bNewTerm = 0;
  1754         -      int nKeep = 0;
  1755         -
  1756         -      /* Search for the end of the position list within the current page. */
  1757         -      u8 *a = pLeaf->p;
  1758         -      int n = pLeaf->szLeaf;
  1759         -
         1880  +  assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE );
         1881  +
         1882  +  /* Search for the end of the position list within the current page. */
         1883  +  u8 *a = pLeaf->p;
         1884  +  int n = pLeaf->szLeaf;
         1885  +
         1886  +  ASSERT_SZLEAF_OK(pLeaf);
         1887  +  iOff = pIter->iLeafOffset + pIter->nPos;
         1888  +
         1889  +  if( iOff<n ){
         1890  +    /* The next entry is on the current page. */
         1891  +    assert_nc( iOff<=pIter->iEndofDoclist );
         1892  +    if( iOff>=pIter->iEndofDoclist ){
         1893  +      bNewTerm = 1;
         1894  +      if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
         1895  +        iOff += fts5GetVarint32(&a[iOff], nKeep);
         1896  +      }
         1897  +    }else{
         1898  +      u64 iDelta;
         1899  +      iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
         1900  +      pIter->iRowid += iDelta;
         1901  +      assert_nc( iDelta>0 );
         1902  +    }
         1903  +    pIter->iLeafOffset = iOff;
         1904  +
         1905  +  }else if( pIter->pSeg==0 ){
         1906  +    const u8 *pList = 0;
         1907  +    const char *zTerm = 0;
         1908  +    int nList = 0;
         1909  +    assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm );
         1910  +    if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){
         1911  +      sqlite3Fts5HashScanNext(p->pHash);
         1912  +      sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
         1913  +    }
         1914  +    if( pList==0 ){
         1915  +      fts5DataRelease(pIter->pLeaf);
         1916  +      pIter->pLeaf = 0;
         1917  +    }else{
         1918  +      pIter->pLeaf->p = (u8*)pList;
         1919  +      pIter->pLeaf->nn = nList;
         1920  +      pIter->pLeaf->szLeaf = nList;
         1921  +      pIter->iEndofDoclist = nList+1;
         1922  +      sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm),
         1923  +          (u8*)zTerm);
         1924  +      pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
         1925  +      *pbNewTerm = 1;
         1926  +    }
         1927  +  }else{
         1928  +    iOff = 0;
         1929  +    /* Next entry is not on the current page */
         1930  +    while( iOff==0 ){
         1931  +      fts5SegIterNextPage(p, pIter);
         1932  +      pLeaf = pIter->pLeaf;
         1933  +      if( pLeaf==0 ) break;
  1760   1934         ASSERT_SZLEAF_OK(pLeaf);
  1761         -      iOff = pIter->iLeafOffset + pIter->nPos;
  1762         -
  1763         -      if( iOff<n ){
  1764         -        /* The next entry is on the current page. */
  1765         -        assert_nc( iOff<=pIter->iEndofDoclist );
  1766         -        if( iOff>=pIter->iEndofDoclist ){
  1767         -          bNewTerm = 1;
  1768         -          if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
  1769         -            iOff += fts5GetVarint32(&a[iOff], nKeep);
  1770         -          }
  1771         -        }else{
  1772         -          u64 iDelta;
  1773         -          iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
  1774         -          pIter->iRowid += iDelta;
  1775         -          assert_nc( iDelta>0 );
  1776         -        }
         1935  +      if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
         1936  +        iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
  1777   1937           pIter->iLeafOffset = iOff;
  1778   1938   
  1779         -      }else if( pIter->pSeg==0 ){
  1780         -        const u8 *pList = 0;
  1781         -        const char *zTerm = 0;
  1782         -        int nList = 0;
  1783         -        assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm );
  1784         -        if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){
  1785         -          sqlite3Fts5HashScanNext(p->pHash);
  1786         -          sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
  1787         -        }
  1788         -        if( pList==0 ){
  1789         -          fts5DataRelease(pIter->pLeaf);
  1790         -          pIter->pLeaf = 0;
  1791         -        }else{
  1792         -          pIter->pLeaf->p = (u8*)pList;
  1793         -          pIter->pLeaf->nn = nList;
  1794         -          pIter->pLeaf->szLeaf = nList;
  1795         -          pIter->iEndofDoclist = nList+1;
  1796         -          sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm),
  1797         -              (u8*)zTerm);
  1798         -          pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid);
  1799         -          *pbNewTerm = 1;
  1800         -        }
  1801         -      }else{
  1802         -        iOff = 0;
  1803         -        /* Next entry is not on the current page */
  1804         -        while( iOff==0 ){
  1805         -          fts5SegIterNextPage(p, pIter);
  1806         -          pLeaf = pIter->pLeaf;
  1807         -          if( pLeaf==0 ) break;
  1808         -          ASSERT_SZLEAF_OK(pLeaf);
  1809         -          if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
  1810         -            iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
  1811         -            pIter->iLeafOffset = iOff;
  1812         -
  1813         -            if( pLeaf->nn>pLeaf->szLeaf ){
  1814         -              pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  1815         -                  &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
         1939  +        if( pLeaf->nn>pLeaf->szLeaf ){
         1940  +          pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
         1941  +              &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
  1816   1942                 );
  1817         -            }
         1943  +        }
  1818   1944   
  1819         -          }
  1820         -          else if( pLeaf->nn>pLeaf->szLeaf ){
  1821         -            pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
  1822         -                &pLeaf->p[pLeaf->szLeaf], iOff
         1945  +      }
         1946  +      else if( pLeaf->nn>pLeaf->szLeaf ){
         1947  +        pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
         1948  +            &pLeaf->p[pLeaf->szLeaf], iOff
  1823   1949               );
  1824         -            pIter->iLeafOffset = iOff;
  1825         -            pIter->iEndofDoclist = iOff;
  1826         -            bNewTerm = 1;
  1827         -          }
  1828         -          if( iOff>=pLeaf->szLeaf ){
  1829         -            p->rc = FTS5_CORRUPT;
  1830         -            return;
  1831         -          }
  1832         -        }
  1833         -      }
  1834         -
  1835         -      /* Check if the iterator is now at EOF. If so, return early. */
  1836         -      if( pIter->pLeaf ){
  1837         -        if( bNewTerm ){
  1838         -          if( pIter->flags & FTS5_SEGITER_ONETERM ){
  1839         -            fts5DataRelease(pIter->pLeaf);
  1840         -            pIter->pLeaf = 0;
  1841         -          }else{
  1842         -            fts5SegIterLoadTerm(p, pIter, nKeep);
  1843         -            fts5SegIterLoadNPos(p, pIter);
  1844         -            if( pbNewTerm ) *pbNewTerm = 1;
  1845         -          }
  1846         -        }else{
  1847         -          /* The following could be done by calling fts5SegIterLoadNPos(). But
  1848         -          ** this block is particularly performance critical, so equivalent
  1849         -          ** code is inlined. */
  1850         -          int nSz;
  1851         -          assert( p->rc==SQLITE_OK );
  1852         -          fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
  1853         -          pIter->bDel = (nSz & 0x0001);
  1854         -          pIter->nPos = nSz>>1;
  1855         -          assert_nc( pIter->nPos>=0 );
  1856         -        }
  1857         -      }
         1950  +        pIter->iLeafOffset = iOff;
         1951  +        pIter->iEndofDoclist = iOff;
         1952  +        bNewTerm = 1;
         1953  +      }
         1954  +      assert_nc( iOff<pLeaf->szLeaf );
         1955  +      if( iOff>pLeaf->szLeaf ){
         1956  +        p->rc = FTS5_CORRUPT;
         1957  +        return;
         1958  +      }
         1959  +    }
         1960  +  }
         1961  +
         1962  +  /* Check if the iterator is now at EOF. If so, return early. */
         1963  +  if( pIter->pLeaf ){
         1964  +    if( bNewTerm ){
         1965  +      if( pIter->flags & FTS5_SEGITER_ONETERM ){
         1966  +        fts5DataRelease(pIter->pLeaf);
         1967  +        pIter->pLeaf = 0;
         1968  +      }else{
         1969  +        fts5SegIterLoadTerm(p, pIter, nKeep);
         1970  +        fts5SegIterLoadNPos(p, pIter);
         1971  +        if( pbNewTerm ) *pbNewTerm = 1;
         1972  +      }
         1973  +    }else{
         1974  +      /* The following could be done by calling fts5SegIterLoadNPos(). But
         1975  +      ** this block is particularly performance critical, so equivalent
         1976  +      ** code is inlined. 
         1977  +      **
         1978  +      ** Later: Switched back to fts5SegIterLoadNPos() because it supports
         1979  +      ** detail=none mode. Not ideal.
         1980  +      */
         1981  +      int nSz;
         1982  +      assert( p->rc==SQLITE_OK );
         1983  +      fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
         1984  +      pIter->bDel = (nSz & 0x0001);
         1985  +      pIter->nPos = nSz>>1;
         1986  +      assert_nc( pIter->nPos>=0 );
  1858   1987       }
  1859   1988     }
  1860   1989   }
  1861   1990   
  1862   1991   #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; }
         1992  +
         1993  +#define fts5IndexSkipVarint(a, iOff) {            \
         1994  +  int iEnd = iOff+9;                              \
         1995  +  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
         1996  +}
  1863   1997   
  1864   1998   /*
  1865   1999   ** Iterator pIter currently points to the first rowid in a doclist. This
  1866   2000   ** function sets the iterator up so that iterates in reverse order through
  1867   2001   ** the doclist.
  1868   2002   */
  1869   2003   static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
................................................................................
  1877   2011       pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast));
  1878   2012     }else{
  1879   2013       Fts5Data *pLeaf = pIter->pLeaf;         /* Current leaf data */
  1880   2014   
  1881   2015       /* Currently, Fts5SegIter.iLeafOffset points to the first byte of
  1882   2016       ** position-list content for the current rowid. Back it up so that it
  1883   2017       ** points to the start of the position-list size field. */
  1884         -    pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel);
         2018  +    int iPoslist;
         2019  +    if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
         2020  +      iPoslist = pIter->iTermLeafOffset;
         2021  +    }else{
         2022  +      iPoslist = 4;
         2023  +    }
         2024  +    fts5IndexSkipVarint(pLeaf->p, iPoslist);
         2025  +    assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==(
         2026  +        pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel)
         2027  +    ));
         2028  +    pIter->iLeafOffset = iPoslist;
  1885   2029   
  1886   2030       /* If this condition is true then the largest rowid for the current
  1887   2031       ** term may not be stored on the current page. So search forward to
  1888   2032       ** see where said rowid really is.  */
  1889   2033       if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
  1890   2034         int pgno;
  1891   2035         Fts5StructureSegment *pSeg = pIter->pSeg;
................................................................................
  1961   2105     ){
  1962   2106       return;
  1963   2107     }
  1964   2108   
  1965   2109     pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
  1966   2110   }
  1967   2111   
  1968         -#define fts5IndexSkipVarint(a, iOff) {            \
  1969         -  int iEnd = iOff+9;                              \
  1970         -  while( (a[iOff++] & 0x80) && iOff<iEnd );       \
  1971         -}
  1972         -
  1973   2112   /*
  1974   2113   ** The iterator object passed as the second argument currently contains
  1975   2114   ** no valid values except for the Fts5SegIter.pLeaf member variable. This
  1976   2115   ** function searches the leaf page for a term matching (pTerm/nTerm).
  1977   2116   **
  1978   2117   ** If the specified term is found on the page, then the iterator is left
  1979   2118   ** pointing to it. If argument bGe is zero and the term is not found,
................................................................................
  2168   2307         }
  2169   2308         if( flags & FTS5INDEX_QUERY_DESC ){
  2170   2309           fts5SegIterReverse(p, pIter);
  2171   2310         }
  2172   2311       }
  2173   2312     }
  2174   2313   
         2314  +  fts5SegIterSetNext(p, pIter);
         2315  +
  2175   2316     /* Either:
  2176   2317     **
  2177   2318     **   1) an error has occurred, or
  2178   2319     **   2) the iterator points to EOF, or
  2179   2320     **   3) the iterator points to an entry with term (pTerm/nTerm), or
  2180   2321     **   4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
  2181   2322     **      to an entry with a term greater than or equal to (pTerm/nTerm).
................................................................................
  2225   2366       sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
  2226   2367       pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
  2227   2368       if( pLeaf==0 ) return;
  2228   2369       pLeaf->p = (u8*)pList;
  2229   2370       pLeaf->nn = pLeaf->szLeaf = nList;
  2230   2371       pIter->pLeaf = pLeaf;
  2231   2372       pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
  2232         -    pIter->iEndofDoclist = pLeaf->nn+1;
         2373  +    pIter->iEndofDoclist = pLeaf->nn;
  2233   2374   
  2234   2375       if( flags & FTS5INDEX_QUERY_DESC ){
  2235   2376         pIter->flags |= FTS5_SEGITER_REVERSE;
  2236   2377         fts5SegIterReverseInitPage(p, pIter);
  2237   2378       }else{
  2238   2379         fts5SegIterLoadNPos(p, pIter);
  2239   2380       }
  2240   2381     }
         2382  +
         2383  +  fts5SegIterSetNext(p, pIter);
  2241   2384   }
  2242   2385   
  2243   2386   /*
  2244   2387   ** Zero the iterator passed as the only argument.
  2245   2388   */
  2246   2389   static void fts5SegIterClear(Fts5SegIter *pIter){
  2247   2390     fts5BufferFree(&pIter->term);
................................................................................
  2477   2620         pIter->iLeafPgno = iLeafPgno+1;
  2478   2621         fts5SegIterReverseNewPage(p, pIter);
  2479   2622         bMove = 0;
  2480   2623       }
  2481   2624     }
  2482   2625   
  2483   2626     do{
  2484         -    if( bMove ) fts5SegIterNext(p, pIter, 0);
         2627  +    if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0);
  2485   2628       if( pIter->pLeaf==0 ) break;
  2486   2629       if( bRev==0 && pIter->iRowid>=iMatch ) break;
  2487   2630       if( bRev!=0 && pIter->iRowid<=iMatch ) break;
  2488   2631       bMove = 1;
  2489   2632     }while( p->rc==SQLITE_OK );
  2490   2633   }
  2491   2634   
................................................................................
  2511   2654     int iChanged,                   /* Index of sub-iterator just advanced */
  2512   2655     int iMinset                     /* Minimum entry in aFirst[] to set */
  2513   2656   ){
  2514   2657     int i;
  2515   2658     for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){
  2516   2659       int iEq;
  2517   2660       if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
  2518         -      fts5SegIterNext(p, &pIter->aSeg[iEq], 0);
         2661  +      Fts5SegIter *pSeg = &pIter->aSeg[iEq];
         2662  +      assert( p->rc==SQLITE_OK );
         2663  +      pSeg->xNext(p, pSeg, 0);
  2519   2664         i = pIter->nSeg + iEq;
  2520   2665       }
  2521   2666     }
  2522   2667   }
  2523   2668   
  2524   2669   /*
  2525   2670   ** Sub-iterator iChanged of iterator pIter has just been advanced. It still
................................................................................
  2598   2743         int iFirst = pIter->aFirst[1].iFirst;
  2599   2744         int bNewTerm = 0;
  2600   2745         Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2601   2746         assert( p->rc==SQLITE_OK );
  2602   2747         if( bUseFrom && pSeg->pDlidx ){
  2603   2748           fts5SegIterNextFrom(p, pSeg, iFrom);
  2604   2749         }else{
  2605         -        fts5SegIterNext(p, pSeg, &bNewTerm);
         2750  +        pSeg->xNext(p, pSeg, &bNewTerm);
  2606   2751         }
  2607   2752   
  2608   2753         if( pSeg->pLeaf==0 || bNewTerm 
  2609   2754          || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
  2610   2755         ){
  2611   2756           fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2612   2757           fts5MultiIterSetEof(pIter);
................................................................................
  2626   2771     assert( pIter->bSkipEmpty );
  2627   2772     if( p->rc==SQLITE_OK ){
  2628   2773       do {
  2629   2774         int iFirst = pIter->aFirst[1].iFirst;
  2630   2775         Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2631   2776         int bNewTerm = 0;
  2632   2777   
  2633         -      fts5SegIterNext(p, pSeg, &bNewTerm);
         2778  +      assert( p->rc==SQLITE_OK );
         2779  +      pSeg->xNext(p, pSeg, &bNewTerm);
  2634   2780         if( pSeg->pLeaf==0 || bNewTerm 
  2635   2781          || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
  2636   2782         ){
  2637   2783           fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2638   2784           fts5MultiIterSetEof(pIter);
  2639   2785           *pbNewTerm = 1;
  2640   2786         }else{
................................................................................
  2746   2892     ** to the first entry in its segment. In this case initialize the 
  2747   2893     ** aFirst[] array. Or, if an error has occurred, free the iterator
  2748   2894     ** object and set the output variable to NULL.  */
  2749   2895     if( p->rc==SQLITE_OK ){
  2750   2896       for(iIter=pNew->nSeg-1; iIter>0; iIter--){
  2751   2897         int iEq;
  2752   2898         if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){
  2753         -        fts5SegIterNext(p, &pNew->aSeg[iEq], 0);
         2899  +        Fts5SegIter *pSeg = &pNew->aSeg[iEq];
         2900  +        if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0);
  2754   2901           fts5MultiIterAdvanced(p, pNew, iEq, iIter);
  2755   2902         }
  2756   2903       }
  2757   2904       fts5MultiIterSetEof(pNew);
  2758   2905       fts5AssertMultiIterSetup(p, pNew);
  2759   2906   
  2760   2907       if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){
................................................................................
  2796   2943         }else{
  2797   2944           fts5SegIterLoadNPos(p, pIter);
  2798   2945         }
  2799   2946         pData = 0;
  2800   2947       }else{
  2801   2948         pNew->bEof = 1;
  2802   2949       }
         2950  +    fts5SegIterSetNext(p, pIter);
  2803   2951   
  2804   2952       *ppOut = pNew;
  2805   2953     }
  2806   2954   
  2807   2955     fts5DataRelease(pData);
  2808   2956   }
  2809   2957   
................................................................................
  2864   3012   ){
  2865   3013     int nRem = pSeg->nPos;          /* Number of bytes still to come */
  2866   3014     Fts5Data *pData = 0;
  2867   3015     u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  2868   3016     int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset);
  2869   3017     int pgno = pSeg->iLeafPgno;
  2870   3018     int pgnoSave = 0;
         3019  +
         3020  +  /* This function does notmwork with detail=none databases. */
         3021  +  assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE );
  2871   3022   
  2872   3023     if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){
  2873   3024       pgnoSave = pgno+1;
  2874   3025     }
  2875   3026   
  2876   3027     while( 1 ){
  2877   3028       xChunk(p, pCtx, pChunk, nChunk);
................................................................................
  3288   3439   
  3289   3440   /*
  3290   3441   ** Append a rowid and position-list size field to the writers output. 
  3291   3442   */
  3292   3443   static void fts5WriteAppendRowid(
  3293   3444     Fts5Index *p, 
  3294   3445     Fts5SegWriter *pWriter,
  3295         -  i64 iRowid,
  3296         -  int nPos
         3446  +  i64 iRowid
  3297   3447   ){
  3298   3448     if( p->rc==SQLITE_OK ){
  3299   3449       Fts5PageWriter *pPage = &pWriter->writer;
  3300   3450   
  3301   3451       if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){
  3302   3452         fts5WriteFlushLeaf(p, pWriter);
  3303   3453       }
................................................................................
  3316   3466       }else{
  3317   3467         assert( p->rc || iRowid>pWriter->iPrevRowid );
  3318   3468         fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid);
  3319   3469       }
  3320   3470       pWriter->iPrevRowid = iRowid;
  3321   3471       pWriter->bFirstRowidInDoclist = 0;
  3322   3472       pWriter->bFirstRowidInPage = 0;
  3323         -
  3324         -    fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos);
  3325   3473     }
  3326   3474   }
  3327   3475   
  3328   3476   static void fts5WriteAppendPoslistData(
  3329   3477     Fts5Index *p, 
  3330   3478     Fts5SegWriter *pWriter, 
  3331   3479     const u8 *aData, 
................................................................................
  3513   3661     Fts5IndexIter *pIter = 0;       /* Iterator to read input data */
  3514   3662     int nRem = pnRem ? *pnRem : 0;  /* Output leaf pages left to write */
  3515   3663     int nInput;                     /* Number of input segments */
  3516   3664     Fts5SegWriter writer;           /* Writer object */
  3517   3665     Fts5StructureSegment *pSeg;     /* Output segment */
  3518   3666     Fts5Buffer term;
  3519   3667     int bOldest;                    /* True if the output segment is the oldest */
         3668  +  int eDetail = p->pConfig->eDetail;
  3520   3669   
  3521   3670     assert( iLvl<pStruct->nLevel );
  3522   3671     assert( pLvl->nMerge<=pLvl->nSeg );
  3523   3672   
  3524   3673     memset(&writer, 0, sizeof(Fts5SegWriter));
  3525   3674     memset(&term, 0, sizeof(Fts5Buffer));
  3526   3675     if( pLvl->nMerge ){
................................................................................
  3582   3731         /* This is a new term. Append a term to the output segment. */
  3583   3732         fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
  3584   3733         fts5BufferSet(&p->rc, &term, nTerm, pTerm);
  3585   3734       }
  3586   3735   
  3587   3736       /* Append the rowid to the output */
  3588   3737       /* WRITEPOSLISTSIZE */
  3589         -    nPos = pSegIter->nPos*2 + pSegIter->bDel;
  3590         -    fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos);
         3738  +    fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));
  3591   3739   
  3592         -    /* Append the position-list data to the output */
  3593         -    fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
         3740  +    if( eDetail==FTS5_DETAIL_NONE ){
         3741  +      if( pSegIter->bDel ){
         3742  +        fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0);
         3743  +        if( pSegIter->nPos>0 ){
         3744  +          fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0);
         3745  +        }
         3746  +      }
         3747  +    }else{
         3748  +      /* Append the position-list data to the output */
         3749  +      nPos = pSegIter->nPos*2 + pSegIter->bDel;
         3750  +      fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos);
         3751  +      fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
         3752  +    }
  3594   3753     }
  3595   3754   
  3596   3755     /* Flush the last leaf page to disk. Set the output segment b-tree height
  3597   3756     ** and last leaf page number at the same time.  */
  3598   3757     fts5WriteFinish(p, &writer, &pSeg->pgnoLast);
  3599   3758   
  3600   3759     if( fts5MultiIterEof(p, pIter) ){
................................................................................
  3774   3933     /* Obtain a reference to the index structure and allocate a new segment-id
  3775   3934     ** for the new level-0 segment.  */
  3776   3935     pStruct = fts5StructureRead(p);
  3777   3936     iSegid = fts5AllocateSegid(p, pStruct);
  3778   3937   
  3779   3938     if( iSegid ){
  3780   3939       const int pgsz = p->pConfig->pgsz;
  3781         -
         3940  +    int eDetail = p->pConfig->eDetail;
  3782   3941       Fts5StructureSegment *pSeg;   /* New segment within pStruct */
  3783   3942       Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
  3784   3943       Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
  3785   3944   
  3786   3945       Fts5SegWriter writer;
  3787   3946       fts5WriteInit(p, &writer, iSegid);
  3788   3947   
................................................................................
  3817   3976           i64 iDelta = 0;
  3818   3977           int iOff = 0;
  3819   3978   
  3820   3979           /* The entire doclist will not fit on this leaf. The following 
  3821   3980           ** loop iterates through the poslists that make up the current 
  3822   3981           ** doclist.  */
  3823   3982           while( p->rc==SQLITE_OK && iOff<nDoclist ){
  3824         -          int nPos;
  3825         -          int nCopy;
  3826         -          int bDummy;
  3827   3983             iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta);
  3828         -          nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
  3829         -          nCopy += nPos;
  3830   3984             iRowid += iDelta;
  3831   3985             
  3832   3986             if( writer.bFirstRowidInPage ){
  3833   3987               fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
  3834   3988               pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
  3835   3989               writer.bFirstRowidInPage = 0;
  3836   3990               fts5WriteDlidxAppend(p, &writer, iRowid);
  3837   3991             }else{
  3838   3992               pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
  3839   3993             }
  3840   3994             assert( pBuf->n<=pBuf->nSpace );
  3841   3995   
  3842         -          if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
  3843         -            /* The entire poslist will fit on the current leaf. So copy
  3844         -            ** it in one go. */
  3845         -            fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
         3996  +          if( eDetail==FTS5_DETAIL_NONE ){
         3997  +            if( iOff<nDoclist && pDoclist[iOff]==0 ){
         3998  +              pBuf->p[pBuf->n++] = 0;
         3999  +              iOff++;
         4000  +              if( iOff<nDoclist && pDoclist[iOff]==0 ){
         4001  +                pBuf->p[pBuf->n++] = 0;
         4002  +                iOff++;
         4003  +              }
         4004  +            }
         4005  +            if( (pBuf->n + pPgidx->n)>=pgsz ){
         4006  +              fts5WriteFlushLeaf(p, &writer);
         4007  +            }
  3846   4008             }else{
  3847         -            /* The entire poslist will not fit on this leaf. So it needs
  3848         -            ** to be broken into sections. The only qualification being
  3849         -            ** that each varint must be stored contiguously.  */
  3850         -            const u8 *pPoslist = &pDoclist[iOff];
  3851         -            int iPos = 0;
  3852         -            while( p->rc==SQLITE_OK ){
  3853         -              int nSpace = pgsz - pBuf->n - pPgidx->n;
  3854         -              int n = 0;
  3855         -              if( (nCopy - iPos)<=nSpace ){
  3856         -                n = nCopy - iPos;
  3857         -              }else{
  3858         -                n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
         4009  +            int bDummy;
         4010  +            int nPos;
         4011  +            int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
         4012  +            nCopy += nPos;
         4013  +            if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
         4014  +              /* The entire poslist will fit on the current leaf. So copy
         4015  +              ** it in one go. */
         4016  +              fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
         4017  +            }else{
         4018  +              /* The entire poslist will not fit on this leaf. So it needs
         4019  +              ** to be broken into sections. The only qualification being
         4020  +              ** that each varint must be stored contiguously.  */
         4021  +              const u8 *pPoslist = &pDoclist[iOff];
         4022  +              int iPos = 0;
         4023  +              while( p->rc==SQLITE_OK ){
         4024  +                int nSpace = pgsz - pBuf->n - pPgidx->n;
         4025  +                int n = 0;
         4026  +                if( (nCopy - iPos)<=nSpace ){
         4027  +                  n = nCopy - iPos;
         4028  +                }else{
         4029  +                  n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
         4030  +                }
         4031  +                assert( n>0 );
         4032  +                fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
         4033  +                iPos += n;
         4034  +                if( (pBuf->n + pPgidx->n)>=pgsz ){
         4035  +                  fts5WriteFlushLeaf(p, &writer);
         4036  +                }
         4037  +                if( iPos>=nCopy ) break;
  3859   4038                 }
  3860         -              assert( n>0 );
  3861         -              fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
  3862         -              iPos += n;
  3863         -              if( (pBuf->n + pPgidx->n)>=pgsz ){
  3864         -                fts5WriteFlushLeaf(p, &writer);
  3865         -              }
  3866         -              if( iPos>=nCopy ) break;
  3867   4039               }
         4040  +            iOff += nCopy;
  3868   4041             }
  3869         -          iOff += nCopy;
  3870   4042           }
  3871   4043         }
  3872   4044   
  3873   4045         /* TODO2: Doclist terminator written here. */
  3874   4046         /* pBuf->p[pBuf->n++] = '\0'; */
  3875   4047         assert( pBuf->n<=pBuf->nSpace );
  3876   4048         sqlite3Fts5HashScanNext(pHash);
................................................................................
  3997   4169   typedef struct PoslistCallbackCtx PoslistCallbackCtx;
  3998   4170   struct PoslistCallbackCtx {
  3999   4171     Fts5Buffer *pBuf;               /* Append to this buffer */
  4000   4172     Fts5Colset *pColset;            /* Restrict matches to this column */
  4001   4173     int eState;                     /* See above */
  4002   4174   };
  4003   4175   
         4176  +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx;
         4177  +struct PoslistOffsetsCtx {
         4178  +  Fts5Buffer *pBuf;               /* Append to this buffer */
         4179  +  Fts5Colset *pColset;            /* Restrict matches to this column */
         4180  +  int iRead;
         4181  +  int iWrite;
         4182  +};
         4183  +
  4004   4184   /*
  4005   4185   ** TODO: Make this more efficient!
  4006   4186   */
  4007   4187   static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
  4008   4188     int i;
  4009   4189     for(i=0; i<pColset->nCol; i++){
  4010   4190       if( pColset->aiCol[i]==iCol ) return 1;
  4011   4191     }
  4012   4192     return 0;
  4013   4193   }
         4194  +
         4195  +static void fts5PoslistOffsetsCallback(
         4196  +  Fts5Index *p, 
         4197  +  void *pContext, 
         4198  +  const u8 *pChunk, int nChunk
         4199  +){
         4200  +  PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext;
         4201  +  assert_nc( nChunk>=0 );
         4202  +  if( nChunk>0 ){
         4203  +    int i = 0;
         4204  +    while( i<nChunk ){
         4205  +      int iVal;
         4206  +      i += fts5GetVarint32(&pChunk[i], iVal);
         4207  +      iVal += pCtx->iRead - 2;
         4208  +      pCtx->iRead = iVal;
         4209  +      if( fts5IndexColsetTest(pCtx->pColset, iVal) ){
         4210  +        fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite);
         4211  +        pCtx->iWrite = iVal;
         4212  +      }
         4213  +    }
         4214  +  }
         4215  +}
  4014   4216   
  4015   4217   static void fts5PoslistFilterCallback(
  4016   4218     Fts5Index *p, 
  4017   4219     void *pContext, 
  4018   4220     const u8 *pChunk, int nChunk
  4019   4221   ){
  4020   4222     PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
................................................................................
  4075   4277     Fts5Colset *pColset,
  4076   4278     Fts5Buffer *pBuf
  4077   4279   ){
  4078   4280     if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
  4079   4281       if( pColset==0 ){
  4080   4282         fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
  4081   4283       }else{
  4082         -      PoslistCallbackCtx sCtx;
  4083         -      sCtx.pBuf = pBuf;
  4084         -      sCtx.pColset = pColset;
  4085         -      sCtx.eState = fts5IndexColsetTest(pColset, 0);
  4086         -      assert( sCtx.eState==0 || sCtx.eState==1 );
  4087         -      fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
         4284  +      if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){
         4285  +        PoslistCallbackCtx sCtx;
         4286  +        sCtx.pBuf = pBuf;
         4287  +        sCtx.pColset = pColset;
         4288  +        sCtx.eState = fts5IndexColsetTest(pColset, 0);
         4289  +        assert( sCtx.eState==0 || sCtx.eState==1 );
         4290  +        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
         4291  +      }else{
         4292  +        PoslistOffsetsCtx sCtx;
         4293  +        memset(&sCtx, 0, sizeof(sCtx));
         4294  +        sCtx.pBuf = pBuf;
         4295  +        sCtx.pColset = pColset;
         4296  +        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
         4297  +      }
  4088   4298       }
  4089   4299     }
  4090   4300   }
  4091   4301   
  4092   4302   /*
  4093   4303   ** IN/OUT parameter (*pa) points to a position list n bytes in size. If
  4094   4304   ** the position list contains entries for column iCol, then (*pa) is set
................................................................................
  4123   4333     assert( (prev & 0x80)==0 );
  4124   4334     while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
  4125   4335       prev = *p++;
  4126   4336     }
  4127   4337     return p - (*pa);
  4128   4338   }
  4129   4339   
         4340  +static int fts5AppendRowid(
         4341  +  Fts5Index *p,
         4342  +  i64 iDelta,
         4343  +  Fts5IndexIter *pMulti,
         4344  +  Fts5Colset *pColset,
         4345  +  Fts5Buffer *pBuf
         4346  +){
         4347  +  fts5BufferAppendVarint(&p->rc, pBuf, iDelta);
         4348  +  return 0;
         4349  +}
  4130   4350   
  4131   4351   /*
  4132   4352   ** Iterator pMulti currently points to a valid entry (not EOF). This
  4133   4353   ** function appends the following to buffer pBuf:
  4134   4354   **
  4135   4355   **   * The varint iDelta, and
  4136   4356   **   * the position list that currently points to, including the size field.
................................................................................
  4150   4370     Fts5Buffer *pBuf
  4151   4371   ){
  4152   4372     if( p->rc==SQLITE_OK ){
  4153   4373       Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  4154   4374       assert( fts5MultiIterEof(p, pMulti)==0 );
  4155   4375       assert( pSeg->nPos>0 );
  4156   4376       if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
  4157         -
  4158         -      if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
         4377  +      if( p->pConfig->eDetail==FTS5_DETAIL_FULL
         4378  +       && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  4159   4379          && (pColset==0 || pColset->nCol==1)
  4160   4380         ){
  4161   4381           const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4162   4382           int nPos;
  4163   4383           if( pColset ){
  4164   4384             nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
  4165   4385             if( nPos==0 ) return 1;
................................................................................
  4196   4416                 int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
  4197   4417                 while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; }
  4198   4418                 sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2);
  4199   4419               }
  4200   4420             }
  4201   4421           }
  4202   4422         }
  4203         -
  4204   4423       }
  4205   4424     }
  4206   4425   
  4207   4426     return 0;
  4208   4427   }
         4428  +
  4209   4429   
  4210   4430   static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  4211   4431     u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
  4212   4432   
  4213   4433     assert( pIter->aPoslist );
  4214   4434     if( p>=pIter->aEof ){
  4215   4435       pIter->aPoslist = 0;
................................................................................
  4262   4482   #endif
  4263   4483   
  4264   4484   #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) {       \
  4265   4485     assert( (pBuf)->n!=0 || (iLastRowid)==0 );                   \
  4266   4486     fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
  4267   4487     (iLastRowid) = (iRowid);                                     \
  4268   4488   }
         4489  +
         4490  +/*
         4491  +** Swap the contents of buffer *p1 with that of *p2.
         4492  +*/
         4493  +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
         4494  +  Fts5Buffer tmp = *p1;
         4495  +  *p1 = *p2;
         4496  +  *p2 = tmp;
         4497  +}
         4498  +
         4499  +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){
         4500  +  int i = *piOff;
         4501  +  if( i>=pBuf->n ){
         4502  +    *piOff = -1;
         4503  +  }else{
         4504  +    u64 iVal;
         4505  +    *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal);
         4506  +    *piRowid += iVal;
         4507  +  }
         4508  +}
         4509  +
         4510  +/*
         4511  +** This is the equivalent of fts5MergePrefixLists() for detail=none mode.
         4512  +** In this case the buffers consist of a delta-encoded list of rowids only.
         4513  +*/
         4514  +static void fts5MergeRowidLists(
         4515  +  Fts5Index *p,                   /* FTS5 backend object */
         4516  +  Fts5Buffer *p1,                 /* First list to merge */
         4517  +  Fts5Buffer *p2                  /* Second list to merge */
         4518  +){
         4519  +  int i1 = 0;
         4520  +  int i2 = 0;
         4521  +  i64 iRowid1 = 0;
         4522  +  i64 iRowid2 = 0;
         4523  +  i64 iOut = 0;
         4524  +
         4525  +  Fts5Buffer out;
         4526  +  memset(&out, 0, sizeof(out));
         4527  +  sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
         4528  +  if( p->rc ) return;
         4529  +
         4530  +  fts5NextRowid(p1, &i1, &iRowid1);
         4531  +  fts5NextRowid(p2, &i2, &iRowid2);
         4532  +  while( i1>=0 || i2>=0 ){
         4533  +    if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){
         4534  +      assert( iOut==0 || iRowid1>iOut );
         4535  +      fts5BufferSafeAppendVarint(&out, iRowid1 - iOut);
         4536  +      iOut = iRowid1;
         4537  +      fts5NextRowid(p1, &i1, &iRowid1);
         4538  +    }else{
         4539  +      assert( iOut==0 || iRowid2>iOut );
         4540  +      fts5BufferSafeAppendVarint(&out, iRowid2 - iOut);
         4541  +      iOut = iRowid2;
         4542  +      if( i1>=0 && iRowid1==iRowid2 ){
         4543  +        fts5NextRowid(p1, &i1, &iRowid1);
         4544  +      }
         4545  +      fts5NextRowid(p2, &i2, &iRowid2);
         4546  +    }
         4547  +  }
         4548  +
         4549  +  fts5BufferSwap(&out, p1);
         4550  +  fts5BufferFree(&out);
         4551  +}
  4269   4552   
  4270   4553   /*
  4271   4554   ** Buffers p1 and p2 contain doclists. This function merges the content
  4272   4555   ** of the two doclists together and sets buffer p1 to the result before
  4273   4556   ** returning.
  4274   4557   **
  4275   4558   ** If an error occurs, an error code is left in p->rc. If an error has
................................................................................
  4331   4614             }else{
  4332   4615               iNew = iPos2;
  4333   4616               sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4334   4617               if( iPos1==iPos2 ){
  4335   4618                 sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
  4336   4619               }
  4337   4620             }
  4338         -          p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
         4621  +          if( iNew!=writer.iPrev || tmp.n==0 ){
         4622  +            p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
         4623  +          }
  4339   4624           }
  4340   4625   
  4341   4626           /* WRITEPOSLISTSIZE */
  4342   4627           fts5BufferSafeAppendVarint(&out, tmp.n * 2);
  4343   4628           fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
  4344   4629           fts5DoclistIterNext(&i1);
  4345   4630           fts5DoclistIterNext(&i2);
................................................................................
  4348   4633   
  4349   4634       fts5BufferSet(&p->rc, p1, out.n, out.p);
  4350   4635       fts5BufferFree(&tmp);
  4351   4636       fts5BufferFree(&out);
  4352   4637     }
  4353   4638   }
  4354   4639   
  4355         -static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
  4356         -  Fts5Buffer tmp = *p1;
  4357         -  *p1 = *p2;
  4358         -  *p2 = tmp;
  4359         -}
  4360         -
  4361   4640   static void fts5SetupPrefixIter(
  4362   4641     Fts5Index *p,                   /* Index to read from */
  4363   4642     int bDesc,                      /* True for "ORDER BY rowid DESC" */
  4364   4643     const u8 *pToken,               /* Buffer containing prefix to match */
  4365   4644     int nToken,                     /* Size of buffer pToken in bytes */
  4366   4645     Fts5Colset *pColset,            /* Restrict matches to these columns */
  4367   4646     Fts5IndexIter **ppIter          /* OUT: New iterator */
  4368   4647   ){
  4369   4648     Fts5Structure *pStruct;
  4370   4649     Fts5Buffer *aBuf;
  4371   4650     const int nBuf = 32;
         4651  +
         4652  +  void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*);
         4653  +  int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*);
         4654  +  if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         4655  +    xMerge = fts5MergeRowidLists;
         4656  +    xAppend = fts5AppendRowid;
         4657  +  }else{
         4658  +    xMerge = fts5MergePrefixLists;
         4659  +    xAppend = fts5AppendPoslist;
         4660  +  }
  4372   4661   
  4373   4662     aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  4374   4663     pStruct = fts5StructureRead(p);
  4375   4664   
  4376   4665     if( aBuf && pStruct ){
  4377   4666       const int flags = FTS5INDEX_QUERY_SCAN;
  4378   4667       int i;
................................................................................
  4398   4687         if( doclist.n>0 && iRowid<=iLastRowid ){
  4399   4688           for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
  4400   4689             assert( i<nBuf );
  4401   4690             if( aBuf[i].n==0 ){
  4402   4691               fts5BufferSwap(&doclist, &aBuf[i]);
  4403   4692               fts5BufferZero(&doclist);
  4404   4693             }else{
  4405         -            fts5MergePrefixLists(p, &doclist, &aBuf[i]);
         4694  +            xMerge(p, &doclist, &aBuf[i]);
  4406   4695               fts5BufferZero(&aBuf[i]);
  4407   4696             }
  4408   4697           }
  4409   4698           iLastRowid = 0;
  4410   4699         }
  4411   4700   
  4412         -      if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
         4701  +      if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
  4413   4702           iLastRowid = iRowid;
  4414   4703         }
  4415   4704       }
  4416   4705   
  4417   4706       for(i=0; i<nBuf; i++){
  4418   4707         if( p->rc==SQLITE_OK ){
  4419         -        fts5MergePrefixLists(p, &doclist, &aBuf[i]);
         4708  +        xMerge(p, &doclist, &aBuf[i]);
  4420   4709         }
  4421   4710         fts5BufferFree(&aBuf[i]);
  4422   4711       }
  4423   4712       fts5MultiIterFree(p, p1);
  4424   4713   
  4425   4714       pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
  4426   4715       if( pData ){
................................................................................
  4442   4731   ** to the document with rowid iRowid.
  4443   4732   */
  4444   4733   int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  4445   4734     assert( p->rc==SQLITE_OK );
  4446   4735   
  4447   4736     /* Allocate the hash table if it has not already been allocated */
  4448   4737     if( p->pHash==0 ){
  4449         -    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
         4738  +    p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData);
  4450   4739     }
  4451   4740   
  4452   4741     /* Flush the hash table to disk if required */
  4453   4742     if( iRowid<p->iWriteRowid 
  4454   4743      || (iRowid==p->iWriteRowid && p->bDelete==0)
  4455   4744      || (p->nPendingData > p->pConfig->nHashSize) 
  4456   4745     ){
................................................................................
  4563   4852   }
  4564   4853   
  4565   4854   /*
  4566   4855   ** Argument p points to a buffer containing utf-8 text that is n bytes in 
  4567   4856   ** size. Return the number of bytes in the nChar character prefix of the
  4568   4857   ** buffer, or 0 if there are less than nChar characters in total.
  4569   4858   */
  4570         -static int fts5IndexCharlenToBytelen(const char *p, int nByte, int nChar){
         4859  +int sqlite3Fts5IndexCharlenToBytelen(
         4860  +  const char *p, 
         4861  +  int nByte, 
         4862  +  int nChar
         4863  +){
  4571   4864     int n = 0;
  4572   4865     int i;
  4573   4866     for(i=0; i<nChar; i++){
  4574   4867       if( n>=nByte ) return 0;      /* Input contains fewer than nChar chars */
  4575   4868       if( (unsigned char)p[n++]>=0xc0 ){
  4576   4869         while( (p[n] & 0xc0)==0x80 ) n++;
  4577   4870       }
................................................................................
  4620   4913   
  4621   4914     /* Add the entry to the main terms index. */
  4622   4915     rc = sqlite3Fts5HashWrite(
  4623   4916         p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken
  4624   4917     );
  4625   4918   
  4626   4919     for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){
  4627         -    int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]);
         4920  +    const int nChar = pConfig->aPrefix[i];
         4921  +    int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
  4628   4922       if( nByte ){
  4629   4923         rc = sqlite3Fts5HashWrite(p->pHash, 
  4630   4924             p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken,
  4631   4925             nByte
  4632   4926         );
  4633   4927       }
  4634   4928     }
................................................................................
  4798   5092     Fts5IndexIter *pIter, 
  4799   5093     Fts5Colset *pColset,            /* Column filter (or NULL) */
  4800   5094     const u8 **pp,                  /* OUT: Pointer to position-list data */
  4801   5095     int *pn,                        /* OUT: Size of position-list in bytes */
  4802   5096     i64 *piRowid                    /* OUT: Current rowid */
  4803   5097   ){
  4804   5098     Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
         5099  +  int eDetail = pIter->pIndex->pConfig->eDetail;
         5100  +
  4805   5101     assert( pIter->pIndex->rc==SQLITE_OK );
  4806   5102     *piRowid = pSeg->iRowid;
  4807         -  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
         5103  +  if( eDetail==FTS5_DETAIL_NONE ){
         5104  +    *pn = pSeg->nPos;
         5105  +  }else
         5106  +  if( eDetail==FTS5_DETAIL_FULL 
         5107  +   && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
         5108  +  ){
  4808   5109       u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4809   5110       if( pColset==0 || pIter->bFiltered ){
  4810   5111         *pn = pSeg->nPos;
  4811   5112         *pp = pPos;
  4812   5113       }else if( pColset->nCol==1 ){
  4813   5114         *pp = pPos;
  4814   5115         *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]);
................................................................................
  4817   5118         fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist);
  4818   5119         *pp = pIter->poslist.p;
  4819   5120         *pn = pIter->poslist.n;
  4820   5121       }
  4821   5122     }else{
  4822   5123       fts5BufferZero(&pIter->poslist);
  4823   5124       fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
  4824         -    *pp = pIter->poslist.p;
         5125  +    if( eDetail==FTS5_DETAIL_FULL ){
         5126  +      *pp = pIter->poslist.p;
         5127  +    }
  4825   5128       *pn = pIter->poslist.n;
  4826   5129     }
  4827   5130     return fts5IndexReturn(pIter->pIndex);
  4828   5131   }
         5132  +
         5133  +int sqlite3Fts5IterCollist(
         5134  +  Fts5IndexIter *pIter, 
         5135  +  const u8 **pp,                  /* OUT: Pointer to position-list data */
         5136  +  int *pn                         /* OUT: Size of position-list in bytes */
         5137  +){
         5138  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         5139  +  *pp = pIter->poslist.p;
         5140  +  *pn = pIter->poslist.n;
         5141  +  return SQLITE_OK;
         5142  +}
  4829   5143   
  4830   5144   /*
  4831   5145   ** This function is similar to sqlite3Fts5IterPoslist(), except that it
  4832   5146   ** copies the position list into the buffer supplied as the second 
  4833   5147   ** argument.
  4834   5148   */
  4835   5149   int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
................................................................................
  4936   5250   ** Below this point is the implementation of the integrity-check 
  4937   5251   ** functionality.
  4938   5252   */
  4939   5253   
  4940   5254   /*
  4941   5255   ** Return a simple checksum value based on the arguments.
  4942   5256   */
  4943         -static u64 fts5IndexEntryCksum(
         5257  +u64 sqlite3Fts5IndexEntryCksum(
  4944   5258     i64 iRowid, 
  4945   5259     int iCol, 
  4946   5260     int iPos, 
  4947   5261     int iIdx,
  4948   5262     const char *pTerm,
  4949   5263     int nTerm
  4950   5264   ){
................................................................................
  5006   5320     Fts5Index *p,                   /* Fts5 index object */
  5007   5321     int iIdx,
  5008   5322     const char *z,                  /* Index key to query for */
  5009   5323     int n,                          /* Size of index key in bytes */
  5010   5324     int flags,                      /* Flags for Fts5IndexQuery */
  5011   5325     u64 *pCksum                     /* IN/OUT: Checksum value */
  5012   5326   ){
         5327  +  int eDetail = p->pConfig->eDetail;
  5013   5328     u64 cksum = *pCksum;
  5014   5329     Fts5IndexIter *pIdxIter = 0;
         5330  +  Fts5Buffer buf = {0, 0, 0};
  5015   5331     int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
  5016   5332   
  5017   5333     while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  5018         -    i64 dummy;
  5019         -    const u8 *pPos;
  5020         -    int nPos;
  5021   5334       i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
  5022         -    rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy);
         5335  +
         5336  +    if( eDetail==FTS5_DETAIL_NONE ){
         5337  +      cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
         5338  +    }else{
         5339  +      rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
         5340  +      if( rc==SQLITE_OK ){
         5341  +        Fts5PoslistReader sReader;
         5342  +        for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
         5343  +            sReader.bEof==0;
         5344  +            sqlite3Fts5PoslistReaderNext(&sReader)
         5345  +        ){
         5346  +          int iCol = FTS5_POS2COLUMN(sReader.iPos);
         5347  +          int iOff = FTS5_POS2OFFSET(sReader.iPos);
         5348  +          cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
         5349  +        }
         5350  +      }
         5351  +    }
  5023   5352       if( rc==SQLITE_OK ){
  5024         -      Fts5PoslistReader sReader;
  5025         -      for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader);
  5026         -          sReader.bEof==0;
  5027         -          sqlite3Fts5PoslistReaderNext(&sReader)
  5028         -      ){
  5029         -        int iCol = FTS5_POS2COLUMN(sReader.iPos);
  5030         -        int iOff = FTS5_POS2OFFSET(sReader.iPos);
  5031         -        cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  5032         -      }
  5033   5353         rc = sqlite3Fts5IterNext(pIdxIter);
  5034   5354       }
  5035   5355     }
  5036   5356     sqlite3Fts5IterClose(pIdxIter);
         5357  +  fts5BufferFree(&buf);
  5037   5358   
  5038   5359     *pCksum = cksum;
  5039   5360     return rc;
  5040   5361   }
  5041   5362   
  5042   5363   
  5043   5364   /*
................................................................................
  5323   5644   #endif
  5324   5645   }
  5325   5646   
  5326   5647   
  5327   5648   /*
  5328   5649   ** Run internal checks to ensure that the FTS index (a) is internally 
  5329   5650   ** consistent and (b) contains entries for which the XOR of the checksums
  5330         -** as calculated by fts5IndexEntryCksum() is cksum.
         5651  +** as calculated by sqlite3Fts5IndexEntryCksum() is cksum.
  5331   5652   **
  5332   5653   ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
  5333   5654   ** checksum does not match. Return SQLITE_OK if all checks pass without
  5334   5655   ** error, or some other SQLite error code if another error (e.g. OOM)
  5335   5656   ** occurs.
  5336   5657   */
  5337   5658   int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
         5659  +  int eDetail = p->pConfig->eDetail;
  5338   5660     u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  5339   5661     Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
  5340   5662     Fts5IndexIter *pIter;           /* Used to iterate through entire index */
  5341   5663     Fts5Structure *pStruct;         /* Index structure */
  5342   5664   
  5343   5665   #ifdef SQLITE_DEBUG
  5344   5666     /* Used by extra internal tests only run if NDEBUG is not defined */
................................................................................
  5382   5704       int iOff = 0;               /* Offset within poslist */
  5383   5705       i64 iRowid = fts5MultiIterRowid(pIter);
  5384   5706       char *z = (char*)fts5MultiIterTerm(pIter, &n);
  5385   5707   
  5386   5708       /* If this is a new term, query for it. Update cksum3 with the results. */
  5387   5709       fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
  5388   5710   
  5389         -    poslist.n = 0;
  5390         -    fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
  5391         -    while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
  5392         -      int iCol = FTS5_POS2COLUMN(iPos);
  5393         -      int iTokOff = FTS5_POS2OFFSET(iPos);
  5394         -      cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
         5711  +    if( eDetail==FTS5_DETAIL_NONE ){
         5712  +      if( 0==fts5MultiIterIsEmpty(p, pIter) ){
         5713  +        cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n);
         5714  +      }
         5715  +    }else{
         5716  +      poslist.n = 0;
         5717  +      fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist);
         5718  +      while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
         5719  +        int iCol = FTS5_POS2COLUMN(iPos);
         5720  +        int iTokOff = FTS5_POS2OFFSET(iPos);
         5721  +        cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
         5722  +      }
  5395   5723       }
  5396   5724     }
  5397   5725     fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
  5398   5726   
  5399   5727     fts5MultiIterFree(p, pIter);
  5400   5728     if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
  5401   5729   
................................................................................
  5403   5731   #ifdef SQLITE_DEBUG
  5404   5732     fts5BufferFree(&term);
  5405   5733   #endif
  5406   5734     fts5BufferFree(&poslist);
  5407   5735     return fts5IndexReturn(p);
  5408   5736   }
  5409   5737   
  5410         -
  5411         -/*
  5412         -** Calculate and return a checksum that is the XOR of the index entry
  5413         -** checksum of all entries that would be generated by the token specified
  5414         -** by the final 5 arguments.
  5415         -*/
  5416         -u64 sqlite3Fts5IndexCksum(
  5417         -  Fts5Config *pConfig,            /* Configuration object */
  5418         -  i64 iRowid,                     /* Document term appears in */
  5419         -  int iCol,                       /* Column term appears in */
  5420         -  int iPos,                       /* Position term appears in */
  5421         -  const char *pTerm, int nTerm    /* Term at iPos */
  5422         -){
  5423         -  u64 ret = 0;                    /* Return value */
  5424         -  int iIdx;                       /* For iterating through indexes */
  5425         -
  5426         -  ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm);
  5427         -
  5428         -  for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){
  5429         -    int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]);
  5430         -    if( nByte ){
  5431         -      ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte);
  5432         -    }
  5433         -  }
  5434         -
  5435         -  return ret;
  5436         -}
  5437         -
  5438   5738   /*************************************************************************
  5439   5739   **************************************************************************
  5440   5740   ** Below this point is the implementation of the fts5_decode() scalar
  5441   5741   ** function only.
  5442   5742   */
  5443   5743   
  5444   5744   /*
................................................................................
  5788   6088     if( rc==SQLITE_OK ){
  5789   6089       rc = sqlite3_create_function(
  5790   6090           db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0
  5791   6091       );
  5792   6092     }
  5793   6093     return rc;
  5794   6094   }
  5795         -

Changes to ext/fts5/fts5_main.c.

   222    222   */
   223    223   #define FTS5CSR_REQUIRE_CONTENT   0x01
   224    224   #define FTS5CSR_REQUIRE_DOCSIZE   0x02
   225    225   #define FTS5CSR_REQUIRE_INST      0x04
   226    226   #define FTS5CSR_EOF               0x08
   227    227   #define FTS5CSR_FREE_ZRANK        0x10
   228    228   #define FTS5CSR_REQUIRE_RESEEK    0x20
          229  +#define FTS5CSR_REQUIRE_POSLIST   0x40
   229    230   
   230    231   #define BitFlagAllTest(x,y) (((x) & (y))==(y))
   231    232   #define BitFlagTest(x,y)    (((x) & (y))!=0)
   232    233   
   233    234   
   234    235   /*
   235    236   ** Macros to Set(), Clear() and Test() cursor flags.
................................................................................
   635    636   ** specific to the previous row stored by the cursor object.
   636    637   */
   637    638   static void fts5CsrNewrow(Fts5Cursor *pCsr){
   638    639     CsrFlagSet(pCsr, 
   639    640         FTS5CSR_REQUIRE_CONTENT 
   640    641       | FTS5CSR_REQUIRE_DOCSIZE 
   641    642       | FTS5CSR_REQUIRE_INST 
          643  +    | FTS5CSR_REQUIRE_POSLIST 
   642    644     );
   643    645   }
   644    646   
   645    647   static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
   646    648     Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
   647    649     Fts5Auxdata *pData;
   648    650     Fts5Auxdata *pNext;
................................................................................
   717    719       int iOff = 0;
   718    720       rc = SQLITE_OK;
   719    721   
   720    722       pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
   721    723       nBlob = sqlite3_column_bytes(pSorter->pStmt, 1);
   722    724       aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1);
   723    725   
   724         -    for(i=0; i<(pSorter->nIdx-1); i++){
   725         -      int iVal;
   726         -      a += fts5GetVarint32(a, iVal);
   727         -      iOff += iVal;
   728         -      pSorter->aIdx[i] = iOff;
          726  +    /* nBlob==0 in detail=none mode. */
          727  +    if( nBlob>0 ){
          728  +      for(i=0; i<(pSorter->nIdx-1); i++){
          729  +        int iVal;
          730  +        a += fts5GetVarint32(a, iVal);
          731  +        iOff += iVal;
          732  +        pSorter->aIdx[i] = iOff;
          733  +      }
          734  +      pSorter->aIdx[i] = &aBlob[nBlob] - a;
          735  +      pSorter->aPoslist = a;
   729    736       }
   730         -    pSorter->aIdx[i] = &aBlob[nBlob] - a;
   731    737   
   732         -    pSorter->aPoslist = a;
   733    738       fts5CsrNewrow(pCsr);
   734    739     }
   735    740   
   736    741     return rc;
   737    742   }
   738    743   
   739    744   
................................................................................
  1163   1168       assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
  1164   1169       assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
  1165   1170       assert( pCsr->iLastRowid==LARGEST_INT64 );
  1166   1171       assert( pCsr->iFirstRowid==SMALLEST_INT64 );
  1167   1172       pCsr->ePlan = FTS5_PLAN_SOURCE;
  1168   1173       pCsr->pExpr = pTab->pSortCsr->pExpr;
  1169   1174       rc = fts5CursorFirst(pTab, pCsr, bDesc);
         1175  +    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  1170   1176     }else if( pMatch ){
  1171   1177       const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1172   1178       if( zExpr==0 ) zExpr = "";
  1173   1179   
  1174   1180       rc = fts5CursorParseRank(pConfig, pCsr, pRank);
  1175   1181       if( rc==SQLITE_OK ){
  1176   1182         if( zExpr[0]=='*' ){
................................................................................
  1592   1598     int rc;
  1593   1599     Fts5Table *pTab = (Fts5Table*)pVtab;
  1594   1600     fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
  1595   1601     rc = sqlite3Fts5StorageRollback(pTab->pStorage);
  1596   1602     return rc;
  1597   1603   }
  1598   1604   
         1605  +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*);
         1606  +
  1599   1607   static void *fts5ApiUserData(Fts5Context *pCtx){
  1600   1608     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1601   1609     return pCsr->pAux->pUserData;
  1602   1610   }
  1603   1611   
  1604   1612   static int fts5ApiColumnCount(Fts5Context *pCtx){
  1605   1613     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
................................................................................
  1641   1649   }
  1642   1650   
  1643   1651   static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
  1644   1652     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1645   1653     return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
  1646   1654   }
  1647   1655   
  1648         -static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
  1649         -  int n;
  1650         -  if( pCsr->pSorter ){
         1656  +static int fts5ApiColumnText(
         1657  +  Fts5Context *pCtx, 
         1658  +  int iCol, 
         1659  +  const char **pz, 
         1660  +  int *pn
         1661  +){
         1662  +  int rc = SQLITE_OK;
         1663  +  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
         1664  +  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
         1665  +    *pz = 0;
         1666  +    *pn = 0;
         1667  +  }else{
         1668  +    rc = fts5SeekCursor(pCsr, 0);
         1669  +    if( rc==SQLITE_OK ){
         1670  +      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
         1671  +      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
         1672  +    }
         1673  +  }
         1674  +  return rc;
         1675  +}
         1676  +
         1677  +static int fts5CsrPoslist(
         1678  +  Fts5Cursor *pCsr, 
         1679  +  int iPhrase, 
         1680  +  const u8 **pa,
         1681  +  int *pn
         1682  +){
         1683  +  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
         1684  +  int rc = SQLITE_OK;
         1685  +  int bLive = (pCsr->pSorter==0);
         1686  +
         1687  +  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
         1688  +
         1689  +    if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
         1690  +      Fts5PoslistPopulator *aPopulator;
         1691  +      int i;
         1692  +      aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
         1693  +      if( aPopulator==0 ) rc = SQLITE_NOMEM;
         1694  +      for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
         1695  +        int n; const char *z;
         1696  +        rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
         1697  +        if( rc==SQLITE_OK ){
         1698  +          rc = sqlite3Fts5ExprPopulatePoslists(
         1699  +              pConfig, pCsr->pExpr, aPopulator, i, z, n
         1700  +          );
         1701  +        }
         1702  +      }
         1703  +      sqlite3_free(aPopulator);
         1704  +
         1705  +      if( pCsr->pSorter ){
         1706  +        sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
         1707  +      }
         1708  +    }
         1709  +    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
         1710  +  }
         1711  +
         1712  +  if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
  1651   1713       Fts5Sorter *pSorter = pCsr->pSorter;
  1652   1714       int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
  1653         -    n = pSorter->aIdx[iPhrase] - i1;
         1715  +    *pn = pSorter->aIdx[iPhrase] - i1;
  1654   1716       *pa = &pSorter->aPoslist[i1];
  1655   1717     }else{
  1656         -    n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
         1718  +    *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
  1657   1719     }
  1658         -  return n;
         1720  +
         1721  +  return rc;
  1659   1722   }
  1660   1723   
  1661   1724   /*
  1662   1725   ** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated
  1663   1726   ** correctly for the current view. Return SQLITE_OK if successful, or an
  1664   1727   ** SQLite error code otherwise.
  1665   1728   */
................................................................................
  1676   1739     aIter = pCsr->aInstIter;
  1677   1740   
  1678   1741     if( aIter ){
  1679   1742       int nInst = 0;                /* Number instances seen so far */
  1680   1743       int i;
  1681   1744   
  1682   1745       /* Initialize all iterators */
  1683         -    for(i=0; i<nIter; i++){
         1746  +    for(i=0; i<nIter && rc==SQLITE_OK; i++){
  1684   1747         const u8 *a;
  1685         -      int n = fts5CsrPoslist(pCsr, i, &a);
         1748  +      int n; 
         1749  +      rc = fts5CsrPoslist(pCsr, i, &a, &n);
  1686   1750         sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
  1687   1751       }
  1688   1752   
  1689         -    while( 1 ){
  1690         -      int *aInst;
  1691         -      int iBest = -1;
  1692         -      for(i=0; i<nIter; i++){
  1693         -        if( (aIter[i].bEof==0) 
  1694         -         && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
  1695         -        ){
  1696         -          iBest = i;
         1753  +    if( rc==SQLITE_OK ){
         1754  +      while( 1 ){
         1755  +        int *aInst;
         1756  +        int iBest = -1;
         1757  +        for(i=0; i<nIter; i++){
         1758  +          if( (aIter[i].bEof==0) 
         1759  +              && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
         1760  +            ){
         1761  +            iBest = i;
         1762  +          }
  1697   1763           }
  1698         -      }
  1699         -      if( iBest<0 ) break;
         1764  +        if( iBest<0 ) break;
  1700   1765   
  1701         -      nInst++;
  1702         -      if( nInst>=pCsr->nInstAlloc ){
  1703         -        pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
  1704         -        aInst = (int*)sqlite3_realloc(
  1705         -            pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
  1706         -        );
  1707         -        if( aInst ){
  1708         -          pCsr->aInst = aInst;
  1709         -        }else{
  1710         -          rc = SQLITE_NOMEM;
  1711         -          break;
         1766  +        nInst++;
         1767  +        if( nInst>=pCsr->nInstAlloc ){
         1768  +          pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
         1769  +          aInst = (int*)sqlite3_realloc(
         1770  +              pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
         1771  +              );
         1772  +          if( aInst ){
         1773  +            pCsr->aInst = aInst;
         1774  +          }else{
         1775  +            rc = SQLITE_NOMEM;
         1776  +            break;
         1777  +          }
  1712   1778           }
         1779  +
         1780  +        aInst = &pCsr->aInst[3 * (nInst-1)];
         1781  +        aInst[0] = iBest;
         1782  +        aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
         1783  +        aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
         1784  +        sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
  1713   1785         }
  1714         -
  1715         -      aInst = &pCsr->aInst[3 * (nInst-1)];
  1716         -      aInst[0] = iBest;
  1717         -      aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
  1718         -      aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
  1719         -      sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
  1720   1786       }
  1721   1787   
  1722   1788       pCsr->nInstCount = nInst;
  1723   1789       CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST);
  1724   1790     }
  1725   1791     return rc;
  1726   1792   }
................................................................................
  1745   1811     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1746   1812     int rc = SQLITE_OK;
  1747   1813     if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
  1748   1814      || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
  1749   1815     ){
  1750   1816       if( iIdx<0 || iIdx>=pCsr->nInstCount ){
  1751   1817         rc = SQLITE_RANGE;
         1818  +#if 0
         1819  +    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
         1820  +      *piPhrase = pCsr->aInst[iIdx*3];
         1821  +      *piCol = pCsr->aInst[iIdx*3 + 2];
         1822  +      *piOff = -1;
         1823  +#endif
  1752   1824       }else{
  1753   1825         *piPhrase = pCsr->aInst[iIdx*3];
  1754   1826         *piCol = pCsr->aInst[iIdx*3 + 1];
  1755   1827         *piOff = pCsr->aInst[iIdx*3 + 2];
  1756   1828       }
  1757   1829     }
  1758   1830     return rc;
  1759   1831   }
  1760   1832   
  1761   1833   static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  1762   1834     return fts5CursorRowid((Fts5Cursor*)pCtx);
  1763   1835   }
  1764   1836   
  1765         -static int fts5ApiColumnText(
  1766         -  Fts5Context *pCtx, 
  1767         -  int iCol, 
  1768         -  const char **pz, 
  1769         -  int *pn
  1770         -){
  1771         -  int rc = SQLITE_OK;
  1772         -  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1773         -  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
  1774         -    *pz = 0;
  1775         -    *pn = 0;
  1776         -  }else{
  1777         -    rc = fts5SeekCursor(pCsr, 0);
  1778         -    if( rc==SQLITE_OK ){
  1779         -      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
  1780         -      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
  1781         -    }
  1782         -  }
  1783         -  return rc;
  1784         -}
  1785         -
  1786   1837   static int fts5ColumnSizeCb(
  1787   1838     void *pContext,                 /* Pointer to int */
  1788   1839     int tflags,
  1789   1840     const char *pToken,             /* Buffer containing token */
  1790   1841     int nToken,                     /* Size of token in bytes */
  1791   1842     int iStart,                     /* Start offset of token */
  1792   1843     int iEnd                        /* End offset of token */
................................................................................
  1923   1974         *piOff = 0;
  1924   1975         pIter->a += fts5GetVarint32(pIter->a, iVal);
  1925   1976       }
  1926   1977       *piOff += (iVal-2);
  1927   1978     }
  1928   1979   }
  1929   1980   
  1930         -static void fts5ApiPhraseFirst(
         1981  +static int fts5ApiPhraseFirst(
  1931   1982     Fts5Context *pCtx, 
  1932   1983     int iPhrase, 
  1933   1984     Fts5PhraseIter *pIter, 
  1934   1985     int *piCol, int *piOff
  1935   1986   ){
  1936   1987     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1937         -  int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
  1938         -  pIter->b = &pIter->a[n];
  1939         -  *piCol = 0;
  1940         -  *piOff = 0;
  1941         -  fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
  1942         -}
         1988  +  int n;
         1989  +  int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
         1990  +  if( rc==SQLITE_OK ){
         1991  +    pIter->b = &pIter->a[n];
         1992  +    *piCol = 0;
         1993  +    *piOff = 0;
         1994  +    fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
         1995  +  }
         1996  +  return rc;
         1997  +}
         1998  +
         1999  +static void fts5ApiPhraseNextColumn(
         2000  +  Fts5Context *pCtx, 
         2001  +  Fts5PhraseIter *pIter, 
         2002  +  int *piCol
         2003  +){
         2004  +  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
         2005  +  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
         2006  +
         2007  +  if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
         2008  +    if( pIter->a>=pIter->b ){
         2009  +      *piCol = -1;
         2010  +    }else{
         2011  +      int iIncr;
         2012  +      pIter->a += fts5GetVarint32(&pIter->a[0], iIncr);
         2013  +      *piCol += (iIncr-2);
         2014  +    }
         2015  +  }else{
         2016  +    while( 1 ){
         2017  +      int dummy;
         2018  +      if( pIter->a>=pIter->b ){
         2019  +        *piCol = -1;
         2020  +        return;
         2021  +      }
         2022  +      if( pIter->a[0]==0x01 ) break;
         2023  +      pIter->a += fts5GetVarint32(pIter->a, dummy);
         2024  +    }
         2025  +    pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
         2026  +  }
         2027  +}
         2028  +
         2029  +static int fts5ApiPhraseFirstColumn(
         2030  +  Fts5Context *pCtx, 
         2031  +  int iPhrase, 
         2032  +  Fts5PhraseIter *pIter, 
         2033  +  int *piCol
         2034  +){
         2035  +  int rc = SQLITE_OK;
         2036  +  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
         2037  +  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
         2038  +
         2039  +  if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
         2040  +    int n;
         2041  +    rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
         2042  +    if( rc==SQLITE_OK ){
         2043  +      pIter->b = &pIter->a[n];
         2044  +      *piCol = 0;
         2045  +      fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
         2046  +    }
         2047  +  }else{
         2048  +    int n;
         2049  +    rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
         2050  +    if( rc==SQLITE_OK ){
         2051  +      pIter->b = &pIter->a[n];
         2052  +      if( n<=0 ){
         2053  +        *piCol = -1;
         2054  +      }else if( pIter->a[0]==0x01 ){
         2055  +        pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
         2056  +      }else{
         2057  +        *piCol = 0;
         2058  +      }
         2059  +    }
         2060  +  }
         2061  +
         2062  +  return rc;
         2063  +}
         2064  +
  1943   2065   
  1944   2066   static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
  1945   2067       int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
  1946   2068   );
  1947   2069   
  1948   2070   static const Fts5ExtensionApi sFts5Api = {
  1949   2071     2,                            /* iVersion */
................................................................................
  1960   2082     fts5ApiColumnText,
  1961   2083     fts5ApiColumnSize,
  1962   2084     fts5ApiQueryPhrase,
  1963   2085     fts5ApiSetAuxdata,
  1964   2086     fts5ApiGetAuxdata,
  1965   2087     fts5ApiPhraseFirst,
  1966   2088     fts5ApiPhraseNext,
         2089  +  fts5ApiPhraseFirstColumn,
         2090  +  fts5ApiPhraseNextColumn,
  1967   2091   };
  1968         -
  1969   2092   
  1970   2093   /*
  1971   2094   ** Implementation of API function xQueryPhrase().
  1972   2095   */
  1973   2096   static int fts5ApiQueryPhrase(
  1974   2097     Fts5Context *pCtx, 
  1975   2098     int iPhrase, 
................................................................................
  2094   2217   static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
  2095   2218     int i;
  2096   2219     int rc = SQLITE_OK;
  2097   2220     int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
  2098   2221     Fts5Buffer val;
  2099   2222   
  2100   2223     memset(&val, 0, sizeof(Fts5Buffer));
  2101         -
  2102         -  /* Append the varints */
  2103         -  for(i=0; i<(nPhrase-1); i++){
  2104         -    const u8 *dummy;
  2105         -    int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
  2106         -    sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
  2107         -  }
  2108         -
  2109         -  /* Append the position lists */
  2110         -  for(i=0; i<nPhrase; i++){
  2111         -    const u8 *pPoslist;
  2112         -    int nPoslist;
  2113         -    nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
  2114         -    sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
         2224  +  switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){
         2225  +    case FTS5_DETAIL_FULL:
         2226  +
         2227  +      /* Append the varints */
         2228  +      for(i=0; i<(nPhrase-1); i++){
         2229  +        const u8 *dummy;
         2230  +        int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
         2231  +        sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
         2232  +      }
         2233  +
         2234  +      /* Append the position lists */
         2235  +      for(i=0; i<nPhrase; i++){
         2236  +        const u8 *pPoslist;
         2237  +        int nPoslist;
         2238  +        nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
         2239  +        sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
         2240  +      }
         2241  +      break;
         2242  +
         2243  +    case FTS5_DETAIL_COLUMNS:
         2244  +
         2245  +      /* Append the varints */
         2246  +      for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){
         2247  +        const u8 *dummy;
         2248  +        int nByte;
         2249  +        rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte);
         2250  +        sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
         2251  +      }
         2252  +
         2253  +      /* Append the position lists */
         2254  +      for(i=0; rc==SQLITE_OK && i<nPhrase; i++){
         2255  +        const u8 *pPoslist;
         2256  +        int nPoslist;
         2257  +        rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist);
         2258  +        sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
         2259  +      }
         2260  +      break;
         2261  +
         2262  +    default:
         2263  +      break;
  2115   2264     }
  2116   2265   
  2117   2266     sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free);
  2118   2267     return rc;
  2119   2268   }
  2120   2269   
  2121   2270   /* 

Changes to ext/fts5/fts5_storage.c.

   821    821   */
   822    822   typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
   823    823   struct Fts5IntegrityCtx {
   824    824     i64 iRowid;
   825    825     int iCol;
   826    826     int szCol;
   827    827     u64 cksum;
          828  +  Fts5Termset *pTermset;
   828    829     Fts5Config *pConfig;
   829    830   };
          831  +
   830    832   
   831    833   /*
   832    834   ** Tokenization callback used by integrity check.
   833    835   */
   834    836   static int fts5StorageIntegrityCallback(
   835         -  void *pContext,                 /* Pointer to Fts5InsertCtx object */
          837  +  void *pContext,                 /* Pointer to Fts5IntegrityCtx object */
   836    838     int tflags,
   837    839     const char *pToken,             /* Buffer containing token */
   838    840     int nToken,                     /* Size of token in bytes */
   839    841     int iStart,                     /* Start offset of token */
   840    842     int iEnd                        /* End offset of token */
   841    843   ){
   842    844     Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
          845  +  Fts5Termset *pTermset = pCtx->pTermset;
          846  +  int bPresent;
          847  +  int ii;
          848  +  int rc = SQLITE_OK;
          849  +  int iPos;
          850  +  int iCol;
          851  +
   843    852     if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
   844    853       pCtx->szCol++;
   845    854     }
   846         -  pCtx->cksum ^= sqlite3Fts5IndexCksum(
   847         -      pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken
   848         -  );
   849         -  return SQLITE_OK;
          855  +
          856  +  switch( pCtx->pConfig->eDetail ){
          857  +    case FTS5_DETAIL_FULL:
          858  +      iPos = pCtx->szCol-1;
          859  +      iCol = pCtx->iCol;
          860  +      break;
          861  +
          862  +    case FTS5_DETAIL_COLUMNS:
          863  +      iPos = pCtx->iCol;
          864  +      iCol = 0;
          865  +      break;
          866  +
          867  +    default:
          868  +      assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
          869  +      iPos = 0;
          870  +      iCol = 0;
          871  +      break;
          872  +  }
          873  +
          874  +  rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
          875  +  if( rc==SQLITE_OK && bPresent==0 ){
          876  +    pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
          877  +        pCtx->iRowid, iCol, iPos, 0, pToken, nToken
          878  +    );
          879  +  }
          880  +
          881  +  for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){
          882  +    const int nChar = pCtx->pConfig->aPrefix[ii];
          883  +    int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
          884  +    if( nByte ){
          885  +      rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
          886  +      if( bPresent==0 ){
          887  +        pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
          888  +            pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
          889  +        );
          890  +      }
          891  +    }
          892  +  }
          893  +
          894  +  return rc;
   850    895   }
   851    896   
   852    897   /*
   853    898   ** Check that the contents of the FTS index match that of the %_content
   854    899   ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
   855    900   ** some other SQLite error code if an error occurs while attempting to
   856    901   ** determine this.
................................................................................
   877    922       int rc2;
   878    923       while( SQLITE_ROW==sqlite3_step(pScan) ){
   879    924         int i;
   880    925         ctx.iRowid = sqlite3_column_int64(pScan, 0);
   881    926         ctx.szCol = 0;
   882    927         if( pConfig->bColumnsize ){
   883    928           rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
          929  +      }
          930  +      if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
          931  +        rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
   884    932         }
   885    933         for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
   886    934           if( pConfig->abUnindexed[i] ) continue;
   887    935           ctx.iCol = i;
   888    936           ctx.szCol = 0;
   889         -        rc = sqlite3Fts5Tokenize(pConfig, 
   890         -            FTS5_TOKENIZE_DOCUMENT,
   891         -            (const char*)sqlite3_column_text(pScan, i+1),
   892         -            sqlite3_column_bytes(pScan, i+1),
   893         -            (void*)&ctx,
   894         -            fts5StorageIntegrityCallback
   895         -        );
   896         -        if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
          937  +        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
          938  +          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
          939  +        }
          940  +        if( rc==SQLITE_OK ){
          941  +          rc = sqlite3Fts5Tokenize(pConfig, 
          942  +              FTS5_TOKENIZE_DOCUMENT,
          943  +              (const char*)sqlite3_column_text(pScan, i+1),
          944  +              sqlite3_column_bytes(pScan, i+1),
          945  +              (void*)&ctx,
          946  +              fts5StorageIntegrityCallback
          947  +          );
          948  +        }
          949  +        if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
   897    950             rc = FTS5_CORRUPT;
   898    951           }
   899    952           aTotalSize[i] += ctx.szCol;
          953  +        if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
          954  +          sqlite3Fts5TermsetFree(ctx.pTermset);
          955  +          ctx.pTermset = 0;
          956  +        }
   900    957         }
          958  +      sqlite3Fts5TermsetFree(ctx.pTermset);
          959  +      ctx.pTermset = 0;
          960  +
   901    961         if( rc!=SQLITE_OK ) break;
   902    962       }
   903    963       rc2 = sqlite3_reset(pScan);
   904    964       if( rc==SQLITE_OK ) rc = rc2;
   905    965     }
   906    966   
   907    967     /* Test that the "totals" (sometimes called "averages") record looks Ok */

Changes to ext/fts5/fts5_tcl.c.

   231    231       { "xColumnText",       1, "COL" },                /*  9 */
   232    232       { "xColumnSize",       1, "COL" },                /* 10 */
   233    233       { "xQueryPhrase",      2, "PHRASE SCRIPT" },      /* 11 */
   234    234       { "xSetAuxdata",       1, "VALUE" },              /* 12 */
   235    235       { "xGetAuxdata",       1, "CLEAR" },              /* 13 */
   236    236       { "xSetAuxdataInt",    1, "INTEGER" },            /* 14 */
   237    237       { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
          238  +    { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
          239  +    { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
   238    240       { 0, 0, 0}
   239    241     };
   240    242   
   241    243     int rc;
   242    244     int iSub = 0;
   243    245     F5tApi *p = (F5tApi*)clientData;
   244    246   
................................................................................
   424    426       }
   425    427       CASE(15, "xGetAuxdataInt") {
   426    428         int iVal;
   427    429         int bClear;
   428    430         if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
   429    431         iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
   430    432         Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
          433  +      break;
          434  +    }
          435  +
          436  +    CASE(16, "xPhraseForeach") {
          437  +      int iPhrase;
          438  +      int iCol;
          439  +      int iOff;
          440  +      const char *zColvar;
          441  +      const char *zOffvar;
          442  +      Tcl_Obj *pScript = objv[5];
          443  +      Fts5PhraseIter iter;
          444  +
          445  +      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
          446  +      zColvar = Tcl_GetString(objv[3]);
          447  +      zOffvar = Tcl_GetString(objv[4]);
          448  +
          449  +      for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
          450  +          iCol>=0;
          451  +          p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff)
          452  +      ){
          453  +        Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
          454  +        Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
          455  +        rc = Tcl_EvalObjEx(interp, pScript, 0);
          456  +        if( rc==TCL_CONTINUE ) rc = TCL_OK;
          457  +        if( rc!=TCL_OK ){
          458  +          if( rc==TCL_BREAK ) rc = TCL_OK;
          459  +          break;
          460  +        }
          461  +      }
          462  +
          463  +      break;
          464  +    }
          465  +
          466  +    CASE(17, "xPhraseColumnForeach") {
          467  +      int iPhrase;
          468  +      int iCol;
          469  +      const char *zColvar;
          470  +      Tcl_Obj *pScript = objv[4];
          471  +      Fts5PhraseIter iter;
          472  +
          473  +      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
          474  +      zColvar = Tcl_GetString(objv[3]);
          475  +
          476  +      for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
          477  +          iCol>=0;
          478  +          p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)
          479  +      ){
          480  +        Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
          481  +        rc = Tcl_EvalObjEx(interp, pScript, 0);
          482  +        if( rc==TCL_CONTINUE ) rc = TCL_OK;
          483  +        if( rc!=TCL_OK ){
          484  +          if( rc==TCL_BREAK ) rc = TCL_OK;
          485  +          break;
          486  +        }
          487  +      }
          488  +
   431    489         break;
   432    490       }
   433    491   
   434    492       default: 
   435    493         assert( 0 );
   436    494         break;
   437    495     }

Changes to ext/fts5/fts5_test_mi.c.

   130    130   ){
   131    131     Fts5PhraseIter iter;
   132    132     int iCol, iOff;
   133    133     u32 *aOut = (u32*)pUserData;
   134    134     int iPrev = -1;
   135    135   
   136    136     for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); 
   137         -      iOff>=0; 
          137  +      iCol>=0; 
   138    138         pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
   139    139     ){
   140    140       aOut[iCol*3+1]++;
   141    141       if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
   142    142       iPrev = iCol;
   143    143     }
   144    144   
................................................................................
   207    207     char f,
   208    208     u32 *aOut
   209    209   ){
   210    210     int i;
   211    211     int rc = SQLITE_OK;
   212    212   
   213    213     switch( f ){
   214         -    case 'b': 
          214  +    case 'b': {
          215  +      int iPhrase;
          216  +      int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
          217  +      for(i=0; i<nInt; i++) aOut[i] = 0;
          218  +
          219  +      for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
          220  +        Fts5PhraseIter iter;
          221  +        int iCol;
          222  +        for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
          223  +            iCol>=0; 
          224  +            pApi->xPhraseNextColumn(pFts, &iter, &iCol)
          225  +        ){
          226  +          aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
          227  +        }
          228  +      }
          229  +
          230  +      break;
          231  +    }
          232  +
   215    233       case 'x':
   216    234       case 'y': {
   217    235         int nMul = (f=='x' ? 3 : 1);
   218    236         int iPhrase;
   219    237   
   220         -      if( f=='b' ){
   221         -        int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
   222         -        for(i=0; i<nInt; i++) aOut[i] = 0;
   223         -      }else{
   224         -        for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
   225         -      }
          238  +      for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
   226    239   
   227    240         for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
   228    241           Fts5PhraseIter iter;
   229    242           int iOff, iCol;
   230    243           for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); 
   231    244               iOff>=0; 
   232    245               pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)

Changes to ext/fts5/fts5_vocab.c.

   375    375     int rc = SQLITE_OK;
   376    376     int nCol = pCsr->pConfig->nCol;
   377    377   
   378    378     pCsr->rowid++;
   379    379   
   380    380     if( pTab->eType==FTS5_VOCAB_COL ){
   381    381       for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
   382         -      if( pCsr->aCnt[pCsr->iCol] ) break;
          382  +      if( pCsr->aDoc[pCsr->iCol] ) break;
   383    383       }
   384    384     }
   385    385   
   386    386     if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
   387    387       if( sqlite3Fts5IterEof(pCsr->pIter) ){
   388    388         pCsr->bEof = 1;
   389    389       }else{
................................................................................
   408    408         assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
   409    409         while( rc==SQLITE_OK ){
   410    410           i64 dummy;
   411    411           const u8 *pPos; int nPos;   /* Position list */
   412    412           i64 iPos = 0;               /* 64-bit position read from poslist */
   413    413           int iOff = 0;               /* Current offset within position list */
   414    414   
   415         -        rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
   416         -        if( rc==SQLITE_OK ){
   417         -          if( pTab->eType==FTS5_VOCAB_ROW ){
   418         -            while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   419         -              pCsr->aCnt[0]++;
          415  +        switch( pCsr->pConfig->eDetail ){
          416  +          case FTS5_DETAIL_FULL:
          417  +            rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
          418  +            if( rc==SQLITE_OK ){
          419  +              if( pTab->eType==FTS5_VOCAB_ROW ){
          420  +                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
          421  +                  pCsr->aCnt[0]++;
          422  +                }
          423  +                pCsr->aDoc[0]++;
          424  +              }else{
          425  +                int iCol = -1;
          426  +                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
          427  +                  int ii = FTS5_POS2COLUMN(iPos);
          428  +                  pCsr->aCnt[ii]++;
          429  +                  if( iCol!=ii ){
          430  +                    pCsr->aDoc[ii]++;
          431  +                    iCol = ii;
          432  +                  }
          433  +                }
          434  +              }
   420    435               }
   421         -            pCsr->aDoc[0]++;
   422         -          }else{
   423         -            int iCol = -1;
   424         -            while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   425         -              int ii = FTS5_POS2COLUMN(iPos);
   426         -              pCsr->aCnt[ii]++;
   427         -              if( iCol!=ii ){
   428         -                pCsr->aDoc[ii]++;
   429         -                iCol = ii;
          436  +            break;
          437  +
          438  +          case FTS5_DETAIL_COLUMNS:
          439  +            if( pTab->eType==FTS5_VOCAB_ROW ){
          440  +              pCsr->aDoc[0]++;
          441  +            }else{
          442  +              Fts5Buffer buf = {0, 0, 0};
          443  +              rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
          444  +              if( rc==SQLITE_OK ){
          445  +                while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
          446  +                  assert_nc( iPos>=0 && iPos<nCol );
          447  +                  if( iPos<nCol ) pCsr->aDoc[iPos]++;
          448  +                }
   430    449                 }
          450  +              sqlite3Fts5BufferFree(&buf);
   431    451               }
   432         -          }
          452  +            break;
          453  +
          454  +          default: 
          455  +            assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
          456  +            pCsr->aDoc[0]++;
          457  +            break;
          458  +        }
          459  +
          460  +        if( rc==SQLITE_OK ){
   433    461             rc = sqlite3Fts5IterNextScan(pCsr->pIter);
   434    462           }
   435    463   
   436    464           if( rc==SQLITE_OK ){
   437    465             zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
   438    466             if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
   439    467               break;
................................................................................
   441    469             if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
   442    470           }
   443    471         }
   444    472       }
   445    473     }
   446    474   
   447    475     if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
   448         -    while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++;
          476  +    while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
   449    477       assert( pCsr->iCol<pCsr->pConfig->nCol );
   450    478     }
   451    479     return rc;
   452    480   }
   453    481   
   454    482   /*
   455    483   ** This is the xFilter implementation for the virtual table.
................................................................................
   521    549   
   522    550   static int fts5VocabColumnMethod(
   523    551     sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
   524    552     sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
   525    553     int iCol                        /* Index of column to read value from */
   526    554   ){
   527    555     Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
          556  +  int eDetail = pCsr->pConfig->eDetail;
          557  +  int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
          558  +  i64 iVal = 0;
   528    559   
   529    560     if( iCol==0 ){
   530    561       sqlite3_result_text(
   531    562           pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
   532    563       );
   533         -  }
   534         -  else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){
          564  +  }else if( eType==FTS5_VOCAB_COL ){
   535    565       assert( iCol==1 || iCol==2 || iCol==3 );
   536    566       if( iCol==1 ){
   537         -      const char *z = pCsr->pConfig->azCol[pCsr->iCol];
   538         -      sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
          567  +      if( eDetail!=FTS5_DETAIL_NONE ){
          568  +        const char *z = pCsr->pConfig->azCol[pCsr->iCol];
          569  +        sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
          570  +      }
   539    571       }else if( iCol==2 ){
   540         -      sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]);
          572  +      iVal = pCsr->aDoc[pCsr->iCol];
   541    573       }else{
   542         -      sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]);
          574  +      iVal = pCsr->aCnt[pCsr->iCol];
   543    575       }
   544    576     }else{
   545    577       assert( iCol==1 || iCol==2 );
   546    578       if( iCol==1 ){
   547         -      sqlite3_result_int64(pCtx, pCsr->aDoc[0]);
          579  +      iVal = pCsr->aDoc[0];
   548    580       }else{
   549         -      sqlite3_result_int64(pCtx, pCsr->aCnt[0]);
          581  +      iVal = pCsr->aCnt[0];
   550    582       }
   551    583     }
          584  +
          585  +  if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
   552    586     return SQLITE_OK;
   553    587   }
   554    588   
   555    589   /* 
   556    590   ** This is the xRowid method. The SQLite core calls this routine to
   557    591   ** retrieve the rowid for the current row of the result set. The
   558    592   ** rowid should be written to *pRowid.

Changes to ext/fts5/test/fts5_common.tcl.

    23     23   proc fts5_test_poslist {cmd} {
    24     24     set res [list]
    25     25     for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    26     26       lappend res [string map {{ } .} [$cmd xInst $i]]
    27     27     }
    28     28     set res
    29     29   }
           30  +
           31  +proc fts5_test_poslist2 {cmd} {
           32  +  set res [list]
           33  +
           34  +  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
           35  +    $cmd xPhraseForeach $i c o {
           36  +      lappend res $i.$c.$o
           37  +    }
           38  +  }
           39  +
           40  +  set res
           41  +}
           42  +
           43  +proc fts5_test_collist {cmd} {
           44  +  set res [list]
           45  +
           46  +  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
           47  +    $cmd xPhraseColumnForeach $i c { lappend res $i.$c }
           48  +  }
           49  +
           50  +  set res
           51  +}
    30     52   
    31     53   proc fts5_test_columnsize {cmd} {
    32     54     set res [list]
    33     55     for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    34     56       lappend res [$cmd xColumnSize $i]
    35     57     }
    36     58     set res
................................................................................
   109    131   
   110    132   proc fts5_aux_test_functions {db} {
   111    133     foreach f {
   112    134       fts5_test_columnsize
   113    135       fts5_test_columntext
   114    136       fts5_test_columntotalsize
   115    137       fts5_test_poslist
          138  +    fts5_test_poslist2
          139  +    fts5_test_collist
   116    140       fts5_test_tokenize
   117    141       fts5_test_rowcount
   118    142       fts5_test_all
   119    143   
   120    144       fts5_test_queryphrase
   121    145       fts5_test_phrasecount
   122    146     } {
................................................................................
   174    198   #   <phrase number> . <column number> . <token offset>
   175    199   #
   176    200   # Options:
   177    201   #
   178    202   #   -near N        (NEAR distance. Default 10)
   179    203   #   -col  C        (List of column indexes to match against)
   180    204   #   -pc   VARNAME  (variable in caller frame to use for phrase numbering)
          205  +#   -dict VARNAME  (array in caller frame to use for synonyms)
   181    206   #
   182    207   proc nearset {aCol args} {
          208  +
          209  +  # Process the command line options.
          210  +  #
   183    211     set O(-near) 10
   184    212     set O(-col)  {}
   185    213     set O(-pc)   ""
          214  +  set O(-dict) ""
   186    215   
   187    216     set nOpt [lsearch -exact $args --]
   188    217     if {$nOpt<0} { error "no -- option" }
          218  +
          219  +  # Set $lPhrase to be a list of phrases. $nPhrase its length.
          220  +  set lPhrase [lrange $args [expr $nOpt+1] end]
          221  +  set nPhrase [llength $lPhrase]
   189    222   
   190    223     foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
   191    224       if {[info exists O($k)]==0} { error "unrecognized option $k" }
   192    225       set O($k) $v
   193    226     }
   194    227   
   195    228     if {$O(-pc) == ""} {
   196    229       set counter 0
   197    230     } else {
   198    231       upvar $O(-pc) counter
   199    232     }
   200    233   
   201         -  # Set $phraselist to be a list of phrases. $nPhrase its length.
   202         -  set phraselist [lrange $args [expr $nOpt+1] end]
   203         -  set nPhrase [llength $phraselist]
          234  +  if {$O(-dict)!=""} { upvar $O(-dict) aDict }
   204    235   
   205    236     for {set j 0} {$j < [llength $aCol]} {incr j} {
   206    237       for {set i 0} {$i < $nPhrase} {incr i} { 
   207    238         set A($j,$i) [list]
   208    239       }
   209    240     }
   210    241   
   211         -  set iCol -1
   212         -  foreach col $aCol {
   213         -    incr iCol
          242  +  # Loop through each column of the current row.
          243  +  for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
          244  +
          245  +    # If there is a column filter, test whether this column is excluded. If
          246  +    # so, skip to the next iteration of this loop. Otherwise, set zCol to the
          247  +    # column value and nToken to the number of tokens that comprise it.
   214    248       if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
   215         -    set nToken [llength $col]
          249  +    set zCol [lindex $aCol $iCol]
          250  +    set nToken [llength $zCol]
   216    251   
   217         -    set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
   218         -    for { } {$iFL < $nToken} {incr iFL} {
   219         -      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
   220         -        set B($iPhrase) [list]
   221         -      }
          252  +    # Each iteration of the following loop searches a substring of the 
          253  +    # column value for phrase matches. The last token of the substring
          254  +    # is token $iLast of the column value. The first token is:
          255  +    #
          256  +    #   iFirst = ($iLast - $O(-near) - 1)
          257  +    #
          258  +    # where $sz is the length of the phrase being searched for. A phrase 
          259  +    # counts as matching the substring if its first token lies on or before
          260  +    # $iLast and its last token on or after $iFirst.
          261  +    #
          262  +    # For example, if the query is "NEAR(a+b c, 2)" and the column value:
          263  +    #
          264  +    #   "x x x x A B x x C x"
          265  +    #    0 1 2 3 4 5 6 7 8 9"
          266  +    #
          267  +    # when (iLast==8 && iFirst=5) the range will contain both phrases and
          268  +    # so both instances can be added to the output poslists.
          269  +    #
          270  +    set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
          271  +    for { } {$iLast < $nToken} {incr iLast} {
          272  +
          273  +      catch { array unset B }
   222    274         
   223    275         for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
   224         -        set p [lindex $phraselist $iPhrase]
          276  +        set p [lindex $lPhrase $iPhrase]
   225    277           set nPm1 [expr {[llength $p] - 1}]
   226         -        set iFirst [expr $iFL - $O(-near) - [llength $p]]
          278  +        set iFirst [expr $iLast - $O(-near) - [llength $p]]
   227    279   
   228         -        for {set i $iFirst} {$i <= $iFL} {incr i} {
   229         -          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
          280  +        for {set i $iFirst} {$i <= $iLast} {incr i} {
          281  +          set lCand [lrange $zCol $i [expr $i+$nPm1]]
          282  +          set bMatch 1
          283  +          foreach tok $p term $lCand {
          284  +            if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break }
          285  +          }
          286  +          if {$bMatch} { lappend B($iPhrase) $i }
   230    287           }
   231         -        if {[llength $B($iPhrase)] == 0} break
          288  +
          289  +        if {![info exists B($iPhrase)]} break
   232    290         }
   233    291   
   234    292         if {$iPhrase==$nPhrase} {
   235    293           for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
   236    294             set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
   237    295             set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
   238    296           }
................................................................................
   248    306         foreach a $A($iCol,$iPhrase) {
   249    307           lappend res "$counter.$iCol.$a"
   250    308         }
   251    309       }
   252    310       incr counter
   253    311     }
   254    312   
   255         -  #puts $res
          313  +  #puts "$aCol -> $res"
   256    314     sort_poslist $res
   257    315   }
          316  +
          317  +proc nearset_match {aDictVar tok term} {
          318  +  if {[string match $tok $term]} { return 1 }
          319  +
          320  +  upvar $aDictVar aDict
          321  +  if {[info exists aDict($tok)]} {
          322  +    foreach s $aDict($tok) {
          323  +      if {[string match $s $term]} { return 1 }
          324  +    }
          325  +  }
          326  +  return 0;
          327  +}
   258    328   
   259    329   #-------------------------------------------------------------------------
   260    330   # Usage:
   261    331   #
   262    332   #   sort_poslist LIST
   263    333   #
   264    334   # Sort a position list of the type returned by command [nearset]
................................................................................
   322    392       lappend ret $word $iOff [expr $iOff+$nToken]
   323    393       incr iOff $nToken
   324    394       incr iOff [gobble_whitespace text]
   325    395     }
   326    396   
   327    397     set ret
   328    398   }
          399  +
          400  +#-------------------------------------------------------------------------
          401  +#
          402  +proc foreach_detail_mode {prefix script} {
          403  +  set saved $::testprefix
          404  +  foreach d [list full col none] {
          405  +    set s [string map [list %DETAIL% $d] $script]
          406  +    set ::detail $d
          407  +    set ::testprefix "$prefix-$d"
          408  +    reset_db
          409  +    uplevel $s
          410  +    unset ::detail
          411  +  }
          412  +  set ::testprefix $saved
          413  +}
          414  +
          415  +proc detail_check {} {
          416  +  if {$::detail != "none" && $::detail!="full" && $::detail!="col"} {
          417  +    error "not in foreach_detail_mode {...} block"
          418  +  }
          419  +}
          420  +proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
          421  +proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
          422  +proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
          423  +
          424  +
          425  +#-------------------------------------------------------------------------
          426  +# Convert a poslist of the type returned by fts5_test_poslist() to a 
          427  +# collist as returned by fts5_test_collist().
          428  +#
          429  +proc fts5_poslist2collist {poslist} {
          430  +  set res [list]
          431  +  foreach h $poslist {
          432  +    regexp {(.*)\.[1234567890]+} $h -> cand
          433  +    lappend res $cand
          434  +  }
          435  +  set res [lsort -command fts5_collist_elem_compare -unique $res]
          436  +  return $res
          437  +}
          438  +
          439  +# Comparison function used by fts5_poslist2collist to sort collist entries.
          440  +proc fts5_collist_elem_compare {a b} {
          441  +  foreach {a1 a2} [split $a .] {}
          442  +  foreach {b1 b2} [split $b .] {}
          443  +
          444  +  if {$a1==$b1} { return [expr $a2 - $b2] }
          445  +  return [expr $a1 - $b1]
          446  +}
          447  +
          448  +
          449  +#--------------------------------------------------------------------------
          450  +# Construct and return a tcl list equivalent to that returned by the SQL
          451  +# query executed against database handle [db]:
          452  +#
          453  +#   SELECT 
          454  +#     rowid, 
          455  +#     fts5_test_poslist($tbl),
          456  +#     fts5_test_collist($tbl) 
          457  +#   FROM $tbl('$expr')
          458  +#   ORDER BY rowid $order;
          459  +#
          460  +proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} {
          461  +
          462  +  # Figure out the set of columns in the FTS5 table. This routine does
          463  +  # not handle tables with UNINDEXED columns, but if it did, it would
          464  +  # have to be here.
          465  +  db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
          466  +
          467  +  set d ""
          468  +  if {$aDictVar != ""} {
          469  +    upvar $aDictVar aDict
          470  +    set d aDict
          471  +  }
          472  +
          473  +  set cols ""
          474  +  foreach e $lCols { append cols ", '$e'" }
          475  +  set tclexpr [db one [subst -novar {
          476  +    SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] )
          477  +  }]]
          478  +
          479  +  set res [list]
          480  +  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
          481  +    set cols [list]
          482  +    foreach col $lCols { lappend cols $x($col) }
          483  +    
          484  +    set ::pc 0
          485  +    set rowdata [eval $tclexpr]
          486  +    if {$rowdata != ""} { 
          487  +      lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
          488  +    }
          489  +  }
          490  +
          491  +  set res
          492  +}
          493  +
          494  +#-------------------------------------------------------------------------
          495  +# Similar to [fts5_query_data], but omit the collist field.
          496  +#
          497  +proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} {
          498  +  set res [list]
          499  +
          500  +  if {$aDictVar!=""} {
          501  +    upvar $aDictVar aDict
          502  +    set dict aDict
          503  +  } else {
          504  +    set dict ""
          505  +  }
          506  +
          507  +  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
          508  +    lappend res $rowid $poslist
          509  +  }
          510  +  set res
          511  +}
          512  +
          513  +#-------------------------------------------------------------------------
          514  +#
          515  +
          516  +# This command will only work inside a [foreach_detail_mode] block. It tests
          517  +# whether or not expression $expr run on FTS5 table $tbl is supported by
          518  +# the current mode. If so, 1 is returned. If not, 0.
          519  +#
          520  +#   detail=full    (all queries supported)
          521  +#   detail=col     (all but phrase queries and NEAR queries)
          522  +#   detail=none    (all but phrase queries, NEAR queries, and column filters)
          523  +#
          524  +proc fts5_expr_ok {expr tbl} {
          525  +
          526  +  if {![detail_is_full]} {
          527  +    set nearset "nearset_rc"
          528  +    if {[detail_is_col]} { set nearset "nearset_rf" }
          529  +
          530  +    set ::expr_not_ok 0
          531  +    db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
          532  +
          533  +    set cols ""
          534  +    foreach e $lCols { append cols ", '$e'" }
          535  +    set ::pc 0
          536  +    set tclexpr [db one [subst -novar {
          537  +      SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
          538  +    }]]
          539  +    eval $tclexpr
          540  +    if {$::expr_not_ok} { return 0 }
          541  +  }
          542  +
          543  +  return 1
          544  +}
          545  +
          546  +# Helper for [fts5_expr_ok]
          547  +proc nearset_rf {aCol args} {
          548  +  set idx [lsearch -exact $args --]
          549  +  if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
          550  +    set ::expr_not_ok 1
          551  +  }
          552  +  list
          553  +}
          554  +
          555  +# Helper for [fts5_expr_ok]
          556  +proc nearset_rc {aCol args} {
          557  +  nearset_rf $aCol {*}$args
          558  +  if {[lsearch $args -col]>=0} { 
          559  +    set ::expr_not_ok 1
          560  +  }
          561  +  list
          562  +}
   329    563   

Changes to ext/fts5/test/fts5aa.test.

    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
    23     23   
           24  +foreach_detail_mode $::testprefix {
           25  +
    24     26   do_execsql_test 1.0 {
    25     27     CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
    26     28     SELECT name, sql FROM sqlite_master;
    27     29   } {
    28     30     t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
    29     31     t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
    30     32     t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID}
................................................................................
    37     39     DROP TABLE t1;
    38     40     SELECT name, sql FROM sqlite_master;
    39     41   } {
    40     42   }
    41     43   
    42     44   #-------------------------------------------------------------------------
    43     45   #
    44         -reset_db
           46  +
    45     47   do_execsql_test 2.0 {
    46         -  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
           48  +  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
    47     49   }
    48     50   do_execsql_test 2.1 {
    49     51     INSERT INTO t1 VALUES('a b c', 'd e f');
    50     52   }
    51     53   
    52     54   do_test 2.2 {
    53     55     execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
................................................................................
    61     63       SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC;
    62     64     } {1}
    63     65   }
    64     66   
    65     67   do_execsql_test 2.4 {
    66     68     INSERT INTO t1(t1) VALUES('integrity-check');
    67     69   }
           70  +
    68     71   
    69     72   #-------------------------------------------------------------------------
    70     73   #
    71     74   reset_db
    72     75   do_execsql_test 3.0 {
    73         -  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
           76  +  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
    74     77   }
    75     78   foreach {i x y} {
    76     79      1  {g f d b f} {h h e i a}
    77     80      2  {f i g j e} {i j c f f}
    78     81      3  {e e i f a} {e h f d f}
    79     82      4  {h j f j i} {h a c f j}
    80     83      5  {d b j c g} {f e i b e}
................................................................................
    89     92     if {[set_test_counter errors]} break
    90     93   }
    91     94   
    92     95   #-------------------------------------------------------------------------
    93     96   #
    94     97   reset_db
    95     98   do_execsql_test 4.0 {
    96         -  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
           99  +  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
    97    100     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    98    101   }
    99    102   foreach {i x y} {
   100    103      1  {g f d b f} {h h e i a}
   101    104      2  {f i g j e} {i j c f f}
   102    105      3  {e e i f a} {e h f d f}
   103    106      4  {h j f j i} {h a c f j}
................................................................................
   113    116     if {[set_test_counter errors]} break
   114    117   }
   115    118   
   116    119   #-------------------------------------------------------------------------
   117    120   #
   118    121   reset_db
   119    122   do_execsql_test 5.0 {
   120         -  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
          123  +  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
   121    124     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   122    125   }
   123    126   foreach {i x y} {
   124    127      1  {dd abc abc abc abcde} {aaa dd ddd ddd aab}
   125    128      2  {dd aab d aaa b} {abcde c aaa aaa aaa}
   126    129      3  {abcde dd b b dd} {abc abc d abc ddddd}
   127    130      4  {aaa abcde dddd dddd abcde} {abc b b abcde abc}
................................................................................
   137    140     if {[set_test_counter errors]} break
   138    141   }
   139    142   
   140    143   #-------------------------------------------------------------------------
   141    144   #
   142    145   reset_db
   143    146   do_execsql_test 6.0 {
   144         -  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
          147  +  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
   145    148     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   146    149   }
   147    150   
   148    151   do_execsql_test 6.1 {
   149    152     INSERT  INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
   150    153     REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
   151    154   }
................................................................................
   272    275   }
   273    276   
   274    277   
   275    278   #-------------------------------------------------------------------------
   276    279   #
   277    280   reset_db
   278    281   do_execsql_test 10.0 {
   279         -  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
          282  +  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
   280    283   }
   281    284   set d10 {
   282    285      1  {g f d b f} {h h e i a}
   283    286      2  {f i g j e} {i j c f f}
   284    287      3  {e e i f a} {e h f d f}
   285    288      4  {h j f j i} {h a c f j}
   286    289      5  {d b j c g} {f e i b e}
................................................................................
   305    308   
   306    309   do_execsql_test 10.4.1 { DELETE FROM t1 }
   307    310   do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
   308    311   
   309    312   #-------------------------------------------------------------------------
   310    313   #
   311    314   do_catchsql_test 11.1 {
   312         -  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
          315  +  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
   313    316   } {1 {reserved fts5 column name: rank}}
   314    317   do_catchsql_test 11.2 {
   315         -  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
          318  +  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
   316    319   } {1 {reserved fts5 table name: rank}}
   317    320   do_catchsql_test 11.3 {
   318         -  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid);
          321  +  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
   319    322   } {1 {reserved fts5 column name: rowid}}
   320    323   
   321    324   #-------------------------------------------------------------------------
   322    325   #
   323    326   do_execsql_test 12.1 {
   324         -  CREATE VIRTUAL TABLE t2 USING fts5(x,y);
          327  +  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
   325    328   } {}
   326    329   
   327    330   do_catchsql_test 12.2 {
   328    331     SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
   329    332   } {1 {unknown special query: stuff}}
   330    333   
   331    334   do_test 12.3 {
................................................................................
   333    336     string is integer $res
   334    337   } {1}
   335    338   
   336    339   #-------------------------------------------------------------------------
   337    340   #
   338    341   reset_db
   339    342   do_execsql_test 13.1 {
   340         -  CREATE VIRTUAL TABLE t1 USING fts5(x);
          343  +  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
   341    344     INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
   342    345   } {}
   343    346   
   344    347   do_execsql_test 13.2 {
   345    348     SELECT rowid FROM t1 WHERE t1 MATCH 'o';
   346    349   } {1 2}
   347    350   
................................................................................
   357    360     SELECT rowid FROM t1 WHERE t1 MATCH '""';
   358    361   } {}
   359    362   
   360    363   #-------------------------------------------------------------------------
   361    364   #
   362    365   reset_db
   363    366   do_execsql_test 14.1 {
   364         -  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
          367  +  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
   365    368     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   366    369     WITH d(x,y) AS (
   367    370       SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
   368    371       UNION ALL 
   369    372       SELECT NULL, 'xyz xyz xyz xyz xyz xyz' FROM d
   370    373     )
   371    374     INSERT INTO t1 SELECT * FROM d LIMIT 200;
   372    375   }
          376  +
          377  +do_execsql_test 15.x {
          378  +  INSERT INTO t1(t1) VALUES('integrity-check');
          379  +}
   373    380   
   374    381   do_test 14.2 {
   375    382     set nRow 0
   376    383     db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } {
   377    384       db eval {
   378    385         BEGIN;
   379    386           CREATE TABLE t2(a, b);
................................................................................
   413    420   #
   414    421   do_execsql_test 16.1 {
   415    422     CREATE VIRTUAL TABLE n1 USING fts5(a);
   416    423     INSERT INTO n1 VALUES('a b c d');
   417    424   }
   418    425   
   419    426   proc funk {} {
          427  +  db eval { UPDATE n1_config SET v=50 WHERE k='version' }
   420    428     set fd [db incrblob main n1_data block 10]
   421    429     fconfigure $fd -encoding binary -translation binary
   422    430     puts -nonewline $fd "\x44\x45"
   423    431     close $fd
   424         -  db eval { UPDATE n1_config SET v=50 WHERE k='version' }
   425    432   }
   426    433   db func funk funk
   427    434   
   428    435   do_catchsql_test 16.2 {
   429    436     SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
   430    437   } {1 {SQL logic error or missing database}}
   431    438   
   432    439   #-------------------------------------------------------------------------
   433    440   #
   434    441   reset_db
   435    442   do_execsql_test 17.1 {
   436         -  CREATE VIRTUAL TABLE b2 USING fts5(x);
          443  +  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
   437    444     INSERT INTO b2 VALUES('a');
   438    445     INSERT INTO b2 VALUES('b');
   439    446     INSERT INTO b2 VALUES('c');
   440    447   }
   441    448   
   442    449   do_test 17.2 {
   443    450     set res [list]
   444    451     db eval { SELECT * FROM b2 ORDER BY rowid ASC } {
   445    452       lappend res [execsql { SELECT * FROM b2 ORDER BY rowid ASC }]
   446    453     }
   447    454     set res
   448    455   } {{a b c} {a b c} {a b c}}
   449    456   
   450         -reset_db
   451         -do_execsql_test 18.1 {
   452         -  CREATE VIRTUAL TABLE c2 USING fts5(x, y);
   453         -  INSERT INTO c2 VALUES('x x x', 'x x x');
   454         -  SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
   455         -} {1}
          457  +if {[string match n* %DETAIL%]==0} {
          458  +  reset_db
          459  +  do_execsql_test 17.3 {
          460  +    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
          461  +    INSERT INTO c2 VALUES('x x x', 'x x x');
          462  +    SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
          463  +  } {1}
          464  +}
   456    465   
   457    466   #-------------------------------------------------------------------------
   458    467   #
   459    468   reset_db
   460    469   do_execsql_test 17.1 {
   461         -  CREATE VIRTUAL TABLE uio USING fts5(ttt);
          470  +  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
   462    471     INSERT INTO uio VALUES(NULL);
   463    472     INSERT INTO uio SELECT NULL FROM uio;
   464    473     INSERT INTO uio SELECT NULL FROM uio;
   465    474     INSERT INTO uio SELECT NULL FROM uio;
   466    475     INSERT INTO uio SELECT NULL FROM uio;
   467    476     INSERT INTO uio SELECT NULL FROM uio;
   468    477     INSERT INTO uio SELECT NULL FROM uio;
................................................................................
   501    510   do_execsql_test 17.9 {
   502    511     SELECT min(rowid), max(rowid), count(*) FROM uio WHERE rowid < 10;
   503    512   } {-9223372036854775808 9 10}
   504    513   
   505    514   #--------------------------------------------------------------------
   506    515   #
   507    516   do_execsql_test 18.1 {
   508         -  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
   509         -  CREATE VIRTUAL TABLE t2 USING fts5(c, d);
          517  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
          518  +  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
   510    519     INSERT INTO t1 VALUES('abc*', NULL);
   511    520     INSERT INTO t2 VALUES(1, 'abcdefg');
   512    521   }
   513    522   do_execsql_test 18.2 {
   514    523     SELECT t1.rowid, t2.rowid FROM t1, t2 WHERE t2 MATCH t1.a AND t1.rowid = t2.c
   515    524   } {1 1}
   516    525   do_execsql_test 18.3 {
................................................................................
   518    527   } {1 1}
   519    528   
   520    529   #--------------------------------------------------------------------
   521    530   # fts5 table in the temp schema.
   522    531   #
   523    532   reset_db
   524    533   do_execsql_test 19.0 {
   525         -  CREATE VIRTUAL TABLE temp.t1 USING fts5(x);
          534  +  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
   526    535     INSERT INTO t1 VALUES('x y z');
   527    536     INSERT INTO t1 VALUES('w x 1');
   528    537     SELECT rowid FROM t1 WHERE t1 MATCH 'x';
   529    538   } {1 2}
   530    539   
   531    540   #--------------------------------------------------------------------
   532    541   # Test that 6 and 7 byte varints can be read.
   533    542   #
   534    543   reset_db
   535    544   do_execsql_test 20.0 {
   536         -  CREATE VIRTUAL TABLE temp.tmp USING fts5(x);
          545  +  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
   537    546   }
   538    547   set ::ids [list \
   539    548     0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
   540    549   ]
   541    550   do_test 20.1 {
   542    551     foreach id $::ids {
   543    552       execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') }
   544    553     }
   545    554     execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
   546    555   } $::ids
   547    556   
          557  +}
   548    558   
   549    559   
   550    560   finish_test
   551    561   
   552    562   

Changes to ext/fts5/test/fts5ab.test.

    18     18   
    19     19   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
           25  +foreach_detail_mode $testprefix {
           26  +
    25     27   do_execsql_test 1.0 {
    26         -  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
           28  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    27     29     INSERT INTO t1 VALUES('hello', 'world');
    28     30     INSERT INTO t1 VALUES('one two', 'three four');
    29     31     INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five');
    30     32   }
    31     33   
    32     34   do_execsql_test 1.1 {
    33     35     SELECT * FROM t1 ORDER BY rowid DESC;
................................................................................
    53     55     SELECT * FROM t1 WHERE rowid=1.99;
    54     56   } {}
    55     57   
    56     58   #-------------------------------------------------------------------------
    57     59   
    58     60   reset_db
    59     61   do_execsql_test 2.1 {
    60         -  CREATE VIRTUAL TABLE t1 USING fts5(x);
           62  +  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
    61     63     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    62     64     INSERT INTO t1 VALUES('one');
    63     65     INSERT INTO t1 VALUES('two');
    64     66     INSERT INTO t1 VALUES('three');
    65     67   }
    66     68   
    67     69   do_catchsql_test 2.2 {
................................................................................
   155    157   }
   156    158   
   157    159   #-------------------------------------------------------------------------
   158    160   # Documents with more than 2M tokens.
   159    161   #
   160    162   
   161    163   do_execsql_test 4.0 {
   162         -  CREATE VIRTUAL TABLE s1 USING fts5(x);
          164  +  CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%);
   163    165   }
   164    166   foreach {tn doc} [list \
   165    167     1 [string repeat {a x } 1500000]       \
   166    168     2 "[string repeat {a a } 1500000] x"   \
   167    169   ] {
   168    170     do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) }
   169    171   }
   170    172   
   171    173   do_execsql_test 4.3 {
   172    174     SELECT rowid FROM s1 WHERE s1 MATCH 'x'
   173    175   } {1 2}
   174    176   
   175         -do_execsql_test 4.4 {
   176         -  SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
          177  +if {[detail_is_full]} {
          178  +  do_execsql_test 4.4 {
          179  +    SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
          180  +  } {1 2}
          181  +}
          182  +
          183  +do_execsql_test 4.5 {
          184  +  SELECT rowid FROM s1 WHERE s1 MATCH 'a x'
   177    185   } {1 2}
   178    186   
   179    187   #-------------------------------------------------------------------------
   180    188   # Check that a special case of segment promotion works. The case is where
   181    189   # a new segment is written to level L, but the oldest segment within level
   182    190   # (L-2) is larger than it.
   183    191   #
   184    192   do_execsql_test 5.0 {
   185         -  CREATE VIRTUAL TABLE s2 USING fts5(x);
          193  +  CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
   186    194     INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
   187    195     INSERT INTO s2(s2, rank) VALUES('automerge', 0);
   188    196   }
   189    197   
   190    198   proc rnddoc {n} {
   191    199     set map [list 0 a  1 b  2 c  3 d  4 e  5 f  6 g  7 h  8 i  9 j]
   192    200     set doc [list]
................................................................................
   218    226   } {8 0 0}
   219    227   
   220    228   # Test also the other type of segment promotion - when a new segment is written
   221    229   # that is larger than segments immediately following it.
   222    230   do_test 5.3 {
   223    231     execsql {
   224    232       DROP TABLE s2;
   225         -    CREATE VIRTUAL TABLE s2 USING fts5(x);
          233  +    CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
   226    234       INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
   227    235       INSERT INTO s2(s2, rank) VALUES('automerge', 0);
   228    236     }
   229    237   
   230    238     for {set i 1} {$i <= 16} {incr i} {
   231    239       execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
   232    240     }
................................................................................
   237    245     execsql { INSERT INTO s2 VALUES(rnddoc(160)) }
   238    246     fts5_level_segs s2
   239    247   } {2 0}
   240    248   
   241    249   #-------------------------------------------------------------------------
   242    250   #
   243    251   do_execsql_test 6.0 {
   244         -  CREATE VIRTUAL TABLE s3 USING fts5(x);
          252  +  CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%);
   245    253     BEGIN;
   246    254       INSERT INTO s3 VALUES('a b c');
   247    255       INSERT INTO s3 VALUES('A B C');
   248    256   }
   249    257   
   250    258   do_execsql_test 6.1.1 {
   251    259     SELECT rowid FROM s3 WHERE s3 MATCH 'a'
................................................................................
   272    280       ROLLBACK;
   273    281     }
   274    282   } {}
   275    283   
   276    284   #-------------------------------------------------------------------------
   277    285   #
   278    286   set doc [string repeat "a b c " 500]
   279         -breakpoint
   280    287   do_execsql_test 7.0 {
   281         -  CREATE VIRTUAL TABLE x1 USING fts5(x);
          288  +  CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%);
   282    289     INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
   283    290     INSERT INTO x1 VALUES($doc);
   284    291   }
   285    292   
          293  +} ;# foreach_detail_mode...
   286    294   
   287    295   
   288    296   finish_test
   289    297   

Changes to ext/fts5/test/fts5ac.test.

    18     18   
    19     19   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
           25  +foreach_detail_mode $testprefix {
           26  +
    25     27   set data {
    26     28       0   {p o q e z k z p n f y u z y n y}   {l o o l v v k}
    27     29       1   {p k h h p y l l h i p v n}         {p p l u r i f a j g e r r x w}
    28     30       2   {l s z j k i m p s}                 {l w e j t j e e i t w r o p o}
    29     31       3   {x g y m y m h p}                   {k j j b r e y y a k y}
    30     32       4   {q m a i y i z}                     {o w a g k x g j m w e u k}
    31     33       5   {k o a w y b s z}                   {s g l m m l m g p}
................................................................................
   120    122       94  {s f e a e t i h h d q p z t q}     {b k m k w h c}
   121    123       95  {h b n j t k i h o q u}             {w n g i t o k c a m y p f l x c p}
   122    124       96  {f c x p y r b m o l m o a}         {p c a q s u n n x d c f a o}
   123    125       97  {u h h k m n k}                     {u b v n u a o c}
   124    126       98  {s p e t c z d f n w f}             {l s f j b l c e s h}
   125    127       99  {r c v w i v h a t a c v c r e}     {h h u m g o f b a e o}
   126    128   }
   127         -
   128         -# Argument $expr is an FTS5 match expression designed to be executed against
   129         -# an FTS5 table with the following schema:
   130         -# 
   131         -#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
   132         -#
   133         -# Assuming the table contains the same records as stored int the global 
   134         -# $::data array (see above), this function returns a list containing one
   135         -# element for each match in the dataset. The elements are themselves lists
   136         -# formatted as follows:
   137         -#
   138         -#   <rowid> {<phrase 0 matches> <phrase 1 matches>...}
   139         -#
   140         -# where each <phrase X matches> element is a list of phrase matches in the
   141         -# same form as returned by auxiliary scalar function fts5_test().
   142         -#
   143         -proc matchdata {bPos expr {bAsc 1}} {
   144         -
   145         -  set tclexpr [db one {
   146         -    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y')
   147         -  }]
   148         -  set res [list]
   149         -
   150         -  #puts $tclexpr
   151         -  foreach {id x y} $::data {
   152         -    set cols [list $x $y]
   153         -    set ::pc 0
   154         -    #set hits [lsort -command instcompare [eval $tclexpr]]
   155         -    set hits [eval $tclexpr]
   156         -    if {[llength $hits]>0} {
   157         -      if {$bPos} {
   158         -        lappend res [list $id $hits]
   159         -      } else {
   160         -        lappend res $id
   161         -      }
   162         -    }
   163         -  }
   164         -
   165         -  if {$bAsc} {
   166         -    set res [lsort -integer -increasing -index 0 $res]
   167         -  } else {
   168         -    set res [lsort -integer -decreasing -index 0 $res]
   169         -  }
   170         -
   171         -  return [concat {*}$res]
   172         -}
   173         -
   174         -#
   175         -# End of test code
   176         -#-------------------------------------------------------------------------
   177         -
   178         -proc fts5_test_poslist {cmd} {
   179         -  set res [list]
   180         -  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
   181         -    lappend res [string map {{ } .} [$cmd xInst $i]]
   182         -  }
   183         -  set res
   184         -}
   185         -
   186    129   
   187    130   foreach {tn2 sql} {
   188    131     1  {}
   189    132     2  {BEGIN}
   190    133   } {
   191    134     reset_db
   192         -  sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
          135  +  fts5_aux_test_functions db
   193    136   
   194         -  do_execsql_test 1.0 {
   195         -    CREATE VIRTUAL TABLE xx USING fts5(x,y);
          137  +  do_execsql_test 1.$tn2.0 {
          138  +    CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%);
   196    139       INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
   197    140     }
   198    141   
   199    142     execsql $sql
   200    143   
   201         -  do_test $tn2.1.1 {
          144  +  do_test 1.$tn2.1.1 {
   202    145       foreach {id x y} $data {
   203    146         execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
   204    147       }
   205    148       execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
   206    149     } {}
   207    150   
   208    151   
   209    152     #-------------------------------------------------------------------------
   210         -  # Test phrase queries.
   211    153     #
   212         -  foreach {tn phrase} {
   213         -    1 "o"
   214         -    2 "b q"
   215         -    3 "e a e"
   216         -    4 "m d g q q b k b w f q q p p"
   217         -    5 "l o o l v v k"
   218         -    6 "a"
   219         -    7 "b"
   220         -    8 "c"
   221         -    9 "no"
   222         -    10 "L O O L V V K"
   223         -  } {
   224         -    set expr "\"$phrase\""
   225         -    set res [matchdata 1 $expr]
   226         -
   227         -    do_execsql_test $tn2.1.2.$tn.[llength $res] { 
   228         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   229         -    } $res
   230         -  }
   231         -
   232         -  #-------------------------------------------------------------------------
   233         -  # Test some AND and OR queries.
   234         -  #
   235         -  foreach {tn expr} {
   236         -    1.1 "a   AND b"
   237         -    1.2 "a+b AND c"
   238         -    1.3 "d+c AND u"
   239         -    1.4 "d+c AND u+d"
   240         -
   241         -    2.1 "a   OR b"
   242         -    2.2 "a+b OR c"
   243         -    2.3 "d+c OR u"
   244         -    2.4 "d+c OR u+d"
   245         -
   246         -    3.1 { a AND b AND c }
   247         -  } {
   248         -    set res [matchdata 1 $expr]
   249         -    do_execsql_test $tn2.2.$tn.[llength $res] { 
   250         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   251         -    } $res
   252         -  }
   253         -
   254         -  #-------------------------------------------------------------------------
   255         -  # Queries on a specific column.
   256         -  #
   257         -  foreach {tn expr} {
   258         -    1.1 "x:a"
   259         -    1.2 "y:a"
   260         -    1.3 "x:b"
   261         -    1.4 "y:b"
   262         -    2.1 "{x}:a"
   263         -    2.2 "{y}:a"
   264         -    2.3 "{x}:b"
   265         -    2.4 "{y}:b"
   266         -
   267         -    3.1 "{x y}:a"
   268         -    3.2 "{y x}:a"
   269         -    3.3 "{x x}:b"
   270         -    3.4 "{y y}:b"
   271         -
   272         -    4.1 {{"x" "y"}:a}
   273         -    4.2 {{"y" x}:a}
   274         -    4.3 {{x "x"}:b}
   275         -    4.4 {{"y" y}:b}
   276         -  } {
   277         -    set res [matchdata 1 $expr]
   278         -    do_execsql_test $tn2.3.$tn.[llength $res] { 
   279         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   280         -    } $res
   281         -  }
   282         -
   283         -  #-------------------------------------------------------------------------
   284         -  # Some NEAR queries.
   285         -  #
   286         -  foreach {tn expr} {
   287         -    1 "NEAR(a b)"
   288         -    2 "NEAR(r c)"
   289         -    2 { NEAR(r c, 5) }
   290         -    3 { NEAR(r c, 3) }
   291         -    4 { NEAR(r c, 2) }
   292         -    5 { NEAR(r c, 0) }
   293         -    6 { NEAR(a b c) }
   294         -    7 { NEAR(a b c, 8) }
   295         -    8  { x : NEAR(r c) }
   296         -    9  { y : NEAR(r c) }
   297         -  } {
   298         -    set res [matchdata 1 $expr]
   299         -    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
   300         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   301         -    } $res
   302         -  }
   303         -
   304         -  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
   305         -  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}
   306         -
   307         -  foreach {tn expr tclexpr} {
   308         -    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
   309         -  } {
   310         -    do_execsql_test $tn2.5.$tn {
   311         -      SELECT fts5_expr_tcl($expr, 'N $x')
   312         -    } [list $tclexpr]
          154  +  do_execsql_test 1.$tn2.integrity {
          155  +    INSERT INTO xx(xx) VALUES('integrity-check');
   313    156     }
   314    157   
   315    158     #-------------------------------------------------------------------------
   316    159     #
   317         -  do_execsql_test $tn2.6.integrity {
   318         -    INSERT INTO xx(xx) VALUES('integrity-check');
   319         -  }
   320         -  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
   321         -  foreach {bAsc sql} {
   322         -    1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
   323         -    0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
          160  +  foreach {tn expr} {
          161  +    1.2 "a   OR b"
          162  +    1.1 "a   AND b"
          163  +    1.3 "o"
          164  +    1.4 "b q"
          165  +    1.5 "e a e"
          166  +    1.6 "m d g q q b k b w f q q p p"
          167  +    1.7 "l o o l v v k"
          168  +    1.8 "a"
          169  +    1.9 "b"
          170  +    1.10 "c"
          171  +    1.11 "no"
          172  +    1.12 "L O O L V V K"
          173  +    1.13 "a AND b AND c"
          174  +    1.14 "x:a"
          175  +
          176  +    2.1 "x:a"
          177  +    2.2 "y:a"
          178  +    2.3 "x:b"
          179  +    2.4 "y:b"
          180  +
          181  +    3.1 "{x}:a"
          182  +    3.2 "{y}:a"
          183  +    3.3 "{x}:b"
          184  +    3.4 "{y}:b"
          185  +
          186  +    4.1 "{x y}:a"
          187  +    4.2 "{y x}:a"
          188  +    4.3 "{x x}:b"
          189  +    4.4 "{y y}:b"
          190  +
          191  +    5.1 {{"x" "y"}:a}
          192  +    5.2 {{"y" x}:a}
          193  +    5.3 {{x "x"}:b}
          194  +    5.4 {{"y" y}:b}
          195  +
          196  +    6.1 "b + q"
          197  +    6.2 "e + a + e"
          198  +    6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
          199  +    6.4 "l + o + o + l + v + v + k"
          200  +    6.5 "L + O + O + L + V + V + K"
          201  +
          202  +    7.1 "a+b AND c"
          203  +    7.2 "d+c AND u"
          204  +    7.3 "d+c AND u+d"
          205  +    7.4 "a+b OR c"
          206  +    7.5 "d+c OR u"
          207  +    7.6 "d+c OR u+d"
          208  +
          209  +    8.1 "NEAR(a b)"
          210  +    8.2 "NEAR(r c)"
          211  +    8.2 { NEAR(r c, 5) }
          212  +    8.3 { NEAR(r c, 3) }
          213  +    8.4 { NEAR(r c, 2) }
          214  +    8.5 { NEAR(r c, 0) }
          215  +    8.6 { NEAR(a b c) }
          216  +    8.7 { NEAR(a b c, 8) }
          217  +    8.8  { x : NEAR(r c) }
          218  +    8.9  { y : NEAR(r c) }
          219  +
          220  +    9.1 { NEAR(r c) }
          221  +    9.2 { NEAR(r c, 5) }
          222  +    9.3 { NEAR(r c, 3) }
          223  +    9.4 { NEAR(r c, 2) }
          224  +    9.5 { NEAR(r c, 0) }
          225  +    9.6 { NEAR(a b c) }
          226  +    9.7 { NEAR(a b c, 8) }
          227  +    9.8  { x : NEAR(r c) }
          228  +    9.9  { y : NEAR(r c) }
          229  +    9.10 { x : "r c" }
          230  +    9.11 { y : "r c" }
          231  +    9.12 { a AND b }
          232  +    9.13 { a AND b AND c }
          233  +    9.14a { a }
          234  +    9.14b { a OR b }
          235  +    9.15 { a OR b AND c }
          236  +    9.16 { c AND b OR a }
          237  +    9.17 { c AND (b OR a) }
          238  +    9.18 { c NOT (b OR a) }
          239  +    9.19 { (c NOT b) OR (a AND d) }
   324    240     } {
   325         -    foreach {tn expr} {
   326         -      0.1 x
   327         -      1 { NEAR(r c) }
   328         -      2 { NEAR(r c, 5) }
   329         -      3 { NEAR(r c, 3) }
   330         -      4 { NEAR(r c, 2) }
   331         -      5 { NEAR(r c, 0) }
   332         -      6 { NEAR(a b c) }
   333         -      7 { NEAR(a b c, 8) }
   334         -      8  { x : NEAR(r c) }
   335         -      9  { y : NEAR(r c) }
   336         -      10 { x : "r c" }
   337         -      11 { y : "r c" }
   338         -      12 { a AND b }
   339         -      13 { a AND b AND c }
   340         -      14a { a }
   341         -      14b { a OR b }
   342         -      15 { a OR b AND c }
   343         -      16 { c AND b OR a }
   344         -      17 { c AND (b OR a) }
   345         -      18 { c NOT (b OR a) }
   346         -      19 { c NOT b OR a AND d }
   347         -    } {
   348         -      set res [matchdata 0 $expr $bAsc]
   349         -      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
          241  +
          242  +    if {[fts5_expr_ok $expr xx]==0} {
          243  +      do_test 1.$tn2.$tn.OMITTED { list } [list]
          244  +      continue
   350    245       }
          246  +
          247  +    set res [fts5_query_data $expr xx]
          248  +    do_execsql_test 1.$tn2.$tn.[llength $res].asc {
          249  +      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
          250  +      FROM xx WHERE xx match $expr
          251  +    } $res
          252  +
          253  +
          254  +    set res [fts5_query_data $expr xx DESC]
          255  +    do_execsql_test 1.$tn2.$tn.[llength $res].desc {
          256  +      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
          257  +      FROM xx WHERE xx match $expr ORDER BY 1 DESC
          258  +    } $res
   351    259     }
   352    260   }
   353    261   
   354         -do_execsql_test 3.1 {
          262  +}
          263  +
          264  +do_execsql_test 2.1 {
   355    265     SELECT fts5_expr_tcl('a AND b');
   356    266   } {{AND [nearset -- {a}] [nearset -- {b}]}}
          267  +
          268  +do_test 2.2.1  { nearset {{a b c}} -- a } {0.0.0}
          269  +do_test 2.2.2  { nearset {{a b c}} -- c } {0.0.2}
          270  +
          271  +foreach {tn expr tclexpr} {
          272  +  1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
          273  +} {
          274  +  do_execsql_test 2.3.$tn {
          275  +    SELECT fts5_expr_tcl($expr, 'N $x')
          276  +  } [list $tclexpr]
          277  +}
   357    278   
   358    279   finish_test
   359    280   

Changes to ext/fts5/test/fts5ad.test.

    18     18   
    19     19   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
           25  +foreach_detail_mode $testprefix {
           26  +
    25     27   do_execsql_test 1.0 {
    26         -  CREATE VIRTUAL TABLE yy USING fts5(x, y);
           28  +  CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%);
    27     29     INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching');
    28     30     INSERT INTO yy VALUES('indices (or all  matching', 'values if -inline is');
    29     31     INSERT INTO yy VALUES('specified as  well.) If', 'indices are returned, the');
    30     32   } {}
    31     33   
    32     34   foreach {tn match res} {
    33     35     1 {c*} {1}
................................................................................
    49     51     do_execsql_test 1.$tn {
    50     52       SELECT rowid FROM yy WHERE yy MATCH $match
    51     53     } $res
    52     54   }
    53     55   
    54     56   foreach {T create} {
    55     57     2 {
    56         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b);
           58  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    57     59       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    58     60     }
    59     61     
    60     62     3 {
    61         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
           63  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
    62     64       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    63     65     }
    64     66   
    65     67     4 {
    66         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b);
           68  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    67     69       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    68     70       BEGIN;
    69     71     }
    70     72     
    71     73     5 {
    72         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
           74  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
    73     75       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    74     76       BEGIN;
    75     77     }
    76     78   
    77     79   } {
    78     80   
    79     81     do_test $T.1 { 
................................................................................
   229    231         set n [llength $res]
   230    232         if {$T==5} breakpoint 
   231    233         do_execsql_test $T.$bAsc.$tn.$n $sql $res
   232    234       }
   233    235     }
   234    236   
   235    237     catchsql COMMIT
          238  +}
          239  +
   236    240   }
   237    241   
   238    242   finish_test
   239    243   

Changes to ext/fts5/test/fts5ae.test.

    18     18   
    19     19   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
           25  +foreach_detail_mode $testprefix {
           26  +
    25     27   do_execsql_test 1.0 {
    26         -  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
           28  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
    27     29     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    28     30   }
    29     31   
    30     32   do_execsql_test 1.1 {
    31     33     INSERT INTO t1 VALUES('hello', 'world');
    32     34     SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
    33     35   } {1}
................................................................................
    51     53   } {1 2 4}
    52     54   
    53     55   fts5_aux_test_functions db
    54     56   
    55     57   #-------------------------------------------------------------------------
    56     58   # 
    57     59   do_execsql_test 2.0 {
    58         -  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
           60  +  CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%);
    59     61     INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
    60     62     INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
    61     63   }
    62     64   
    63     65   do_execsql_test 2.1 {
    64     66     SELECT rowid, fts5_test_poslist(t2) FROM t2 
    65     67     WHERE t2 MATCH 'm' ORDER BY rowid;
................................................................................
    72     74     SELECT rowid, fts5_test_poslist(t2) FROM t2 
    73     75     WHERE t2 MATCH 'u OR q' ORDER BY rowid;
    74     76   } {
    75     77     1 {0.0.0}
    76     78     2 {1.0.2 1.0.10}
    77     79   }
    78     80   
    79         -do_execsql_test 2.3 {
    80         -  SELECT rowid, fts5_test_poslist(t2) FROM t2 
    81         -  WHERE t2 MATCH 'y:o' ORDER BY rowid;
    82         -} {
    83         -  1 {0.1.3 0.1.7}
           81  +if {[detail_is_full]} {
           82  +  do_execsql_test 2.3 {
           83  +    SELECT rowid, fts5_test_poslist(t2) FROM t2 
           84  +      WHERE t2 MATCH 'y:o' ORDER BY rowid;
           85  +  } {
           86  +    1 {0.1.3 0.1.7}
           87  +  }
    84     88   }
    85     89   
    86     90   #-------------------------------------------------------------------------
    87     91   # 
    88     92   do_execsql_test 3.0 {
    89         -  CREATE VIRTUAL TABLE t3 USING fts5(x, y);
           93  +  CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%);
    90     94     INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
    91     95     INSERT INTO t3 VALUES( 'r c', '');
    92     96   }
    93     97   
    94         -do_execsql_test 3.1 {
    95         -  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
    96         -} {
    97         -  1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
    98         -}
           98  +if {[detail_is_full]} {
           99  +  do_execsql_test 3.1 {
          100  +    SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
          101  +  } {
          102  +    1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
          103  +  }
    99    104   
   100         -do_execsql_test 3.2 {
   101         -  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
   102         -} {
   103         -  2 {0.0.0 1.0.1}
          105  +  do_execsql_test 3.2 {
          106  +    SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
          107  +  } {
          108  +    2 {0.0.0 1.0.1}
          109  +  }
   104    110   }
   105    111   
   106    112   do_execsql_test 3.3 {
   107    113     INSERT INTO t3 
   108    114     VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
   109    115     SELECT rowid, fts5_test_poslist(t3) 
   110    116     FROM t3 WHERE t3 MATCH 'a OR b AND c';
................................................................................
   112    118     1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2}
   113    119     3 0.0.5 
   114    120   }
   115    121   
   116    122   #-------------------------------------------------------------------------
   117    123   # 
   118    124   do_execsql_test 4.0 {
   119         -  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
          125  +  CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
   120    126     INSERT INTO t4 
   121    127     VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
   122    128   }
   123    129   
   124    130   do_execsql_test 4.1 {
   125    131     SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c';
   126    132   } {
................................................................................
   130    136   #-------------------------------------------------------------------------
   131    137   # Test that the xColumnSize() and xColumnAvgsize() APIs work.
   132    138   #
   133    139   reset_db
   134    140   fts5_aux_test_functions db
   135    141   
   136    142   do_execsql_test 5.1 {
   137         -  CREATE VIRTUAL TABLE t5 USING fts5(x, y);
          143  +  CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%);
   138    144     INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
   139    145     INSERT INTO t5 VALUES('', 'a');
   140    146     INSERT INTO t5 VALUES('a', '');
   141    147   }
   142    148   do_execsql_test 5.2 {
   143    149     SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a'
   144    150     ORDER BY rowid DESC;
................................................................................
   178    184   
   179    185   #-------------------------------------------------------------------------
   180    186   # Test the xTokenize() API
   181    187   #
   182    188   reset_db
   183    189   fts5_aux_test_functions db
   184    190   do_execsql_test 6.1 {
   185         -  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
          191  +  CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%);
   186    192     INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
   187    193     INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
   188    194   }
   189    195   
   190    196   do_execsql_test 6.2 {
   191    197     SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*'
   192    198   } {
................................................................................
   196    202   
   197    203   #-------------------------------------------------------------------------
   198    204   # Test the xQueryPhrase() API
   199    205   #
   200    206   reset_db
   201    207   fts5_aux_test_functions db
   202    208   do_execsql_test 7.1 {
   203         -  CREATE VIRTUAL TABLE t7 USING fts5(x, y);
          209  +  CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%);
   204    210   }
   205    211   do_test 7.2 {
   206    212     foreach {x y} {
   207    213       {q i b w s a a e l o} {i b z a l f p t e u}
   208    214       {b a z t a l o x d i} {b p a d b f h d w y}
   209    215       {z m h n p p u i e g} {v h d v b x j j c z}
   210    216       {a g i m v a u c b i} {p k s o t l r t b m}
................................................................................
   236    242   #  SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a';
   237    243   #} {5 5 5 5}
   238    244   #
   239    245   
   240    246   #-------------------------------------------------------------------------
   241    247   #
   242    248   do_test 8.1 {
   243         -  execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) }
          249  +  execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) }
   244    250     foreach {rowid x y} {
   245    251        0 {A o}   {o o o C o o o o o o o o}
   246    252        1 {o o B} {o o o C C o o o o o o o}
   247    253        2 {A o o} {o o o o D D o o o o o o}
   248    254        3 {o B}   {o o o o o D o o o o o o}
   249    255        4 {E o G} {H o o o o o o o o o o o}
   250    256        5 {F o G} {I o J o o o o o o o o o}
................................................................................
   294    300     2 {a OR b}       2
   295    301     3 {a OR b OR c}  3
   296    302     4 {NEAR(a b)}    2
   297    303   } {
   298    304     do_execsql_test 9.2.$tn {
   299    305       SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
   300    306     } $cnt
          307  +}
          308  +
   301    309   }
   302    310   
   303    311   finish_test
   304    312   

Changes to ext/fts5/test/fts5af.test.

    20     20   
    21     21   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    22     22   ifcapable !fts5 {
    23     23     finish_test
    24     24     return
    25     25   }
    26     26   
           27  +foreach_detail_mode $testprefix {
    27     28   
    28     29   do_execsql_test 1.0 {
    29         -  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
           30  +  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
    30     31   }
    31     32   
    32     33   proc do_snippet_test {tn doc match res} {
    33     34   
    34     35     uplevel #0 [list set v1 $doc]
    35     36     uplevel #0 [list set v2 $match]
    36     37   
................................................................................
   107    108     7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
   108    109     7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
   109    110     7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
   110    111   } {
   111    112     do_snippet_test 1.$tn $doc X $res
   112    113   }
   113    114   
   114         -foreach {tn doc res} {
   115         -  1.1 {X Y o o o o o} {[X Y] o o o o o}
   116         -  1.2 {o X Y o o o o} {o [X Y] o o o o}
   117         -  1.3 {o o X Y o o o} {o o [X Y] o o o}
   118         -  1.4 {o o o X Y o o} {o o o [X Y] o o}
   119         -  1.5 {o o o o X Y o} {o o o o [X Y] o}
   120         -  1.6 {o o o o o X Y} {o o o o o [X Y]}
          115  +if {[detail_is_full]} {
          116  +  foreach {tn doc res} {
          117  +    1.1 {X Y o o o o o} {[X Y] o o o o o}
          118  +    1.2 {o X Y o o o o} {o [X Y] o o o o}
          119  +    1.3 {o o X Y o o o} {o o [X Y] o o o}
          120  +    1.4 {o o o X Y o o} {o o o [X Y] o o}
          121  +    1.5 {o o o o X Y o} {o o o o [X Y] o}
          122  +    1.6 {o o o o o X Y} {o o o o o [X Y]}
   121    123   
   122         -  2.1 {X Y o o o o o o} {[X Y] o o o o o...}
   123         -  2.2 {o X Y o o o o o} {o [X Y] o o o o...}
   124         -  2.3 {o o X Y o o o o} {o o [X Y] o o o...}
   125         -  2.4 {o o o X Y o o o} {...o o [X Y] o o o}
   126         -  2.5 {o o o o X Y o o} {...o o o [X Y] o o}
   127         -  2.6 {o o o o o X Y o} {...o o o o [X Y] o}
   128         -  2.7 {o o o o o o X Y} {...o o o o o [X Y]}
          124  +    2.1 {X Y o o o o o o} {[X Y] o o o o o...}
          125  +    2.2 {o X Y o o o o o} {o [X Y] o o o o...}
          126  +    2.3 {o o X Y o o o o} {o o [X Y] o o o...}
          127  +    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
          128  +    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
          129  +    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
          130  +    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
   129    131   
   130         -  3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
   131         -  3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
   132         -  3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
   133         -  3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
   134         -  3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
   135         -  3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
   136         -  3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
   137         -  3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
   138         -
   139         -} {
   140         -  do_snippet_test 2.$tn $doc "X + Y" $res
          132  +    3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
          133  +    3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
          134  +    3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
          135  +    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
          136  +    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
          137  +    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
          138  +    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
          139  +    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
          140  +  } {
          141  +    do_snippet_test 2.$tn $doc "X + Y" $res
          142  +  }
   141    143   }
          144  +
          145  +} ;# foreach_detail_mode 
   142    146   
   143    147   finish_test
   144    148   

Changes to ext/fts5/test/fts5ag.test.

    29     29   #      ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC]
    30     30   #
    31     31   # and
    32     32   #
    33     33   #      ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC]
    34     34   #
    35     35   
           36  +foreach_detail_mode $testprefix {
           37  +
    36     38   do_execsql_test 1.0 {
    37         -  CREATE VIRTUAL TABLE t1 USING fts5(x, y, z);
           39  +  CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%);
    38     40   }
    39     41   
    40     42   do_test 1.1 {
    41     43     foreach {x y z} {
    42     44       {j s m y m r n l u k} {z k f u z g h s w g} {r n o s s b v n w w}
    43     45       {m v g n d x q r r s} {q t d a q a v l h j} {s k l f s i n v q v}
    44     46       {m f f d h h s o h a} {y e v r q i u m h d} {b c k q m z l z h n}
................................................................................
   115    117   
   116    118   foreach {tn expr} {
   117    119     2.1 a
   118    120     2.2 b
   119    121     2.3 c
   120    122     2.4 d
   121    123   
   122         -  2.5 {"m m"}
   123         -  2.6 {e + s}
   124         -
   125    124     3.0 {a AND b}
   126    125     3.1 {a OR b}
   127    126     3.2 {b OR c AND d}
   128         -  3.3 {NEAR(c d)}
   129    127   } {
   130    128     do_fts5ag_test $tn $expr
          129  +}
   131    130   
   132         -  if {[set_test_counter errors]} break
          131  +if {[detail_is_full]} {
          132  +  foreach {tn expr} {
          133  +    4.1 {"m m"}
          134  +    4.2 {e + s}
          135  +    4.3 {NEAR(c d)}
          136  +  } {
          137  +    do_fts5ag_test $tn $expr
          138  +  }
   133    139   }
   134    140   
          141  +} ;# foreach_detail_mode
   135    142   
   136    143   
   137    144   finish_test
   138    145   

Changes to ext/fts5/test/fts5ah.test.

    16     16   set testprefix fts5ah
    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
           23  +
           24  +foreach_detail_mode $testprefix {
    23     25   
    24     26   #-------------------------------------------------------------------------
    25     27   # This file contains tests for very large doclists.
    26     28   #
    27     29   
           30  +set Y [list]
           31  +set W [list]
    28     32   do_test 1.0 {
    29         -  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
           33  +  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) }
    30     34     execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }
    31     35     set v {w w w w w w w w w w w w w w w w w w w w}
    32     36     execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) }
    33     37     for {set i 1} {$i <= 10000} {incr i} {
    34     38       set v {x x x x x x x x x x x x x x x x x x x x}
    35     39       if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
    36     40       if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
................................................................................
    66     70     expr [reads] - $nRead
    67     71   }
    68     72   
    69     73   do_test 1.4 {
    70     74     set nRead [reads]
    71     75     execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
    72     76     set nReadX [expr [reads] - $nRead]
    73         -  expr $nReadX>1000
           77  +  #puts -nonewline "(nReadX=$nReadX)"
           78  +  if {[detail_is_full]} { set expect 1000 }
           79  +  if {[detail_is_col]}  { set expect 250 }
           80  +  if {[detail_is_none]} { set expect 80 }
           81  +
           82  +  expr $nReadX>$expect
    74     83   } {1}
    75     84   
    76     85   do_test 1.5 {
    77     86     set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }]
    78     87     set bwd [execsql_reads {
    79     88       SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC 
    80     89     }]
................................................................................
    83     92   
    84     93   foreach {tn q res} "
    85     94     1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x'   }  [list $W]
    86     95     2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w'   }  [list $W]
    87     96     3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' }  [list $W]
    88     97     4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' }  [list $Y]
    89     98   " {
           99  +  if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue
          100  +
          101  +  if {[detail_is_full]} { set ratio 8 }
          102  +  if {[detail_is_col]}  { set ratio 4 }
          103  +  if {[detail_is_none]} { set ratio 2 }
    90    104   
    91    105     do_test 1.6.$tn.1 {
    92    106       set n [execsql_reads $q]
    93    107       #puts -nonewline "(n=$n nReadX=$nReadX)"
    94         -    expr {$n < ($nReadX / 8)}
          108  +    expr {$n < ($nReadX / $ratio)}
    95    109     } {1}
    96    110   
    97    111     do_test 1.6.$tn.2 {
    98    112       set n [execsql_reads "$q ORDER BY rowid DESC"]
    99    113       #puts -nonewline "(n=$n nReadX=$nReadX)"
   100         -    expr {$n < ($nReadX / 8)}
          114  +    expr {$n < ($nReadX / $ratio)}
   101    115     } {1}
   102    116   
   103    117     do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res]
   104    118     do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res]
   105    119   }
   106    120   
   107    121   #-------------------------------------------------------------------------
   108    122   # Now test that adding range constraints on the rowid field reduces the
   109    123   # number of pages loaded from disk.
   110    124   #
   111    125   foreach {tn fraction tail cnt} {
   112         -  1 0.6 {rowid > 5000} 5000
   113         -  2 0.2 {rowid > 9000} 1000
   114         -  3 0.2 {rowid < 1000}  999
   115         -  4 0.2 {rowid BETWEEN 4000 AND 5000}  1001
   116         -  5 0.6 {rowid >= 5000} 5001
   117         -  6 0.2 {rowid >= 9000} 1001
   118         -  7 0.2 {rowid <= 1000} 1000
   119         -  8 0.6 {rowid > '5000'} 5000
   120         -  9 0.2 {rowid > '9000'} 1000
          126  +  1  0.6 {rowid > 5000} 5000
          127  +  2  0.2 {rowid > 9000} 1000
          128  +  3  0.2 {rowid < 1000}  999
          129  +  4  0.2 {rowid BETWEEN 4000 AND 5000}  1001
          130  +  5  0.6 {rowid >= 5000} 5001
          131  +  6  0.2 {rowid >= 9000} 1001
          132  +  7  0.2 {rowid <= 1000} 1000
          133  +  8  0.6 {rowid > '5000'} 5000
          134  +  9  0.2 {rowid > '9000'} 1000
   121    135     10 0.1 {rowid = 444} 1
   122    136   } {
   123    137     set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail"
   124    138     set n [execsql_reads $q]
   125    139     set ret [llength [execsql $q]]
   126    140   
          141  +  # Because the position lists for 'x' are quite long in this db, the 
          142  +  # advantage is a bit smaller in detail=none mode. Update $fraction to 
          143  +  # reflect this.
          144  +  if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] }
          145  +
   127    146     do_test "1.7.$tn.asc.(n=$n ret=$ret)" {
   128    147       expr {$n < ($fraction*$nReadX) && $ret==$cnt}
   129    148     } {1}
   130    149   
   131    150     set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail ORDER BY rowid DESC"
   132    151     set n [execsql_reads $q]
   133    152     set ret [llength [execsql $q]]
................................................................................
   139    158   do_execsql_test 1.8.1 {
   140    159     SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND +rowid < 'text';
   141    160   } {10000}
   142    161   do_execsql_test 1.8.2 {
   143    162     SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
   144    163   } {10000}
   145    164   
          165  +} ;# foreach_detail_mode
   146    166   
   147    167   #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
   148    168   
   149    169   finish_test
   150    170   

Changes to ext/fts5/test/fts5ai.test.

    19     19   
    20     20   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    21     21   ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
           26  +foreach_detail_mode $testprefix {
           27  +
    26     28   do_execsql_test 1.0 {
    27         -  CREATE VIRTUAL TABLE t1 USING fts5(a);
           29  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%);
    28     30   } {}
    29     31   
    30     32   do_execsql_test 1.1 {
    31     33     BEGIN;
    32     34       INSERT INTO t1 VALUES('a b c');
    33     35       INSERT INTO t1 VALUES('d e f');
    34     36       SAVEPOINT one;
................................................................................
    44     46         INSERT INTO t1 VALUES('s t u');
    45     47       ROLLBACK TO one;
    46     48     COMMIT;
    47     49   }
    48     50   
    49     51   do_execsql_test 1.2 {
    50     52     INSERT INTO t1(t1) VALUES('integrity-check');
           53  +}
    51     54   }
    52     55   
    53     56   
    54     57   finish_test
    55     58   

Changes to ext/fts5/test/fts5ak.test.

    19     19   
    20     20   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    21     21   ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
           26  +foreach_detail_mode $testprefix {
           27  +
    26     28   do_execsql_test 1.1 {
    27         -  CREATE VIRTUAL TABLE ft1 USING fts5(x);
           29  +  CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
    28     30     INSERT INTO ft1 VALUES('i d d a g i b g d d');
    29     31     INSERT INTO ft1 VALUES('h d b j c c g a c a');
    30     32     INSERT INTO ft1 VALUES('e j a e f h b f h h');
    31     33     INSERT INTO ft1 VALUES('j f h d g h i b d f');
    32     34     INSERT INTO ft1 VALUES('d c j d c j b c g e');
    33     35     INSERT INTO ft1 VALUES('i a d e g j g d a a');
    34     36     INSERT INTO ft1 VALUES('j f c e d a h j d b');
    35     37     INSERT INTO ft1 VALUES('i c c f a d g h j e');
    36     38     INSERT INTO ft1 VALUES('i d i g c d c h b f');
    37     39     INSERT INTO ft1 VALUES('g d a e h a b c f j');
           40  +
           41  +  CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%);
           42  +  INSERT INTO ft2 VALUES('a b c d e f g h i j');
    38     43   }
    39     44   
    40     45   do_execsql_test 1.2 {
    41     46     SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e';
    42     47   } {
    43     48     {[e] j a [e] f h b f h h}
    44     49     {d c j d c j b c g [e]}
................................................................................
    45     50     {i a d [e] g j g d a a}
    46     51     {j f c [e] d a h j d b}
    47     52     {i c c f a d g h j [e]}
    48     53     {g d a [e] h a b c f j}
    49     54   }
    50     55   
    51     56   do_execsql_test 1.3 {
    52         -  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d';
    53         -} {
    54         -  {[h d] b j c c g a c a}
    55         -  {j f [h d] g h i b d f} 
    56         -}
    57         -
    58         -do_execsql_test 1.4 {
    59         -  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d';
    60         -} {
    61         -  {i [d d] a g i b g [d d]}
    62         -}
    63         -
    64         -do_execsql_test 1.5 {
    65     57     SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e'
    66     58   } {
    67     59     {[e] j a [e] f h b f h h}
    68     60     {d c j d c j b c g [e]}
    69     61     {i a d [e] g j g d a a}
    70     62     {j f c [e] d a h j d b}
    71     63     {i c c f a d g h j [e]}
    72     64     {g d a [e] h a b c f j}
    73     65   }
    74     66   
    75         -do_execsql_test 1.6 {
           67  +do_execsql_test 1.4 {
           68  +  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
           69  +} {
           70  +  {a b c [d] e [f] g h i j}
           71  +}
           72  +
           73  +do_execsql_test 1.5 {
           74  +  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
           75  +} {
           76  +  {a b c [d] e [f] g h i j}
           77  +}
           78  +
           79  +#-------------------------------------------------------------------------
           80  +# Tests below this point require detail=full.
           81  +#-------------------------------------------------------------------------
           82  +if {[detail_is_full]==0} continue
           83  +
           84  +
           85  +do_execsql_test 2.1 {
           86  +  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d';
           87  +} {
           88  +  {[h d] b j c c g a c a}
           89  +  {j f [h d] g h i b d f} 
           90  +}
           91  +
           92  +do_execsql_test 2.2 {
           93  +  SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d';
           94  +} {
           95  +  {i [d d] a g i b g [d d]}
           96  +}
           97  +
           98  +do_execsql_test 2.3 {
    76     99     SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d';
    77    100   } {
    78    101     {i [d d] a g i b g [d d]}
    79    102   }
    80    103   
    81         -do_execsql_test 2.1 {
    82         -  CREATE VIRTUAL TABLE ft2 USING fts5(x);
    83         -  INSERT INTO ft2 VALUES('a b c d e f g h i j');
    84         -}
    85         -
    86         -do_execsql_test 2.2 {
          104  +do_execsql_test 2.4 {
    87    105     SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e'
    88    106   } {{a [b c d e] f g h i j}}
    89    107   
    90         -do_execsql_test 2.3 {
          108  +do_execsql_test 2.5 {
    91    109     SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g'
    92    110   } {
    93    111     {a [b c d] [e f g] h i j}
    94    112   }
    95    113   
    96         -do_execsql_test 2.4 {
          114  +do_execsql_test 2.6 {
    97    115     SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c'
    98    116   } {
    99    117     {a [b c d] e f g h i j}
   100    118   }
   101    119   
   102         -do_execsql_test 2.5 {
          120  +do_execsql_test 2.7 {
   103    121     SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
   104    122   } {
   105    123     {a [b c d e] f g h i j}
   106    124   }
   107    125   
   108         -do_execsql_test 2.6.1 {
   109         -  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
   110         -} {
   111         -  {a b c [d] e [f] g h i j}
   112         -}
   113         -
   114         -do_execsql_test 2.6.2 {
   115         -  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
   116         -} {
   117         -  {a b c [d] e [f] g h i j}
   118         -}
   119         -
   120    126   #-------------------------------------------------------------------------
   121    127   # The example from the docs.
   122    128   #
   123    129   do_execsql_test 3.1 {
   124    130     -- Assuming this:
   125         -  CREATE VIRTUAL TABLE ft USING fts5(a);
          131  +  CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%);
   126    132     INSERT INTO ft VALUES('a b c x c d e');
   127    133     INSERT INTO ft VALUES('a b c c d e');
   128    134     INSERT INTO ft VALUES('a b c d e');
   129    135   
   130    136     -- The following SELECT statement returns these three rows:
   131    137     --   '[a b c] x [c d e]'
   132    138     --   '[a b c] [c d e]'
................................................................................
   134    140     SELECT highlight(ft, 0, '[', ']') FROM ft WHERE ft MATCH 'a+b+c AND c+d+e';
   135    141   } {
   136    142     {[a b c] x [c d e]}
   137    143     {[a b c] [c d e]}
   138    144     {[a b c d e]}
   139    145   }
   140    146   
          147  +}
   141    148   
   142    149   finish_test
   143    150   

Changes to ext/fts5/test/fts5al.test.

    19     19   
    20     20   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    21     21   ifcapable !fts5 {
    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
           26  +foreach_detail_mode $testprefix {
           27  +
    26     28   do_execsql_test 1.1 {
    27         -  CREATE VIRTUAL TABLE ft1 USING fts5(x);
           29  +  CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
    28     30     SELECT * FROM ft1_config;
    29     31   } {version 4}
    30     32   
    31     33   do_execsql_test 1.2 {
    32     34     INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32);
    33     35     SELECT * FROM ft1_config;
    34     36   } {pgsz 32 version 4}
................................................................................
    79     81   }
    80     82   
    81     83   #-------------------------------------------------------------------------
    82     84   # Assorted tests of the tcl interface for creating extension functions.
    83     85   #
    84     86   
    85     87   do_execsql_test 3.1 {
    86         -  CREATE VIRTUAL TABLE t1 USING fts5(x);
           88  +  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
    87     89     INSERT INTO t1 VALUES('q w e r t y');
    88     90     INSERT INTO t1 VALUES('y t r e w q');
    89     91   }
    90     92   
    91     93   proc argtest {cmd args} { return $args }
    92     94   sqlite3_fts5_create_function db argtest argtest
    93     95   
................................................................................
   118    120   do_execsql_test 3.4.1 {
   119    121     SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'q'
   120    122   } {
   121    123     {{0 0 0}}
   122    124     {{0 0 5}} 
   123    125   }
   124    126   
   125         -do_execsql_test 3.4.2 {
   126         -  SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w'
   127         -} {
   128         -  {{1 0 1}}
   129         -  {{0 0 2} {1 0 4}} 
          127  +if {[detail_is_full]} {
          128  +  do_execsql_test 3.4.2 {
          129  +    SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w'
          130  +  } {
          131  +    {{1 0 1}}
          132  +    {{0 0 2} {1 0 4}} 
          133  +  }
   130    134   }
   131    135   
   132    136   proc coltest {cmd} {
   133    137     list [$cmd xColumnSize 0] [$cmd xColumnText 0]
   134    138   }
   135    139   sqlite3_fts5_create_function db coltest coltest
   136    140   
................................................................................
   145    149   # Tests for remapping the "rank" column.
   146    150   #
   147    151   #   4.1.*: Mapped to a function with no arguments.
   148    152   #   4.2.*: Mapped to a function with one or more arguments.
   149    153   #
   150    154   
   151    155   do_execsql_test 4.0 {
   152         -  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
          156  +  CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
   153    157     INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e');
   154    158     INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q');
   155    159     INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t');
   156    160     INSERT INTO t2 VALUES('h j g t r e l s g s', 'k q k c i i c k n s');
   157    161     INSERT INTO t2 VALUES('b l k h d n n n m i', 'p t i a r b t q o l');
   158    162     INSERT INTO t2 VALUES('k r i l j b g i p a', 't q c h a i m g n l');
   159    163     INSERT INTO t2 VALUES('a e c q n m o m d g', 'l c t g i s q g q e');
................................................................................
   214    218   }
   215    219   
   216    220   proc rowidplus {cmd ival} { 
   217    221     expr [$cmd xRowid] + $ival
   218    222   }
   219    223   sqlite3_fts5_create_function db rowidplus rowidplus
   220    224   
   221         -do_execsql_test 4.2.1 {
   222         -  INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) ');
   223         -  SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
   224         -} {
   225         -  10 110
   226         -}
   227         -do_execsql_test 4.2.2 {
   228         -  INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) ');
   229         -  SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
   230         -} {
   231         -  10 121
   232         -}
          225  +if {[detail_is_full]} {
          226  +  do_execsql_test 4.2.1 {
          227  +    INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) ');
          228  +    SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
          229  +  } {
          230  +    10 110
          231  +  }
          232  +  do_execsql_test 4.2.2 {
          233  +    INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) ');
          234  +    SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
          235  +  } {
          236  +    10 121
          237  +  }
   233    238   
   234         -do_execsql_test 4.2.3 {
   235         -  SELECT rowid, rank FROM t2 
   236         -  WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)'
   237         -} {
   238         -  10 122
          239  +  do_execsql_test 4.2.3 {
          240  +    SELECT rowid, rank FROM t2 
          241  +      WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)'
          242  +  } {
          243  +    10 122
          244  +  }
   239    245   }
   240    246   
   241    247   proc rowidmod {cmd imod} { 
   242    248     expr [$cmd xRowid] % $imod
   243    249   }
   244    250   sqlite3_fts5_create_function db rowidmod rowidmod
   245    251   do_execsql_test 4.3.1 {
   246         -  CREATE VIRTUAL TABLE t3 USING fts5(x);
          252  +  CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%);
   247    253     INSERT INTO t3 VALUES('a one');
   248    254     INSERT INTO t3 VALUES('a two');
   249    255     INSERT INTO t3 VALUES('a three');
   250    256     INSERT INTO t3 VALUES('a four');
   251    257     INSERT INTO t3 VALUES('a five');
   252    258     INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()');
   253    259   }
................................................................................
   283    289   do_catchsql_test 4.4.3 {
   284    290     SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 
   285    291   } {1 {no such function: xyz}}
   286    292   do_catchsql_test 4.4.4 {
   287    293     SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
   288    294   } {1 {parse error in rank function: }}
   289    295   
          296  +} ;# foreach_detail_mode
   290    297   
   291    298   
   292    299   finish_test
   293    300   

Changes to ext/fts5/test/fts5auto.test.

    18     18   
    19     19   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25         -
    26     25   set data {
    27     26       -4026076
    28     27       {n x w k b p x b n t t d s}     {f j j s p j o}               
    29     28       {w v i y r}                     {i p y s}                     
    30     29       {a o q v e n q r}               {q v g u c y a z y}           
    31     30       3995120
    32     31       {c}                             {e e w d t}                   
................................................................................
   228    227   
   229    228   do_execsql_test 1.0 {
   230    229     CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f);
   231    230   } {}
   232    231   
   233    232   fts5_aux_test_functions db
   234    233   
   235         -proc matchdata {expr tbl collist {order ASC}} {
   236         -
   237         -  set cols ""
   238         -  foreach e $collist {
   239         -    append cols ", '$e'"
   240         -  }
   241         -
   242         -  set tclexpr [db one [subst -novar {
   243         -    SELECT fts5_expr_tcl(
   244         -      $expr, 'nearset $cols -pc ::pc' [set cols]
   245         -    )
   246         -  }]]
   247         -  set res [list]
   248         -
   249         -  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
   250         -    set cols [list]
   251         -    foreach col $x(*) {
   252         -      if {$col != "rowid"} { lappend cols $x($col) }
   253         -    }
   254         -    # set cols [list $a $b $c $d $e $f]
   255         -    set ::pc 0
   256         -    set rowdata [eval $tclexpr]
   257         -    if {$rowdata != ""} { lappend res $x(rowid) $rowdata }
   258         -  }
   259         -
   260         -  set res
   261         -}
   262         -
   263         -proc do_auto_test {tn tbl cols expr} { 
          234  +proc do_auto_test {tn tbl expr} {
   264    235     foreach order {asc desc} {
   265         -    set res [matchdata $expr $tbl $cols $order]
          236  +    set res [fts5_poslist_data $expr $tbl $order]
   266    237       set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"
   267    238   
   268    239       set ::autotest_expr $expr
   269    240       do_execsql_test $testname [subst -novar {
   270    241         SELECT rowid, fts5_test_poslist([set tbl]) FROM [set tbl] 
   271    242         WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order]
   272    243       }] $res
   273    244     }
   274         -
   275         -
   276    245   }
   277    246   
   278    247   #-------------------------------------------------------------------------
   279    248   #
   280    249   
   281    250   for {set fold 0} {$fold < 3} {incr fold} {
   282    251     switch $fold {
................................................................................
   328    297       B.4 { a OR (b AND {a b c}:c) }
   329    298       B.5 { a OR "b c" }
   330    299       B.6 { a OR b OR c }
   331    300   
   332    301       C.1 { a OR (b AND "b c") }
   333    302       C.2 { a OR (b AND "z c") }
   334    303     } {
   335         -    do_auto_test 3.$fold.$tn tt {a b c d e f} $expr
          304  +    do_auto_test 3.$fold.$tn tt $expr
   336    305     }
   337    306   }
   338    307   
   339    308   proc replace_elems {list args} {
   340    309     set ret $list
   341    310     foreach {idx elem} $args {
   342    311       set ret [lreplace $ret $idx $idx $elem]
................................................................................
   362    331     1 x    
   363    332     2 y    
   364    333     3 z
   365    334   
   366    335     4 {c1 : x} 5 {c2 : x} 6 {c3 : x}
   367    336     7 {c1 : y} 8 {c2 : y} 9 {c3 : y}
   368    337     10 {c1 : z} 11 {c2 : z} 12 {c3 : z}
   369         -
   370         -
   371    338   } {
   372         -breakpoint
   373         -  do_auto_test 4.$tn yy {c1 c2 c3} $expr
          339  +  do_auto_test 4.$tn yy $expr
   374    340   }
   375    341   
   376    342   
   377    343   
   378    344   finish_test
   379    345   

Added ext/fts5/test/fts5detail.test.

            1  +# 2015 December 18
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5detail
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +fts5_aux_test_functions db
           25  +
           26  +#--------------------------------------------------------------------------
           27  +# Simple tests.
           28  +#
           29  +do_execsql_test 1.0 {
           30  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
           31  +  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
           32  +  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
           33  +  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
           34  +  INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
           35  +  INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
           36  +  INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
           37  +  INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
           38  +  INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8
           39  +  INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9
           40  +  INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10
           41  +}
           42  +
           43  +do_execsql_test 1.1 {
           44  +  INSERT INTO t1(t1) VALUES('integrity-check');
           45  +}
           46  +
           47  +foreach {tn match res} {
           48  +  1 "a:a" {9}
           49  +  2 "b:g" {1 4 6}
           50  +  3 "c:h" {1 3 6 8 9 10}
           51  +} {
           52  +  do_execsql_test 1.2.$tn.1 {
           53  +    SELECT rowid FROM t1($match);
           54  +  } $res
           55  +
           56  +  do_execsql_test 1.2.$tn.2 {
           57  +    SELECT rowid FROM t1($match || '*');
           58  +  } $res
           59  +}
           60  +
           61  +do_catchsql_test 1.3.1 {
           62  +  SELECT rowid FROM t1('h + d');
           63  +} {1 {fts5: phrase queries are not supported (detail!=full)}}
           64  +
           65  +do_catchsql_test 1.3.2 {
           66  +  SELECT rowid FROM t1('NEAR(h d)');
           67  +} {1 {fts5: NEAR queries are not supported (detail!=full)}}
           68  +
           69  +
           70  +#-------------------------------------------------------------------------
           71  +# integrity-check with both detail= and prefix= options.
           72  +#
           73  +do_execsql_test 2.0 {
           74  +  CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1");
           75  +  INSERT INTO t2(a) VALUES('aa ab');
           76  +}
           77  +
           78  +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r}
           79  +
           80  +do_execsql_test 2.1 {
           81  +  INSERT INTO t2(t2) VALUES('integrity-check');
           82  +}
           83  +
           84  +do_execsql_test 2.2 {
           85  +  SELECT fts5_test_poslist(t2) FROM t2('aa');
           86  +} {0.0.0}
           87  +
           88  +set ::pc 0
           89  +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
           90  +#exit
           91  +
           92  +#-------------------------------------------------------------------------
           93  +# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs
           94  +# work with detail=col tables.
           95  +#
           96  +set data {
           97  +  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
           98  +  2  {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc}
           99  +  3  {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca}
          100  +  4  {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca}
          101  +  5  {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb}
          102  +  6  {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb}
          103  +  7  {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc}
          104  +  8  {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb}
          105  +  9  {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb}
          106  +  10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa}
          107  +  11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac}
          108  +  12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc}
          109  +  13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc}
          110  +  14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa}
          111  +  15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc}
          112  +  16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc}
          113  +  17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca}
          114  +  18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc}
          115  +  19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc}
          116  +  20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc}
          117  +  21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb}
          118  +  22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc}
          119  +  23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc}
          120  +  24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb}
          121  +  25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc}
          122  +  26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa}
          123  +  27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac}
          124  +  28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa}
          125  +  29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac}
          126  +  30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac}
          127  +  31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba}
          128  +  32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc}
          129  +  33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa}
          130  +  34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac}
          131  +  35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb}
          132  +  36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa}
          133  +  37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb}
          134  +  38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac}
          135  +  39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba}
          136  +  40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca}
          137  +}
          138  +
          139  +set data {
          140  +  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
          141  +}
          142  +
          143  +proc matchdata {expr {bAsc 1}} {
          144  +
          145  +  set tclexpr [db one {
          146  +    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z')
          147  +  }]
          148  +  set res [list]
          149  +
          150  +  #puts "$expr -> $tclexpr"
          151  +  foreach {id x y z} $::data {
          152  +    set cols [list $x $y $z]
          153  +    set ::pc 0
          154  +    #set hits [lsort -command instcompare [eval $tclexpr]]
          155  +    set hits [eval $tclexpr]
          156  +    if {[llength $hits]>0} {
          157  +      lappend res [list $id $hits]
          158  +    }
          159  +  }
          160  +
          161  +  if {$bAsc} {
          162  +    set res [lsort -integer -increasing -index 0 $res]
          163  +  } else {
          164  +    set res [lsort -integer -decreasing -index 0 $res]
          165  +  }
          166  +
          167  +  return [concat {*}$res]
          168  +}
          169  +
          170  +foreach {tn tbl} {
          171  +  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) }
          172  +  2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) }
          173  +} {
          174  +  reset_db
          175  +  fts5_aux_test_functions db
          176  +  execsql $tbl
          177  +  foreach {id x y z} $data {
          178  +    execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) }
          179  +  }
          180  +  foreach {tn2 expr} {
          181  +    1 aaa    2 ccc    3 bab    4 aac
          182  +    5 aa*    6 cc*    7 ba*    8 aa*
          183  +    9 a*     10 b*   11 c*
          184  +  } {
          185  +
          186  +    set res [matchdata $expr]
          187  +
          188  +    do_execsql_test 3.$tn.$tn2.1 {
          189  +      SELECT rowid, fts5_test_poslist(t3) FROM t3($expr)
          190  +    } $res
          191  +
          192  +    do_execsql_test 3.$tn.$tn2.2 {
          193  +      SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr)
          194  +    } $res
          195  +  }
          196  +}
          197  +
          198  +#-------------------------------------------------------------------------
          199  +# Simple tests for detail=none tables.
          200  +#
          201  +do_execsql_test 4.0 {
          202  +  CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none);
          203  +  INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g');
          204  +  INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9');
          205  +}
          206  +
          207  +do_catchsql_test 4.1 {
          208  +  SELECT * FROM t4('a:a')
          209  +} {1 {fts5: column queries are not supported (detail=none)}}
          210  +
          211  +#-------------------------------------------------------------------------
          212  +# Test that for the same content detail=none uses less space than 
          213  +# detail=col, and that detail=col uses less space than detail=full
          214  +#
          215  +reset_db
          216  +do_test 5.1 {
          217  +  foreach {tbl detail} {t1 none t2 col t3 full} {
          218  +    execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)"
          219  +    foreach {rowid x y z} $::data {
          220  +      execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)"
          221  +    }
          222  +  }
          223  +} {}
          224  +
          225  +do_execsql_test 5.2 {
          226  +  SELECT 
          227  +    (SELECT sum(length(block)) from t1_data) <
          228  +    (SELECT sum(length(block)) from t2_data)
          229  +} {1}
          230  +
          231  +do_execsql_test 5.3 {
          232  +  SELECT 
          233  +    (SELECT sum(length(block)) from t2_data) <
          234  +    (SELECT sum(length(block)) from t3_data)
          235  +} {1}
          236  +
          237  +
          238  +
          239  +finish_test
          240  +

Changes to ext/fts5/test/fts5dlidx.test.

    22     22   }
    23     23   
    24     24   if { $tcl_platform(wordSize)<8 } {
    25     25     finish_test
    26     26     return
    27     27   }
    28     28   
    29         -if 1 {
           29  +foreach_detail_mode $testprefix {
    30     30   
    31     31   proc do_fb_test {tn sql res} {
    32     32     set res2 [lsort -integer -decr $res]
    33     33     uplevel [list do_execsql_test $tn.1 $sql $res]
    34     34     uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
    35     35   }
    36     36   
    37         -# This test populates the FTS5 table containing $nEntry entries. Rows are 
           37  +# This test populates the FTS5 table with $nEntry entries. Rows are 
    38     38   # numbered from 0 to ($nEntry-1). The rowid for row $i is:
    39     39   #
    40     40   #   ($iFirst + $i*$nStep)
    41     41   #
    42     42   # Each document is of the form "a b c a b c a b c...". If the row number ($i)
    43     43   # is an integer multiple of $spc1, then an "x" token is appended to the
    44     44   # document. If it is *also* a multiple of $spc2, a "y" token is also appended.
................................................................................
    73     73     
    74     74     do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
    75     75     do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
    76     76     
    77     77     do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
    78     78     do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
    79     79     
    80         -  do_fb_test $tn.5.1 { 
    81         -    SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
    82         -  do_fb_test $tn.5.2 { 
    83         -    SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
           80  +  if {[detail_is_full]} {
           81  +    do_fb_test $tn.5.1 { 
           82  +      SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
           83  +    do_fb_test $tn.5.2 { 
           84  +      SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
           85  +  }
    84     86   }
    85     87   
    86     88   
    87     89   foreach {tn pgsz} {
    88     90     1 32
    89     91     2 200
    90     92   } {
    91     93     do_execsql_test $tn.0 { 
    92     94       DROP TABLE IF EXISTS t1;
    93         -    CREATE VIRTUAL TABLE t1 USING fts5(x);
           95  +    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
    94     96       INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
    95     97     }
    96     98   
    97     99     do_dlidx_test1 1.$tn.1     10 100 10000 0 1000
    98    100     do_dlidx_test1 1.$tn.2     10 10  10000 0 128
    99    101     do_dlidx_test1 1.$tn.3     10 10  66    0 36028797018963970
   100    102     do_dlidx_test1 1.$tn.4     10 10  50    0 150000000000000000
................................................................................
   103    105   }
   104    106   
   105    107   proc do_dlidx_test2 {tn nEntry iFirst nStep} {
   106    108     set str [string repeat "a " 500]
   107    109     execsql {
   108    110       BEGIN;
   109    111       DROP TABLE IF EXISTS t1;
   110         -    CREATE VIRTUAL TABLE t1 USING fts5(x);
          112  +    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
   111    113       INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
   112    114       INSERT INTO t1 VALUES('b a');
   113    115   
   114    116       WITH iii(ii, i) AS (
   115    117         SELECT 1,     $iFirst UNION ALL 
   116    118         SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry
   117    119       )
................................................................................
   126    128     do_execsql_test $tn.2 {
   127    129       SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
   128    130     } {1}
   129    131   }
   130    132   
   131    133   do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
   132    134   
   133         -}
   134         -
   135    135   #--------------------------------------------------------------------
   136    136   #
   137    137   reset_db
   138    138   
   139    139   set ::vocab [list \
   140    140     IteratorpItercurrentlypointstothefirstrowidofadoclist \
   141    141     Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
................................................................................
   154    154       lappend ret [lindex $vocab [expr $i % $nVocab]]
   155    155     }
   156    156     set ret
   157    157   }
   158    158   db func rnddoc rnddoc
   159    159   
   160    160   do_execsql_test 3.1 {
   161         -  CREATE VIRTUAL TABLE abc USING fts5(a);
          161  +  CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%);
   162    162     INSERT INTO abc(abc, rank) VALUES('pgsz', 32);
   163    163   
   164    164     INSERT INTO abc VALUES ( rnddoc() );
   165    165     INSERT INTO abc VALUES ( rnddoc() );
   166    166     INSERT INTO abc VALUES ( rnddoc() );
   167    167     INSERT INTO abc VALUES ( rnddoc() );
   168    168   
................................................................................
   187    187   set v [lindex $vocab 0]
   188    188   set i 0
   189    189   foreach v $vocab {
   190    190     do_execsql_test 3.3.[incr i] {
   191    191       SELECT rowid FROM abc WHERE abc MATCH $v
   192    192     } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
   193    193   }
          194  +
          195  +} ;# foreach_detail_mode
          196  +
   194    197   
   195    198   
   196    199   finish_test
   197    200   

Changes to ext/fts5/test/fts5eb.test.

    38     38     6  {abc AND ""}                     {"abc"}
    39     39     7  {"" OR abc}                      {"abc"}
    40     40     8  {"" NOT abc}                     {"abc"}
    41     41     9  {"" AND abc}                     {"abc"}
    42     42     10 {abc + "" + def}                 {"abc" + "def"}
    43     43     11 {abc "" def}                     {"abc" AND "def"}
    44     44     12 {r+e OR w}                       {"r" + "e" OR "w"}
           45  +
           46  +  13 {a AND b NOT c}                  {"a" AND ("b" NOT "c")}
           47  +  14 {a OR b NOT c}                   {"a" OR ("b" NOT "c")}
           48  +  15 {a NOT b AND c}                  {("a" NOT "b") AND "c"}
           49  +  16 {a NOT b OR c}                   {("a" NOT "b") OR "c"}
           50  +
           51  +  17 {a AND b OR c}                   {("a" AND "b") OR "c"}
           52  +  18 {a OR b AND c}                   {"a" OR ("b" AND "c")}
           53  +
    45     54   } {
    46     55     do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
    47     56   }
    48     57   
    49     58   do_catchsql_test 2.1 {
    50     59     SELECT fts5_expr()
    51     60   } {1 {wrong number of arguments to function fts5_expr}}

Added ext/fts5/test/fts5fault8.test.

            1  +# 2015 September 3
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +# This file is focused on OOM errors.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +source $testdir/malloc_common.tcl
           17  +set testprefix fts5fault8
           18  +
           19  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           20  +ifcapable !fts5 {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +foreach_detail_mode $testprefix {
           26  +
           27  +if {[detail_is_none]==0} continue
           28  +
           29  +fts5_aux_test_functions db
           30  +do_execsql_test 1.0 {
           31  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
           32  +  INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
           33  +  INSERT INTO t1 VALUES('a b a b', NULL);
           34  +  INSERT INTO t1 VALUES(NULL, '1 2 1 2');
           35  +}
           36  +
           37  +do_faultsim_test 1 -faults oom-t* -body {
           38  +  execsql { 
           39  +    SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 
           40  +  }
           41  +} -test {
           42  +  faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
           43  +                       {1 SQLITE_NOMEM}
           44  +}
           45  +
           46  +}
           47  +
           48  +finish_test
           49  +

Changes to ext/fts5/test/fts5matchinfo.test.

    12     12   
    13     13   source [file join [file dirname [info script]] fts5_common.tcl]
    14     14   set testprefix fts5matchinfo
    15     15   
    16     16   # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
    17     17   ifcapable !fts5 { finish_test ; return }
    18     18   
           19  +foreach_detail_mode $testprefix {
           20  +
    19     21   proc mit {blob} {
    20     22     set scan(littleEndian) i*
    21     23     set scan(bigEndian) I*
    22     24     binary scan $blob $scan($::tcl_platform(byteOrder)) r
    23     25     return $r
    24     26   }
    25     27   db func mit mit
    26     28   
    27     29   sqlite3_fts5_register_matchinfo db
    28     30   
    29     31   do_execsql_test 1.0 {
    30         -  CREATE VIRTUAL TABLE t1 USING fts5(content);
           32  +  CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%);
    31     33   } 
    32     34   
    33     35   do_execsql_test 1.1 {
    34     36     INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');
    35     37     INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');
    36     38     INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');
    37     39     INSERT INTO t1(content) VALUES('A host, of golden daffodils,');
    38     40     SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';
    39     41   } {{1 1 1 2 2} {1 1 1 2 2}}
    40     42   
    41     43   # Now create an FTS4 table that does not specify matchinfo=fts3.
    42     44   #
    43     45   do_execsql_test 1.2 {
    44         -  CREATE VIRTUAL TABLE t2 USING fts5(content);
           46  +  CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%);
    45     47     INSERT INTO t2 SELECT * FROM t1;
    46     48     SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
    47     49   } {{1 1 1 2 2} {1 1 1 2 2}}
    48     50   
    49     51   
    50     52   #--------------------------------------------------------------------------
    51     53   # Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
................................................................................
   145    147     set res [list]
   146    148     foreach elem $list_of_lists {
   147    149       lappend res [list {*}$elem]
   148    150     }
   149    151     return $res
   150    152   }
   151    153   
          154  +# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5
          155  +# mode is not detail=full.
          156  +#
          157  +proc do_matchinfo_p_test {tn tbl expr results} {
          158  +  if {[detail_is_full]} {
          159  +    uplevel [list do_matchinfo_test $tn $tbl $expr $results]
          160  +  }
          161  +}
   152    162   
   153    163   do_execsql_test 4.1.0 {
   154         -  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
          164  +  CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
   155    165     INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
   156    166     INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
   157    167   }
   158    168   
   159    169   do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
   160    170     s {{3 0} {0 3}}
   161    171   }
................................................................................
   181    191   
   182    192     s {{3 0} {0 3}}
   183    193   
   184    194     xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
   185    195     xpxsscplax -
   186    196   }
   187    197   
   188         -do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
          198  +do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
   189    199     p {1 1}
   190    200     c {2 2}
   191    201     x {
   192    202       {0 1 1   1 1 1}
   193    203       {1 1 1   0 1 1}
   194    204     }
   195    205     n {2 2}
................................................................................
   199    209     s {{0 1} {1 0}}
   200    210   
   201    211     xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
   202    212     sxsxs -
   203    213   }
   204    214   
   205    215   do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'}     { s {{2 0} {0 2}} }
   206         -do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
   207         -do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
          216  +do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
          217  +do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
   208    218   do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'}     { s {{1 0} {0 1}} }
   209    219   do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
   210    220     x { 
   211    221       {0 1 1  1 1 1  0 0 0  0 0 0} 
   212    222       {1 1 1  0 1 1  0 0 0  0 0 0}
   213    223     }
   214    224   }
................................................................................
   216    226     x { 
   217    227       {0 1 1  1 1 1  0 0 0  0 0 0}
   218    228       {1 1 1  0 1 1  0 0 0  0 0 0}
   219    229     }
   220    230   }
   221    231   
   222    232   do_execsql_test 4.2.0 {
   223         -  CREATE VIRTUAL TABLE t5 USING fts5(content);
          233  +  CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%);
   224    234     INSERT INTO t5 VALUES('a a a a a');
   225    235     INSERT INTO t5 VALUES('a b a b a');
   226    236     INSERT INTO t5 VALUES('c b c b c');
   227    237     INSERT INTO t5 VALUES('x x x x x');
   228    238   }
   229    239   do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'}         { 
   230    240     x {{5 8 2   5 8 2} {3 8 2   3 8 2}}
   231    241     s {2 1} 
   232    242   }
   233    243   do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'}         { s {2} }
   234    244   do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
   235    245   do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
   236         -do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
          246  +do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
   237    247   do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }
   238    248   
   239    249   do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
   240    250   
   241    251   # It used to be that the second 'a' token would be deferred. That doesn't
   242    252   # work any longer.
   243    253   if 0 {
................................................................................
   246    256       s {2 1} 
   247    257     }
   248    258   }
   249    259   
   250    260   do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
   251    261   do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
   252    262   do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
   253         -do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
          263  +do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
   254    264   do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }
   255    265   
   256    266   do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }
   257    267   
   258    268   do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
   259    269   do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
   260    270   do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
   261    271   do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
   262    272   do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
   263         -do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
          273  +do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
   264    274   
   265    275   do_execsql_test 4.5.0 {
   266         -  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c);
          276  +  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%);
   267    277     INSERT INTO t6 VALUES('a', 'b', 'c');
   268    278   }
   269    279   do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }
   270    280   
   271    281   
   272    282   #-------------------------------------------------------------------------
   273    283   # Test the outcome of matchinfo() when used within a query that does not
   274    284   # use the full-text index (i.e. lookup by rowid or full-table scan).
   275    285   #
   276    286   do_execsql_test 7.1 {
   277         -  CREATE VIRTUAL TABLE t10 USING fts5(content);
          287  +  CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%);
   278    288     INSERT INTO t10 VALUES('first record');
   279    289     INSERT INTO t10 VALUES('second record');
   280    290   }
   281    291   do_execsql_test 7.2 {
   282    292     SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10;
   283    293   } {blob 8 blob 8}
   284    294   do_execsql_test 7.3 {
................................................................................
   295    305   # This was causing a problem at one point in the obscure case where the
   296    306   # total number of bytes of data stored in an fts3 table was greater than
   297    307   # the number of rows. i.e. when the following query returns true:
   298    308   #
   299    309   #   SELECT sum(length(content)) < count(*) FROM fts4table;
   300    310   #
   301    311   do_execsql_test 8.1 {
   302         -  CREATE VIRTUAL TABLE t11 USING fts5(content);
          312  +  CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%);
   303    313     INSERT INTO t11(t11, rank) VALUES('pgsz', 32);
   304    314     INSERT INTO t11 VALUES('quitealongstringoftext');
   305    315     INSERT INTO t11 VALUES('anotherquitealongstringoftext');
   306    316     INSERT INTO t11 VALUES('athirdlongstringoftext');
   307    317     INSERT INTO t11 VALUES('andonemoreforgoodluck');
   308    318   }
   309    319   do_test 8.2 {
................................................................................
   314    324   } {}
   315    325   do_execsql_test 8.3 {
   316    326     SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
   317    327   } {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}}
   318    328   
   319    329   #-------------------------------------------------------------------------
   320    330   
   321         -do_execsql_test 9.1 {
   322         -  CREATE VIRTUAL TABLE t12 USING fts5(content);
   323         -  INSERT INTO t12 VALUES('a b c d');
   324         -  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
   325         -} {{0 1 1 0 1 1 1 1 1}}
   326         -do_execsql_test 9.2 {
   327         -  INSERT INTO t12 VALUES('a d c d');
   328         -  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
   329         -} {
   330         -  {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
   331         -}
   332         -do_execsql_test 9.3 {
   333         -  INSERT INTO t12 VALUES('a d d a');
   334         -  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
   335         -} {
   336         -  {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
          331  +if {[detail_is_full]} {
          332  +  do_execsql_test 9.1 {
          333  +    CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%);
          334  +    INSERT INTO t12 VALUES('a b c d');
          335  +    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
          336  +  } {{0 1 1 0 1 1 1 1 1}}
          337  +  do_execsql_test 9.2 {
          338  +    INSERT INTO t12 VALUES('a d c d');
          339  +    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
          340  +  } {
          341  +    {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
          342  +  }
          343  +  do_execsql_test 9.3 {
          344  +    INSERT INTO t12 VALUES('a d d a');
          345  +    SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
          346  +  } {
          347  +    {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
          348  +  }
   337    349   }
   338    350   
   339    351   #---------------------------------------------------------------------------
   340    352   # Test for a memory leak
   341    353   #
   342    354   do_execsql_test 10.1 {
   343    355     DROP TABLE t10;
   344         -  CREATE VIRTUAL TABLE t10 USING fts5(idx, value);
          356  +  CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%);
   345    357     INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
   346    358     SELECT t10.rowid, t10.*
   347    359       FROM t10
   348    360       JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x
   349    361      WHERE t10 MATCH x.idx
   350    362        AND matchinfo(t10) not null
   351    363      GROUP BY t10.rowid
................................................................................
   354    366     
   355    367   #---------------------------------------------------------------------------
   356    368   # Test the 'y' matchinfo flag
   357    369   #
   358    370   reset_db
   359    371   sqlite3_fts5_register_matchinfo db
   360    372   do_execsql_test 11.0 {
   361         -  CREATE VIRTUAL TABLE tt USING fts5(x, y);
          373  +  CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%);
   362    374     INSERT INTO tt VALUES('c d a c d d', 'e a g b d a');   -- 1
   363    375     INSERT INTO tt VALUES('c c g a e b', 'c g d g e c');   -- 2
   364    376     INSERT INTO tt VALUES('b e f d e g', 'b a c b c g');   -- 3
   365    377     INSERT INTO tt VALUES('a c f f g d', 'd b f d e g');   -- 4
   366    378     INSERT INTO tt VALUES('g a c f c f', 'd g g b c c');   -- 5
   367    379     INSERT INTO tt VALUES('g a c e b b', 'd b f b g g');   -- 6
   368    380     INSERT INTO tt VALUES('f d a a f c', 'e e a d c f');   -- 7
................................................................................
   406    418     7 "a OR (a AND b)" {
   407    419         1 {1 2 1 2 0 1}   2 {1 0 1 0 1 0}   3 {0 1 0 1 1 2}   4 {1 0 1 0 0 1}   
   408    420         5 {1 0 1 0 0 1}   6 {1 0 1 0 2 2}   7 {2 1 0 0 0 0}   8 {1 2 1 2 2 1}   
   409    421         9 {1 1 1 1 1 3}  10 {1 3 0 0 0 0}
   410    422     }
   411    423   
   412    424   } {
          425  +
          426  +  if {[string match *:* $expr] && [detail_is_none]} continue
   413    427     do_execsql_test 11.1.$tn.1  {
   414    428       SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
   415    429     } $res
   416    430   
   417    431     set r2 [list]
   418    432     foreach {rowid L} $res {
   419    433       lappend r2 $rowid
................................................................................
   439    453   reset_db
   440    454   sqlite3_fts5_register_matchinfo db
   441    455   db func mit mit
   442    456   
   443    457   do_test 12.0 {
   444    458     set cols [list]
   445    459     for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
   446         -  execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])"
          460  +  execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)"
   447    461   } {}
   448    462   
   449    463   do_execsql_test 12.1 {
   450    464     INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc');
   451    465     SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
   452    466   } [list [list [expr 1<<4] [expr 1<<(45-32)]]]
          467  +
          468  +} ;# foreach_detail_mode
   453    469   
   454    470   finish_test
   455    471   

Changes to ext/fts5/test/fts5simple.test.

    14     14   set testprefix fts5simple
    15     15   
    16     16   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    17     17   ifcapable !fts5 {
    18     18     finish_test
    19     19     return
    20     20   }
    21         - 
           21  +
           22  + if 1 {
           23  +
    22     24   #-------------------------------------------------------------------------
    23     25   #
    24     26   set doc "x x [string repeat {y } 50]z z"
    25     27   do_execsql_test 1.0 {
    26     28     CREATE VIRTUAL TABLE t1 USING fts5(x);
    27     29     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    28     30     BEGIN;
................................................................................
   346    348     INSERT INTO x1 SELECT rnddoc(5) FROM ii;
   347    349   }
   348    350   
   349    351   do_execsql_test 4.1 {
   350    352     SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
   351    353   } {0 {} 4}
   352    354   
          355  +#-------------------------------------------------------------------------
          356  +reset_db
          357  +do_execsql_test 15.0 {
          358  +  CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1);
          359  +  INSERT INTO x2 VALUES('ab');
          360  +}
          361  +
          362  +do_execsql_test 15.1 {
          363  +  INSERT INTO x2(x2) VALUES('integrity-check');
          364  +}
          365  +
          366  +}
          367  +
          368  +#-------------------------------------------------------------------------
          369  +foreach_detail_mode $testprefix {
          370  +  reset_db
          371  +  fts5_aux_test_functions db
          372  +  do_execsql_test 16.0 {
          373  +    CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%);
          374  +    INSERT INTO x3 VALUES('a b c d e f');
          375  +  }
          376  +  do_execsql_test 16.1 {
          377  +    SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c');
          378  +  } {2.0.2}
          379  +
          380  +  do_execsql_test 16.1 {
          381  +    SELECT fts5_test_poslist(x3) FROM x3('a OR c');
          382  +  } {{0.0.0 1.0.2}}
          383  +}
   353    384   
   354    385   finish_test
   355    386   

Added ext/fts5/test/fts5simple2.test.

            1  +# 2015 September 05
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +set testprefix fts5simple2
           15  +
           16  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           17  +ifcapable !fts5 {
           18  +  finish_test
           19  +  return
           20  +}
           21  +
           22  +if 1 {
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           26  +  INSERT INTO t1 VALUES('a b c');
           27  +}
           28  +do_execsql_test 1.1 {
           29  +  SELECT rowid FROM t1('c a b')
           30  +} {1}
           31  +
           32  +#-------------------------------------------------------------------------
           33  +#
           34  +reset_db
           35  +do_execsql_test 2.0 {
           36  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           37  +  BEGIN;
           38  +    INSERT INTO t1 VALUES('b c d');
           39  +    INSERT INTO t1 VALUES('b c d');
           40  +  COMMIT;
           41  +}
           42  +do_execsql_test 2.1 {
           43  +  SELECT rowid FROM t1('b c d')
           44  +} {1 2}
           45  +
           46  +#-------------------------------------------------------------------------
           47  +#
           48  +reset_db
           49  +do_execsql_test 3.0 {
           50  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           51  +  BEGIN;
           52  +    INSERT INTO t1 VALUES('b c d');
           53  +    INSERT INTO t1 VALUES('b c d');
           54  +}
           55  +do_execsql_test 3.1 {
           56  +  SELECT rowid FROM t1('b c d'); COMMIT;
           57  +} {1 2}
           58  +
           59  +#-------------------------------------------------------------------------
           60  +#
           61  +reset_db
           62  +do_execsql_test 4.0 {
           63  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           64  +  BEGIN;
           65  +    INSERT INTO t1 VALUES('a1 b1 c1');
           66  +    INSERT INTO t1 VALUES('a2 b2 c2');
           67  +    INSERT INTO t1 VALUES('a3 b3 c3');
           68  +  COMMIT;
           69  +}
           70  +do_execsql_test 4.1 {
           71  +  SELECT rowid FROM t1('b*');
           72  +} {1 2 3}
           73  +
           74  +
           75  +#-------------------------------------------------------------------------
           76  +#
           77  +reset_db
           78  +do_execsql_test 5.0 {
           79  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           80  +  BEGIN;
           81  +  INSERT INTO t1 VALUES('a1 b1 c1');
           82  +  INSERT INTO t1 VALUES('a2 b2 c2');
           83  +  INSERT INTO t1 VALUES('a1 b1 c1');
           84  +  COMMIT;
           85  +}
           86  +do_execsql_test 5.1 { SELECT rowid FROM t1('b*') } {1 2 3}
           87  +
           88  +#-------------------------------------------------------------------------
           89  +#
           90  +reset_db
           91  +do_execsql_test 6.0 {
           92  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=full);
           93  +  BEGIN;
           94  +  INSERT INTO t1 VALUES('a1 b1 c1');
           95  +  INSERT INTO t1 VALUES('a1 b1 c1');
           96  +  INSERT INTO t1 VALUES('a1 b1 c1');
           97  +  COMMIT;
           98  +}
           99  +
          100  +do_execsql_test 6.1 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 2 1}
          101  +do_execsql_test 6.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC } {3 2 1}
          102  +do_execsql_test 6.3 { SELECT rowid FROM t1('c1') ORDER BY rowid DESC } {3 2 1}
          103  +
          104  +#-------------------------------------------------------------------------
          105  +#
          106  +reset_db
          107  +do_execsql_test 7.0 {
          108  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
          109  +  BEGIN;
          110  +  INSERT INTO t1 VALUES('a1 b1');
          111  +  INSERT INTO t1 VALUES('a1 b2');
          112  +  COMMIT;
          113  +}
          114  +do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
          115  +do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}
          116  +
          117  +#-------------------------------------------------------------------------
          118  +#
          119  +reset_db
          120  +do_execsql_test 8.0 {
          121  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
          122  +  INSERT INTO t1 VALUES('a1 b1 c1');
          123  +  INSERT INTO t1 VALUES('a2 b2 c2');
          124  +  INSERT INTO t1 VALUES('a1 b1 c1');
          125  +}
          126  +do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
          127  +do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3}
          128  +do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2}
          129  +
          130  +do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
          131  +do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
          132  +do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}
          133  +
          134  +do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') }
          135  +
          136  +do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
          137  +do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3}
          138  +do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2}
          139  +
          140  +do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
          141  +do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
          142  +do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}
          143  +
          144  +#--------------------------------------------------------------------------
          145  +#
          146  +reset_db
          147  +do_execsql_test 9.0.0 {
          148  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
          149  +  INSERT INTO t1 VALUES('a1 b1 c1');
          150  +  INSERT INTO t1 VALUES('a2 b2 c2');
          151  +  INSERT INTO t1 VALUES('a1 b1 c1');
          152  +}
          153  +do_execsql_test 9.0.1 {
          154  +  INSERT INTO t1(t1) VALUES('integrity-check');
          155  +} {}
          156  +
          157  +reset_db
          158  +do_execsql_test 9.1.0 {
          159  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
          160  +  INSERT INTO t1 VALUES('a1 b1 c1', 'x y z');
          161  +  INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3');
          162  +  INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z');
          163  +}
          164  +do_execsql_test 9.2.1 {
          165  +  INSERT INTO t1(t1) VALUES('integrity-check');
          166  +} {}
          167  +
          168  +#--------------------------------------------------------------------------
          169  +#
          170  +reset_db
          171  +do_execsql_test 10.0 {
          172  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
          173  +  INSERT INTO t1 VALUES('b1');
          174  +  INSERT INTO t1 VALUES('b1');
          175  +  DELETE FROM t1 WHERE rowid=1;
          176  +}
          177  +
          178  +do_execsql_test 10.1 {
          179  +  SELECT rowid FROM t1('b1');
          180  +} {2}
          181  +
          182  +do_execsql_test 10.2 {
          183  +  SELECT rowid FROM t1('b1') ORDER BY rowid DESC;
          184  +} {2}
          185  +
          186  +do_execsql_test 10.3 {
          187  +  INSERT INTO t1(t1) VALUES('integrity-check');
          188  +} {}
          189  +
          190  +#--------------------------------------------------------------------------
          191  +#
          192  +reset_db
          193  +do_execsql_test 11.1 {
          194  +  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none);
          195  +  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
          196  +  WITH d(x,y) AS (
          197  +    SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d
          198  +  )
          199  +  INSERT INTO t1 SELECT * FROM d LIMIT 23;
          200  +}
          201  +
          202  +#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" }
          203  +do_execsql_test 11.2 {
          204  +  SELECT rowid FROM t1;
          205  +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23}
          206  +
          207  +do_execsql_test 11.3 {
          208  +  SELECT rowid FROM t1('xyz');
          209  +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23}
          210  +
          211  +do_execsql_test 11.4 {
          212  +  INSERT INTO t1(t1) VALUES('integrity-check');
          213  +}
          214  +
          215  +#-------------------------------------------------------------------------
          216  +#
          217  +reset_db
          218  +do_execsql_test 12.0 {
          219  +  CREATE VIRTUAL TABLE yy USING fts5(x, detail=none);
          220  +  INSERT INTO yy VALUES('in if');
          221  +  INSERT INTO yy VALUES('if');
          222  +} {}
          223  +
          224  +do_execsql_test 12.1 {
          225  +  SELECT rowid FROM yy('i*');
          226  +} {1 2}
          227  +
          228  +#-------------------------------------------------------------------------
          229  +#
          230  +reset_db
          231  +do_execsql_test 13.0 {
          232  +  CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none);
          233  +} {}
          234  +foreach {rowid a} {
          235  +  0   {f}
          236  +  1   {u}
          237  +  2   {k}
          238  +  3   {a}
          239  +  4   {a}
          240  +  5   {u}
          241  +  6   {u}
          242  +  7   {u}
          243  +  8   {f}
          244  +  9   {f}
          245  +  10  {a}
          246  +  11  {p}
          247  +  12  {f}
          248  +  13  {u}
          249  +  14  {a}
          250  +  15  {a}
          251  +} {
          252  +  do_execsql_test 13.1.$rowid {
          253  +    INSERT INTO t1(rowid, a) VALUES($rowid, $a);
          254  +  }
          255  +}
          256  +
          257  +#-------------------------------------------------------------------------
          258  +#
          259  +reset_db
          260  +fts5_aux_test_functions db
          261  +do_execsql_test 14.0 {
          262  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
          263  +  INSERT INTO t1 VALUES('a b c d');
          264  +} {}
          265  +
          266  +do_execsql_test 14.1 {
          267  +  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
          268  +} {0.0.1}
          269  +
          270  +}
          271  +
          272  +#-------------------------------------------------------------------------
          273  +#
          274  +reset_db
          275  +do_execsql_test 15.1 {
          276  +  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
          277  +  BEGIN;
          278  +    INSERT INTO t1(rowid, x) VALUES(1, 'sqlite');
          279  +    INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); 
          280  +  COMMIT;
          281  +} {}
          282  +
          283  +do_test 15.1 {
          284  +  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
          285  +} {}
          286  +
          287  +do_test 15.2 {
          288  +  execsql { DELETE FROM t1 }
          289  +} {}
          290  +
          291  +do_execsql_test 15.3.1 {
          292  +  SELECT rowid FROM t1('sqlite');
          293  +} {}
          294  +
          295  +do_execsql_test 15.3.2 {
          296  +  SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
          297  +} {}
          298  +
          299  +do_test 15.4 {
          300  +  execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
          301  +} {}
          302  +  
          303  +finish_test
          304  +

Added ext/fts5/test/fts5synonym2.test.

            1  +# 2014 Dec 20
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Tests focusing on custom tokenizers that support synonyms.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5synonym2
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +#-------------------------------------------------------------------------
           25  +# Code for a simple Tcl tokenizer that supports synonyms at query time.
           26  +#
           27  +foreach SYNDICT {
           28  +  {zero  0}
           29  +  {one   1 i}
           30  +  {two   2 ii}
           31  +  {three 3 iii}
           32  +  {four  4 iv}
           33  +  {five  5 v}
           34  +  {six   6 vi}
           35  +  {seven 7 vii}
           36  +  {eight 8 viii}
           37  +  {nine  9 ix}
           38  +} {
           39  +  foreach s $SYNDICT {
           40  +    set o [list]
           41  +    foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
           42  +    set ::syn($s) $o
           43  +  }
           44  +}
           45  +
           46  +proc tcl_tokenize {tflags text} {
           47  +  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
           48  +    sqlite3_fts5_token $w $iStart $iEnd
           49  +    if {$tflags == "query"} {
           50  +      foreach s $::syn($w)  { sqlite3_fts5_token -colo $s $iStart $iEnd }
           51  +    }
           52  +  }
           53  +}
           54  +
           55  +proc tcl_create {args} {
           56  +  return "tcl_tokenize"
           57  +}
           58  +
           59  +#
           60  +# End of tokenizer code.
           61  +#-------------------------------------------------------------------------
           62  +
           63  +foreach_detail_mode $testprefix {
           64  +
           65  +sqlite3_fts5_create_tokenizer db tcl tcl_create
           66  +fts5_aux_test_functions db
           67  +
           68  +do_execsql_test 1.0 {
           69  +  CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
           70  +  INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
           71  +  INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
           72  +  INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
           73  +  INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7');
           74  +  INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii');
           75  +  INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4');
           76  +  INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4');
           77  +  INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii');
           78  +  INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five');
           79  +  INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine');
           80  +  INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven');
           81  +  INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three');
           82  +  INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9');
           83  +  INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight');
           84  +  INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven');
           85  +  INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four');
           86  +  INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2');
           87  +  INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six');
           88  +  INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi');
           89  +  INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi');
           90  +  INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii');
           91  +  INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii');
           92  +  INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4');
           93  +  INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii');
           94  +  INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4');
           95  +  INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii');
           96  +  INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four');
           97  +  INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five');
           98  +  INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one');
           99  +  INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8');
          100  +  INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i');
          101  +  INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9');
          102  +  INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five');
          103  +  INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6');
          104  +  INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4');
          105  +  INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv');
          106  +  INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine');
          107  +  INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five');
          108  +  INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi');
          109  +  INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii');
          110  +  INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two');
          111  +  INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one');
          112  +  INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii');
          113  +  INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4');
          114  +  INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii');
          115  +  INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii');
          116  +  INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4');
          117  +  INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii');
          118  +  INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one');
          119  +}
          120  +
          121  +foreach {tn expr} {
          122  +  1.1 "one"   1.2 "two"   1.3 "three"   1.4 "four"
          123  +  1.5 "v"     1.6 "vi"    1.7 "vii"     1.8 "viii"
          124  +  1.9 "9"    1.10 "0"    1.11 "1"      1.12 "2"
          125  +
          126  +  2.1 "one OR two OR three OR four"
          127  +  2.2 "(one AND two) OR (three AND four)"
          128  +  2.3 "(one AND two) OR (three AND four) NOT five"
          129  +  2.4 "(one AND two) NOT 6"
          130  +
          131  +  3.1 "b:one AND a:two"
          132  +  3.2 "b:one OR a:two"
          133  +  3.3 "a:one OR b:1 OR {a b} : i"
          134  +
          135  +  4.1 "NEAR(one two, 2)"
          136  +  4.2 "NEAR(one two three, 2)"
          137  +  4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
          138  +} {
          139  +  if {[fts5_expr_ok $expr ss]==0} {
          140  +    do_test 1.$tn.OMITTED { list } [list]
          141  +    continue
          142  +  }
          143  +
          144  +  set res [fts5_query_data $expr ss ASC ::syn]
          145  +  breakpoint
          146  +  do_execsql_test 1.$tn.[llength $res].asc {
          147  +    SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
          148  +  } $res
          149  +}
          150  +
          151  +}
          152  +
          153  +finish_test
          154  +

Changes to ext/fts5/test/fts5vocab.test.

    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
    23     23   
           24  +foreach_detail_mode $testprefix {
           25  +
           26  +proc null_list_entries {iFirst nInterval L} {
           27  +  for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} {
           28  +    lset L $i {}
           29  +  }
           30  +  return $L
           31  +}
           32  +
           33  +proc star_from_row {L} {
           34  +  if {[detail_is_full]==0} {
           35  +    set L [null_list_entries 2 3 $L]
           36  +  }
           37  +  return $L
           38  +}
           39  +
           40  +proc star_from_col {L} {
           41  +  if {[detail_is_col]} {
           42  +    set L [null_list_entries 3 4 $L]
           43  +  }
           44  +  if {[detail_is_none]} {
           45  +    set L [null_list_entries 1 4 $L]
           46  +    set L [null_list_entries 3 4 $L]
           47  +  }
           48  +  return $L
           49  +}
           50  +
           51  +proc row_to_col {L} {
           52  +  if {[detail_is_none]==0} { error "this is for detail=none mode" }
           53  +  set ret [list]
           54  +  foreach {a b c} $L {
           55  +    lappend ret $a {} $b {}
           56  +  }
           57  +  set ret
           58  +}
    24     59   
    25     60   do_execsql_test 1.1.1 {
    26         -  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1);
           61  +  CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
    27     62     CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
    28     63     PRAGMA table_info = v1;
    29     64   } {
    30     65     0 term {} 0 {} 0
    31     66     1 doc {} 0 {} 0
    32     67     2 cnt {} 0 {} 0
    33     68   }
................................................................................
    48     83   do_execsql_test 1.3 {
    49     84     INSERT INTO t1 VALUES('x y z');
    50     85     INSERT INTO t1 VALUES('x x x');
    51     86   }
    52     87   
    53     88   do_execsql_test 1.4.1 {
    54     89     SELECT * FROM v1;
    55         -} {x 2 4  y 1 1  z 1 1}
           90  +} [star_from_row {x 2 4  y 1 1  z 1 1}]
    56     91   
    57     92   do_execsql_test 1.4.2 {
    58     93     SELECT * FROM v2;
    59         -} {x one 2 4  y one 1 1  z one 1 1}
           94  +} [star_from_col {x one 2 4  y one 1 1  z one 1 1}]
    60     95   
    61     96   do_execsql_test 1.5.1 {
    62     97     BEGIN;
    63     98       INSERT INTO t1 VALUES('a b c');
    64     99       SELECT * FROM v1 WHERE term<'d';
    65         -} {a 1 1   b 1 1   c 1 1}
          100  +} [star_from_row {a 1 1   b 1 1   c 1 1}]
    66    101   
    67    102   do_execsql_test 1.5.2 {
    68    103       SELECT * FROM v2 WHERE term<'d';
    69    104     COMMIT;
    70         -} {a one 1 1  b one 1 1  c one 1 1}
          105  +} [star_from_col {a one 1 1  b one 1 1  c one 1 1}]
    71    106   
    72    107   do_execsql_test 1.6 {
    73    108     DELETE FROM t1 WHERE one = 'a b c';
    74    109     SELECT * FROM v1;
    75         -} {x 2 4  y 1 1  z 1 1}
          110  +} [star_from_row {x 2 4  y 1 1  z 1 1}]
    76    111   
    77    112   #-------------------------------------------------------------------------
    78    113   #
    79    114   do_execsql_test 2.0 {
    80         -  CREATE VIRTUAL TABLE tt USING fts5(a, b);
          115  +  CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%);
    81    116     INSERT INTO tt VALUES('d g b f d f', 'f c e c d a');
    82    117     INSERT INTO tt VALUES('f a e a a b', 'e d c f d d');
    83    118     INSERT INTO tt VALUES('b c a a a b', 'f f c c b c');
    84    119     INSERT INTO tt VALUES('f d c a c e', 'd g d e g d');
    85    120     INSERT INTO tt VALUES('g d e f a g x', 'f f d a a b');
    86    121     INSERT INTO tt VALUES('g c f b c g', 'a g f d c b');
    87    122     INSERT INTO tt VALUES('c e c f g b', 'f e d b g a');
    88    123     INSERT INTO tt VALUES('g d e f d e', 'a c d b a g');
    89    124     INSERT INTO tt VALUES('e f a c c b', 'b f e a f d y');
    90    125     INSERT INTO tt VALUES('c c a a c f', 'd g a e b g');
    91    126   }
    92    127   
    93         -set res_col {
          128  +set res_row [star_from_row {
          129  +  a 10 20   b 9 14   c 9 20   d 9 19   
          130  +  e 8 13   f 10 20   g 7 14   x 1 1   
          131  +  y 1 1
          132  +}]
          133  +set res_col [star_from_col {
    94    134     a a 6 11    a b 7 9
    95    135     b a 6 7     b b 7 7 
    96    136     c a 6 12    c b 5 8 
    97    137     d a 4 6     d b 9 13 
    98    138     e a 6 7     e b 6 6 
    99    139     f a 9 10    f b 7 10 
   100    140     g a 5 7     g b 5 7
   101    141     x a 1 1     y b 1 1
   102         -}
   103         -set res_row {
   104         -  a 10 20   b 9 14   c 9 20   d 9 19   
   105         -  e 8 13   f 10 20   g 7 14   x 1 1   
   106         -  y 1 1
          142  +}]
          143  +if {[detail_is_none]} {
          144  +  set res_col [row_to_col $res_row]
   107    145   }
   108    146   
   109    147   foreach {tn tbl resname} {
   110    148     1 "fts5vocab(tt, 'col')" res_col
   111    149     2 "fts5vocab(tt, 'row')" res_row
   112    150     3 "fts5vocab(tt, \"row\")" res_row
   113    151     4 "fts5vocab(tt, [row])" res_row
................................................................................
   149    187   #-------------------------------------------------------------------------
   150    188   # Test fts5vocab tables created in the temp schema. 
   151    189   #
   152    190   reset_db
   153    191   forcedelete test.db2
   154    192   do_execsql_test 5.0 {
   155    193     ATTACH 'test.db2' AS aux;
   156         -  CREATE VIRTUAL TABLE t1 USING fts5(x);
   157         -  CREATE VIRTUAL TABLE temp.t1 USING fts5(x);
   158         -  CREATE VIRTUAL TABLE aux.t1 USING fts5(x);
          194  +  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
          195  +  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
          196  +  CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%);
   159    197   
   160    198     INSERT INTO main.t1 VALUES('a b c');
   161    199     INSERT INTO main.t1 VALUES('d e f');
   162    200     INSERT INTO main.t1 VALUES('a e c');
   163    201   
   164    202     INSERT INTO temp.t1 VALUES('1 2 3');
   165    203     INSERT INTO temp.t1 VALUES('4 5 6');
................................................................................
   174    212   do_execsql_test 5.1 {
   175    213     CREATE VIRTUAL TABLE temp.vm  USING fts5vocab(main, t1, row);
   176    214     CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row);
   177    215     CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row);
   178    216     CREATE VIRTUAL TABLE temp.va  USING fts5vocab(aux, t1, row);
   179    217   }
   180    218   
   181         -do_execsql_test 5.2 { SELECT * FROM vm } {
          219  +do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row {
   182    220     a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1
   183         -}
   184         -do_execsql_test 5.3 { SELECT * FROM vt1 } {
          221  +}]
          222  +do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row {
   185    223     1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1
   186         -}
   187         -do_execsql_test 5.4 { SELECT * FROM vt2 } {
          224  +}]
          225  +do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row {
   188    226     1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1
   189         -}
   190         -do_execsql_test 5.5 { SELECT * FROM va } {
          227  +}]
          228  +do_execsql_test 5.5 { SELECT * FROM va } [star_from_row {
   191    229     m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2
   192         -}
          230  +}]
   193    231   
   194    232   #-------------------------------------------------------------------------
   195    233   #
   196    234   do_execsql_test 6.0 {
   197    235     CREATE TABLE iii(iii);
   198    236     CREATE TABLE jjj(x);
   199    237   }
................................................................................
   214    252   } {1 {no such fts5 table: main.lll}}
   215    253   
   216    254   #-------------------------------------------------------------------------
   217    255   # Test single term queries on fts5vocab tables (i.e. those with term=?
   218    256   # constraints in the WHERE clause).
   219    257   #
   220    258   do_execsql_test 7.0 {
   221         -  CREATE VIRTUAL TABLE tx USING fts5(one, two);
          259  +  CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%);
   222    260     INSERT INTO tx VALUES('g a ggg g a b eee',      'cc d aa ff g ee');
   223    261     INSERT INTO tx VALUES('dd fff i a i jjj',       'f fff hh jj e f');
   224    262     INSERT INTO tx VALUES('ggg a f f fff dd aa',    'd ggg f f j gg ddd');
   225    263     INSERT INTO tx VALUES('e bb h jjj ii gg',       'e aa e f c fff');
   226    264     INSERT INTO tx VALUES('j ff aa a h',            'h a j bbb bb');
   227    265     INSERT INTO tx VALUES('cc i ff c d f',          'dd ii fff f c cc d');
   228    266     INSERT INTO tx VALUES('jjj g i bb cc eee',      'hhh iii aaa b bbb aaa');
................................................................................
   272    310   
   273    311     set r2 [db eval {
   274    312       SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx
   275    313     }]
   276    314     if {[lindex $r2 2]==0} {set r2 [list]}
   277    315   
   278    316     set resc [concat $r1 $r2]
          317  +
          318  +  set resc [star_from_col $resc]
          319  +  set resr [star_from_row $resr]
          320  +  if {[detail_is_none]} { set resc [row_to_col $resr] }
   279    321     do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc
   280    322     do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr
   281    323   }
   282    324   
   283    325   do_execsql_test 7.1 {
   284    326     CREATE TABLE txr_c AS SELECT * FROM txr;
   285    327     CREATE TABLE txc_c AS SELECT * FROM txc;
................................................................................
   336    378     } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}]
   337    379   }
   338    380   
   339    381   do_execsql_test 7.3.1 {
   340    382     SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term;
   341    383   } {30}
   342    384   
   343         -do_execsql_test 7.3.2 {
   344         -  SELECT count(*) FROM txc, txc_c 
   345         -  WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
   346         -} {57}
          385  +if {![detail_is_none]} {
          386  +  do_execsql_test 7.3.2 {
          387  +    SELECT count(*) FROM txc, txc_c
          388  +      WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
          389  +  } {57}
          390  +}
          391  +
          392  +}
   347    393   
   348    394   finish_test
   349    395   

Added ext/fts5/tool/fts5speed.tcl.

            1  +
            2  +
            3  +set Q {
            4  +  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"}
            5  +  {25  "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"}
            6  +  {300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"}
            7  +  {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"}
            8  +  {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"}
            9  +  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"}
           10  +
           11  +  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"}
           12  +  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"}
           13  +  {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"}
           14  +}
           15  +
           16  +proc usage {} {
           17  +  global Q
           18  +  puts stderr "Usage: $::argv0 DATABASE QUERY"
           19  +  puts stderr ""
           20  +  for {set i 1} {$i <= [llength $Q]} {incr i} {
           21  +    puts stderr "       $i. [lindex $Q [expr $i-1]]"
           22  +  }
           23  +  puts stderr ""
           24  +  exit -1
           25  +}
           26  +
           27  +
           28  +set nArg [llength $argv]
           29  +if {$nArg!=2 && $nArg!=3} usage
           30  +set database [lindex $argv 0]
           31  +set iquery [lindex $argv 1]
           32  +if {$iquery<1 || $iquery>[llength $Q]} usage
           33  +set nRepeat 0
           34  +if {$nArg==3} { set nRepeat [lindex $argv 2] }
           35  +
           36  +
           37  +sqlite3 db $database
           38  +catch { load_static_extension db fts5 }
           39  +
           40  +incr iquery -1
           41  +set sql [lindex $Q $iquery 1]
           42  +if {$nRepeat==0} {
           43  +  set nRepeat [lindex $Q $iquery 0]
           44  +}
           45  +
           46  +puts "sql:     $sql"
           47  +puts "nRepeat: $nRepeat"
           48  +if {[regexp matchinfo $sql]} {
           49  +  sqlite3_fts5_register_matchinfo db
           50  +  db eval $sql 
           51  +} else {
           52  +  puts "result:  [db eval $sql]"
           53  +}
           54  +
           55  +for {set i 1} {$i < $nRepeat} {incr i} {
           56  +  db eval $sql
           57  +}
           58  +
           59  +

Changes to ext/rbu/rbu.c.

    71     71     int nStep = 0;                  /* Maximum number of step() calls */
    72     72     int rc;
    73     73     sqlite3_int64 nProgress = 0;
    74     74   
    75     75     /* Process command line arguments. Following this block local variables 
    76     76     ** zTarget, zRbu and nStep are all set. */
    77     77     if( argc==5 ){
    78         -    int nArg1 = strlen(argv[1]);
           78  +    size_t nArg1 = strlen(argv[1]);
    79     79       if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]);
    80     80       nStep = atoi(argv[2]);
    81     81     }else if( argc!=3 ){
    82     82       usage(argv[0]);
    83     83     }
    84     84     zTarget = argv[argc-2];
    85     85     zRbu = argv[argc-1];
................................................................................
    99     99     /* Let the user know what happened. */
   100    100     switch( rc ){
   101    101       case SQLITE_OK:
   102    102         sqlite3_snprintf(sizeof(zBuf), zBuf,
   103    103             "SQLITE_OK: rbu update incomplete (%lld operations so far)\n",
   104    104             nProgress
   105    105         );
   106         -      fprintf(stdout, zBuf);
          106  +      fprintf(stdout, "%s", zBuf);
   107    107         break;
   108    108   
   109    109       case SQLITE_DONE:
   110    110         sqlite3_snprintf(sizeof(zBuf), zBuf,
   111    111             "SQLITE_DONE: rbu update completed (%lld operations)\n",
   112    112             nProgress
   113    113         );
   114         -      fprintf(stdout, zBuf);
          114  +      fprintf(stdout, "%s", zBuf);
   115    115         break;
   116    116   
   117    117       default:
   118    118         fprintf(stderr, "error=%d: %s\n", rc, zErrmsg);
   119    119         break;
   120    120     }
   121    121   
   122    122     sqlite3_free(zErrmsg);
   123    123     return (rc==SQLITE_OK || rc==SQLITE_DONE) ? 0 : 1;
   124    124   }
   125         -

Changes to ext/rbu/sqlite3rbu.c.

   931    931   ** immediately without attempting the allocation or modifying the stored
   932    932   ** error code.
   933    933   */
   934    934   static void *rbuMalloc(sqlite3rbu *p, int nByte){
   935    935     void *pRet = 0;
   936    936     if( p->rc==SQLITE_OK ){
   937    937       assert( nByte>0 );
   938         -    pRet = sqlite3_malloc(nByte);
          938  +    pRet = sqlite3_malloc64(nByte);
   939    939       if( pRet==0 ){
   940    940         p->rc = SQLITE_NOMEM;
   941    941       }else{
   942    942         memset(pRet, 0, nByte);
   943    943       }
   944    944     }
   945    945     return pRet;
................................................................................
   977    977   ** if the allocation succeeds, (*pRc) is left unchanged.
   978    978   */
   979    979   static char *rbuStrndup(const char *zStr, int *pRc){
   980    980     char *zRet = 0;
   981    981   
   982    982     assert( *pRc==SQLITE_OK );
   983    983     if( zStr ){
   984         -    int nCopy = strlen(zStr) + 1;
   985         -    zRet = (char*)sqlite3_malloc(nCopy);
          984  +    size_t nCopy = strlen(zStr) + 1;
          985  +    zRet = (char*)sqlite3_malloc64(nCopy);
   986    986       if( zRet ){
   987    987         memcpy(zRet, zStr, nCopy);
   988    988       }else{
   989    989         *pRc = SQLITE_NOMEM;
   990    990       }
   991    991     }
   992    992   
................................................................................
  2326   2326       return SQLITE_INTERNAL;
  2327   2327     }
  2328   2328   
  2329   2329     pRbu->pgsz = iAmt;
  2330   2330     if( pRbu->nFrame==pRbu->nFrameAlloc ){
  2331   2331       int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2;
  2332   2332       RbuFrame *aNew;
  2333         -    aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame));
         2333  +    aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame));
  2334   2334       if( aNew==0 ) return SQLITE_NOMEM;
  2335   2335       pRbu->aFrame = aNew;
  2336   2336       pRbu->nFrameAlloc = nNew;
  2337   2337     }
  2338   2338   
  2339   2339     iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1;
  2340   2340     if( pRbu->iMaxFrame<iFrame ) pRbu->iMaxFrame = iFrame;
................................................................................
  2391   2391     int nChar;
  2392   2392     LPWSTR zWideFilename;
  2393   2393   
  2394   2394     nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0);
  2395   2395     if( nChar==0 ){
  2396   2396       return 0;
  2397   2397     }
  2398         -  zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) );
         2398  +  zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) );
  2399   2399     if( zWideFilename==0 ){
  2400   2400       return 0;
  2401   2401     }
  2402   2402     memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0]));
  2403   2403     nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename,
  2404   2404                                   nChar);
  2405   2405     if( nChar==0 ){
................................................................................
  3025   3025   */
  3026   3026   sqlite3rbu *sqlite3rbu_open(
  3027   3027     const char *zTarget, 
  3028   3028     const char *zRbu,
  3029   3029     const char *zState
  3030   3030   ){
  3031   3031     sqlite3rbu *p;
  3032         -  int nTarget = strlen(zTarget);
  3033         -  int nRbu = strlen(zRbu);
  3034         -  int nState = zState ? strlen(zState) : 0;
         3032  +  size_t nTarget = strlen(zTarget);
         3033  +  size_t nRbu = strlen(zRbu);
         3034  +  size_t nState = zState ? strlen(zState) : 0;
         3035  +  size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1;
  3035   3036   
  3036         -  p = (sqlite3rbu*)sqlite3_malloc(sizeof(sqlite3rbu)+nTarget+1+nRbu+1+nState+1);
         3037  +  p = (sqlite3rbu*)sqlite3_malloc64(nByte);
  3037   3038     if( p ){
  3038   3039       RbuState *pState = 0;
  3039   3040   
  3040   3041       /* Create the custom VFS. */
  3041   3042       memset(p, 0, sizeof(sqlite3rbu));
  3042   3043       rbuCreateVfs(p);
  3043   3044   
................................................................................
  3166   3167   ** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT,
  3167   3168   ** then edit any error message string so as to remove all occurrences of
  3168   3169   ** the pattern "rbu_imp_[0-9]*".
  3169   3170   */
  3170   3171   static void rbuEditErrmsg(sqlite3rbu *p){
  3171   3172     if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){
  3172   3173       int i;
  3173         -    int nErrmsg = strlen(p->zErrmsg);
         3174  +    size_t nErrmsg = strlen(p->zErrmsg);
  3174   3175       for(i=0; i<(nErrmsg-8); i++){
  3175   3176         if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){
  3176   3177           int nDel = 8;
  3177   3178           while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++;
  3178   3179           memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel);
  3179   3180           nErrmsg -= nDel;
  3180   3181         }
................................................................................
  3630   3631     /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this
  3631   3632     ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space 
  3632   3633     ** instead of a file on disk.  */
  3633   3634     assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) );
  3634   3635     if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){
  3635   3636       if( iRegion<=p->nShm ){
  3636   3637         int nByte = (iRegion+1) * sizeof(char*);
  3637         -      char **apNew = (char**)sqlite3_realloc(p->apShm, nByte);
         3638  +      char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte);
  3638   3639         if( apNew==0 ){
  3639   3640           rc = SQLITE_NOMEM;
  3640   3641         }else{
  3641   3642           memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm));
  3642   3643           p->apShm = apNew;
  3643   3644           p->nShm = iRegion+1;
  3644   3645         }
  3645   3646       }
  3646   3647   
  3647   3648       if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){
  3648         -      char *pNew = (char*)sqlite3_malloc(szRegion);
         3649  +      char *pNew = (char*)sqlite3_malloc64(szRegion);
  3649   3650         if( pNew==0 ){
  3650   3651           rc = SQLITE_NOMEM;
  3651   3652         }else{
  3652   3653           memset(pNew, 0, szRegion);
  3653   3654           p->apShm[iRegion] = pNew;
  3654   3655         }
  3655   3656       }
................................................................................
  3751   3752     if( zName ){
  3752   3753       if( flags & SQLITE_OPEN_MAIN_DB ){
  3753   3754         /* A main database has just been opened. The following block sets
  3754   3755         ** (pFd->zWal) to point to a buffer owned by SQLite that contains
  3755   3756         ** the name of the *-wal file this db connection will use. SQLite
  3756   3757         ** happens to pass a pointer to this buffer when using xAccess()
  3757   3758         ** or xOpen() to operate on the *-wal file.  */
  3758         -      int n = strlen(zName);
         3759  +      int n = (int)strlen(zName);
  3759   3760         const char *z = &zName[n];
  3760   3761         if( flags & SQLITE_OPEN_URI ){
  3761   3762           int odd = 0;
  3762   3763           while( 1 ){
  3763   3764             if( z[0]==0 ){
  3764   3765               odd = 1 - odd;
  3765   3766               if( odd && z[1]==0 ) break;
................................................................................
  3777   3778         rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName);
  3778   3779         if( pDb ){
  3779   3780           if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
  3780   3781             /* This call is to open a *-wal file. Intead, open the *-oal. This
  3781   3782             ** code ensures that the string passed to xOpen() is terminated by a
  3782   3783             ** pair of '\0' bytes in case the VFS attempts to extract a URI 
  3783   3784             ** parameter from it.  */
  3784         -          int nCopy = strlen(zName);
  3785         -          char *zCopy = sqlite3_malloc(nCopy+2);
         3785  +          size_t nCopy = strlen(zName);
         3786  +          char *zCopy = sqlite3_malloc64(nCopy+2);
  3786   3787             if( zCopy ){
  3787   3788               memcpy(zCopy, zName, nCopy);
  3788   3789               zCopy[nCopy-3] = 'o';
  3789   3790               zCopy[nCopy] = '\0';
  3790   3791               zCopy[nCopy+1] = '\0';
  3791   3792               zOpen = (const char*)(pFd->zDel = zCopy);
  3792   3793             }else{
................................................................................
  4007   4008       rbuVfsCurrentTime,            /* xCurrentTime */
  4008   4009       rbuVfsGetLastError,           /* xGetLastError */
  4009   4010       0,                            /* xCurrentTimeInt64 (version 2) */
  4010   4011       0, 0, 0                       /* Unimplemented version 3 methods */
  4011   4012     };
  4012   4013   
  4013   4014     rbu_vfs *pNew = 0;              /* Newly allocated VFS */
  4014         -  int nName;
  4015   4015     int rc = SQLITE_OK;
         4016  +  size_t nName;
         4017  +  size_t nByte;
  4016   4018   
  4017         -  int nByte;
  4018   4019     nName = strlen(zName);
  4019   4020     nByte = sizeof(rbu_vfs) + nName + 1;
  4020         -  pNew = (rbu_vfs*)sqlite3_malloc(nByte);
         4021  +  pNew = (rbu_vfs*)sqlite3_malloc64(nByte);
  4021   4022     if( pNew==0 ){
  4022   4023       rc = SQLITE_NOMEM;
  4023   4024     }else{
  4024   4025       sqlite3_vfs *pParent;           /* Parent VFS */
  4025   4026       memset(pNew, 0, nByte);
  4026   4027       pParent = sqlite3_vfs_find(zParent);
  4027   4028       if( pParent==0 ){

Changes to src/btree.c.

  1081   1081   static void btreeParseCellPtrNoPayload(
  1082   1082     MemPage *pPage,         /* Page containing the cell */
  1083   1083     u8 *pCell,              /* Pointer to the cell text. */
  1084   1084     CellInfo *pInfo         /* Fill in this structure */
  1085   1085   ){
  1086   1086     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1087   1087     assert( pPage->leaf==0 );
  1088         -  assert( pPage->noPayload );
  1089   1088     assert( pPage->childPtrSize==4 );
  1090   1089   #ifndef SQLITE_DEBUG
  1091   1090     UNUSED_PARAMETER(pPage);
  1092   1091   #endif
  1093   1092     pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey);
  1094   1093     pInfo->nPayload = 0;
  1095   1094     pInfo->nLocal = 0;
................................................................................
  1103   1102   ){
  1104   1103     u8 *pIter;              /* For scanning through pCell */
  1105   1104     u32 nPayload;           /* Number of bytes of cell payload */
  1106   1105     u64 iKey;               /* Extracted Key value */
  1107   1106   
  1108   1107     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1109   1108     assert( pPage->leaf==0 || pPage->leaf==1 );
  1110         -  assert( pPage->intKeyLeaf || pPage->noPayload );
  1111         -  assert( pPage->noPayload==0 );
  1112   1109     assert( pPage->intKeyLeaf );
  1113   1110     assert( pPage->childPtrSize==0 );
  1114   1111     pIter = pCell;
  1115   1112   
  1116   1113     /* The next block of code is equivalent to:
  1117   1114     **
  1118   1115     **     pIter += getVarint32(pIter, nPayload);
................................................................................
  1173   1170   ){
  1174   1171     u8 *pIter;              /* For scanning through pCell */
  1175   1172     u32 nPayload;           /* Number of bytes of cell payload */
  1176   1173   
  1177   1174     assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  1178   1175     assert( pPage->leaf==0 || pPage->leaf==1 );
  1179   1176     assert( pPage->intKeyLeaf==0 );
  1180         -  assert( pPage->noPayload==0 );
  1181   1177     pIter = pCell + pPage->childPtrSize;
  1182   1178     nPayload = *pIter;
  1183   1179     if( nPayload>=0x80 ){
  1184   1180       u8 *pEnd = &pIter[8];
  1185   1181       nPayload &= 0x7f;
  1186   1182       do{
  1187   1183         nPayload = (nPayload<<7) | (*++pIter & 0x7f);
................................................................................
  1234   1230     ** the (CellInfo.nSize) value found by doing a full parse of the
  1235   1231     ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of
  1236   1232     ** this function verifies that this invariant is not violated. */
  1237   1233     CellInfo debuginfo;
  1238   1234     pPage->xParseCell(pPage, pCell, &debuginfo);
  1239   1235   #endif
  1240   1236   
  1241         -  assert( pPage->noPayload==0 );
  1242   1237     nSize = *pIter;
  1243   1238     if( nSize>=0x80 ){
  1244   1239       pEnd = &pIter[8];
  1245   1240       nSize &= 0x7f;
  1246   1241       do{
  1247   1242         nSize = (nSize<<7) | (*++pIter & 0x7f);
  1248   1243       }while( *(pIter)>=0x80 && pIter<pEnd );
................................................................................
  1692   1687       assert( (PTF_LEAFDATA|PTF_INTKEY)==5 );
  1693   1688       /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf
  1694   1689       ** table b-tree page. */
  1695   1690       assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 );
  1696   1691       pPage->intKey = 1;
  1697   1692       if( pPage->leaf ){
  1698   1693         pPage->intKeyLeaf = 1;
  1699         -      pPage->noPayload = 0;
  1700   1694         pPage->xParseCell = btreeParseCellPtr;
  1701   1695       }else{
  1702   1696         pPage->intKeyLeaf = 0;
  1703         -      pPage->noPayload = 1;
  1704   1697         pPage->xCellSize = cellSizePtrNoPayload;
  1705   1698         pPage->xParseCell = btreeParseCellPtrNoPayload;
  1706   1699       }
  1707   1700       pPage->maxLocal = pBt->maxLeaf;
  1708   1701       pPage->minLocal = pBt->minLeaf;
  1709   1702     }else if( flagByte==PTF_ZERODATA ){
  1710   1703       /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior
................................................................................
  1711   1704       ** index b-tree page. */
  1712   1705       assert( (PTF_ZERODATA)==2 );
  1713   1706       /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf
  1714   1707       ** index b-tree page. */
  1715   1708       assert( (PTF_ZERODATA|PTF_LEAF)==10 );
  1716   1709       pPage->intKey = 0;
  1717   1710       pPage->intKeyLeaf = 0;
  1718         -    pPage->noPayload = 0;
  1719   1711       pPage->xParseCell = btreeParseCellPtrIndex;
  1720   1712       pPage->maxLocal = pBt->maxLocal;
  1721   1713       pPage->minLocal = pBt->minLocal;
  1722   1714     }else{
  1723   1715       /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is
  1724   1716       ** an error. */
  1725   1717       return SQLITE_CORRUPT_BKPT;

Changes to src/btreeInt.h.

   272    272   ** stored in MemPage.pBt->mutex.
   273    273   */
   274    274   struct MemPage {
   275    275     u8 isInit;           /* True if previously initialized. MUST BE FIRST! */
   276    276     u8 nOverflow;        /* Number of overflow cell bodies in aCell[] */
   277    277     u8 intKey;           /* True if table b-trees.  False for index b-trees */
   278    278     u8 intKeyLeaf;       /* True if the leaf of an intKey table */
   279         -  u8 noPayload;        /* True if internal intKey page (thus w/o data) */
   280    279     u8 leaf;             /* True if a leaf page */
   281    280     u8 hdrOffset;        /* 100 for page 1.  0 otherwise */
   282    281     u8 childPtrSize;     /* 0 if leaf==1.  4 if leaf==0 */
   283    282     u8 max1bytePayload;  /* min(maxLocal,127) */
   284    283     u8 bBusy;            /* Prevent endless loops on corrupt database files */
   285    284     u16 maxLocal;        /* Copy of BtShared.maxLocal or BtShared.maxLeaf */
   286    285     u16 minLocal;        /* Copy of BtShared.minLocal or BtShared.minLeaf */

Changes to src/main.c.

  3539   3539       assert( fd!=0 );
  3540   3540       if( op==SQLITE_FCNTL_FILE_POINTER ){
  3541   3541         *(sqlite3_file**)pArg = fd;
  3542   3542         rc = SQLITE_OK;
  3543   3543       }else if( op==SQLITE_FCNTL_VFS_POINTER ){
  3544   3544         *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager);
  3545   3545         rc = SQLITE_OK;
         3546  +    }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){
         3547  +      *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager);
         3548  +      rc = SQLITE_OK;
  3546   3549       }else if( fd->pMethods ){
  3547   3550         rc = sqlite3OsFileControl(fd, op, pArg);
  3548   3551   #ifndef SQLITE_OMIT_WAL
  3549   3552         if( (rc==SQLITE_OK)&&(op==SQLITE_FCNTL_LAST_ERRNO)&&(*(int *)pArg==0) ){
  3550   3553           sqlite3_file *pWalFd = sqlite3PagerWalFile(pPager);
  3551   3554           if( pWalFd&&(pWalFd->pMethods) ){
  3552   3555             rc = sqlite3OsFileControl(pWalFd, op, pArg);

Changes to src/os_unix.c.

   714    714   
   715    715     { "mkdir",        (sqlite3_syscall_ptr)mkdir,           0 },
   716    716   #define osMkdir     ((int(*)(const char*,mode_t))aSyscall[18].pCurrent)
   717    717   
   718    718     { "rmdir",        (sqlite3_syscall_ptr)rmdir,           0 },
   719    719   #define osRmdir     ((int(*)(const char*))aSyscall[19].pCurrent)
   720    720   
          721  +#if defined(HAVE_FCHOWN)
   721    722     { "fchown",       (sqlite3_syscall_ptr)fchown,          0 },
          723  +#else
          724  +  { "fchown",       (sqlite3_syscall_ptr)0,               0 },
          725  +#endif
   722    726   #define osFchown    ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent)
   723    727   
   724    728     { "geteuid",      (sqlite3_syscall_ptr)geteuid,         0 },
   725    729   #define osGeteuid   ((uid_t(*)(void))aSyscall[21].pCurrent)
   726    730   
   727    731   #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
   728    732     { "mmap",         (sqlite3_syscall_ptr)mmap,            0 },
................................................................................
   748    752   #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0
   749    753     { "getpagesize",  (sqlite3_syscall_ptr)unixGetpagesize, 0 },
   750    754   #else
   751    755     { "getpagesize",  (sqlite3_syscall_ptr)0,               0 },
   752    756   #endif
   753    757   #define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent)
   754    758   
          759  +#if defined(HAVE_READLINK)
   755    760     { "readlink",     (sqlite3_syscall_ptr)readlink,        0 },
          761  +#else
          762  +  { "readlink",     (sqlite3_syscall_ptr)0,               0 },
          763  +#endif
   756    764   #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent)
   757    765   
   758    766   
   759    767   }; /* End of the overrideable system calls */
   760    768   
   761    769   
   762    770   /*
   763    771   ** On some systems, calls to fchown() will trigger a message in a security
   764    772   ** log if they come from non-root processes.  So avoid calling fchown() if
   765    773   ** we are not running as root.
   766    774   */
   767    775   static int robustFchown(int fd, uid_t uid, gid_t gid){
   768         -#if OS_VXWORKS
   769         -  return 0;
   770         -#else
          776  +#if defined(HAVE_FCHOWN)
   771    777     return osGeteuid() ? 0 : osFchown(fd,uid,gid);
          778  +#else
          779  +  return 0;
   772    780   #endif
   773    781   }
   774    782   
   775    783   /*
   776    784   ** This is the xSetSystemCall() method of sqlite3_vfs for all of the
   777    785   ** "unix" VFSes.  Return SQLITE_OK opon successfully updating the
   778    786   ** system call pointer, or SQLITE_NOTFOUND if there is no configurable
................................................................................
  7150   7158     ** current working directory has been unlinked.
  7151   7159     */
  7152   7160     SimulateIOError( return SQLITE_ERROR );
  7153   7161   
  7154   7162     assert( pVfs->mxPathname==MAX_PATHNAME );
  7155   7163     UNUSED_PARAMETER(pVfs);
  7156   7164   
         7165  +#if defined(HAVE_READLINK)
  7157   7166     /* Attempt to resolve the path as if it were a symbolic link. If it is
  7158   7167     ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if
  7159   7168     ** the identified file is not a symbolic link or does not exist, then
  7160   7169     ** zPath is copied directly into zOut. Either way, nByte is left set to
  7161   7170     ** the size of the string copied into zOut[] in bytes.  */
  7162   7171     nByte = osReadlink(zPath, zOut, nOut-1);
  7163   7172     if( nByte<0 ){
................................................................................
  7165   7174         return unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zPath);
  7166   7175       }
  7167   7176       sqlite3_snprintf(nOut, zOut, "%s", zPath);
  7168   7177       nByte = sqlite3Strlen30(zOut);
  7169   7178     }else{
  7170   7179       zOut[nByte] = '\0';
  7171   7180     }
         7181  +#endif
  7172   7182   
  7173   7183     /* If buffer zOut[] now contains an absolute path there is nothing more
  7174   7184     ** to do. If it contains a relative path, do the following:
  7175   7185     **
  7176   7186     **   * move the relative path string so that it is at the end of th
  7177   7187     **     zOut[] buffer.
  7178   7188     **   * Call getcwd() to read the path of the current working directory 

Changes to src/pager.c.

  5649   5649         ** exclusive lock on the database is not already held, obtain it now.
  5650   5650         */
  5651   5651         if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){
  5652   5652           rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
  5653   5653           if( rc!=SQLITE_OK ){
  5654   5654             return rc;
  5655   5655           }
  5656         -        sqlite3WalExclusiveMode(pPager->pWal, 1);
         5656  +        (void)sqlite3WalExclusiveMode(pPager->pWal, 1);
  5657   5657         }
  5658   5658   
  5659   5659         /* Grab the write lock on the log file. If successful, upgrade to
  5660   5660         ** PAGER_RESERVED state. Otherwise, return an error code to the caller.
  5661   5661         ** The busy-handler is not invoked if another connection already
  5662   5662         ** holds the write-lock. If possible, the upper layer will call it.
  5663   5663         */
................................................................................
  6714   6714   ** with the pager.  This might return NULL if the file has
  6715   6715   ** not yet been opened.
  6716   6716   */
  6717   6717   sqlite3_file *sqlite3PagerFile(Pager *pPager){
  6718   6718     return pPager->fd;
  6719   6719   }
  6720   6720   
         6721  +#if !defined(SQLITE_OMIT_WAL)
  6721   6722   /*
  6722   6723    ** Return the file handle for the WAL journal file associated
  6723   6724    ** with the pager.  This might return NULL if the file has
  6724   6725    ** not yet been opened.
  6725   6726    */
  6726   6727   sqlite3_file *sqlite3PagerWalFile(Pager *pPager){
  6727   6728     return ((pPager->pWal) ? sqlite3WalFile(pPager->pWal) : (NULL));
         6729  +}
         6730  +#endif
         6731  +
         6732  +
         6733  +/*
         6734  +** Return the file handle for the journal file (if it exists).
         6735  +** This will be either the rollback journal or the WAL file.
         6736  +*/
         6737  +sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){
         6738  +#if SQLITE_OMIT_WAL
         6739  +  return pPager->jfd;
         6740  +#else
         6741  +  return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd;
         6742  +#endif
  6728   6743   }
  6729   6744   
  6730   6745   /*
  6731   6746   ** Return the full pathname of the journal file.
  6732   6747   */
  6733   6748   const char *sqlite3PagerJournalname(Pager *pPager){
  6734   6749     return pPager->zJournal;

Changes to src/pager.h.

   185    185     int sqlite3PagerRefcount(Pager*);
   186    186   #endif
   187    187   int sqlite3PagerMemUsed(Pager*);
   188    188   const char *sqlite3PagerFilename(Pager*, int);
   189    189   sqlite3_vfs *sqlite3PagerVfs(Pager*);
   190    190   sqlite3_file *sqlite3PagerFile(Pager*);
   191    191   sqlite3_file *sqlite3PagerWalFile(Pager *pPager);
          192  +sqlite3_file *sqlite3PagerJrnlFile(Pager*);
   192    193   const char *sqlite3PagerJournalname(Pager*);
   193    194   int sqlite3PagerNosync(Pager*);
   194    195   void *sqlite3PagerTempSpace(Pager*);
   195    196   int sqlite3PagerIsMemdb(Pager*);
   196    197   void sqlite3PagerCacheStat(Pager *, int, int, int *);
   197    198   void sqlite3PagerClearCache(Pager *);
   198    199   int sqlite3SectorSize(sqlite3_file *);

Changes to src/select.c.

    50     50   struct SortCtx {
    51     51     ExprList *pOrderBy;   /* The ORDER BY (or GROUP BY clause) */
    52     52     int nOBSat;           /* Number of ORDER BY terms satisfied by indices */
    53     53     int iECursor;         /* Cursor number for the sorter */
    54     54     int regReturn;        /* Register holding block-output return address */
    55     55     int labelBkOut;       /* Start label for the block-output subroutine */
    56     56     int addrSortIndex;    /* Address of the OP_SorterOpen or OP_OpenEphemeral */
           57  +  int labelDone;        /* Jump here when done, ex: LIMIT reached */
    57     58     u8 sortFlags;         /* Zero or more SORTFLAG_* bits */
    58     59   };
    59     60   #define SORTFLAG_UseSorter  0x01   /* Use SorterOpen instead of OpenEphemeral */
    60     61   
    61     62   /*
    62     63   ** Delete all the content of a Select structure.  Deallocate the structure
    63     64   ** itself only if bFree is true.
................................................................................
   120    121       pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0));
   121    122     }
   122    123     pNew->pEList = pEList;
   123    124     pNew->op = TK_SELECT;
   124    125     pNew->selFlags = selFlags;
   125    126     pNew->iLimit = 0;
   126    127     pNew->iOffset = 0;
          128  +#if SELECTTRACE_ENABLED
          129  +  pNew->zSelName[0] = 0;
          130  +#endif
   127    131     pNew->addrOpenEphm[0] = -1;
   128    132     pNew->addrOpenEphm[1] = -1;
   129    133     pNew->nSelectRow = 0;
   130    134     if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc));
   131    135     pNew->pSrc = pSrc;
   132    136     pNew->pWhere = pWhere;
   133    137     pNew->pGroupBy = pGroupBy;
................................................................................
   509    513     int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0);
   510    514     int nExpr = pSort->pOrderBy->nExpr;              /* No. of ORDER BY terms */
   511    515     int nBase = nExpr + bSeq + nData;                /* Fields in sorter record */
   512    516     int regBase;                                     /* Regs for sorter record */
   513    517     int regRecord = ++pParse->nMem;                  /* Assembled sorter record */
   514    518     int nOBSat = pSort->nOBSat;                      /* ORDER BY terms to skip */
   515    519     int op;                            /* Opcode to add sorter record to sorter */
          520  +  int iLimit;                        /* LIMIT counter */
   516    521   
   517    522     assert( bSeq==0 || bSeq==1 );
   518    523     assert( nData==1 || regData==regOrigData );
   519    524     if( nPrefixReg ){
   520    525       assert( nPrefixReg==nExpr+bSeq );
   521    526       regBase = regData - nExpr - bSeq;
   522    527     }else{
   523    528       regBase = pParse->nMem + 1;
   524    529       pParse->nMem += nBase;
   525    530     }
          531  +  assert( pSelect->iOffset==0 || pSelect->iLimit!=0 );
          532  +  iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit;
          533  +  pSort->labelDone = sqlite3VdbeMakeLabel(v);
   526    534     sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData,
   527    535                             SQLITE_ECEL_DUP|SQLITE_ECEL_REF);
   528    536     if( bSeq ){
   529    537       sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr);
   530    538     }
   531    539     if( nPrefixReg==0 ){
   532    540       sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData);
   533    541     }
   534         -
   535    542     sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord);
   536    543     if( nOBSat>0 ){
   537    544       int regPrevKey;   /* The first nOBSat columns of the previous row */
   538    545       int addrFirst;    /* Address of the OP_IfNot opcode */
   539    546       int addrJmp;      /* Address of the OP_Jump opcode */
   540    547       VdbeOp *pOp;      /* Opcode that opens the sorter */
   541    548       int nKey;         /* Number of sorting key columns, including OP_Sequence */
................................................................................
   562    569                                              pKI->nXField-1);
   563    570       addrJmp = sqlite3VdbeCurrentAddr(v);
   564    571       sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v);
   565    572       pSort->labelBkOut = sqlite3VdbeMakeLabel(v);
   566    573       pSort->regReturn = ++pParse->nMem;
   567    574       sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
   568    575       sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor);
          576  +    if( iLimit ){
          577  +      sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone);
          578  +      VdbeCoverage(v);
          579  +    }
   569    580       sqlite3VdbeJumpHere(v, addrFirst);
   570    581       sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat);
   571    582       sqlite3VdbeJumpHere(v, addrJmp);
   572    583     }
   573    584     if( pSort->sortFlags & SORTFLAG_UseSorter ){
   574    585       op = OP_SorterInsert;
   575    586     }else{
   576    587       op = OP_IdxInsert;
   577    588     }
   578    589     sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord);
   579         -  if( pSelect->iLimit ){
          590  +  if( iLimit ){
   580    591       int addr;
   581         -    int iLimit;
   582         -    if( pSelect->iOffset ){
   583         -      iLimit = pSelect->iOffset+1;
   584         -    }else{
   585         -      iLimit = pSelect->iLimit;
   586         -    }
   587    592       addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v);
   588    593       sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor);
   589    594       sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor);
   590    595       sqlite3VdbeJumpHere(v, addr);
   591    596     }
   592    597   }
   593    598   
................................................................................
  1183   1188     Parse *pParse,    /* Parsing context */
  1184   1189     Select *p,        /* The SELECT statement */
  1185   1190     SortCtx *pSort,   /* Information on the ORDER BY clause */
  1186   1191     int nColumn,      /* Number of columns of data */
  1187   1192     SelectDest *pDest /* Write the sorted results here */
  1188   1193   ){
  1189   1194     Vdbe *v = pParse->pVdbe;                     /* The prepared statement */
  1190         -  int addrBreak = sqlite3VdbeMakeLabel(v);     /* Jump here to exit loop */
         1195  +  int addrBreak = pSort->labelDone;            /* Jump here to exit loop */
  1191   1196     int addrContinue = sqlite3VdbeMakeLabel(v);  /* Jump here for next cycle */
  1192   1197     int addr;
  1193   1198     int addrOnce = 0;
  1194   1199     int iTab;
  1195   1200     ExprList *pOrderBy = pSort->pOrderBy;
  1196   1201     int eDest = pDest->eDest;
  1197   1202     int iParm = pDest->iSDParm;
................................................................................
  1202   1207     int nSortData;                  /* Trailing values to read from sorter */
  1203   1208     int i;
  1204   1209     int bSeq;                       /* True if sorter record includes seq. no. */
  1205   1210   #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
  1206   1211     struct ExprList_item *aOutEx = p->pEList->a;
  1207   1212   #endif
  1208   1213   
         1214  +  assert( addrBreak<0 );
  1209   1215     if( pSort->labelBkOut ){
  1210   1216       sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
  1211   1217       sqlite3VdbeGoto(v, addrBreak);
  1212   1218       sqlite3VdbeResolveLabel(v, pSort->labelBkOut);
  1213   1219     }
  1214   1220     iTab = pSort->iECursor;
  1215   1221     if( eDest==SRT_Output || eDest==SRT_Coroutine ){

Changes to src/shell.c.

  1917   1917     ".tables ?TABLE?        List names of tables\n"
  1918   1918     "                         If TABLE specified, only list tables matching\n"
  1919   1919     "                         LIKE pattern TABLE.\n"
  1920   1920     ".timeout MS            Try opening locked tables for MS milliseconds\n"
  1921   1921     ".timer on|off          Turn SQL timer on or off\n"
  1922   1922     ".trace FILE|off        Output each SQL statement as it is run\n"
  1923   1923     ".vfsinfo ?AUX?         Information about the top-level VFS\n"
         1924  +  ".vfslist               List all available VFSes\n"
  1924   1925     ".vfsname ?AUX?         Print the name of the VFS stack\n"
  1925   1926     ".width NUM1 NUM2 ...   Set column widths for \"column\" mode\n"
  1926   1927     "                         Negative values right-justify\n"
  1927   1928   ;
  1928   1929   
  1929   1930   /* Forward reference */
  1930   1931   static int process_input(ShellState *p, FILE *in);
................................................................................
  4163   4164         sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs);
  4164   4165         if( pVfs ){
  4165   4166           utf8_printf(p->out, "vfs.zName      = \"%s\"\n", pVfs->zName);
  4166   4167           raw_printf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
  4167   4168           raw_printf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
  4168   4169           raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
  4169   4170         }
         4171  +    }
         4172  +  }else
         4173  +
         4174  +  if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){
         4175  +    sqlite3_vfs *pVfs;
         4176  +    sqlite3_vfs *pCurrent = 0;
         4177  +    if( p->db ){
         4178  +      sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent);
         4179  +    }
         4180  +    for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){
         4181  +      utf8_printf(p->out, "vfs.zName      = \"%s\"%s\n", pVfs->zName,
         4182  +           pVfs==pCurrent ? "  <--- CURRENT" : "");
         4183  +      raw_printf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
         4184  +      raw_printf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
         4185  +      raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
         4186  +      if( pVfs->pNext ){
         4187  +        raw_printf(p->out, "-----------------------------------\n");
         4188  +      }
  4170   4189       }
  4171   4190     }else
  4172   4191   
  4173   4192     if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){
  4174   4193       const char *zDbName = nArg==2 ? azArg[1] : "main";
  4175   4194       char *zVfsName = 0;
  4176   4195       if( p->db ){

Changes to src/sqlite.h.in.

   791    791   ** for the nominated database. Allocating database file space in large
   792    792   ** chunks (say 1MB at a time), may reduce file-system fragmentation and
   793    793   ** improve performance on some systems.
   794    794   **
   795    795   ** <li>[[SQLITE_FCNTL_FILE_POINTER]]
   796    796   ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer
   797    797   ** to the [sqlite3_file] object associated with a particular database
   798         -** connection.  See the [sqlite3_file_control()] documentation for
   799         -** additional information.
          798  +** connection.  See also [SQLITE_FCNTL_JOURNAL_POINTER].
          799  +**
          800  +** <li>[[SQLITE_FCNTL_JOURNAL_POINTER]]
          801  +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer
          802  +** to the [sqlite3_file] object associated with the journal file (either
          803  +** the [rollback journal] or the [write-ahead log]) for a particular database
          804  +** connection.  See also [SQLITE_FCNTL_FILE_POINTER].
   800    805   **
   801    806   ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]]
   802    807   ** No longer in use.
   803    808   **
   804    809   ** <li>[[SQLITE_FCNTL_SYNC]]
   805    810   ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and
   806    811   ** sent to the VFS immediately before the xSync method is invoked on a
................................................................................
  1007   1012   #define SQLITE_FCNTL_SYNC                   21
  1008   1013   #define SQLITE_FCNTL_COMMIT_PHASETWO        22
  1009   1014   #define SQLITE_FCNTL_WIN32_SET_HANDLE       23
  1010   1015   #define SQLITE_FCNTL_WAL_BLOCK              24
  1011   1016   #define SQLITE_FCNTL_ZIPVFS                 25
  1012   1017   #define SQLITE_FCNTL_RBU                    26
  1013   1018   #define SQLITE_FCNTL_VFS_POINTER            27
         1019  +#define SQLITE_FCNTL_JOURNAL_POINTER        28
  1014   1020   
  1015   1021   /* deprecated names */
  1016   1022   #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
  1017   1023   #define SQLITE_SET_LOCKPROXYFILE      SQLITE_FCNTL_SET_LOCKPROXYFILE
  1018   1024   #define SQLITE_LAST_ERRNO             SQLITE_FCNTL_LAST_ERRNO
  1019   1025   
  1020   1026   

Changes to src/vdbe.c.

  2371   2371     const u8 *zData;   /* Part of the record being decoded */
  2372   2372     const u8 *zHdr;    /* Next unparsed byte of the header */
  2373   2373     const u8 *zEndHdr; /* Pointer to first byte after the header */
  2374   2374     u32 offset;        /* Offset into the data */
  2375   2375     u64 offset64;      /* 64-bit offset */
  2376   2376     u32 avail;         /* Number of bytes of available data */
  2377   2377     u32 t;             /* A type code from the record header */
  2378         -  u16 fx;            /* pDest->flags value */
  2379   2378     Mem *pReg;         /* PseudoTable input register */
  2380   2379   
  2381   2380     p2 = pOp->p2;
  2382   2381     assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
  2383   2382     pDest = &aMem[pOp->p3];
  2384   2383     memAboutToChange(p, pDest);
  2385   2384     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
................................................................................
  2549   2548     ** all valid.
  2550   2549     */
  2551   2550     assert( p2<pC->nHdrParsed );
  2552   2551     assert( rc==SQLITE_OK );
  2553   2552     assert( sqlite3VdbeCheckMemInvariants(pDest) );
  2554   2553     if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest);
  2555   2554     assert( t==pC->aType[p2] );
         2555  +  pDest->enc = encoding;
  2556   2556     if( pC->szRow>=aOffset[p2+1] ){
  2557   2557       /* This is the common case where the desired content fits on the original
  2558   2558       ** page - where the content is not on an overflow page */
  2559         -    sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest);
         2559  +    zData = pC->aRow + aOffset[p2];
         2560  +    if( t<12 ){
         2561  +      sqlite3VdbeSerialGet(zData, t, pDest);
         2562  +    }else{
         2563  +      /* If the column value is a string, we need a persistent value, not
         2564  +      ** a MEM_Ephem value.  This branch is a fast short-cut that is equivalent
         2565  +      ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize().
         2566  +      */
         2567  +      static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term };
         2568  +      pDest->n = len = (t-12)/2;
         2569  +      if( pDest->szMalloc < len+2 ){
         2570  +        pDest->flags = MEM_Null;
         2571  +        if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem;
         2572  +      }else{
         2573  +        pDest->z = pDest->zMalloc;
         2574  +      }
         2575  +      memcpy(pDest->z, zData, len);
         2576  +      pDest->z[len] = 0;
         2577  +      pDest->z[len+1] = 0;
         2578  +      pDest->flags = aFlag[t&1];
         2579  +    }
  2560   2580     }else{
  2561   2581       /* This branch happens only when content is on overflow pages */
  2562   2582       if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
  2563   2583             && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
  2564   2584        || (len = sqlite3VdbeSerialTypeLen(t))==0
  2565   2585       ){
  2566   2586         /* Content is irrelevant for
  2567   2587         **    1. the typeof() function,
  2568   2588         **    2. the length(X) function if X is a blob, and
  2569   2589         **    3. if the content length is zero.
  2570   2590         ** So we might as well use bogus content rather than reading
  2571         -      ** content from disk.  NULL will work for the value for strings
  2572         -      ** and blobs and whatever is in the payloadSize64 variable
  2573         -      ** will work for everything else. */
  2574         -      sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest);
         2591  +      ** content from disk. */
         2592  +      static u8 aZero[8];  /* This is the bogus content */
         2593  +      sqlite3VdbeSerialGet(aZero, t, pDest);
  2575   2594       }else{
  2576   2595         rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable,
  2577   2596                                      pDest);
  2578         -      if( rc!=SQLITE_OK ){
  2579         -        goto op_column_error;
         2597  +      if( rc==SQLITE_OK ){
         2598  +        sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
         2599  +        pDest->flags &= ~MEM_Ephem;
  2580   2600         }
  2581         -      sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
  2582         -      pDest->flags &= ~MEM_Ephem;
  2583   2601       }
  2584   2602     }
  2585         -  pDest->enc = encoding;
  2586   2603   
  2587   2604   op_column_out:
  2588         -  /* If the column value is an ephemeral string, go ahead and persist
  2589         -  ** that string in case the cursor moves before the column value is
  2590         -  ** used.  The following code does the equivalent of Deephemeralize()
  2591         -  ** but does it faster. */
  2592         -  if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){
  2593         -    fx = pDest->flags & (MEM_Str|MEM_Blob);
  2594         -    assert( fx!=0 );
  2595         -    zData = (const u8*)pDest->z;
  2596         -    len = pDest->n;
  2597         -    if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem;
  2598         -    memcpy(pDest->z, zData, len);
  2599         -    pDest->z[len] = 0;
  2600         -    pDest->z[len+1] = 0;
  2601         -    pDest->flags = fx|MEM_Term;
  2602         -  }
  2603   2605   op_column_error:
  2604   2606     UPDATE_MAX_BLOBSIZE(pDest);
  2605   2607     REGISTER_TRACE(pOp->p3, pDest);
  2606   2608     break;
  2607   2609   }
  2608   2610   
  2609   2611   /* Opcode: Affinity P1 P2 * P4 *

Changes to src/vdbeaux.c.

   301    301   ** as having been used.
   302    302   **
   303    303   ** The zWhere string must have been obtained from sqlite3_malloc().
   304    304   ** This routine will take ownership of the allocated memory.
   305    305   */
   306    306   void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){
   307    307     int j;
   308         -  int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0);
   309         -  sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC);
          308  +  sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
   310    309     for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
   311    310   }
   312    311   
   313    312   /*
   314    313   ** Add an opcode that includes the p4 value as an integer.
   315    314   */
   316    315   int sqlite3VdbeAddOp4Int(
................................................................................
   801    800   ** opcodes contained within. If aOp is not NULL it is assumed to contain 
   802    801   ** nOp entries. 
   803    802   */
   804    803   static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){
   805    804     if( aOp ){
   806    805       Op *pOp;
   807    806       for(pOp=aOp; pOp<&aOp[nOp]; pOp++){
   808         -      freeP4(db, pOp->p4type, pOp->p4.p);
          807  +      if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p);
   809    808   #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
   810    809         sqlite3DbFree(db, pOp->zComment);
   811    810   #endif     
   812    811       }
   813    812     }
   814    813     sqlite3DbFree(db, aOp);
   815    814   }
................................................................................
   863    862   ** 
   864    863   ** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points
   865    864   ** to a string or structure that is guaranteed to exist for the lifetime of
   866    865   ** the Vdbe. In these cases we can just copy the pointer.
   867    866   **
   868    867   ** If addr<0 then change P4 on the most recently inserted instruction.
   869    868   */
          869  +static void SQLITE_NOINLINE vdbeChangeP4Full(
          870  +  Vdbe *p,
          871  +  Op *pOp,
          872  +  const char *zP4,
          873  +  int n
          874  +){
          875  +  if( pOp->p4type ){
          876  +    freeP4(p->db, pOp->p4type, pOp->p4.p);
          877  +    pOp->p4type = 0;
          878  +    pOp->p4.p = 0;
          879  +  }
          880  +  if( n<0 ){
          881  +    sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n);
          882  +  }else{
          883  +    if( n==0 ) n = sqlite3Strlen30(zP4);
          884  +    pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n);
          885  +    pOp->p4type = P4_DYNAMIC;
          886  +  }
          887  +}
   870    888   void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){
   871    889     Op *pOp;
   872    890     sqlite3 *db;
   873    891     assert( p!=0 );
   874    892     db = p->db;
   875    893     assert( p->magic==VDBE_MAGIC_INIT );
   876         -  if( p->aOp==0 || db->mallocFailed ){
   877         -    if( n!=P4_VTAB ){
   878         -      freeP4(db, n, (void*)*(char**)&zP4);
   879         -    }
          894  +  assert( p->aOp!=0 || db->mallocFailed );
          895  +  if( db->mallocFailed ){
          896  +    if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4);
   880    897       return;
   881    898     }
   882    899     assert( p->nOp>0 );
   883    900     assert( addr<p->nOp );
   884    901     if( addr<0 ){
   885    902       addr = p->nOp - 1;
   886    903     }
   887    904     pOp = &p->aOp[addr];
   888         -  assert( pOp->p4type==P4_NOTUSED
   889         -       || pOp->p4type==P4_INT32
   890         -       || pOp->p4type==P4_KEYINFO );
   891         -  freeP4(db, pOp->p4type, pOp->p4.p);
   892         -  pOp->p4.p = 0;
          905  +  if( n>=0 || pOp->p4type ){
          906  +    vdbeChangeP4Full(p, pOp, zP4, n);
          907  +    return;
          908  +  }
   893    909     if( n==P4_INT32 ){
   894    910       /* Note: this cast is safe, because the origin data point was an int
   895    911       ** that was cast to a (const char *). */
   896    912       pOp->p4.i = SQLITE_PTR_TO_INT(zP4);
   897    913       pOp->p4type = P4_INT32;
   898         -  }else if( zP4==0 ){
   899         -    pOp->p4.p = 0;
   900         -    pOp->p4type = P4_NOTUSED;
   901         -  }else if( n==P4_KEYINFO ){
   902         -    pOp->p4.p = (void*)zP4;
   903         -    pOp->p4type = P4_KEYINFO;
   904         -#ifdef SQLITE_ENABLE_CURSOR_HINTS
   905         -  }else if( n==P4_EXPR ){
   906         -    /* Responsibility for deleting the Expr tree is handed over to the
   907         -    ** VDBE by this operation.  The caller should have already invoked
   908         -    ** sqlite3ExprDup() or whatever other routine is needed to make a 
   909         -    ** private copy of the tree. */
   910         -    pOp->p4.pExpr = (Expr*)zP4;
   911         -    pOp->p4type = P4_EXPR;
   912         -#endif
   913         -  }else if( n==P4_VTAB ){
   914         -    pOp->p4.p = (void*)zP4;
   915         -    pOp->p4type = P4_VTAB;
   916         -    sqlite3VtabLock((VTable *)zP4);
   917         -    assert( ((VTable *)zP4)->db==p->db );
   918         -  }else if( n<0 ){
          914  +  }else if( zP4!=0 ){
          915  +    assert( n<0 );
   919    916       pOp->p4.p = (void*)zP4;
   920    917       pOp->p4type = (signed char)n;
   921         -  }else{
   922         -    if( n==0 ) n = sqlite3Strlen30(zP4);
   923         -    pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n);
   924         -    pOp->p4type = P4_DYNAMIC;
          918  +    if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4);
   925    919     }
   926    920   }
   927    921   
   928    922   /*
   929    923   ** Set the P4 on the most recently added opcode to the KeyInfo for the
   930    924   ** index given.
   931    925   */

Changes to src/vdbesort.c.

   733    733     assert( pFile->iEof>iStart );
   734    734     assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 );
   735    735     assert( pReadr->aBuffer==0 );
   736    736     assert( pReadr->aMap==0 );
   737    737   
   738    738     rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart);
   739    739     if( rc==SQLITE_OK ){
   740         -    u64 nByte;                    /* Size of PMA in bytes */
          740  +    u64 nByte = 0;                 /* Size of PMA in bytes */
   741    741       rc = vdbePmaReadVarint(pReadr, &nByte);
   742    742       pReadr->iEof = pReadr->iReadOff + nByte;
   743    743       *pnByte += nByte;
   744    744     }
   745    745   
   746    746     if( rc==SQLITE_OK ){
   747    747       rc = vdbePmaReaderNext(pReadr);

Changes to src/vxworks.h.

    22     22   #define SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1
    23     23   #define SQLITE_OMIT_LOAD_EXTENSION 1
    24     24   #define SQLITE_ENABLE_LOCKING_STYLE 0
    25     25   #define HAVE_UTIME 1
    26     26   #else
    27     27   /* This is not VxWorks. */
    28     28   #define OS_VXWORKS 0
           29  +#define HAVE_FCHOWN 1
           30  +#define HAVE_READLINK 1
    29     31   #endif /* defined(_WRS_KERNEL) */

Changes to src/wal.c.

   704    704   
   705    705       nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
   706    706       walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum);
   707    707       walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
   708    708   
   709    709       sqlite3Put4byte(&aFrame[16], aCksum[0]);
   710    710       sqlite3Put4byte(&aFrame[20], aCksum[1]);
          711  +  }else{
          712  +    memset(&aFrame[8], 0, 16);
   711    713     }
   712    714   }
   713    715   
   714    716   /*
   715    717   ** Check to see if the frame with header in aFrame[] and content
   716    718   ** in aData[] is valid.  If it is a valid frame, fill *piPage and
   717    719   ** *pnTruncate and return true.  Return if the frame is not valid.
................................................................................
  2619   2621   Pgno sqlite3WalDbsize(Wal *pWal){
  2620   2622     if( pWal && ALWAYS(pWal->readLock>=0) ){
  2621   2623       return pWal->hdr.nPage;
  2622   2624     }
  2623   2625     return 0;
  2624   2626   }
  2625   2627   
  2626         -/* 
  2627         - ** Return the file for this Wal journal (or zero, if unknown).
  2628         - */
  2629         -sqlite3_file *sqlite3WalFile(Wal *pWal){
  2630         -  if( pWal ){
  2631         -    return pWal->pWalFd;
  2632         -  }
  2633         -  return 0;
  2634         -}
  2635         -
  2636   2628   /* 
  2637   2629   ** This function starts a write transaction on the WAL.
  2638   2630   **
  2639   2631   ** A read transaction must have already been started by a prior call
  2640   2632   ** to sqlite3WalBeginReadTransaction().
  2641   2633   **
  2642   2634   ** If another thread or process has written into the database since
................................................................................
  3445   3437   ** or zero if it is not (or if pWal is NULL).
  3446   3438   */
  3447   3439   int sqlite3WalFramesize(Wal *pWal){
  3448   3440     assert( pWal==0 || pWal->readLock>=0 );
  3449   3441     return (pWal ? pWal->szPage : 0);
  3450   3442   }
  3451   3443   #endif
         3444  +
         3445  +/* Return the sqlite3_file object for the WAL file
         3446  +*/
         3447  +sqlite3_file *sqlite3WalFile(Wal *pWal){
         3448  +  return pWal->pWalFd;
         3449  +}
  3452   3450   
  3453   3451   #endif /* #ifndef SQLITE_OMIT_WAL */

Changes to src/wal.h.

    40     40   # define sqlite3WalFrames(u,v,w,x,y,z)           0
    41     41   # define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0
    42     42   # define sqlite3WalCallback(z)                   0
    43     43   # define sqlite3WalExclusiveMode(y,z)            0
    44     44   # define sqlite3WalHeapMemory(z)                 0
    45     45   # define sqlite3WalFramesize(z)                  0
    46     46   # define sqlite3WalFindFrame(x,y,z)              0
           47  +# define sqlite3WalFile(x)                       0
    47     48   #else
    48     49   
    49     50   #define WAL_SAVEPOINT_NDATA 4
    50     51   
    51     52   /* Connection to a write-ahead log (WAL) file. 
    52     53   ** There is one object of this type for each pager. 
    53     54   */
................................................................................
   136    137   
   137    138   #ifdef SQLITE_ENABLE_ZIPVFS
   138    139   /* If the WAL file is not empty, return the number of bytes of content
   139    140   ** stored in each frame (i.e. the db page-size when the WAL was created).
   140    141   */
   141    142   int sqlite3WalFramesize(Wal *pWal);
   142    143   #endif
          144  +
          145  +/* Return the sqlite3_file object for the WAL file */
          146  +sqlite3_file *sqlite3WalFile(Wal *pWal);
   143    147   
   144    148   #endif /* ifndef SQLITE_OMIT_WAL */
   145    149   #endif /* _WAL_H_ */

Changes to src/walker.c.

    32     32   **
    33     33   **    WRC_Abort         Do no more callbacks.  Unwind the stack and
    34     34   **                      return the top-level walk call.
    35     35   **
    36     36   ** The return value from this routine is WRC_Abort to abandon the tree walk
    37     37   ** and WRC_Continue to continue.
    38     38   */
    39         -int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){
           39  +static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){
    40     40     int rc;
    41         -  if( pExpr==0 ) return WRC_Continue;
    42     41     testcase( ExprHasProperty(pExpr, EP_TokenOnly) );
    43     42     testcase( ExprHasProperty(pExpr, EP_Reduced) );
    44     43     rc = pWalker->xExprCallback(pWalker, pExpr);
    45     44     if( rc==WRC_Continue
    46     45                 && !ExprHasProperty(pExpr,EP_TokenOnly) ){
    47     46       if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
    48     47       if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort;
................................................................................
    50     49         if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort;
    51     50       }else{
    52     51         if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort;
    53     52       }
    54     53     }
    55     54     return rc & WRC_Abort;
    56     55   }
           56  +int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){
           57  +  return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue;
           58  +}
    57     59   
    58     60   /*
    59     61   ** Call sqlite3WalkExpr() for every expression in list p or until
    60     62   ** an abort request is seen.
    61     63   */
    62     64   int sqlite3WalkExprList(Walker *pWalker, ExprList *p){
    63     65     int i;

Changes to src/where.c.

  4314   4314         assert( pTabItem->iCursor==pLevel->iTabCur );
  4315   4315         testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
  4316   4316         testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
  4317   4317         if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol<BMS && HasRowid(pTab) ){
  4318   4318           Bitmask b = pTabItem->colUsed;
  4319   4319           int n = 0;
  4320   4320           for(; b; b=b>>1, n++){}
  4321         -        sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, 
  4322         -                            SQLITE_INT_TO_PTR(n), P4_INT32);
         4321  +        sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32);
  4323   4322           assert( n<=pTab->nCol );
  4324   4323         }
  4325   4324   #ifdef SQLITE_ENABLE_CURSOR_HINTS
  4326   4325         if( pLoop->u.btree.pIndex!=0 ){
  4327   4326           sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
  4328   4327         }else
  4329   4328   #endif

Changes to test/orderby1.test.

   523    523     INSERT INTO t1 SELECT i%2, randomblob(500) FROM cnt;
   524    524   }
   525    525   
   526    526   do_test 8.3 {
   527    527     db eval { SELECT * FROM t1 ORDER BY a, b } { incr res $a }
   528    528     set res
   529    529   } 5000
          530  +
          531  +#---------------------------------------------------------------------------
          532  +# https://www.sqlite.org/src/tktview/cb3aa0641d9a413841c004293a4fc06cdc122029
          533  +#
          534  +# Adverse interaction between scalar subqueries and the partial-sorting
          535  +# logic.
          536  +#
          537  +do_execsql_test 9.0 {
          538  +  DROP TABLE IF EXISTS t1;
          539  +  CREATE TABLE t1(x INTEGER PRIMARY KEY);
          540  +  INSERT INTO t1 VALUES(1),(2);
          541  +  DROP TABLE IF EXISTS t2;
          542  +  CREATE TABLE t2(y);
          543  +  INSERT INTO t2 VALUES(9),(8),(3),(4);
          544  +  SELECT (SELECT x||y FROM t2, t1 ORDER BY x, y);
          545  +} {13}
          546  +
   530    547   
   531    548   finish_test

Changes to test/waloverwrite.test.

   150    150     do_test 1.$tn.9 {
   151    151       db2 close
   152    152       forcecopy test.db-wal test.db2-wal
   153    153       sqlite3 db2 test.db2
   154    154       execsql { SELECT sum(length(y)) FROM t1 } db2
   155    155     } [expr 20*798]
   156    156   
   157         -  do_test 1.$tn.9 {
          157  +  do_test 1.$tn.10 {
   158    158       execsql { PRAGMA integrity_check } db2
   159    159     } ok
   160    160     db2 close
   161    161   }
   162    162   
   163    163   finish_test
   164    164   

Changes to tool/sqldiff.c.

   990    990     if( lenSrc<=NHASH ){
   991    991       putInt(lenOut, &zDelta);
   992    992       *(zDelta++) = ':';
   993    993       memcpy(zDelta, zOut, lenOut);
   994    994       zDelta += lenOut;
   995    995       putInt(checksum(zOut, lenOut), &zDelta);
   996    996       *(zDelta++) = ';';
   997         -    return zDelta - zOrigDelta;
          997  +    return (int)(zDelta - zOrigDelta);
   998    998     }
   999    999   
  1000   1000     /* Compute the hash table used to locate matching sections in the
  1001   1001     ** source file.
  1002   1002     */
  1003   1003     nHash = lenSrc/NHASH;
  1004   1004     collide = sqlite3_malloc( nHash*2*sizeof(int) );
................................................................................
  1137   1137       memcpy(zDelta, &zOut[base], lenOut-base);
  1138   1138       zDelta += lenOut-base;
  1139   1139     }
  1140   1140     /* Output the final checksum record. */
  1141   1141     putInt(checksum(zOut, lenOut), &zDelta);
  1142   1142     *(zDelta++) = ';';
  1143   1143     sqlite3_free(collide);
  1144         -  return zDelta - zOrigDelta;
         1144  +  return (int)(zDelta - zOrigDelta);
  1145   1145   }
  1146   1146   
  1147   1147   /*
  1148   1148   ** End of code copied from fossil.
  1149   1149   **************************************************************************/
  1150   1150   
  1151   1151   static void strPrintfArray(