/ Check-in [06bb80ee]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest wal2 changes into this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent-wal2
Files: files | file ages | folders
SHA3-256: 06bb80eeb84f57b1e8109a35f5b14992a2e23f2bf5a8921fa29087f96cb44d10
User & Date: dan 2018-12-15 20:59:14
Wiki:begin-concurrent-wal2
Context
2018-12-17
15:22
Merge changes from wal2 branch. check-in: 85f49f0d user: dan tags: begin-concurrent-wal2
2018-12-15
20:59
Merge latest wal2 changes into this branch. check-in: 06bb80ee user: dan tags: begin-concurrent-wal2
20:20
Further test cases for wal2 mode. check-in: 9cb5f8da user: dan tags: wal2
2018-12-07
20:25
Add multi-threaded performance test program "tserver" to this branch. Fix bugs in the begin-concurrent/wal2 integration revealed by the same. check-in: 7bd3b356 user: dan tags: begin-concurrent-wal2
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

  1302   1302   
  1303   1303   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1304   1304   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1305   1305   
  1306   1306   showshm$(TEXE):	$(TOP)/tool/showshm.c
  1307   1307   	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
  1308   1308   
         1309  +index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
         1310  +	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
         1311  +
  1309   1312   changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
  1310   1313   	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
  1311   1314   
  1312   1315   changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
  1313   1316   	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
  1314   1317   
  1315   1318   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo

Changes to Makefile.msc.

  2440   2440   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  2441   2441   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2442   2442   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2443   2443   
  2444   2444   showshm.exe:	$(TOP)\tool\showshm.c
  2445   2445   	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
  2446   2446   
         2447  +index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
         2448  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2449  +		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2450  +
  2447   2451   changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
  2448   2452   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2449   2453   		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
  2450   2454   		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2451   2455   
  2452   2456   changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
  2453   2457   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \

Changes to VERSION.

     1         -3.26.0
            1  +3.27.0

Changes to autoconf/Makefile.msc.

   279    279   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
   280    280   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
   281    281   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
   282    282   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
   283    283   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
   284    284   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
   285    285   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
          286  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
   286    287   !ENDIF
   287    288   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   288    289   !ENDIF
   289    290   
   290    291   # Should the session extension be enabled?  If so, add compilation options
   291    292   # to enable it.
   292    293   #
................................................................................
   933    934   # Additional compiler options for the shell.  These are only effective
   934    935   # when the shell is not being dynamically linked.
   935    936   #
   936    937   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   937    938   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
   938    939   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
   939    940   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
          941  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
   940    942   !ENDIF
   941    943   
   942    944   
   943    945   # This is the default Makefile target.  The objects listed here
   944    946   # are what get build when you type just "make" with no arguments.
   945    947   #
   946    948   core:	dll shell

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.27.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.26.0'
   730         -PACKAGE_STRING='sqlite 3.26.0'
          729  +PACKAGE_VERSION='3.27.0'
          730  +PACKAGE_STRING='sqlite 3.27.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1462   1462   #
  1463   1463   # Report the --help message.
  1464   1464   #
  1465   1465   if test "$ac_init_help" = "long"; then
  1466   1466     # Omit some internal or obsolete options to make the list less imposing.
  1467   1467     # This message is too long to be a string in the A/UX 3.1 sh.
  1468   1468     cat <<_ACEOF
  1469         -\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.
         1469  +\`configure' configures sqlite 3.27.0 to adapt to many kinds of systems.
  1470   1470   
  1471   1471   Usage: $0 [OPTION]... [VAR=VALUE]...
  1472   1472   
  1473   1473   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1474   1474   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1475   1475   
  1476   1476   Defaults for the options are specified in brackets.
................................................................................
  1527   1527     --build=BUILD     configure for building on BUILD [guessed]
  1528   1528     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1529   1529   _ACEOF
  1530   1530   fi
  1531   1531   
  1532   1532   if test -n "$ac_init_help"; then
  1533   1533     case $ac_init_help in
  1534         -     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
         1534  +     short | recursive ) echo "Configuration of sqlite 3.27.0:";;
  1535   1535      esac
  1536   1536     cat <<\_ACEOF
  1537   1537   
  1538   1538   Optional Features:
  1539   1539     --disable-option-checking  ignore unrecognized --enable/--with options
  1540   1540     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1541   1541     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1653   1653       cd "$ac_pwd" || { ac_status=$?; break; }
  1654   1654     done
  1655   1655   fi
  1656   1656   
  1657   1657   test -n "$ac_init_help" && exit $ac_status
  1658   1658   if $ac_init_version; then
  1659   1659     cat <<\_ACEOF
  1660         -sqlite configure 3.26.0
         1660  +sqlite configure 3.27.0
  1661   1661   generated by GNU Autoconf 2.69
  1662   1662   
  1663   1663   Copyright (C) 2012 Free Software Foundation, Inc.
  1664   1664   This configure script is free software; the Free Software Foundation
  1665   1665   gives unlimited permission to copy, distribute and modify it.
  1666   1666   _ACEOF
  1667   1667     exit
................................................................................
  2072   2072     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2073   2073   
  2074   2074   } # ac_fn_c_check_header_mongrel
  2075   2075   cat >config.log <<_ACEOF
  2076   2076   This file contains any messages produced by compilers while
  2077   2077   running configure, to aid debugging if configure makes a mistake.
  2078   2078   
  2079         -It was created by sqlite $as_me 3.26.0, which was
         2079  +It was created by sqlite $as_me 3.27.0, which was
  2080   2080   generated by GNU Autoconf 2.69.  Invocation command line was
  2081   2081   
  2082   2082     $ $0 $@
  2083   2083   
  2084   2084   _ACEOF
  2085   2085   exec 5>>config.log
  2086   2086   {
................................................................................
 12228  12228   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12229  12229   
 12230  12230   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12231  12231   # Save the log message, to keep $0 and so on meaningful, and to
 12232  12232   # report actual input values of CONFIG_FILES etc. instead of their
 12233  12233   # values after options handling.
 12234  12234   ac_log="
 12235         -This file was extended by sqlite $as_me 3.26.0, which was
        12235  +This file was extended by sqlite $as_me 3.27.0, which was
 12236  12236   generated by GNU Autoconf 2.69.  Invocation command line was
 12237  12237   
 12238  12238     CONFIG_FILES    = $CONFIG_FILES
 12239  12239     CONFIG_HEADERS  = $CONFIG_HEADERS
 12240  12240     CONFIG_LINKS    = $CONFIG_LINKS
 12241  12241     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12242  12242     $ $0 $@
................................................................................
 12294  12294   
 12295  12295   Report bugs to the package provider."
 12296  12296   
 12297  12297   _ACEOF
 12298  12298   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12299  12299   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12300  12300   ac_cs_version="\\
 12301         -sqlite config.status 3.26.0
        12301  +sqlite config.status 3.27.0
 12302  12302   configured by $0, generated by GNU Autoconf 2.69,
 12303  12303     with options \\"\$ac_cs_config\\"
 12304  12304   
 12305  12305   Copyright (C) 2012 Free Software Foundation, Inc.
 12306  12306   This config.status script is free software; the Free Software Foundation
 12307  12307   gives unlimited permission to copy, distribute and modify it."
 12308  12308   

Changes to ext/fts3/fts3_unicode.c.

    78     78   #endif /* ifndef SQLITE_AMALGAMATION */
    79     79   
    80     80   typedef struct unicode_tokenizer unicode_tokenizer;
    81     81   typedef struct unicode_cursor unicode_cursor;
    82     82   
    83     83   struct unicode_tokenizer {
    84     84     sqlite3_tokenizer base;
    85         -  int bRemoveDiacritic;
           85  +  int eRemoveDiacritic;
    86     86     int nException;
    87     87     int *aiException;
    88     88   };
    89     89   
    90     90   struct unicode_cursor {
    91     91     sqlite3_tokenizer_cursor base;
    92     92     const unsigned char *aInput;    /* Input text being tokenized */
................................................................................
   223    223     unicode_tokenizer *pNew;        /* New tokenizer object */
   224    224     int i;
   225    225     int rc = SQLITE_OK;
   226    226   
   227    227     pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   228    228     if( pNew==NULL ) return SQLITE_NOMEM;
   229    229     memset(pNew, 0, sizeof(unicode_tokenizer));
   230         -  pNew->bRemoveDiacritic = 1;
          230  +  pNew->eRemoveDiacritic = 1;
   231    231   
   232    232     for(i=0; rc==SQLITE_OK && i<nArg; i++){
   233    233       const char *z = azArg[i];
   234    234       int n = (int)strlen(z);
   235    235   
   236    236       if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
   237         -      pNew->bRemoveDiacritic = 1;
          237  +      pNew->eRemoveDiacritic = 1;
   238    238       }
   239    239       else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
   240         -      pNew->bRemoveDiacritic = 0;
          240  +      pNew->eRemoveDiacritic = 0;
          241  +    }
          242  +    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
          243  +      pNew->eRemoveDiacritic = 2;
   241    244       }
   242    245       else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
   243    246         rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
   244    247       }
   245    248       else if( n>=11 && memcmp("separators=", z, 11)==0 ){
   246    249         rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
   247    250       }
................................................................................
   346    349         zOut = &zNew[zOut - pCsr->zToken];
   347    350         pCsr->zToken = zNew;
   348    351         pCsr->nAlloc += 64;
   349    352       }
   350    353   
   351    354       /* Write the folded case of the last character read to the output */
   352    355       zEnd = z;
   353         -    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
          356  +    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
   354    357       if( iOut ){
   355    358         WRITE_UTF8(zOut, iOut);
   356    359       }
   357    360   
   358    361       /* If the cursor is not at EOF, read the next character */
   359    362       if( z>=zTerm ) break;
   360    363       READ_UTF8(z, zTerm, iCode);

Changes to ext/fts3/fts3_unicode2.c.

   155    155   ** If the argument is a codepoint corresponding to a lowercase letter
   156    156   ** in the ASCII range with a diacritic added, return the codepoint
   157    157   ** of the ASCII letter only. For example, if passed 235 - "LATIN
   158    158   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
   159    159   ** E"). The resuls of passing a codepoint that corresponds to an
   160    160   ** uppercase letter are undefined.
   161    161   */
   162         -static int remove_diacritic(int c){
          162  +static int remove_diacritic(int c, int bComplex){
   163    163     unsigned short aDia[] = {
   164    164           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
   165    165        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
   166    166        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
   167    167        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
   168         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
   169         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
   170         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
   171         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
   172         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
   173         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
   174         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
   175         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
   176         -    62924, 63050, 63082, 63274, 63390, 
          168  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
          169  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
          170  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
          171  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
          172  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
          173  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
          174  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
          175  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
          176  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
          177  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
          178  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
          179  +    63182, 63242, 63274, 63310, 63368, 63390, 
   177    180     };
   178    181     char aChar[] = {
   179         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
   180         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
   181         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
   182         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
   183         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
   184         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
   185         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
   186         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
   187         -    'e',  'i',  'o',  'u',  'y',  
          182  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
          183  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
          184  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
          185  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
          186  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
          187  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
          188  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
          189  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
          190  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
          191  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
          192  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
          193  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
          194  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
          195  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
          196  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
          197  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
          198  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
          199  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
          200  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
          201  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
          202  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   188    203     };
   189    204   
   190    205     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   191    206     int iRes = 0;
   192    207     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
   193    208     int iLo = 0;
   194    209     while( iHi>=iLo ){
................................................................................
   197    212         iRes = iTest;
   198    213         iLo = iTest+1;
   199    214       }else{
   200    215         iHi = iTest-1;
   201    216       }
   202    217     }
   203    218     assert( key>=aDia[iRes] );
   204         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
          219  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
          220  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
   205    221   }
   206    222   
   207    223   
   208    224   /*
   209    225   ** Return true if the argument interpreted as a unicode codepoint
   210    226   ** is a diacritical modifier character.
   211    227   */
................................................................................
   224    240   ** is an upper case character that has a lower case equivalent,
   225    241   ** return the codepoint corresponding to the lower case version.
   226    242   ** Otherwise, return a copy of the argument.
   227    243   **
   228    244   ** The results are undefined if the value passed to this function
   229    245   ** is less than zero.
   230    246   */
   231         -int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
          247  +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
   232    248     /* Each entry in the following array defines a rule for folding a range
   233    249     ** of codepoints to lower case. The rule applies to a range of nRange
   234    250     ** codepoints starting at codepoint iCode.
   235    251     **
   236    252     ** If the least significant bit in flags is clear, then the rule applies
   237    253     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   238    254     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   347    363       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   348    364       p = &aEntry[iRes];
   349    365       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   350    366         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   351    367         assert( ret>0 );
   352    368       }
   353    369   
   354         -    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
          370  +    if( eRemoveDiacritic ){
          371  +      ret = remove_diacritic(ret, eRemoveDiacritic==2);
          372  +    }
   355    373     }
   356    374     
   357    375     else if( c>=66560 && c<66600 ){
   358    376       ret = c + 40;
   359    377     }
   360    378   
   361    379     return ret;
   362    380   }
   363    381   #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
   364    382   #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/unicode/mkunicode.tcl.

     5      5     global tl_lookup_table
     6      6     set aChar [list]
     7      7     set lRange [list]
     8      8   
     9      9     set nRange 1
    10     10     set iFirst  [lindex $map 0 0]
    11     11     set cPrev   [lindex $map 0 1]
           12  +  set fPrev   [lindex $map 0 2]
    12     13   
    13     14     foreach m [lrange $map 1 end] {
    14         -    foreach {i c} $m {}
           15  +    foreach {i c f} $m {}
    15     16   
    16         -    if {$cPrev == $c} {
           17  +    if {$cPrev == $c && $fPrev==$f} {
    17     18         for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
    18     19           if {[info exists tl_lookup_table($j)]==0} break
    19     20         }
    20     21   
    21     22         if {$j==$i} {
    22     23           set nNew [expr {(1 + $i - $iFirst)}]
    23     24           if {$nNew<=8} {
................................................................................
    25     26             continue
    26     27           }
    27     28         }
    28     29       }
    29     30   
    30     31       lappend lRange [list $iFirst $nRange]
    31     32       lappend aChar  $cPrev
           33  +    lappend aFlag  $fPrev
    32     34   
    33     35       set iFirst $i
    34     36       set cPrev  $c
           37  +    set fPrev  $f
    35     38       set nRange 1
    36     39     }
    37     40     lappend lRange [list $iFirst $nRange]
    38     41     lappend aChar $cPrev
           42  +  lappend aFlag $fPrev
    39     43   
    40     44     puts "/*"
    41     45     puts "** If the argument is a codepoint corresponding to a lowercase letter"
    42     46     puts "** in the ASCII range with a diacritic added, return the codepoint"
    43     47     puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
    44     48     puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
    45     49     puts "** E\"). The resuls of passing a codepoint that corresponds to an"
    46     50     puts "** uppercase letter are undefined."
    47     51     puts "*/"
    48         -  puts "static int ${::remove_diacritic}(int c)\{"
           52  +  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
    49     53     puts "  unsigned short aDia\[\] = \{"
    50     54     puts -nonewline "        0, "
    51     55     set i 1
    52     56     foreach r $lRange {
    53     57       foreach {iCode nRange} $r {}
    54     58       if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
    55     59       incr i
................................................................................
    56     60   
    57     61       puts -nonewline [format "%5d" [expr ($iCode<<3) + $nRange-1]]
    58     62       puts -nonewline ", "
    59     63     }
    60     64     puts ""
    61     65     puts "  \};"
    62     66     puts "  char aChar\[\] = \{"
    63         -  puts -nonewline "    '\\0', "
           67  +  puts -nonewline "    '\\0',      "
    64     68     set i 1
    65         -  foreach c $aChar {
    66         -    set str "'$c',  "
    67         -    if {$c == ""} { set str "'\\0', " }
           69  +  foreach c $aChar f $aFlag {
           70  +    if { $f } {
           71  +      set str "'$c'|0x80,  "
           72  +    } else {
           73  +      set str "'$c'|0x00,  "
           74  +    }
           75  +    if {$c == ""} { set str "'\\0',      " }
    68     76   
    69         -    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
           77  +    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    70     78       incr i
    71     79       puts -nonewline "$str"
    72     80     }
    73     81     puts ""
    74     82     puts "  \};"
    75     83     puts {
    76     84     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
................................................................................
    83     91         iRes = iTest;
    84     92         iLo = iTest+1;
    85     93       }else{
    86     94         iHi = iTest-1;
    87     95       }
    88     96     }
    89     97     assert( key>=aDia[iRes] );
    90         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
           98  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           99  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
    91    100     puts "\}"
    92    101   }
    93    102   
    94    103   proc print_isdiacritic {zFunc map} {
    95    104   
    96    105     set lCode [list]
    97    106     foreach m $map {
    98         -    foreach {code char} $m {}
          107  +    foreach {code char flag} $m {}
          108  +    if {$flag} continue
    99    109       if {$code && $char == ""} { lappend lCode $code }
   100    110     }
   101    111     set lCode [lsort -integer $lCode]
   102    112     set iFirst [lindex $lCode 0]
   103    113     set iLast [lindex $lCode end]
   104    114   
   105    115     set i1 0
................................................................................
   468    478     puts "** is an upper case character that has a lower case equivalent,"
   469    479     puts "** return the codepoint corresponding to the lower case version."
   470    480     puts "** Otherwise, return a copy of the argument."
   471    481     puts "**"
   472    482     puts "** The results are undefined if the value passed to this function"
   473    483     puts "** is less than zero."
   474    484     puts "*/"
   475         -  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
          485  +  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
   476    486   
   477    487     set liOff [tl_generate_ioff_table $lRecord]
   478    488     tl_print_table_header
   479    489     foreach entry $lRecord { 
   480    490       if {[tl_print_table_entry toggle $entry $liOff]} { 
   481    491         lappend lHigh $entry 
   482    492       } 
................................................................................
   512    522       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   513    523       p = &aEntry[iRes];
   514    524       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   515    525         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   516    526         assert( ret>0 );
   517    527       }
   518    528   
   519         -    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
          529  +    if( eRemoveDiacritic ){
          530  +      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
          531  +    }
   520    532     }
   521    533     }]
   522    534   
   523    535     foreach entry $lHigh {
   524    536       tl_print_if_entry $entry
   525    537     }
   526    538   
................................................................................
   601    613     set caseN [categories_switch C N {d l o}]
   602    614     set caseP [categories_switch C P {c d e f i o s}]
   603    615     set caseS [categories_switch C S {c k m o}]
   604    616     set caseZ [categories_switch C Z {l p s}]
   605    617   
   606    618     set nCat [expr [llength [array names C]] + 1]
   607    619     puts [code {
   608         -    int sqlite3Fts5UnicodeNCat(void) { 
   609         -      return $nCat;
   610         -    }
   611         -
   612    620       int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   613    621         aArray[0] = 1;
   614    622         switch( zCat[0] ){
   615    623           $caseC
   616    624           $caseL
   617    625           $caseM
   618    626           $caseN

Changes to ext/fts3/unicode/parseunicode.tcl.

     3      3   # Parameter $zName must be a path to the file UnicodeData.txt. This command
     4      4   # reads the file and returns a list of mappings required to remove all
     5      5   # diacritical marks from a unicode string. Each mapping is itself a list
     6      6   # consisting of two elements - the unicode codepoint and the single ASCII
     7      7   # character that it should be replaced with, or an empty string if the 
     8      8   # codepoint should simply be removed from the input. Examples:
     9      9   #
    10         -#   { 224 a  }     (replace codepoint 224 to "a")
    11         -#   { 769 "" }     (remove codepoint 769 from input)
           10  +#   { 224 a  0 }     (replace codepoint 224 to "a")
           11  +#   { 769 "" 0 }     (remove codepoint 769 from input)
    12     12   #
    13     13   # Mappings are only returned for non-upper case codepoints. It is assumed
    14     14   # that the input has already been folded to lower case.
           15  +#
           16  +# The third value in the list is always either 0 or 1. 0 if the 
           17  +# UnicodeData.txt file maps the codepoint to a single ASCII character and
           18  +# a diacritic, or 1 if the mapping is indirect. For example, consider the 
           19  +# two entries:
           20  +#
           21  +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
           22  +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
           23  +#
           24  +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
           25  +# diacritic). The second is an indirect mapping, as it maps to the
           26  +# first codepoint plus 0302 (a diacritic).
    15     27   #
    16     28   proc rd_load_unicodedata_text {zName} {
    17     29     global tl_lookup_table
    18     30   
    19     31     set fd [open $zName]
    20     32     set lField {
    21     33       code
................................................................................
    49     61         continue
    50     62       }
    51     63   
    52     64       set iCode  [expr "0x$code"]
    53     65       set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    54     66       set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
    55     67   
           68  +    # Filter out upper-case characters, as they will be mapped to their
           69  +    # lower-case equivalents before this data is used.
    56     70       if {[info exists tl_lookup_table($iCode)]} continue
           71  +
           72  +    # Check if this is an indirect mapping. If so, set bIndirect to true
           73  +    # and change $iAscii to the indirectly mappped ASCII character.
           74  +    set bIndirect 0
           75  +    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
           76  +      set iAscii $mapping($iAscii)
           77  +      set bIndirect 1
           78  +    }
    57     79   
    58     80       if { ($iAscii >= 97 && $iAscii <= 122)
    59     81         || ($iAscii >= 65 && $iAscii <= 90)
    60     82       } {
    61         -      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
           83  +      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
           84  +      set mapping($iCode) $iAscii
    62     85         set dia($iDia) 1
    63     86       }
    64     87     }
    65     88   
    66     89     foreach d [array names dia] {
    67         -    lappend lRet [list $d ""]
           90  +    lappend lRet [list $d "" 0]
    68     91     }
    69     92     set lRet [lsort -integer -index 0 $lRet]
    70     93   
    71     94     close $fd
    72     95     set lRet
    73     96   }
    74     97   

Changes to ext/fts5/fts5_tokenize.c.

   230    230   #endif /* ifndef SQLITE_AMALGAMATION */
   231    231   
   232    232   typedef struct Unicode61Tokenizer Unicode61Tokenizer;
   233    233   struct Unicode61Tokenizer {
   234    234     unsigned char aTokenChar[128];  /* ASCII range token characters */
   235    235     char *aFold;                    /* Buffer to fold text into */
   236    236     int nFold;                      /* Size of aFold[] in bytes */
   237         -  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
          237  +  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   238    238     int nException;
   239    239     int *aiException;
   240    240   
   241    241     unsigned char aCategory[32];    /* True for token char categories */
   242    242   };
          243  +
          244  +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
          245  +#define FTS5_REMOVE_DIACRITICS_NONE    0
          246  +#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
          247  +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
   243    248   
   244    249   static int fts5UnicodeAddExceptions(
   245    250     Unicode61Tokenizer *p,          /* Tokenizer object */
   246    251     const char *z,                  /* Characters to treat as exceptions */
   247    252     int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
   248    253   ){
   249    254     int rc = SQLITE_OK;
................................................................................
   357    362     }else{
   358    363       p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
   359    364       if( p ){
   360    365         const char *zCat = "L* N* Co";
   361    366         int i;
   362    367         memset(p, 0, sizeof(Unicode61Tokenizer));
   363    368   
   364         -      p->bRemoveDiacritic = 1;
          369  +      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
   365    370         p->nFold = 64;
   366    371         p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
   367    372         if( p->aFold==0 ){
   368    373           rc = SQLITE_NOMEM;
   369    374         }
   370    375   
   371    376         /* Search for a "categories" argument */
................................................................................
   378    383         if( rc==SQLITE_OK ){
   379    384           rc = unicodeSetCategories(p, zCat);
   380    385         }
   381    386   
   382    387         for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
   383    388           const char *zArg = azArg[i+1];
   384    389           if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
   385         -          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
          390  +          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
   386    391               rc = SQLITE_ERROR;
          392  +          }else{
          393  +            p->eRemoveDiacritic = (zArg[0] - '0');
          394  +            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
          395  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
          396  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
          397  +            );
   387    398             }
   388         -          p->bRemoveDiacritic = (zArg[0]=='1');
   389    399           }else
   390    400           if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
   391    401             rc = fts5UnicodeAddExceptions(p, zArg, 1);
   392    402           }else
   393    403           if( 0==sqlite3_stricmp(azArg[i], "separators") ){
   394    404             rc = fts5UnicodeAddExceptions(p, zArg, 0);
   395    405           }else
................................................................................
   495    505   
   496    506         if( *zCsr & 0x80 ){
   497    507           /* An non-ascii-range character. Fold it into the output buffer if
   498    508           ** it is a token character, or break out of the loop if it is not. */
   499    509           READ_UTF8(zCsr, zTerm, iCode);
   500    510           if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
   501    511    non_ascii_tokenchar:
   502         -          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
          512  +          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
   503    513             if( iCode ) WRITE_UTF8(zOut, iCode);
   504    514           }else{
   505    515             break;
   506    516           }
   507    517         }else if( a[*zCsr]==0 ){
   508    518           /* An ascii-range separator character. End of token. */
   509    519           break; 

Changes to ext/fts5/fts5_unicode2.c.

    24     24   ** If the argument is a codepoint corresponding to a lowercase letter
    25     25   ** in the ASCII range with a diacritic added, return the codepoint
    26     26   ** of the ASCII letter only. For example, if passed 235 - "LATIN
    27     27   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
    28     28   ** E"). The resuls of passing a codepoint that corresponds to an
    29     29   ** uppercase letter are undefined.
    30     30   */
    31         -static int fts5_remove_diacritic(int c){
           31  +static int fts5_remove_diacritic(int c, int bComplex){
    32     32     unsigned short aDia[] = {
    33     33           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
    34     34        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
    35     35        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
    36     36        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
    37         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
    38         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
    39         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
    40         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
    41         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
    42         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
    43         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
    44         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
    45         -    62924, 63050, 63082, 63274, 63390, 
           37  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
           38  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
           39  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
           40  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
           41  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
           42  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
           43  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
           44  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
           45  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
           46  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
           47  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
           48  +    63182, 63242, 63274, 63310, 63368, 63390, 
    46     49     };
    47     50     char aChar[] = {
    48         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
    49         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
    50         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
    51         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
    52         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
    53         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
    54         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
    55         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
    56         -    'e',  'i',  'o',  'u',  'y',  
           51  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
           52  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
           53  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
           54  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
           55  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
           56  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
           57  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
           58  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
           59  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
           60  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
           61  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
           62  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
           63  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
           64  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
           65  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
           66  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
           67  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
           68  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
           69  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
           70  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
           71  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
    57     72     };
    58     73   
    59     74     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
    60     75     int iRes = 0;
    61     76     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
    62     77     int iLo = 0;
    63     78     while( iHi>=iLo ){
................................................................................
    66     81         iRes = iTest;
    67     82         iLo = iTest+1;
    68     83       }else{
    69     84         iHi = iTest-1;
    70     85       }
    71     86     }
    72     87     assert( key>=aDia[iRes] );
    73         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
           88  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           89  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
    74     90   }
    75     91   
    76     92   
    77     93   /*
    78     94   ** Return true if the argument interpreted as a unicode codepoint
    79     95   ** is a diacritical modifier character.
    80     96   */
................................................................................
    93    109   ** is an upper case character that has a lower case equivalent,
    94    110   ** return the codepoint corresponding to the lower case version.
    95    111   ** Otherwise, return a copy of the argument.
    96    112   **
    97    113   ** The results are undefined if the value passed to this function
    98    114   ** is less than zero.
    99    115   */
   100         -int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
          116  +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   101    117     /* Each entry in the following array defines a rule for folding a range
   102    118     ** of codepoints to lower case. The rule applies to a range of nRange
   103    119     ** codepoints starting at codepoint iCode.
   104    120     **
   105    121     ** If the least significant bit in flags is clear, then the rule applies
   106    122     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   107    123     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   216    232       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   217    233       p = &aEntry[iRes];
   218    234       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   219    235         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   220    236         assert( ret>0 );
   221    237       }
   222    238   
   223         -    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
          239  +    if( eRemoveDiacritic ){
          240  +      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
          241  +    }
   224    242     }
   225    243     
   226    244     else if( c>=66560 && c<66600 ){
   227    245       ret = c + 40;
   228    246     }
   229    247   
   230    248     return ret;
   231    249   }
   232    250   
   233         -
   234         -#if 0
   235         -int sqlite3Fts5UnicodeNCat(void) { 
   236         -  return 32;
   237         -}
   238         -#endif
   239         -
   240    251   int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   241    252     aArray[0] = 1;
   242    253     switch( zCat[0] ){
   243    254       case 'C':
   244    255             switch( zCat[1] ){
   245    256               case 'c': aArray[1] = 1; break;
   246    257               case 'f': aArray[2] = 1; break;
................................................................................
   752    763   void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
   753    764     int i = 0;
   754    765     int iTbl = 0;
   755    766     while( i<128 ){
   756    767       int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
   757    768       int n = (aFts5UnicodeData[iTbl] >> 5) + i;
   758    769       for(; i<128 && i<n; i++){
   759         -      aAscii[i] = (u8)bToken;
          770  +      aAscii[i] = bToken;
   760    771       }
   761    772       iTbl++;
   762    773     }
   763    774   }
   764         -

Changes to ext/fts5/test/fts5tokenizer.test.

   185    185     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 tokenchars');
   186    186   } {1 {error in tokenizer constructor}}
   187    187   do_catchsql_test 6.2 {
   188    188     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
   189    189   } {1 {error in tokenizer constructor}}
   190    190   do_catchsql_test 6.3 {
   191    191     CREATE VIRTUAL TABLE a3 USING fts5(
   192         -    x, y, tokenize = 'unicode61 remove_diacritics 2'
          192  +    x, y, tokenize = 'unicode61 remove_diacritics 3'
   193    193     );
   194    194   } {1 {error in tokenizer constructor}}
   195    195   do_catchsql_test 6.4 {
   196    196     CREATE VIRTUAL TABLE a3 USING fts5(
   197    197       x, y, tokenize = 'unicode61 remove_diacritics 10'
   198    198     );
   199    199   } {1 {error in tokenizer constructor}}

Added ext/fts5/test/fts5umlaut.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5umlaut
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts5(
           27  +      x, 
           28  +      tokenize="unicode61 remove_diacritics 2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1($q)
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1('Ha Noi')
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2($q)
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2('Ha Noi')
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to ext/fts5/test/fts5unicode3.test.

    32     32   
    33     33   tl_load_casefolding_txt $CF
    34     34   foreach x [an_load_unicodedata_text $UD] {
    35     35     set aNotAlnum($x) 1
    36     36   }
    37     37   
    38     38   foreach {y} [rd_load_unicodedata_text $UD] {
    39         -  foreach {code ascii} $y {}
           39  +  foreach {code ascii f} $y {}
    40     40     if {$ascii==""} {
    41     41       set int 0
    42     42     } else {
    43     43       binary scan $ascii c int
    44     44     }
    45         -  set aDiacritic($code) $int
           45  +  set aDiacritic($code,$f) $int
           46  +  if {$f==0} { set aDiacritic($code,1) $int }
    46     47   }
    47     48   
    48     49   proc tcl_fold {i {bRemoveDiacritic 0}} {
    49     50     global tl_lookup_table
    50     51     global aDiacritic
           52  +  set f [expr $bRemoveDiacritic==2]
    51     53   
    52     54     if {[info exists tl_lookup_table($i)]} {
    53     55       set i $tl_lookup_table($i)
    54     56     }
    55         -  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
    56         -    set i $aDiacritic($i)
           57  +  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
           58  +    set i $aDiacritic($i,$f)
    57     59     }
    58     60     expr $i
    59     61   }
    60     62   db func tcl_fold tcl_fold
    61     63   
    62     64   proc tcl_isalnum {i} {
    63     65     global aNotAlnum
................................................................................
    81     83       SELECT -1
    82     84       UNION ALL
    83     85       SELECT i+1 FROM ii WHERE i<100000
    84     86     )
    85     87     SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
    86     88   } {0 {}}
    87     89   
    88         -do_execsql_test 1.2 {
           90  +do_execsql_test 1.2.1 {
    89     91     WITH ii(i) AS (
    90     92       SELECT -1
    91     93       UNION ALL
    92     94       SELECT i+1 FROM ii WHERE i<100000
    93     95     )
    94     96     SELECT count(*), min(i) FROM ii 
    95     97     WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
    96     98   } {0 {}}
           99  +
          100  +do_execsql_test 1.2.2 {
          101  +  WITH ii(i) AS (
          102  +    SELECT -1
          103  +    UNION ALL
          104  +    SELECT i+1 FROM ii WHERE i<100000
          105  +  )
          106  +  SELECT count(*), min(i) FROM ii 
          107  +  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
          108  +} {0 {}}
    97    109   
    98    110   do_execsql_test 1.3 {
    99    111     WITH ii(i) AS (
   100    112       SELECT -1
   101    113       UNION ALL
   102    114       SELECT i+1 FROM ii WHERE i<100000
   103    115     )

Changes to ext/rbu/rbu_common.tcl.

    82     82       sqlite3 tmpdb $rbu
    83     83       tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
    84     84       tmpdb close
    85     85     }
    86     86     set rc
    87     87   }
    88     88   
    89         -proc do_rbu_vacuum_test {tn step} {
    90         -  forcedelete state.db
    91         -  uplevel [list do_test $tn.1 {
    92         -    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
           89  +proc do_rbu_vacuum_test {tn step {statedb state.db}} {
           90  +  forcedelete $statedb
           91  +  if {$statedb=="" && $step==1} breakpoint
           92  +  uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
           93  +    if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
    93     94       while 1 {
    94         -      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
           95  +      if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
    95     96         set state [rbu state]
    96     97         check_prestep_state test.db $state
    97     98         set rc [rbu step]
    98     99         check_poststep_state $rc test.db $state
    99    100         if {$rc!="SQLITE_OK"} break
   100    101         if {$step==1} { rbu close }
   101    102       }
   102    103       rbu close
   103         -  } {SQLITE_DONE}]
          104  +  }] {SQLITE_DONE}]
   104    105   
   105    106     uplevel [list do_execsql_test $tn.2 {
   106    107       PRAGMA integrity_check
   107    108     } ok]
   108    109   }
   109    110   

Changes to ext/rbu/rbuvacuum2.test.

    12     12   # This file contains tests for the RBU module. More specifically, it
    13     13   # contains tests to ensure that the sqlite3rbu_vacuum() API works as
    14     14   # expected.
    15     15   #
    16     16   
    17     17   source [file join [file dirname [info script]] rbu_common.tcl]
    18     18   
    19         -foreach step {0 1} {
    20         -  set ::testprefix rbuvacuum2-$step
           19  +foreach {step} {0 1} {
           20  +foreach {ttt state} {
           21  +  s state.db t test.db-vacuum n {}
           22  +} {
           23  +  set ::testprefix rbuvacuum2-$step$ttt
    21     24     
    22     25     #-------------------------------------------------------------------------
    23     26     # Test that a database that contains fts3 tables can be vacuumed.
    24     27     #
    25     28     ifcapable fts3 {
    26     29       reset_db
    27     30       do_execsql_test 1.1 {
    28     31         CREATE VIRTUAL TABLE t1 USING fts3(z, y);
    29     32         INSERT INTO t1 VALUES('fix this issue', 'at some point');
    30     33       }
    31     34     
    32         -    do_rbu_vacuum_test 1.2 $step
           35  +    do_rbu_vacuum_test 1.2 $step $state
    33     36     
    34     37       do_execsql_test 1.3 {
    35     38         SELECT * FROM t1;
    36     39       } {{fix this issue} {at some point}}
    37     40     
    38     41       do_execsql_test 1.4 {
    39     42         SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
................................................................................
    42     45       do_execsql_test 1.5 {
    43     46         INSERT INTO t1 VALUES('a b c', 'd e f');
    44     47         INSERT INTO t1 VALUES('l h i', 'd e f');
    45     48         DELETE FROM t1 WHERE docid = 2;
    46     49         INSERT INTO t1 VALUES('a b c', 'x y z');
    47     50       }
    48     51   
    49         -    do_rbu_vacuum_test 1.6 $step
           52  +    do_rbu_vacuum_test 1.6 $step $state
    50     53       do_execsql_test 1.7 {
    51     54         INSERT INTO t1(t1) VALUES('integrity-check');
    52     55         SELECT * FROM t1;
    53     56       } {
    54     57         {fix this issue} {at some point}
    55     58         {l h i} {d e f}
    56     59         {a b c} {x y z}
................................................................................
    63     66     ifcapable fts5 {
    64     67       reset_db
    65     68       do_execsql_test 2.1 {
    66     69         CREATE VIRTUAL TABLE t1 USING fts5(z, y);
    67     70         INSERT INTO t1 VALUES('fix this issue', 'at some point');
    68     71       }
    69     72     
    70         -    do_rbu_vacuum_test 2.2 $step
           73  +    do_rbu_vacuum_test 2.2 $step $state
    71     74     
    72     75       do_execsql_test 2.3 {
    73     76         SELECT * FROM t1;
    74     77       } {{fix this issue} {at some point}}
    75     78     
    76     79       do_execsql_test 2.4 {
    77     80         SELECT rowid FROM t1 ('fix');
................................................................................
    80     83       do_execsql_test 2.5 {
    81     84         INSERT INTO t1 VALUES('a b c', 'd e f');
    82     85         INSERT INTO t1 VALUES('l h i', 'd e f');
    83     86         DELETE FROM t1 WHERE rowid = 2;
    84     87         INSERT INTO t1 VALUES('a b c', 'x y z');
    85     88       }
    86     89   
    87         -    do_rbu_vacuum_test 2.6 $step
           90  +    do_rbu_vacuum_test 2.6 $step $state
    88     91       do_execsql_test 2.7 {
    89     92         INSERT INTO t1(t1) VALUES('integrity-check');
    90     93         SELECT * FROM t1;
    91     94       } {
    92     95         {fix this issue} {at some point}
    93     96         {l h i} {d e f}
    94     97         {a b c} {x y z}
................................................................................
   103    106       do_execsql_test 3.1 {
   104    107         CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
   105    108         INSERT INTO rt VALUES(1, 45, 55);
   106    109         INSERT INTO rt VALUES(2, 50, 60);
   107    110         INSERT INTO rt VALUES(3, 55, 65);
   108    111       }
   109    112     
   110         -    do_rbu_vacuum_test 3.2 $step
          113  +    do_rbu_vacuum_test 3.2 $step $state
   111    114     
   112    115       do_execsql_test 3.3 {
   113    116         SELECT * FROM rt;
   114    117       } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
   115    118     
   116    119       do_execsql_test 3.4.1 {
   117    120         SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
   118    121       } {1 2}
   119    122       do_execsql_test 3.4.2 {
   120    123         SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
   121    124       } {2 3}
   122    125   
   123         -    do_rbu_vacuum_test 3.5 $step
          126  +    do_rbu_vacuum_test 3.5 $step $state
   124    127   
   125    128       do_execsql_test 3.6.1 {
   126    129         SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
   127    130       } {1 2}
   128    131       do_execsql_test 3.6.2 {
   129    132         SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
   130    133       } {2 3}
................................................................................
   143    146         SELECT * FROM sqlite_master;
   144    147       } {
   145    148       table t1 t1 2 {CREATE TABLE t1(a, b, c)}
   146    149       view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
   147    150       trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
   148    151       }
   149    152   
   150         -    do_rbu_vacuum_test 4.3 $step
          153  +    do_rbu_vacuum_test 4.3 $step $state
   151    154       do_execsql_test 4.4 {
   152    155         SELECT * FROM sqlite_master;
   153    156       } {
   154    157       table t1 t1 2 {CREATE TABLE t1(a, b, c)}
   155    158       view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
   156    159       trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
   157    160       }
   158    161     }
          162  +}
   159    163   }
   160    164     
   161    165   #-------------------------------------------------------------------------
   162    166   # Test that passing a NULL value as the second argument to 
   163    167   # sqlite3rbu_vacuum() causes it to:
   164    168   #
   165    169   #   * Use <database>-vacuum as the state db, and
................................................................................
   227    231   do_test 6.3 {
   228    232     sqlite3rbu_vacuum rbu test.db test.db2
   229    233     while {[rbu step]!="SQLITE_DONE"} { rbu step }
   230    234     rbu close
   231    235     execsql { PRAGMA integrity_check }
   232    236   } {ok}
   233    237   
          238  +do_test 6.4 {
          239  +  sqlite3rbu_vacuum rbu test.db test.db-vactmp
          240  +  list [catch { rbu close } msg] $msg
          241  +} {1 SQLITE_MISUSE}
          242  +
   234    243   finish_test

Changes to ext/rbu/sqlite3rbu.c.

  2473   2473           zExtra = &p->zRbu[5];
  2474   2474           while( *zExtra ){
  2475   2475             if( *zExtra++=='?' ) break;
  2476   2476           }
  2477   2477           if( *zExtra=='\0' ) zExtra = 0;
  2478   2478         }
  2479   2479   
  2480         -      zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", 
         2480  +      zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", 
  2481   2481             sqlite3_db_filename(p->dbRbu, "main"),
  2482   2482             (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
  2483   2483         );
  2484   2484   
  2485   2485         if( zTarget==0 ){
  2486   2486           p->rc = SQLITE_NOMEM;
  2487   2487           return;
................................................................................
  3739   3739   ** Open a handle to begin or resume an RBU VACUUM operation.
  3740   3740   */
  3741   3741   sqlite3rbu *sqlite3rbu_vacuum(
  3742   3742     const char *zTarget, 
  3743   3743     const char *zState
  3744   3744   ){
  3745   3745     if( zTarget==0 ){ return rbuMisuseError(); }
         3746  +  if( zState ){
         3747  +    int n = strlen(zState);
         3748  +    if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
         3749  +      return rbuMisuseError();
         3750  +    }
         3751  +  }
  3746   3752     /* TODO: Check that both arguments are non-NULL */
  3747   3753     return openRbuHandle(0, zTarget, zState);
  3748   3754   }
  3749   3755   
  3750   3756   /*
  3751   3757   ** Return the database handle used by pRbu.
  3752   3758   */

Changes to ext/rbu/sqlite3rbu.h.

   329    329   ** The vacuum can be resumed by calling this function to open a new RBU
   330    330   ** handle specifying the same target and state databases.
   331    331   **
   332    332   ** If the second argument passed to this function is NULL, then the
   333    333   ** name of the state database is "<database>-vacuum", where <database>
   334    334   ** is the name of the target database file. In this case, on UNIX, if the
   335    335   ** state database is not already present in the file-system, it is created
   336         -** with the same permissions as the target db is made.
          336  +** with the same permissions as the target db is made. 
          337  +**
          338  +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the 
          339  +** state database ends with "-vactmp". This name is reserved for internal 
          340  +** use.
   337    341   **
   338    342   ** This function does not delete the state database after an RBU vacuum
   339    343   ** is completed, even if it created it. However, if the call to
   340    344   ** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
   341    345   ** of the state tables within the state database are zeroed. This way,
   342    346   ** the next call to sqlite3rbu_vacuum() opens a handle that starts a 
   343    347   ** new RBU vacuum operation.

Changes to ext/rbu/test_rbu.c.

   269    269     if( objc!=3 && objc!=4 ){
   270    270       Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?");
   271    271       return TCL_ERROR;
   272    272     }
   273    273     zCmd = Tcl_GetString(objv[1]);
   274    274     zTarget = Tcl_GetString(objv[2]);
   275    275     if( objc==4 ) zStateDb = Tcl_GetString(objv[3]);
          276  +  if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0;
   276    277   
   277    278     pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
   278    279     Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
   279    280     Tcl_SetObjResult(interp, objv[1]);
   280    281     return TCL_OK;
   281    282   }
   282    283   

Changes to main.mk.

   995    995   
   996    996   showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
   997    997   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
   998    998   		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
   999    999   
  1000   1000   showshm$(EXE):	$(TOP)/tool/showshm.c
  1001   1001   	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
         1002  +
         1003  +index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
         1004  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
         1005  +		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)
  1002   1006   
  1003   1007   changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
  1004   1008   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
  1005   1009   		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
  1006   1010   
  1007   1011   changesetfuzz$(EXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.o
  1008   1012   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changesetfuzz$(EXE) \

Changes to src/alter.c.

   775    775         pToken->pNext = pCtx->pList;
   776    776         pCtx->pList = pToken;
   777    777         pCtx->nList++;
   778    778         break;
   779    779       }
   780    780     }
   781    781   }
          782  +
          783  +/*
          784  +** Iterate through the Select objects that are part of WITH clauses attached
          785  +** to select statement pSelect.
          786  +*/
          787  +static void renameWalkWith(Walker *pWalker, Select *pSelect){
          788  +  if( pSelect->pWith ){
          789  +    int i;
          790  +    for(i=0; i<pSelect->pWith->nCte; i++){
          791  +      Select *p = pSelect->pWith->a[i].pSelect;
          792  +      NameContext sNC;
          793  +      memset(&sNC, 0, sizeof(sNC));
          794  +      sNC.pParse = pWalker->pParse;
          795  +      sqlite3SelectPrep(sNC.pParse, p, &sNC);
          796  +      sqlite3WalkSelect(pWalker, p);
          797  +    }
          798  +  }
          799  +}
   782    800   
   783    801   /*
   784    802   ** This is a Walker select callback. It does nothing. It is only required
   785    803   ** because without a dummy callback, sqlite3WalkExpr() and similar do not
   786    804   ** descend into sub-select statements.
   787    805   */
   788    806   static int renameColumnSelectCb(Walker *pWalker, Select *p){
   789         -  UNUSED_PARAMETER(pWalker);
   790         -  UNUSED_PARAMETER(p);
          807  +  renameWalkWith(pWalker, p);
   791    808     return WRC_Continue;
   792    809   }
   793    810   
   794    811   /*
   795    812   ** This is a Walker expression callback.
   796    813   **
   797    814   ** For every TK_COLUMN node in the expression tree, search to see
................................................................................
  1354   1371   /*
  1355   1372   ** Walker select callback used by "RENAME TABLE". 
  1356   1373   */
  1357   1374   static int renameTableSelectCb(Walker *pWalker, Select *pSelect){
  1358   1375     int i;
  1359   1376     RenameCtx *p = pWalker->u.pRename;
  1360   1377     SrcList *pSrc = pSelect->pSrc;
         1378  +  if( pSrc==0 ){
         1379  +    assert( pWalker->pParse->db->mallocFailed );
         1380  +    return WRC_Abort;
         1381  +  }
  1361   1382     for(i=0; i<pSrc->nSrc; i++){
  1362   1383       struct SrcList_item *pItem = &pSrc->a[i];
  1363   1384       if( pItem->pTab==p->pTab ){
  1364   1385         renameTokenFind(pWalker->pParse, p, pItem->zName);
  1365   1386       }
  1366   1387     }
         1388  +  renameWalkWith(pWalker, pSelect);
  1367   1389   
  1368   1390     return WRC_Continue;
  1369   1391   }
  1370   1392   
  1371   1393   
  1372   1394   /*
  1373   1395   ** This C function implements an SQL user function that is used by SQL code

Changes to src/btree.c.

 10325  10325     }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){
 10326  10326       checkAppendMsg(&sCheck,
 10327  10327         "incremental_vacuum enabled with a max rootpage of zero"
 10328  10328       );
 10329  10329     }
 10330  10330   #endif
 10331  10331     testcase( pBt->db->flags & SQLITE_CellSizeCk );
 10332         -  pBt->db->flags &= ~SQLITE_CellSizeCk;
        10332  +  pBt->db->flags &= ~(u64)SQLITE_CellSizeCk;
 10333  10333     for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
 10334  10334       i64 notUsed;
 10335  10335       if( aRoot[i]==0 ) continue;
 10336  10336   #ifndef SQLITE_OMIT_AUTOVACUUM
 10337  10337       if( pBt->autoVacuum && aRoot[i]>1 ){
 10338  10338         checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
 10339  10339       }

Changes to src/build.c.

   225    225   
   226    226   
   227    227     /* Get the VDBE program ready for execution
   228    228     */
   229    229     if( v && pParse->nErr==0 && !db->mallocFailed ){
   230    230       /* A minimum of one cursor is required if autoincrement is used
   231    231       *  See ticket [a696379c1f08866] */
   232         -    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
          232  +    assert( pParse->pAinc==0 || pParse->nTab>0 );
   233    233       sqlite3VdbeMakeReady(v, pParse);
   234    234       pParse->rc = SQLITE_DONE;
   235    235     }else{
   236    236       pParse->rc = SQLITE_ERROR;
   237    237     }
   238    238   }
   239    239   
................................................................................
   634    634     }
   635    635   
   636    636     /* Delete any foreign keys attached to this table. */
   637    637     sqlite3FkDelete(db, pTable);
   638    638   
   639    639     /* Delete the Table structure itself.
   640    640     */
   641         -#ifdef SQLITE_ENABLE_NORMALIZE
   642         -  if( pTable->pColHash ){
   643         -    sqlite3HashClear(pTable->pColHash);
   644         -    sqlite3_free(pTable->pColHash);
   645         -  }
   646         -#endif
   647    641     sqlite3DeleteColumnNames(db, pTable);
   648    642     sqlite3DbFree(db, pTable->zName);
   649    643     sqlite3DbFree(db, pTable->zColAff);
   650    644     sqlite3SelectDelete(db, pTable->pSelect);
   651    645     sqlite3ExprListDelete(db, pTable->pCheck);
   652    646   #ifndef SQLITE_OMIT_VIRTUALTABLE
   653    647     sqlite3VtabClear(db, pTable);

Changes to src/callback.c.

   279    279     return match;
   280    280   }
   281    281   
   282    282   /*
   283    283   ** Search a FuncDefHash for a function with the given name.  Return
   284    284   ** a pointer to the matching FuncDef if found, or 0 if there is no match.
   285    285   */
   286         -static FuncDef *functionSearch(
          286  +FuncDef *sqlite3FunctionSearch(
   287    287     int h,               /* Hash of the name */
   288    288     const char *zFunc    /* Name of function */
   289    289   ){
   290    290     FuncDef *p;
   291    291     for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
   292    292       if( sqlite3StrICmp(p->zName, zFunc)==0 ){
   293    293         return p;
   294    294       }
   295    295     }
   296    296     return 0;
   297    297   }
   298         -#ifdef SQLITE_ENABLE_NORMALIZE
   299         -FuncDef *sqlite3FunctionSearchN(
   300         -  int h,               /* Hash of the name */
   301         -  const char *zFunc,   /* Name of function */
   302         -  int nFunc            /* Length of the name */
   303         -){
   304         -  FuncDef *p;
   305         -  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
   306         -    if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
   307         -      return p;
   308         -    }
   309         -  }
   310         -  return 0;
   311         -}
   312         -#endif /* SQLITE_ENABLE_NORMALIZE */
   313    298   
   314    299   /*
   315    300   ** Insert a new FuncDef into a FuncDefHash hash table.
   316    301   */
   317    302   void sqlite3InsertBuiltinFuncs(
   318    303     FuncDef *aDef,      /* List of global functions to be inserted */
   319    304     int nDef            /* Length of the apDef[] list */
................................................................................
   321    306     int i;
   322    307     for(i=0; i<nDef; i++){
   323    308       FuncDef *pOther;
   324    309       const char *zName = aDef[i].zName;
   325    310       int nName = sqlite3Strlen30(zName);
   326    311       int h = SQLITE_FUNC_HASH(zName[0], nName);
   327    312       assert( zName[0]>='a' && zName[0]<='z' );
   328         -    pOther = functionSearch(h, zName);
          313  +    pOther = sqlite3FunctionSearch(h, zName);
   329    314       if( pOther ){
   330    315         assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] );
   331    316         aDef[i].pNext = pOther->pNext;
   332    317         pOther->pNext = &aDef[i];
   333    318       }else{
   334    319         aDef[i].pNext = 0;
   335    320         aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h];
................................................................................
   399    384     ** have fields overwritten with new information appropriate for the
   400    385     ** new function.  But the FuncDefs for built-in functions are read-only.
   401    386     ** So we must not search for built-ins when creating a new function.
   402    387     */ 
   403    388     if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
   404    389       bestScore = 0;
   405    390       h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
   406         -    p = functionSearch(h, zName);
          391  +    p = sqlite3FunctionSearch(h, zName);
   407    392       while( p ){
   408    393         int score = matchQuality(p, nArg, enc);
   409    394         if( score>bestScore ){
   410    395           pBest = p;
   411    396           bestScore = score;
   412    397         }
   413    398         p = p->pNext;

Changes to src/expr.c.

  1325   1325       }
  1326   1326     }
  1327   1327     return pRet;
  1328   1328   }
  1329   1329   #else
  1330   1330   # define withDup(x,y) 0
  1331   1331   #endif
         1332  +
         1333  +#ifndef SQLITE_OMIT_WINDOWFUNC
         1334  +/*
         1335  +** The gatherSelectWindows() procedure and its helper routine
         1336  +** gatherSelectWindowsCallback() are used to scan all the expressions
         1337  +** an a newly duplicated SELECT statement and gather all of the Window
         1338  +** objects found there, assembling them onto the linked list at Select->pWin.
         1339  +*/
         1340  +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){
         1341  +  if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){
         1342  +    assert( ExprHasProperty(pExpr, EP_WinFunc) );
         1343  +    pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin;
         1344  +    pWalker->u.pSelect->pWin = pExpr->y.pWin;
         1345  +  }
         1346  +  return WRC_Continue;
         1347  +}
         1348  +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){
         1349  +  return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune;
         1350  +}
         1351  +static void gatherSelectWindows(Select *p){
         1352  +  Walker w;
         1353  +  w.xExprCallback = gatherSelectWindowsCallback;
         1354  +  w.xSelectCallback = gatherSelectWindowsSelectCallback;
         1355  +  w.xSelectCallback2 = 0;
         1356  +  w.u.pSelect = p;
         1357  +  sqlite3WalkSelect(&w, p);
         1358  +}
         1359  +#endif
         1360  +
  1332   1361   
  1333   1362   /*
  1334   1363   ** The following group of routines make deep copies of expressions,
  1335   1364   ** expression lists, ID lists, and select statements.  The copies can
  1336   1365   ** be deleted (by being passed to their respective ...Delete() routines)
  1337   1366   ** without effecting the originals.
  1338   1367   **
................................................................................
  1493   1522       pNew->addrOpenEphm[0] = -1;
  1494   1523       pNew->addrOpenEphm[1] = -1;
  1495   1524       pNew->nSelectRow = p->nSelectRow;
  1496   1525       pNew->pWith = withDup(db, p->pWith);
  1497   1526   #ifndef SQLITE_OMIT_WINDOWFUNC
  1498   1527       pNew->pWin = 0;
  1499   1528       pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn);
         1529  +    if( p->pWin ) gatherSelectWindows(pNew);
  1500   1530   #endif
  1501   1531       pNew->selId = p->selId;
  1502   1532       *pp = pNew;
  1503   1533       pp = &pNew->pPrior;
  1504   1534       pNext = pNew;
  1505   1535     }
  1506   1536   
................................................................................
  2145   2175   */
  2146   2176   int sqlite3IsRowid(const char *z){
  2147   2177     if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
  2148   2178     if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
  2149   2179     if( sqlite3StrICmp(z, "OID")==0 ) return 1;
  2150   2180     return 0;
  2151   2181   }
  2152         -#ifdef SQLITE_ENABLE_NORMALIZE
  2153         -int sqlite3IsRowidN(const char *z, int n){
  2154         -  if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1;
  2155         -  if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1;
  2156         -  if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1;
  2157         -  return 0;
  2158         -}
  2159         -#endif
  2160   2182   
  2161   2183   /*
  2162   2184   ** pX is the RHS of an IN operator.  If pX is a SELECT statement 
  2163   2185   ** that can be simplified to a direct table access, then return
  2164   2186   ** a pointer to the SELECT statement.  If pX is not a SELECT statement,
  2165   2187   ** or if the SELECT statement needs to be manifested into a transient
  2166   2188   ** table, then return NULL.
................................................................................
  2417   2439   
  2418   2440         if( affinity_ok ){
  2419   2441           /* Search for an existing index that will work for this IN operator */
  2420   2442           for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){
  2421   2443             Bitmask colUsed;      /* Columns of the index used */
  2422   2444             Bitmask mCol;         /* Mask for the current column */
  2423   2445             if( pIdx->nColumn<nExpr ) continue;
         2446  +          if( pIdx->pPartIdxWhere!=0 ) continue;
  2424   2447             /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute
  2425   2448             ** BITMASK(nExpr) without overflowing */
  2426   2449             testcase( pIdx->nColumn==BMS-2 );
  2427   2450             testcase( pIdx->nColumn==BMS-1 );
  2428   2451             if( pIdx->nColumn>=BMS-1 ) continue;
  2429   2452             if( mustBeUnique ){
  2430   2453               if( pIdx->nKeyCol>nExpr

Changes to src/hash.c.

    60     60       ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    61     61       ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    62     62       h += sqlite3UpperToLower[c];
    63     63       h *= 0x9e3779b1;
    64     64     }
    65     65     return h;
    66     66   }
    67         -#ifdef SQLITE_ENABLE_NORMALIZE
    68         -static unsigned int strHashN(const char *z, int n){
    69         -  unsigned int h = 0;
    70         -  int i;
    71         -  for(i=0; i<n; i++){
    72         -    /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
    73         -    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    74         -    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    75         -    h += sqlite3UpperToLower[z[i]];
    76         -    h *= 0x9e3779b1;
    77         -  }
    78         -  return h;
    79         -}
    80         -#endif /* SQLITE_ENABLE_NORMALIZE */
    81     67   
    82     68   
    83     69   /* Link pNew element into the hash table pH.  If pEntry!=0 then also
    84     70   ** insert pNew into the pEntry hash bucket.
    85     71   */
    86     72   static void insertElement(
    87     73     Hash *pH,              /* The complete hash table */
................................................................................
   185    171       if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ 
   186    172         return elem;
   187    173       }
   188    174       elem = elem->next;
   189    175     }
   190    176     return &nullElement;
   191    177   }
   192         -#ifdef SQLITE_ENABLE_NORMALIZE
   193         -static HashElem *findElementWithHashN(
   194         -  const Hash *pH,     /* The pH to be searched */
   195         -  const char *pKey,   /* The key we are searching for */
   196         -  int nKey,           /* Number of key bytes to use */
   197         -  unsigned int *pHash /* Write the hash value here */
   198         -){
   199         -  HashElem *elem;                /* Used to loop thru the element list */
   200         -  int count;                     /* Number of elements left to test */
   201         -  unsigned int h;                /* The computed hash */
   202         -  static HashElem nullElement = { 0, 0, 0, 0 };
   203         -
   204         -  if( pH->ht ){   /*OPTIMIZATION-IF-TRUE*/
   205         -    struct _ht *pEntry;
   206         -    h = strHashN(pKey, nKey) % pH->htsize;
   207         -    pEntry = &pH->ht[h];
   208         -    elem = pEntry->chain;
   209         -    count = pEntry->count;
   210         -  }else{
   211         -    h = 0;
   212         -    elem = pH->first;
   213         -    count = pH->count;
   214         -  }
   215         -  if( pHash ) *pHash = h;
   216         -  while( count-- ){
   217         -    assert( elem!=0 );
   218         -    if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ 
   219         -      return elem;
   220         -    }
   221         -    elem = elem->next;
   222         -  }
   223         -  return &nullElement;
   224         -}
   225         -#endif /* SQLITE_ENABLE_NORMALIZE */
   226    178   
   227    179   /* Remove a single entry from the hash table given a pointer to that
   228    180   ** element and a hash on the element's key.
   229    181   */
   230    182   static void removeElementGivenHash(
   231    183     Hash *pH,         /* The pH containing "elem" */
   232    184     HashElem* elem,   /* The element to be removed from the pH */
................................................................................
   263    215   ** found, or NULL if there is no match.
   264    216   */
   265    217   void *sqlite3HashFind(const Hash *pH, const char *pKey){
   266    218     assert( pH!=0 );
   267    219     assert( pKey!=0 );
   268    220     return findElementWithHash(pH, pKey, 0)->data;
   269    221   }
   270         -#ifdef SQLITE_ENABLE_NORMALIZE
   271         -void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
   272         -  assert( pH!=0 );
   273         -  assert( pKey!=0 );
   274         -  assert( nKey>=0 );
   275         -  return findElementWithHashN(pH, pKey, nKey, 0)->data;
   276         -}
   277         -#endif /* SQLITE_ENABLE_NORMALIZE */
   278    222   
   279    223   /* Insert an element into the hash table pH.  The key is pKey
   280    224   ** and the data is "data".
   281    225   **
   282    226   ** If no element exists with a matching key, then a new
   283    227   ** element is created and NULL is returned.
   284    228   **

Changes to src/insert.c.

   315    315       aOp[3].p5 = SQLITE_JUMPIFNULL;
   316    316       aOp[4].p2 = memId+1;
   317    317       aOp[5].p3 = memId;
   318    318       aOp[6].p1 = memId;
   319    319       aOp[7].p2 = memId+2;
   320    320       aOp[7].p1 = memId;
   321    321       aOp[10].p2 = memId;
          322  +    if( pParse->nTab==0 ) pParse->nTab = 1;
   322    323     }
   323    324   }
   324    325   
   325    326   /*
   326    327   ** Update the maximum rowid for an autoincrement calculation.
   327    328   **
   328    329   ** This routine should be called when the regRowid register holds a

Changes to src/loadext.c.

   646    646   ** default so as not to open security holes in older applications.
   647    647   */
   648    648   int sqlite3_enable_load_extension(sqlite3 *db, int onoff){
   649    649     sqlite3_mutex_enter(db->mutex);
   650    650     if( onoff ){
   651    651       db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc;
   652    652     }else{
   653         -    db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
          653  +    db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
   654    654     }
   655    655     sqlite3_mutex_leave(db->mutex);
   656    656     return SQLITE_OK;
   657    657   }
   658    658   
   659    659   #endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */
   660    660   

Changes to src/main.c.

   839    839         };
   840    840         unsigned int i;
   841    841         rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
   842    842         for(i=0; i<ArraySize(aFlagOp); i++){
   843    843           if( aFlagOp[i].op==op ){
   844    844             int onoff = va_arg(ap, int);
   845    845             int *pRes = va_arg(ap, int*);
   846         -          u32 oldFlags = db->flags;
          846  +          u64 oldFlags = db->flags;
   847    847             if( onoff>0 ){
   848    848               db->flags |= aFlagOp[i].mask;
   849    849             }else if( onoff==0 ){
   850         -            db->flags &= ~aFlagOp[i].mask;
          850  +            db->flags &= ~(u64)aFlagOp[i].mask;
   851    851             }
   852    852             if( oldFlags!=db->flags ){
   853    853               sqlite3ExpirePreparedStatements(db, 0);
   854    854             }
   855    855             if( pRes ){
   856    856               *pRes = (db->flags & aFlagOp[i].mask)!=0;
   857    857             }
................................................................................
  1306   1306       sqlite3ResetAllSchemasOfConnection(db);
  1307   1307     }
  1308   1308     sqlite3BtreeLeaveAll(db);
  1309   1309   
  1310   1310     /* Any deferred constraint violations have now been resolved. */
  1311   1311     db->nDeferredCons = 0;
  1312   1312     db->nDeferredImmCons = 0;
  1313         -  db->flags &= ~SQLITE_DeferFKs;
         1313  +  db->flags &= ~(u64)SQLITE_DeferFKs;
  1314   1314   
  1315   1315     /* If one has been configured, invoke the rollback-hook callback */
  1316   1316     if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
  1317   1317       db->xRollbackCallback(db->pRollbackArg);
  1318   1318     }
  1319   1319   }
  1320   1320   
................................................................................
  2048   2048       return 0;
  2049   2049     }
  2050   2050   #endif
  2051   2051     sqlite3_mutex_enter(db->mutex);
  2052   2052     pOld = db->pProfileArg;
  2053   2053     db->xProfile = xProfile;
  2054   2054     db->pProfileArg = pArg;
         2055  +  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
         2056  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  2055   2057     sqlite3_mutex_leave(db->mutex);
  2056   2058     return pOld;
  2057   2059   }
  2058   2060   #endif /* SQLITE_OMIT_DEPRECATED */
  2059   2061   #endif /* SQLITE_OMIT_TRACE */
  2060   2062   
  2061   2063   /*
................................................................................
  2399   2401       return sqlite3ErrStr(SQLITE_MISUSE_BKPT);
  2400   2402     }
  2401   2403     sqlite3_mutex_enter(db->mutex);
  2402   2404     if( db->mallocFailed ){
  2403   2405       z = sqlite3ErrStr(SQLITE_NOMEM_BKPT);
  2404   2406     }else{
  2405   2407       testcase( db->pErr==0 );
  2406         -    z = (char*)sqlite3_value_text(db->pErr);
         2408  +    z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0;
  2407   2409       assert( !db->mallocFailed );
  2408   2410       if( z==0 ){
  2409   2411         z = sqlite3ErrStr(db->errCode);
  2410   2412       }
  2411   2413     }
  2412   2414     sqlite3_mutex_leave(db->mutex);
  2413   2415     return z;

Changes to src/parse.y.

   675    675         A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,N,U);
   676    676         if( A ){
   677    677           struct SrcList_item *pNew = &A->a[A->nSrc-1];
   678    678           struct SrcList_item *pOld = F->a;
   679    679           pNew->zName = pOld->zName;
   680    680           pNew->zDatabase = pOld->zDatabase;
   681    681           pNew->pSelect = pOld->pSelect;
          682  +        if( pOld->fg.isTabFunc ){
          683  +          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
          684  +          pOld->u1.pFuncArg = 0;
          685  +          pOld->fg.isTabFunc = 0;
          686  +          pNew->fg.isTabFunc = 1;
          687  +        }
   682    688           pOld->zName = pOld->zDatabase = 0;
   683    689           pOld->pSelect = 0;
   684    690         }
   685    691         sqlite3SrcListDelete(pParse->db, F);
   686    692       }else{
   687    693         Select *pSubquery;
   688    694         sqlite3SrcListShiftJoinType(F);
................................................................................
  1372   1378   //
  1373   1379   cmd ::= DROP INDEX ifexists(E) fullname(X).   {sqlite3DropIndex(pParse, X, E);}
  1374   1380   
  1375   1381   ///////////////////////////// The VACUUM command /////////////////////////////
  1376   1382   //
  1377   1383   %ifndef SQLITE_OMIT_VACUUM
  1378   1384   %ifndef SQLITE_OMIT_ATTACH
  1379         -cmd ::= VACUUM.                {sqlite3Vacuum(pParse,0);}
  1380         -cmd ::= VACUUM nm(X).          {sqlite3Vacuum(pParse,&X);}
         1385  +%type vinto {Expr*}
         1386  +%destructor vinto {sqlite3ExprDelete(pParse->db, $$);}
         1387  +cmd ::= VACUUM vinto(Y).                {sqlite3Vacuum(pParse,0,Y);}
         1388  +cmd ::= VACUUM nm(X) vinto(Y).          {sqlite3Vacuum(pParse,&X,Y);}
         1389  +vinto(A) ::= INTO expr(X).              {A = X;}
         1390  +vinto(A) ::= .                          {A = 0;}
  1381   1391   %endif  SQLITE_OMIT_ATTACH
  1382   1392   %endif  SQLITE_OMIT_VACUUM
  1383   1393   
  1384   1394   ///////////////////////////// The PRAGMA command /////////////////////////////
  1385   1395   //
  1386   1396   %ifndef SQLITE_OMIT_PRAGMA
  1387   1397   cmd ::= PRAGMA nm(X) dbnm(Z).                {sqlite3Pragma(pParse,&X,&Z,0,0);}

Changes to src/pcache1.c.

    98     98     unsigned int iKey;             /* Key value (page number) */
    99     99     u8 isBulkLocal;                /* This page from bulk local storage */
   100    100     u8 isAnchor;                   /* This is the PGroup.lru element */
   101    101     PgHdr1 *pNext;                 /* Next in hash table chain */
   102    102     PCache1 *pCache;               /* Cache that currently owns this page */
   103    103     PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
   104    104     PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
          105  +                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
   105    106   };
   106    107   
   107    108   /*
   108    109   ** A page is pinned if it is not on the LRU list.  To be "pinned" means
   109    110   ** that the page is in active use and must not be deallocated.
   110    111   */
   111    112   #define PAGE_IS_PINNED(p)    ((p)->pLruNext==0)
................................................................................
   566    567     assert( PAGE_IS_UNPINNED(pPage) );
   567    568     assert( pPage->pLruNext );
   568    569     assert( pPage->pLruPrev );
   569    570     assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
   570    571     pPage->pLruPrev->pLruNext = pPage->pLruNext;
   571    572     pPage->pLruNext->pLruPrev = pPage->pLruPrev;
   572    573     pPage->pLruNext = 0;
   573         -  pPage->pLruPrev = 0;
          574  +  /* pPage->pLruPrev = 0;
          575  +  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
   574    576     assert( pPage->isAnchor==0 );
   575    577     assert( pPage->pCache->pGroup->lru.isAnchor==1 );
   576    578     pPage->pCache->nRecyclable--;
   577    579     return pPage;
   578    580   }
   579    581   
   580    582   
................................................................................
   904    906   
   905    907     if( pPage ){
   906    908       unsigned int h = iKey % pCache->nHash;
   907    909       pCache->nPage++;
   908    910       pPage->iKey = iKey;
   909    911       pPage->pNext = pCache->apHash[h];
   910    912       pPage->pCache = pCache;
   911         -    pPage->pLruPrev = 0;
   912    913       pPage->pLruNext = 0;
          914  +    /* pPage->pLruPrev = 0;
          915  +    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
   913    916       *(void **)pPage->page.pExtra = 0;
   914    917       pCache->apHash[h] = pPage;
   915    918       if( iKey>pCache->iMaxKey ){
   916    919         pCache->iMaxKey = iKey;
   917    920       }
   918    921     }
   919    922     return pPage;
................................................................................
  1065   1068    
  1066   1069     assert( pPage->pCache==pCache );
  1067   1070     pcache1EnterMutex(pGroup);
  1068   1071   
  1069   1072     /* It is an error to call this function if the page is already 
  1070   1073     ** part of the PGroup LRU list.
  1071   1074     */
  1072         -  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
         1075  +  assert( pPage->pLruNext==0 );
  1073   1076     assert( PAGE_IS_PINNED(pPage) );
  1074   1077   
  1075   1078     if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
  1076   1079       pcache1RemoveFromHash(pPage, 1);
  1077   1080     }else{
  1078   1081       /* Add the page to the PGroup LRU list. */
  1079   1082       PgHdr1 **ppFirst = &pGroup->lru.pLruNext;

Changes to src/pragma.c.

   813    813         int size = 1;
   814    814         if( sqlite3GetInt32(zRight, &size) ){
   815    815           sqlite3BtreeSetSpillSize(pDb->pBt, size);
   816    816         }
   817    817         if( sqlite3GetBoolean(zRight, size!=0) ){
   818    818           db->flags |= SQLITE_CacheSpill;
   819    819         }else{
   820         -        db->flags &= ~SQLITE_CacheSpill;
          820  +        db->flags &= ~(u64)SQLITE_CacheSpill;
   821    821         }
   822    822         setAllPagerFlags(db);
   823    823       }
   824    824       break;
   825    825     }
   826    826   
   827    827     /*

Changes to src/prepare.c.

   289    289   
   290    290     /* Ticket #2804:  When we open a database in the newer file format,
   291    291     ** clear the legacy_file_format pragma flag so that a VACUUM will
   292    292     ** not downgrade the database and thus invalidate any descending
   293    293     ** indices that the user might have created.
   294    294     */
   295    295     if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){
   296         -    db->flags &= ~SQLITE_LegacyFileFmt;
          296  +    db->flags &= ~(u64)SQLITE_LegacyFileFmt;
   297    297     }
   298    298   
   299    299     /* Read the schema information out of the schema tables
   300    300     */
   301    301     assert( db->init.busy );
   302    302     {
   303    303       char *zSql;
................................................................................
   705    705     sqlite3BtreeLeaveAll(db);
   706    706     rc = sqlite3ApiExit(db, rc);
   707    707     assert( (rc&db->errMask)==rc );
   708    708     sqlite3_mutex_leave(db->mutex);
   709    709     return rc;
   710    710   }
   711    711   
   712         -#ifdef SQLITE_ENABLE_NORMALIZE
   713         -/*
   714         -** Checks if the specified token is a table, column, or function name,
   715         -** based on the databases associated with the statement being prepared.
   716         -** If the function fails, zero is returned and pRc is filled with the
   717         -** error code.
   718         -*/
   719         -static int shouldTreatAsIdentifier(
   720         -  sqlite3 *db,        /* Database handle. */
   721         -  const char *zToken, /* Pointer to start of token to be checked */
   722         -  int nToken,         /* Length of token to be checked */
   723         -  int *pRc            /* Pointer to error code upon failure */
   724         -){
   725         -  int bFound = 0;     /* Non-zero if token is an identifier name. */
   726         -  int i, j;           /* Database and column loop indexes. */
   727         -  Schema *pSchema;    /* Schema for current database. */
   728         -  Hash *pHash;        /* Hash table of tables for current database. */
   729         -  HashElem *e;        /* Hash element for hash table iteration. */
   730         -  Table *pTab;        /* Database table for columns being checked. */
   731         -
   732         -  if( sqlite3IsRowidN(zToken, nToken) ){
   733         -    return 1;
   734         -  }
   735         -  if( nToken>0 ){
   736         -    int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
   737         -    if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
   738         -  }
   739         -  assert( db!=0 );
   740         -  sqlite3_mutex_enter(db->mutex);
   741         -  sqlite3BtreeEnterAll(db);
   742         -  for(i=0; i<db->nDb; i++){
   743         -    pHash = &db->aFunc;
   744         -    if( sqlite3HashFindN(pHash, zToken, nToken) ){
   745         -      bFound = 1;
   746         -      break;
   747         -    }
   748         -    pSchema = db->aDb[i].pSchema;
   749         -    if( pSchema==0 ) continue;
   750         -    pHash = &pSchema->tblHash;
   751         -    if( sqlite3HashFindN(pHash, zToken, nToken) ){
   752         -      bFound = 1;
   753         -      break;
   754         -    }
   755         -    for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
   756         -      pTab = sqliteHashData(e);
   757         -      if( pTab==0 ) continue;
   758         -      pHash = pTab->pColHash;
   759         -      if( pHash==0 ){
   760         -        pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
   761         -        if( pHash ){
   762         -          sqlite3HashInit(pHash);
   763         -          for(j=0; j<pTab->nCol; j++){
   764         -            Column *pCol = &pTab->aCol[j];
   765         -            sqlite3HashInsert(pHash, pCol->zName, pCol);
   766         -          }
   767         -        }else{
   768         -          *pRc = SQLITE_NOMEM_BKPT;
   769         -          bFound = 0;
   770         -          goto done;
   771         -        }
   772         -      }
   773         -      if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
   774         -        bFound = 1;
   775         -        goto done;
   776         -      }
   777         -    }
   778         -  }
   779         -done:
   780         -  sqlite3BtreeLeaveAll(db);
   781         -  sqlite3_mutex_leave(db->mutex);
   782         -  return bFound;
   783         -}
   784         -
   785         -/*
   786         -** Attempt to estimate the final output buffer size needed for the fully
   787         -** normalized version of the specified SQL string.  This should take into
   788         -** account any potential expansion that could occur (e.g. via IN clauses
   789         -** being expanded, etc).  This size returned is the total number of bytes
   790         -** including the NUL terminator.
   791         -*/
   792         -static int estimateNormalizedSize(
   793         -  const char *zSql, /* The original SQL string */
   794         -  int nSql,         /* Length of original SQL string */
   795         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
   796         -){
   797         -  int nOut = nSql + 4;
   798         -  const char *z = zSql;
   799         -  while( nOut<nSql*5 ){
   800         -    while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
   801         -    if( z[0]==0 ) break;
   802         -    z++;
   803         -    if( z[0]!='N' && z[0]!='n' ) break;
   804         -    z++;
   805         -    while( sqlite3Isspace(z[0]) ){ z++; }
   806         -    if( z[0]!='(' ) break;
   807         -    z++;
   808         -    nOut += 5; /* ?,?,? */
   809         -  }
   810         -  return nOut;
   811         -}
   812         -
   813         -/*
   814         -** Copy the current token into the output buffer while dealing with quoted
   815         -** identifiers.  By default, all letters will be converted into lowercase.
   816         -** If the bUpper flag is set, uppercase will be used.  The piOut argument
   817         -** will be used to update the target index into the output string.
   818         -*/
   819         -static void copyNormalizedToken(
   820         -  const char *zSql, /* The original SQL string */
   821         -  int iIn,          /* Current index into the original SQL string */
   822         -  int nToken,       /* Number of bytes in the current token */
   823         -  int tokenFlags,   /* Flags returned by the tokenizer */
   824         -  char *zOut,       /* The output string */
   825         -  int *piOut        /* Pointer to target index into the output string */
   826         -){
   827         -  int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
   828         -  int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
   829         -  int j = *piOut, k = 0;
   830         -  for(; k<nToken; k++){
   831         -    if( bQuoted ){
   832         -      if( k==0 && iIn>0 ){
   833         -        zOut[j++] = '"';
   834         -        continue;
   835         -      }else if( k==nToken-1 ){
   836         -        zOut[j++] = '"';
   837         -        continue;
   838         -      }
   839         -    }
   840         -    if( bKeyword ){
   841         -      zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
   842         -    }else{
   843         -      zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
   844         -    }
   845         -  }
   846         -  *piOut = j;
   847         -}
   848         -
   849         -/*
   850         -** Perform normalization of the SQL contained in the prepared statement and
   851         -** store the result in the zNormSql field.  The schema for the associated
   852         -** databases are consulted while performing the normalization in order to
   853         -** determine if a token appears to be an identifier.  All identifiers are
   854         -** left intact in the normalized SQL and all literals are replaced with a
   855         -** single '?'.
   856         -*/
   857         -void sqlite3Normalize(
   858         -  Vdbe *pVdbe,      /* VM being reprepared */
   859         -  const char *zSql, /* The original SQL string */
   860         -  int nSql,         /* Size of the input string in bytes */
   861         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
   862         -){
   863         -  sqlite3 *db;           /* Database handle. */
   864         -  char *z;               /* The output string */
   865         -  int nZ;                /* Size of the output string in bytes */
   866         -  int i;                 /* Next character to read from zSql[] */
   867         -  int j;                 /* Next character to fill in on z[] */
   868         -  int tokenType = 0;     /* Type of the next token */
   869         -  int prevTokenType = 0; /* Type of the previous token, except spaces */
   870         -  int n;                 /* Size of the next token */
   871         -  int nParen = 0;        /* Nesting level of parenthesis */
   872         -  Hash inHash;           /* Table of parenthesis levels to output index. */
   873         -
   874         -  db = sqlite3VdbeDb(pVdbe);
   875         -  assert( db!=0 );
   876         -  assert( pVdbe->zNormSql==0 );
   877         -  if( zSql==0 ) return;
   878         -  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
   879         -  z = sqlite3DbMallocRawNN(db, nZ);
   880         -  if( z==0 ) return;
   881         -  sqlite3HashInit(&inHash);
   882         -  for(i=j=0; i<nSql && zSql[i]; i+=n){
   883         -    int flags = 0;
   884         -    if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
   885         -    n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
   886         -    switch( tokenType ){
   887         -      case TK_SPACE: {
   888         -        break;
   889         -      }
   890         -      case TK_ILLEGAL: {
   891         -        sqlite3DbFree(db, z);
   892         -        sqlite3HashClear(&inHash);
   893         -        return;
   894         -      }
   895         -      case TK_STRING:
   896         -      case TK_INTEGER:
   897         -      case TK_FLOAT:
   898         -      case TK_VARIABLE:
   899         -      case TK_BLOB: {
   900         -        z[j++] = '?';
   901         -        break;
   902         -      }
   903         -      case TK_LP:
   904         -      case TK_RP: {
   905         -        if( tokenType==TK_LP ){
   906         -          nParen++;
   907         -          if( prevTokenType==TK_IN ){
   908         -            assert( nParen<nSql );
   909         -            sqlite3HashInsert(&inHash, zSql+nParen, SQLITE_INT_TO_PTR(j));
   910         -          }
   911         -        }else{
   912         -          int jj;
   913         -          assert( nParen<nSql );
   914         -          jj = SQLITE_PTR_TO_INT(sqlite3HashFind(&inHash, zSql+nParen));
   915         -          if( jj>0 ){
   916         -            sqlite3HashInsert(&inHash, zSql+nParen, 0);
   917         -            assert( jj+6<nZ );
   918         -            memcpy(z+jj+1, "?,?,?", 5);
   919         -            j = jj+6;
   920         -            assert( nZ-1-j>=0 );
   921         -            assert( nZ-1-j<nZ );
   922         -            memset(z+j, 0, nZ-1-j);
   923         -          }
   924         -          nParen--;
   925         -        }
   926         -        assert( nParen>=0 );
   927         -        /* Fall through */
   928         -      }
   929         -      case TK_MINUS:
   930         -      case TK_SEMI:
   931         -      case TK_PLUS:
   932         -      case TK_STAR:
   933         -      case TK_SLASH:
   934         -      case TK_REM:
   935         -      case TK_EQ:
   936         -      case TK_LE:
   937         -      case TK_NE:
   938         -      case TK_LSHIFT:
   939         -      case TK_LT:
   940         -      case TK_RSHIFT:
   941         -      case TK_GT:
   942         -      case TK_GE:
   943         -      case TK_BITOR:
   944         -      case TK_CONCAT:
   945         -      case TK_COMMA:
   946         -      case TK_BITAND:
   947         -      case TK_BITNOT:
   948         -      case TK_DOT:
   949         -      case TK_IN:
   950         -      case TK_IS:
   951         -      case TK_NOT:
   952         -      case TK_NULL:
   953         -      case TK_ID: {
   954         -        if( tokenType==TK_NULL ){
   955         -          if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
   956         -            /* NULL is a keyword in this case, not a literal value */
   957         -          }else{
   958         -            /* Here the NULL is a literal value */
   959         -            z[j++] = '?';
   960         -            break;
   961         -          }
   962         -        }
   963         -        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
   964         -          z[j++] = ' ';
   965         -        }
   966         -        if( tokenType==TK_ID ){
   967         -          int i2 = i, n2 = n, rc = SQLITE_OK;
   968         -          if( nParen>0 ){
   969         -            assert( nParen<nSql );
   970         -            sqlite3HashInsert(&inHash, zSql+nParen, 0);
   971         -          }
   972         -          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
   973         -          if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
   974         -            if( rc!=SQLITE_OK ){
   975         -              sqlite3DbFree(db, z);
   976         -              sqlite3HashClear(&inHash);
   977         -              return;
   978         -            }
   979         -            if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
   980         -              z[j++] = '?';
   981         -              break;
   982         -            }
   983         -          }
   984         -        }
   985         -        copyNormalizedToken(zSql, i, n, flags, z, &j);
   986         -        break;
   987         -      }
   988         -    }
   989         -  }
   990         -  assert( j<nZ && "one" );
   991         -  while( j>0 && z[j-1]==' ' ){ j--; }
   992         -  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   993         -  z[j] = 0;
   994         -  assert( j<nZ && "two" );
   995         -  pVdbe->zNormSql = z;
   996         -  sqlite3HashClear(&inHash);
   997         -}
   998         -#endif /* SQLITE_ENABLE_NORMALIZE */
   999    712   
  1000    713   /*
  1001    714   ** Rerun the compilation of a statement after a schema change.
  1002    715   **
  1003    716   ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,
  1004    717   ** if the statement cannot be recompiled because another connection has
  1005    718   ** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error

Changes to src/resolve.c.

    76     76     db = pParse->db;
    77     77     pDup = sqlite3ExprDup(db, pOrig, 0);
    78     78     if( pDup!=0 ){
    79     79       if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
    80     80       if( pExpr->op==TK_COLLATE ){
    81     81         pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
    82     82       }
    83         -    ExprSetProperty(pDup, EP_Alias);
           83  +//    ExprSetProperty(pDup, EP_Alias);
    84     84   
    85     85       /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
    86     86       ** prevents ExprDelete() from deleting the Expr structure itself,
    87     87       ** allowing it to be repopulated by the memcpy() on the following line.
    88     88       ** The pExpr->u.zToken might point into memory that will be freed by the
    89     89       ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
    90     90       ** make a copy of the token before doing the sqlite3DbFree().
................................................................................
   470    470     **
   471    471     ** Because no reference was made to outer contexts, the pNC->nRef
   472    472     ** fields are not changed in any context.
   473    473     */
   474    474     if( cnt==0 && zTab==0 ){
   475    475       assert( pExpr->op==TK_ID );
   476    476       if( ExprHasProperty(pExpr,EP_DblQuoted) ){
          477  +      /* If a double-quoted identifier does not match any known column name,
          478  +      ** then treat it as a string.
          479  +      **
          480  +      ** This hack was added in the early days of SQLite in a misguided attempt
          481  +      ** to be compatible with MySQL 3.x, which used double-quotes for strings.
          482  +      ** I now sorely regret putting in this hack. The effect of this hack is
          483  +      ** that misspelled identifier names are silently converted into strings
          484  +      ** rather than causing an error, to the frustration of countless
          485  +      ** programmers. To all those frustrated programmers, my apologies.
          486  +      **
          487  +      ** Someday, I hope to get rid of this hack. Unfortunately there is
          488  +      ** a huge amount of legacy SQL that uses it. So for now, we just
          489  +      ** issue a warning.
          490  +      */
          491  +      sqlite3_log(SQLITE_WARNING,
          492  +        "double-quoted string literal: \"%w\"", zCol);
          493  +#ifdef SQLITE_ENABLE_NORMALIZE
          494  +      sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol);
          495  +#endif
   477    496         pExpr->op = TK_STRING;
   478    497         pExpr->y.pTab = 0;
   479    498         return WRC_Prune;
   480    499       }
   481    500       if( sqlite3ExprIdToTrueFalse(pExpr) ){
   482    501         return WRC_Prune;
   483    502       }

Changes to src/select.c.

  2073   2073   /*
  2074   2074   ** Given a SELECT statement, generate a Table structure that describes
  2075   2075   ** the result set of that SELECT.
  2076   2076   */
  2077   2077   Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){
  2078   2078     Table *pTab;
  2079   2079     sqlite3 *db = pParse->db;
  2080         -  int savedFlags;
         2080  +  u64 savedFlags;
  2081   2081   
  2082   2082     savedFlags = db->flags;
  2083         -  db->flags &= ~SQLITE_FullColNames;
         2083  +  db->flags &= ~(u64)SQLITE_FullColNames;
  2084   2084     db->flags |= SQLITE_ShortColNames;
  2085   2085     sqlite3SelectPrep(pParse, pSelect, 0);
  2086   2086     if( pParse->nErr ) return 0;
  2087   2087     while( pSelect->pPrior ) pSelect = pSelect->pPrior;
  2088   2088     db->flags = savedFlags;
  2089   2089     pTab = sqlite3DbMallocZero(db, sizeof(Table) );
  2090   2090     if( pTab==0 ){
................................................................................
  3457   3457           if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
  3458   3458             memset(&ifNullRow, 0, sizeof(ifNullRow));
  3459   3459             ifNullRow.op = TK_IF_NULL_ROW;
  3460   3460             ifNullRow.pLeft = pCopy;
  3461   3461             ifNullRow.iTable = pSubst->iNewTable;
  3462   3462             pCopy = &ifNullRow;
  3463   3463           }
         3464  +        testcase( ExprHasProperty(pCopy, EP_Subquery) );
  3464   3465           pNew = sqlite3ExprDup(db, pCopy, 0);
  3465   3466           if( pNew && pSubst->isLeftJoin ){
  3466   3467             ExprSetProperty(pNew, EP_CanBeNull);
  3467   3468           }
  3468   3469           if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){
  3469   3470             pNew->iRightJoinTable = pExpr->iRightJoinTable;
  3470   3471             ExprSetProperty(pNew, EP_FromJoin);
................................................................................
  4021   4022         for(i=0; i<pOrderBy->nExpr; i++){
  4022   4023           pOrderBy->a[i].u.x.iOrderByCol = 0;
  4023   4024         }
  4024   4025         assert( pParent->pOrderBy==0 );
  4025   4026         pParent->pOrderBy = pOrderBy;
  4026   4027         pSub->pOrderBy = 0;
  4027   4028       }
  4028         -    pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
         4029  +    pWhere = pSub->pWhere;
         4030  +    pSub->pWhere = 0;
  4029   4031       if( isLeftJoin>0 ){
  4030   4032         setJoinExpr(pWhere, iNewParent);
  4031   4033       }
  4032   4034       pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
  4033   4035       if( db->mallocFailed==0 ){
  4034   4036         SubstContext x;
  4035   4037         x.pParse = pParse;

Changes to src/shell.c.in.

  1003   1003     u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  1004   1004     u8 autoEQPtest;        /* autoEQP is in test mode */
  1005   1005     u8 statsOn;            /* True to display memory stats before each finalize */
  1006   1006     u8 scanstatsOn;        /* True to display scan stats before each finalize */
  1007   1007     u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  1008   1008     u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
  1009   1009     u8 nEqpLevel;          /* Depth of the EQP output graph */
         1010  +  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
  1010   1011     unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  1011   1012     int outCount;          /* Revert to stdout when reaching zero */
  1012   1013     int cnt;               /* Number of records displayed so far */
  1013   1014     FILE *out;             /* Write results here */
  1014   1015     FILE *traceOut;        /* Output for sqlite3_trace() */
  1015   1016     int nErr;              /* Number of errors seen */
  1016   1017     int mode;              /* An output mode setting */
................................................................................
  1062   1063   #define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
  1063   1064   #define SHELL_OPEN_NORMAL      1      /* Normal database file */
  1064   1065   #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
  1065   1066   #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
  1066   1067   #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
  1067   1068   #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
  1068   1069   
         1070  +/* Allowed values for ShellState.eTraceType
         1071  +*/
         1072  +#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
         1073  +#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
         1074  +#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
         1075  +
  1069   1076   /*
  1070   1077   ** These are the allowed shellFlgs values
  1071   1078   */
  1072   1079   #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
  1073   1080   #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
  1074   1081   #define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
  1075   1082   #define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
................................................................................
  3366   3373     "      http://sqlite.org/cli.html#sqlar_archive_support",
  3367   3374   #endif
  3368   3375   #ifndef SQLITE_OMIT_AUTHORIZATION
  3369   3376     ".auth ON|OFF             Show authorizer callbacks",
  3370   3377   #endif
  3371   3378     ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
  3372   3379     "       --append            Use the appendvfs",
         3380  +  "       --async             Write to FILE without a journal and without fsync()",
  3373   3381     ".bail on|off             Stop after hitting an error.  Default OFF",
  3374   3382     ".binary on|off           Turn binary output on or off.  Default OFF",
  3375   3383     ".cd DIRECTORY            Change the working directory to DIRECTORY",
  3376   3384     ".changes on|off          Show number of rows changed by SQL",
  3377   3385     ".check GLOB              Fail if output since .testcase does not match",
  3378   3386     ".clone NEWDB             Clone data into NEWDB from the existing database",
  3379   3387     ".databases               List names and files of attached databases",
................................................................................
  3488   3496   #ifndef SQLITE_NOHAVE_SYSTEM
  3489   3497     ".system CMD ARGS...      Run CMD ARGS... in a system shell",
  3490   3498   #endif
  3491   3499     ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
  3492   3500     ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
  3493   3501     ".timeout MS              Try opening locked tables for MS milliseconds",
  3494   3502     ".timer on|off            Turn SQL timer on or off",
  3495         -  ".trace FILE|off          Output each SQL statement as it is run",
         3503  +#ifndef SQLITE_OMIT_TRACE
         3504  +  ".trace ?OPTIONS?         Output each SQL statement as it is run",
         3505  +  "    FILE                    Send output to FILE",
         3506  +  "    stdout                  Send output to stdout",
         3507  +  "    stderr                  Send output to stderr",
         3508  +  "    off                     Disable tracing",
         3509  +  "    --expanded              Expand query parameters",
         3510  +#ifdef SQLITE_ENABLE_NORMALIZE
         3511  +  "    --normalized            Normal the SQL statements",
         3512  +#endif
         3513  +  "    --plain                 Show SQL as it is input",
         3514  +  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
         3515  +  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
         3516  +  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
         3517  +  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
         3518  +#endif /* SQLITE_OMIT_TRACE */
  3496   3519     ".vfsinfo ?AUX?           Information about the top-level VFS",
  3497   3520     ".vfslist                 List all available VFSes",
  3498   3521     ".vfsname ?AUX?           Print the name of the VFS stack",
  3499   3522     ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
  3500   3523     "     Negative values right-justify",
  3501   3524   };
  3502   3525   
................................................................................
  3995   4018       if( f==0 ){
  3996   4019         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
  3997   4020       }
  3998   4021     }
  3999   4022     return f;
  4000   4023   }
  4001   4024   
  4002         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
         4025  +#ifndef SQLITE_OMIT_TRACE
  4003   4026   /*
  4004   4027   ** A routine for handling output from sqlite3_trace().
  4005   4028   */
  4006   4029   static int sql_trace_callback(
  4007         -  unsigned mType,
  4008         -  void *pArg,
  4009         -  void *pP,
  4010         -  void *pX
  4011         -){
  4012         -  FILE *f = (FILE*)pArg;
  4013         -  UNUSED_PARAMETER(mType);
  4014         -  UNUSED_PARAMETER(pP);
  4015         -  if( f ){
  4016         -    const char *z = (const char*)pX;
  4017         -    int i = strlen30(z);
  4018         -    while( i>0 && z[i-1]==';' ){ i--; }
  4019         -    utf8_printf(f, "%.*s;\n", i, z);
         4030  +  unsigned mType,         /* The trace type */
         4031  +  void *pArg,             /* The ShellState pointer */
         4032  +  void *pP,               /* Usually a pointer to sqlite_stmt */
         4033  +  void *pX                /* Auxiliary output */
         4034  +){
         4035  +  ShellState *p = (ShellState*)pArg;
         4036  +  sqlite3_stmt *pStmt;
         4037  +  const char *zSql;
         4038  +  int nSql;
         4039  +  if( p->traceOut==0 ) return 0;
         4040  +  if( mType==SQLITE_TRACE_CLOSE ){
         4041  +    utf8_printf(p->traceOut, "-- closing database connection\n");
         4042  +    return 0;
         4043  +  }
         4044  +  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
         4045  +    zSql = (const char*)pX;
         4046  +  }else{
         4047  +    pStmt = (sqlite3_stmt*)pP;
         4048  +    switch( p->eTraceType ){
         4049  +      case SHELL_TRACE_EXPANDED: {
         4050  +        zSql = sqlite3_expanded_sql(pStmt);
         4051  +        break;
         4052  +      }
         4053  +#ifdef SQLITE_ENABLE_NORMALIZE
         4054  +      case SHELL_TRACE_NORMALIZED: {
         4055  +        zSql = sqlite3_normalized_sql(pStmt);
         4056  +        break;
         4057  +      }
         4058  +#endif
         4059  +      default: {
         4060  +        zSql = sqlite3_sql(pStmt);
         4061  +        break;
         4062  +      }
         4063  +    }
         4064  +  }
         4065  +  if( zSql==0 ) return 0;
         4066  +  nSql = strlen30(zSql);
         4067  +  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
         4068  +  switch( mType ){
         4069  +    case SQLITE_TRACE_ROW:
         4070  +    case SQLITE_TRACE_STMT: {
         4071  +      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
         4072  +      break;
         4073  +    }
         4074  +    case SQLITE_TRACE_PROFILE: {
         4075  +      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
         4076  +      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
         4077  +      break;
         4078  +    }
  4020   4079     }
  4021   4080     return 0;
  4022   4081   }
  4023   4082   #endif
  4024   4083   
  4025   4084   /*
  4026   4085   ** A no-op routine that runs with the ".breakpoint" doc-command.  This is
................................................................................
  5778   5837      || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
  5779   5838     ){
  5780   5839       const char *zDestFile = 0;
  5781   5840       const char *zDb = 0;
  5782   5841       sqlite3 *pDest;
  5783   5842       sqlite3_backup *pBackup;
  5784   5843       int j;
         5844  +    int bAsync = 0;
  5785   5845       const char *zVfs = 0;
  5786   5846       for(j=1; j<nArg; j++){
  5787   5847         const char *z = azArg[j];
  5788   5848         if( z[0]=='-' ){
  5789   5849           if( z[1]=='-' ) z++;
  5790   5850           if( strcmp(z, "-append")==0 ){
  5791   5851             zVfs = "apndvfs";
         5852  +        }else
         5853  +        if( strcmp(z, "-async")==0 ){
         5854  +          bAsync = 1;
  5792   5855           }else
  5793   5856           {
  5794   5857             utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
  5795   5858             return 1;
  5796   5859           }
  5797   5860         }else if( zDestFile==0 ){
  5798   5861           zDestFile = azArg[j];
  5799   5862         }else if( zDb==0 ){
  5800   5863           zDb = zDestFile;
  5801   5864           zDestFile = azArg[j];
  5802   5865         }else{
  5803         -        raw_printf(stderr, "Usage: .backup ?DB? ?--append? FILENAME\n");
         5866  +        raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
  5804   5867           return 1;
  5805   5868         }
  5806   5869       }
  5807   5870       if( zDestFile==0 ){
  5808   5871         raw_printf(stderr, "missing FILENAME argument on .backup\n");
  5809   5872         return 1;
  5810   5873       }
................................................................................
  5812   5875       rc = sqlite3_open_v2(zDestFile, &pDest, 
  5813   5876                     SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs);
  5814   5877       if( rc!=SQLITE_OK ){
  5815   5878         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
  5816   5879         close_db(pDest);
  5817   5880         return 1;
  5818   5881       }
         5882  +    if( bAsync ){
         5883  +      sqlite3_exec(pDest, "PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;",
         5884  +                   0, 0, 0);
         5885  +    }
  5819   5886       open_db(p, 0);
  5820   5887       pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
  5821   5888       if( pBackup==0 ){
  5822   5889         utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
  5823   5890         close_db(pDest);
  5824   5891         return 1;
  5825   5892       }
................................................................................
  7834   7901         }
  7835   7902       }else{
  7836   7903         raw_printf(stderr, "Usage: .timer on|off\n");
  7837   7904         rc = 1;
  7838   7905       }
  7839   7906     }else
  7840   7907   
         7908  +#ifndef SQLITE_OMIT_TRACE
  7841   7909     if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
         7910  +    int mType = 0;
         7911  +    int jj;
  7842   7912       open_db(p, 0);
  7843         -    if( nArg!=2 ){
  7844         -      raw_printf(stderr, "Usage: .trace FILE|off\n");
  7845         -      rc = 1;
  7846         -      goto meta_command_exit;
         7913  +    for(jj=1; jj<nArg; jj++){
         7914  +      const char *z = azArg[jj];
         7915  +      if( z[0]=='-' ){
         7916  +        if( optionMatch(z, "expanded") ){
         7917  +          p->eTraceType = SHELL_TRACE_EXPANDED;
         7918  +        }
         7919  +#ifdef SQLITE_ENABLE_NORMALIZE
         7920  +        else if( optionMatch(z, "normalized") ){
         7921  +          p->eTraceType = SHELL_TRACE_NORMALIZED;
         7922  +        }
         7923  +#endif
         7924  +        else if( optionMatch(z, "plain") ){
         7925  +          p->eTraceType = SHELL_TRACE_PLAIN;
         7926  +        }
         7927  +        else if( optionMatch(z, "profile") ){
         7928  +          mType |= SQLITE_TRACE_PROFILE;
         7929  +        }
         7930  +        else if( optionMatch(z, "row") ){
         7931  +          mType |= SQLITE_TRACE_ROW;
         7932  +        }
         7933  +        else if( optionMatch(z, "stmt") ){
         7934  +          mType |= SQLITE_TRACE_STMT;
         7935  +        }
         7936  +        else if( optionMatch(z, "close") ){
         7937  +          mType |= SQLITE_TRACE_CLOSE;
         7938  +        }
         7939  +        else {
         7940  +          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
         7941  +          rc = 1;
         7942  +          goto meta_command_exit;
         7943  +        }
         7944  +      }else{
         7945  +        output_file_close(p->traceOut);
         7946  +        p->traceOut = output_file_open(azArg[1], 0);
         7947  +      }
  7847   7948       }
  7848         -    output_file_close(p->traceOut);
  7849         -    p->traceOut = output_file_open(azArg[1], 0);
  7850         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
  7851   7949       if( p->traceOut==0 ){
  7852   7950         sqlite3_trace_v2(p->db, 0, 0, 0);
  7853   7951       }else{
  7854         -      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
         7952  +      if( mType==0 ) mType = SQLITE_TRACE_STMT;
         7953  +      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
  7855   7954       }
  7856         -#endif
  7857   7955     }else
         7956  +#endif /* !defined(SQLITE_OMIT_TRACE) */
  7858   7957   
  7859   7958   #if SQLITE_USER_AUTHENTICATION
  7860   7959     if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
  7861   7960       if( nArg<2 ){
  7862   7961         raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
  7863   7962         rc = 1;
  7864   7963         goto meta_command_exit;

Changes to src/sqlite.h.in.

  2987   2987   ** ^The callback function registered by sqlite3_profile() is invoked
  2988   2988   ** as each SQL statement finishes.  ^The profile callback contains
  2989   2989   ** the original statement text and an estimate of wall-clock time
  2990   2990   ** of how long that statement took to run.  ^The profile callback
  2991   2991   ** time is in units of nanoseconds, however the current implementation
  2992   2992   ** is only capable of millisecond resolution so the six least significant
  2993   2993   ** digits in the time are meaningless.  Future versions of SQLite
  2994         -** might provide greater resolution on the profiler callback.  The
  2995         -** sqlite3_profile() function is considered experimental and is
  2996         -** subject to change in future versions of SQLite.
         2994  +** might provide greater resolution on the profiler callback.  Invoking
         2995  +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the
         2996  +** profile callback.
  2997   2997   */
  2998   2998   SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
  2999   2999      void(*xTrace)(void*,const char*), void*);
  3000   3000   SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
  3001   3001      void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
  3002   3002   
  3003   3003   /*
................................................................................
  3625   3625   ** and [sqlite3_prepare16_v3()] assume that the prepared statement will 
  3626   3626   ** be used just once or at most a few times and then destroyed using
  3627   3627   ** [sqlite3_finalize()] relatively soon. The current implementation acts
  3628   3628   ** on this hint by avoiding the use of [lookaside memory] so as not to
  3629   3629   ** deplete the limited store of lookaside memory. Future versions of
  3630   3630   ** SQLite may act on this hint differently.
  3631   3631   **
  3632         -** [[SQLITE_PREPARE_NORMALIZE]] ^(<dt>SQLITE_PREPARE_NORMALIZE</dt>
  3633         -** <dd>The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized
  3634         -** representation of the SQL statement should be calculated and then
  3635         -** associated with the prepared statement, which can be obtained via
  3636         -** the [sqlite3_normalized_sql()] interface.)^  The semantics used to
  3637         -** normalize a SQL statement are unspecified and subject to change.
  3638         -** At a minimum, literal values will be replaced with suitable
  3639         -** placeholders.
         3632  +** [[SQLITE_PREPARE_NORMALIZE]] <dt>SQLITE_PREPARE_NORMALIZE</dt>
         3633  +** <dd>The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used
         3634  +** to be required for any prepared statement that wanted to use the
         3635  +** [sqlite3_normalized_sql()] interface.  However, the
         3636  +** [sqlite3_normalized_sql()] interface is now available to all
         3637  +** prepared statements, regardless of whether or not they use this
         3638  +** flag.
  3640   3639   ** </dl>
  3641   3640   */
  3642   3641   #define SQLITE_PREPARE_PERSISTENT              0x01
  3643   3642   #define SQLITE_PREPARE_NORMALIZE               0x02
  3644   3643   
  3645   3644   /*
  3646   3645   ** CAPI3REF: Compiling An SQL Statement

Changes to src/sqliteInt.h.

  1352   1352                                  const char*);
  1353   1353   #endif
  1354   1354   
  1355   1355   #ifndef SQLITE_OMIT_DEPRECATED
  1356   1356   /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
  1357   1357   ** in the style of sqlite3_trace()
  1358   1358   */
  1359         -#define SQLITE_TRACE_LEGACY  0x80
         1359  +#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
         1360  +#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
  1360   1361   #else
  1361         -#define SQLITE_TRACE_LEGACY  0
         1362  +#define SQLITE_TRACE_LEGACY          0
         1363  +#define SQLITE_TRACE_XPROFILE        0
  1362   1364   #endif /* SQLITE_OMIT_DEPRECATED */
         1365  +#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */
  1363   1366   
  1364   1367   
  1365   1368   /*
  1366   1369   ** Each database connection is an instance of the following structure.
  1367   1370   */
  1368   1371   struct sqlite3 {
  1369   1372     sqlite3_vfs *pVfs;            /* OS Interface */
................................................................................
  1415   1418     int nVdbeWrite;               /* Number of active VDBEs that read and write */
  1416   1419     int nVdbeExec;                /* Number of nested calls to VdbeExec() */
  1417   1420     int nVDestroy;                /* Number of active OP_VDestroy operations */
  1418   1421     int nExtension;               /* Number of loaded extensions */
  1419   1422     void **aExtension;            /* Array of shared library handles */
  1420   1423     int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
  1421   1424     void *pTraceArg;                          /* Argument to the trace function */
         1425  +#ifndef SQLITE_OMIT_DEPRECATED
  1422   1426     void (*xProfile)(void*,const char*,u64);  /* Profiling function */
  1423   1427     void *pProfileArg;                        /* Argument to profile function */
         1428  +#endif
  1424   1429     void *pCommitArg;                 /* Argument to xCommitCallback() */
  1425   1430     int (*xCommitCallback)(void*);    /* Invoked at every commit. */
  1426   1431     void *pRollbackArg;               /* Argument to xRollbackCallback() */
  1427   1432     void (*xRollbackCallback)(void*); /* Invoked at every commit. */
  1428   1433     void *pUpdateArg;
  1429   1434     void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
  1430   1435   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
................................................................................
  1949   1954   /*
  1950   1955   ** The schema for each SQL table and view is represented in memory
  1951   1956   ** by an instance of the following structure.
  1952   1957   */
  1953   1958   struct Table {
  1954   1959     char *zName;         /* Name of the table or view */
  1955   1960     Column *aCol;        /* Information about each column */
  1956         -#ifdef SQLITE_ENABLE_NORMALIZE
  1957         -  Hash *pColHash;      /* All columns indexed by name */
  1958         -#endif
  1959   1961     Index *pIndex;       /* List of SQL indexes on this table. */
  1960   1962     Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
  1961   1963     FKey *pFKey;         /* Linked list of all foreign keys in this table */
  1962   1964     char *zColAff;       /* String defining the affinity of each column */
  1963   1965     ExprList *pCheck;    /* All CHECK constraints */
  1964   1966                          /*   ... also used as column name list in a VIEW */
  1965   1967     int tnum;            /* Root BTree page for this table */
................................................................................
  3980   3982   #define LOCATE_VIEW    0x01
  3981   3983   #define LOCATE_NOERR   0x02
  3982   3984   Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
  3983   3985   Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
  3984   3986   Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
  3985   3987   void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
  3986   3988   void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
  3987         -void sqlite3Vacuum(Parse*,Token*);
  3988         -int sqlite3RunVacuum(char**, sqlite3*, int);
         3989  +void sqlite3Vacuum(Parse*,Token*,Expr*);
         3990  +int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*);
  3989   3991   char *sqlite3NameFromToken(sqlite3*, Token*);
  3990   3992   int sqlite3ExprCompare(Parse*,Expr*, Expr*, int);
  3991   3993   int sqlite3ExprCompareSkip(Expr*, Expr*, int);
  3992   3994   int sqlite3ExprListCompare(ExprList*, ExprList*, int);
  3993   3995   int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int);
  3994   3996   int sqlite3ExprImpliesNonNullRow(Expr*,int);
  3995   3997   void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
................................................................................
  4019   4021   #ifdef SQLITE_ENABLE_CURSOR_HINTS
  4020   4022   int sqlite3ExprContainsSubquery(Expr*);
  4021   4023   #endif
  4022   4024   int sqlite3ExprIsInteger(Expr*, int*);
  4023   4025   int sqlite3ExprCanBeNull(const Expr*);
  4024   4026   int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
  4025   4027   int sqlite3IsRowid(const char*);
  4026         -#ifdef SQLITE_ENABLE_NORMALIZE
  4027         -int sqlite3IsRowidN(const char*, int);
  4028         -#endif
  4029   4028   void sqlite3GenerateRowDelete(
  4030   4029       Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
  4031   4030   void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
  4032   4031   int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
  4033   4032   void sqlite3ResolvePartIdxLabel(Parse*,int);
  4034   4033   int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int);
  4035   4034   void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
................................................................................
  4048   4047   void sqlite3UniqueConstraint(Parse*, int, Index*);
  4049   4048   void sqlite3RowidConstraint(Parse*, int, Table*);
  4050   4049   Expr *sqlite3ExprDup(sqlite3*,Expr*,int);
  4051   4050   ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
  4052   4051   SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
  4053   4052   IdList *sqlite3IdListDup(sqlite3*,IdList*);
  4054   4053   Select *sqlite3SelectDup(sqlite3*,Select*,int);
  4055         -#ifdef SQLITE_ENABLE_NORMALIZE
  4056         -FuncDef *sqlite3FunctionSearchN(int,const char*,int);
  4057         -#endif
         4054  +FuncDef *sqlite3FunctionSearch(int,const char*);
  4058   4055   void sqlite3InsertBuiltinFuncs(FuncDef*,int);
  4059   4056   FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
  4060   4057   void sqlite3RegisterBuiltinFunctions(void);
  4061   4058   void sqlite3RegisterDateTimeFunctions(void);
  4062   4059   void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
  4063   4060   int sqlite3SafetyCheckOk(sqlite3*);
  4064   4061   int sqlite3SafetyCheckSickOrOk(sqlite3*);
................................................................................
  4255   4252   #endif
  4256   4253   void sqlite3RootPageMoved(sqlite3*, int, int, int);
  4257   4254   void sqlite3Reindex(Parse*, Token*, Token*);
  4258   4255   void sqlite3AlterFunctions(void);
  4259   4256   void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
  4260   4257   void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
  4261   4258   int sqlite3GetToken(const unsigned char *, int *);
  4262         -#ifdef SQLITE_ENABLE_NORMALIZE
  4263         -int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
  4264         -#endif
  4265   4259   void sqlite3NestedParse(Parse*, const char*, ...);
  4266   4260   void sqlite3ExpirePreparedStatements(sqlite3*, int);
  4267   4261   int sqlite3CodeSubselect(Parse*, Expr *, int, int);
  4268   4262   void sqlite3SelectPrep(Parse*, Select*, NameContext*);
  4269   4263   void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
  4270   4264   int sqlite3MatchSpanName(const char*, const char*, const char*, const char*);
  4271   4265   int sqlite3ResolveExprNames(NameContext*, Expr*);
................................................................................
  4416   4410   int sqlite3VtabBegin(sqlite3 *, VTable *);
  4417   4411   FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
  4418   4412   sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
  4419   4413   int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
  4420   4414   int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
  4421   4415   void sqlite3ParserReset(Parse*);
  4422   4416   #ifdef SQLITE_ENABLE_NORMALIZE
  4423         -void sqlite3Normalize(Vdbe*, const char*, int, u8);
         4417  +char *sqlite3Normalize(Vdbe*, const char*);
  4424   4418   #endif
  4425   4419   int sqlite3Reprepare(Vdbe*);
  4426   4420   void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
  4427   4421   CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
  4428   4422   int sqlite3TempInMemory(const sqlite3*);
  4429   4423   const char *sqlite3JournalModename(int);
  4430   4424   #ifndef SQLITE_OMIT_WAL

Changes to src/test_hexio.c.

   186    186     sqlite3_free(aOut);
   187    187     fclose(out);
   188    188     Tcl_SetObjResult(interp, Tcl_NewIntObj(written));
   189    189     return TCL_OK;
   190    190   }
   191    191   
   192    192   /*
   193         -** USAGE:   hexio_get_int   HEXDATA
          193  +** USAGE:   hexio_get_int [-littleendian] HEXDATA
   194    194   **
   195    195   ** Interpret the HEXDATA argument as a big-endian integer.  Return
   196    196   ** the value of that integer.  HEXDATA can contain between 2 and 8
   197    197   ** hexadecimal digits.
   198    198   */
   199    199   static int SQLITE_TCLAPI hexio_get_int(
   200    200     void * clientData,
................................................................................
   203    203     Tcl_Obj *CONST objv[]
   204    204   ){
   205    205     int val;
   206    206     int nIn, nOut;
   207    207     const unsigned char *zIn;
   208    208     unsigned char *aOut;
   209    209     unsigned char aNum[4];
          210  +  int bLittle = 0;
   210    211   
   211         -  if( objc!=2 ){
   212         -    Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA");
          212  +  if( objc==3 ){
          213  +    int n;
          214  +    char *z = Tcl_GetStringFromObj(objv[1], &n);
          215  +    if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){
          216  +      bLittle = 1;
          217  +    }
          218  +  }
          219  +  if( (objc-bLittle)!=2 ){
          220  +    Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA");
   213    221       return TCL_ERROR;
   214    222     }
   215         -  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
          223  +  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn);
   216    224     aOut = sqlite3_malloc( nIn/2 );
   217    225     if( aOut==0 ){
   218    226       return TCL_ERROR;
   219    227     }
   220    228     nOut = sqlite3TestHexToBin(zIn, nIn, aOut);
   221    229     if( nOut>=4 ){
   222    230       memcpy(aNum, aOut, 4);
   223    231     }else{
   224    232       memset(aNum, 0, sizeof(aNum));
   225    233       memcpy(&aNum[4-nOut], aOut, nOut);
   226    234     }
   227    235     sqlite3_free(aOut);
   228         -  val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
          236  +  if( bLittle ){
          237  +    val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
          238  +  }else{
          239  +    val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
          240  +  }
   229    241     Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
   230    242     return TCL_OK;
   231    243   }
   232    244   
   233    245   
   234    246   /*
   235    247   ** USAGE:   hexio_render_int16   INTEGER

Changes to src/tokenize.c.

   541    541       }
   542    542     }
   543    543     while( IdChar(z[i]) ){ i++; }
   544    544     *tokenType = TK_ID;
   545    545     return i;
   546    546   }
   547    547   
   548         -#ifdef SQLITE_ENABLE_NORMALIZE
   549         -/*
   550         -** Return the length (in bytes) of the token that begins at z[0].
   551         -** Store the token type in *tokenType before returning.  If flags has
   552         -** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
   553         -** for keywords.  Add SQLITE_TOKEN_QUOTED to flags if the token was
   554         -** actually a quoted identifier.  Add SQLITE_TOKEN_KEYWORD to flags
   555         -** if the token was recognized as a keyword; this is useful when the
   556         -** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
   557         -** to differentiate between a keyword being treated as an identifier
   558         -** (for normalization purposes) and an actual identifier.
   559         -*/
   560         -int sqlite3GetTokenNormalized(
   561         -  const unsigned char *z,
   562         -  int *tokenType,
   563         -  int *flags
   564         -){
   565         -  int n;
   566         -  unsigned char iClass = aiClass[*z];
   567         -  if( iClass==CC_KYWD ){
   568         -    int i;
   569         -    for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
   570         -    if( IdChar(z[i]) ){
   571         -      /* This token started out using characters that can appear in keywords,
   572         -      ** but z[i] is a character not allowed within keywords, so this must
   573         -      ** be an identifier instead */
   574         -      i++;
   575         -      while( IdChar(z[i]) ){ i++; }
   576         -      *tokenType = TK_ID;
   577         -      return i;
   578         -    }
   579         -    *tokenType = TK_ID;
   580         -    n = keywordCode((char*)z, i, tokenType);
   581         -    /* If the token is no longer considered to be an identifier, then it is a
   582         -    ** keyword of some kind.  Make the token back into an identifier and then
   583         -    ** set the SQLITE_TOKEN_KEYWORD flag.  Several non-identifier tokens are
   584         -    ** used verbatim, including IN, IS, NOT, and NULL. */
   585         -    switch( *tokenType ){
   586         -      case TK_ID: {
   587         -        /* do nothing, handled by caller */
   588         -        break;
   589         -      }
   590         -      case TK_IN:
   591         -      case TK_IS:
   592         -      case TK_NOT:
   593         -      case TK_NULL: {
   594         -        *flags |= SQLITE_TOKEN_KEYWORD;
   595         -        break;
   596         -      }
   597         -      default: {
   598         -        *tokenType = TK_ID;
   599         -        *flags |= SQLITE_TOKEN_KEYWORD;
   600         -        break;
   601         -      }
   602         -    }
   603         -  }else{
   604         -    n = sqlite3GetToken(z, tokenType);
   605         -    /* If the token is considered to be an identifier and the character class
   606         -    ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
   607         -    if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
   608         -      *flags |= SQLITE_TOKEN_QUOTED;
   609         -    }
   610         -  }
   611         -  return n;
   612         -}
   613         -#endif /* SQLITE_ENABLE_NORMALIZE */
   614         -
   615    548   /*
   616    549   ** Run the parser on the given SQL string.  The parser structure is
   617    550   ** passed in.  An SQLITE_ status code is returned.  If an error occurs
   618    551   ** then an and attempt is made to write an error message into 
   619    552   ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
   620    553   ** error message.
   621    554   */
................................................................................
   777    710       Table *p = pParse->pZombieTab;
   778    711       pParse->pZombieTab = p->pNextZombie;
   779    712       sqlite3DeleteTable(db, p);
   780    713     }
   781    714     assert( nErr==0 || pParse->rc!=SQLITE_OK );
   782    715     return nErr;
   783    716   }
          717  +
          718  +
          719  +#ifdef SQLITE_ENABLE_NORMALIZE
          720  +/*
          721  +** Insert a single space character into pStr if the current string
          722  +** ends with an identifier
          723  +*/
          724  +static void addSpaceSeparator(sqlite3_str *pStr){
          725  +  if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){
          726  +    sqlite3_str_append(pStr, " ", 1);
          727  +  }
          728  +}
          729  +
          730  +/*
          731  +** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
          732  +** the normalization in space obtained from sqlite3DbMalloc().  Or return
          733  +** NULL if anything goes wrong or if zSql is NULL.
          734  +*/
          735  +char *sqlite3Normalize(
          736  +  Vdbe *pVdbe,       /* VM being reprepared */
          737  +  const char *zSql   /* The original SQL string */
          738  +){
          739  +  sqlite3 *db;       /* The database connection */
          740  +  int i;             /* Next unread byte of zSql[] */
          741  +  int n;             /* length of current token */
          742  +  int tokenType;     /* type of current token */
          743  +  int prevType;      /* Previous non-whitespace token */
          744  +  int nParen;        /* Number of nested levels of parentheses */
          745  +  int iStartIN;      /* Start of RHS of IN operator in z[] */
          746  +  int nParenAtIN;    /* Value of nParent at start of RHS of IN operator */
          747  +  int j;             /* Bytes of normalized SQL generated so far */
          748  +  sqlite3_str *pStr; /* The normalized SQL string under construction */
          749  +
          750  +  db = sqlite3VdbeDb(pVdbe);
          751  +  tokenType = -1;
          752  +  nParen = iStartIN = nParenAtIN = 0;
          753  +  pStr = sqlite3_str_new(db);
          754  +  assert( pStr!=0 );  /* sqlite3_str_new() never returns NULL */
          755  +  for(i=0; zSql[i] && pStr->accError==0; i+=n){
          756  +    if( tokenType!=TK_SPACE ){
          757  +      prevType = tokenType;
          758  +    }
          759  +    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
          760  +    if( NEVER(n<=0) ) break;
          761  +    switch( tokenType ){
          762  +      case TK_SPACE: {
          763  +        break;
          764  +      }
          765  +      case TK_NULL: {
          766  +        if( prevType==TK_IS || prevType==TK_NOT ){
          767  +          sqlite3_str_append(pStr, " NULL", 5);
          768  +          break;
          769  +        }
          770  +        /* Fall through */
          771  +      }
          772  +      case TK_STRING:
          773  +      case TK_INTEGER:
          774  +      case TK_FLOAT:
          775  +      case TK_VARIABLE:
          776  +      case TK_BLOB: {
          777  +        sqlite3_str_append(pStr, "?", 1);
          778  +        break;
          779  +      }
          780  +      case TK_LP: {
          781  +        nParen++;
          782  +        if( prevType==TK_IN ){
          783  +          iStartIN = pStr->nChar;
          784  +          nParenAtIN = nParen;
          785  +        }
          786  +        sqlite3_str_append(pStr, "(", 1);
          787  +        break;
          788  +      }
          789  +      case TK_RP: {
          790  +        if( iStartIN>0 && nParen==nParenAtIN ){
          791  +          assert( pStr->nChar>=iStartIN );
          792  +          pStr->nChar = iStartIN+1;
          793  +          sqlite3_str_append(pStr, "?,?,?", 5);
          794  +          iStartIN = 0;
          795  +        }
          796  +        nParen--;
          797  +        sqlite3_str_append(pStr, ")", 1);
          798  +        break;
          799  +      }
          800  +      case TK_ID: {
          801  +        iStartIN = 0;
          802  +        j = pStr->nChar;
          803  +        if( sqlite3Isquote(zSql[i]) ){
          804  +          char *zId = sqlite3DbStrNDup(db, zSql+i, n);
          805  +          int nId;
          806  +          int eType = 0;
          807  +          if( zId==0 ) break;
          808  +          sqlite3Dequote(zId);
          809  +          if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){
          810  +            sqlite3_str_append(pStr, "?", 1);
          811  +            sqlite3DbFree(db, zId);
          812  +            break;
          813  +          }
          814  +          nId = sqlite3Strlen30(zId);
          815  +          if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){
          816  +            addSpaceSeparator(pStr);
          817  +            sqlite3_str_append(pStr, zId, nId);
          818  +          }else{
          819  +            sqlite3_str_appendf(pStr, "\"%w\"", zId);
          820  +          }
          821  +          sqlite3DbFree(db, zId);
          822  +        }else{
          823  +          addSpaceSeparator(pStr);
          824  +          sqlite3_str_append(pStr, zSql+i, n);
          825  +        }
          826  +        while( j<pStr->nChar ){
          827  +          pStr->zText[j] = sqlite3Tolower(pStr->zText[j]);
          828  +          j++;
          829  +        }
          830  +        break;
          831  +      }
          832  +      case TK_SELECT: {
          833  +        iStartIN = 0;
          834  +        /* fall through */
          835  +      }
          836  +      default: {
          837  +        if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr);
          838  +        j = pStr->nChar;
          839  +        sqlite3_str_append(pStr, zSql+i, n);
          840  +        while( j<pStr->nChar ){
          841  +          pStr->zText[j] = sqlite3Toupper(pStr->zText[j]);
          842  +          j++;
          843  +        }
          844  +        break;
          845  +      }
          846  +    }
          847  +  }
          848  +  if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1);
          849  +  return sqlite3_str_finish(pStr);
          850  +}
          851  +#endif /* SQLITE_ENABLE_NORMALIZE */

Changes to src/vacuum.c.

    98     98   ** the copy of step (3) were replaced by deleting the original database
    99     99   ** and renaming the transient database as the original.  But that will
   100    100   ** not work if other processes are attached to the original database.
   101    101   ** And a power loss in between deleting the original and renaming the
   102    102   ** transient would cause the database file to appear to be deleted
   103    103   ** following reboot.
   104    104   */
   105         -void sqlite3Vacuum(Parse *pParse, Token *pNm){
          105  +void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){
   106    106     Vdbe *v = sqlite3GetVdbe(pParse);
   107    107     int iDb = 0;
   108         -  if( v==0 ) return;
          108  +  if( v==0 ) goto build_vacuum_end;
   109    109     if( pNm ){
   110    110   #ifndef SQLITE_BUG_COMPATIBLE_20160819
   111    111       /* Default behavior:  Report an error if the argument to VACUUM is
   112    112       ** not recognized */
   113    113       iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm);
   114         -    if( iDb<0 ) return;
          114  +    if( iDb<0 ) goto build_vacuum_end;
   115    115   #else
   116    116       /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
   117    117       ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
   118    118       ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
   119    119       ** The buggy behavior is required for binary compatibility with some
   120    120       ** legacy applications. */
   121    121       iDb = sqlite3FindDb(pParse->db, pNm);
   122    122       if( iDb<0 ) iDb = 0;
   123    123   #endif
   124    124     }
   125    125     if( iDb!=1 ){
   126         -    sqlite3VdbeAddOp1(v, OP_Vacuum, iDb);
          126  +    int iIntoReg = 0;
          127  +    if( pInto ){
          128  +      iIntoReg = ++pParse->nMem;
          129  +      sqlite3ExprCode(pParse, pInto, iIntoReg);
          130  +    }
          131  +    sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg);
   127    132       sqlite3VdbeUsesBtree(v, iDb);
   128    133     }
          134  +build_vacuum_end:
          135  +  sqlite3ExprDelete(pParse->db, pInto);
   129    136     return;
   130    137   }
   131    138   
   132    139   /*
   133    140   ** This routine implements the OP_Vacuum opcode of the VDBE.
   134    141   */
   135         -int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){
          142  +int sqlite3RunVacuum(
          143  +  char **pzErrMsg,        /* Write error message here */
          144  +  sqlite3 *db,            /* Database connection */
          145  +  int iDb,                /* Which attached DB to vacuum */
          146  +  sqlite3_value *pOut     /* Write results here, if not NULL */
          147  +){
   136    148     int rc = SQLITE_OK;     /* Return code from service routines */
   137    149     Btree *pMain;           /* The database being vacuumed */
   138    150     Btree *pTemp;           /* The temporary database we vacuum into */
   139         -  u16 saved_mDbFlags;     /* Saved value of db->mDbFlags */
   140         -  u32 saved_flags;        /* Saved value of db->flags */
          151  +  u32 saved_mDbFlags;     /* Saved value of db->mDbFlags */
          152  +  u64 saved_flags;        /* Saved value of db->flags */
   141    153     int saved_nChange;      /* Saved value of db->nChange */
   142    154     int saved_nTotalChange; /* Saved value of db->nTotalChange */
   143    155     u8 saved_mTrace;        /* Saved trace settings */
   144    156     Db *pDb = 0;            /* Database to detach at end of vacuum */
   145    157     int isMemDb;            /* True if vacuuming a :memory: database */
   146    158     int nRes;               /* Bytes of reserved space at the end of each page */
   147    159     int nDb;                /* Number of attached databases */
   148    160     const char *zDbMain;    /* Schema name of database to vacuum */
          161  +  const char *zOut;       /* Name of output file */
   149    162   
   150    163     if( !db->autoCommit ){
   151    164       sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
   152    165       return SQLITE_ERROR;
   153    166     }
   154    167     if( db->nVdbeActive>1 ){
   155    168       sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress");
   156    169       return SQLITE_ERROR;
   157    170     }
          171  +  if( pOut ){
          172  +    if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){
          173  +      sqlite3SetString(pzErrMsg, db, "non-text filename");
          174  +      return SQLITE_ERROR;
          175  +    }
          176  +    zOut = (const char*)sqlite3_value_text(pOut);
          177  +  }else{
          178  +    zOut = "";
          179  +  }
   158    180   
   159    181     /* Save the current value of the database flags so that it can be 
   160    182     ** restored before returning. Then set the writable-schema flag, and
   161    183     ** disable CHECK and foreign key constraints.  */
   162    184     saved_flags = db->flags;
   163    185     saved_mDbFlags = db->mDbFlags;
   164    186     saved_nChange = db->nChange;
   165    187     saved_nTotalChange = db->nTotalChange;
   166    188     saved_mTrace = db->mTrace;
   167    189     db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
   168    190     db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
   169         -  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder
          191  +  db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder
   170    192                      | SQLITE_Defensive | SQLITE_CountRows);
   171    193     db->mTrace = 0;
   172    194   
   173    195     zDbMain = db->aDb[iDb].zDbSName;
   174    196     pMain = db->aDb[iDb].pBt;
   175    197     isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));
   176    198   
................................................................................
   185    207     ** that actually made the VACUUM run slower.  Very little journalling
   186    208     ** actually occurs when doing a vacuum since the vacuum_db is initially
   187    209     ** empty.  Only the journal header is written.  Apparently it takes more
   188    210     ** time to parse and run the PRAGMA to turn journalling off than it does
   189    211     ** to write the journal header file.
   190    212     */
   191    213     nDb = db->nDb;
   192         -  rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db");
          214  +  rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut);
   193    215     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   194    216     assert( (db->nDb-1)==nDb );
   195    217     pDb = &db->aDb[nDb];
   196    218     assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
   197    219     pTemp = pDb->pBt;
   198         -
   199         -  /* The call to execSql() to attach the temp database has left the file
   200         -  ** locked (as there was more than one active statement when the transaction
   201         -  ** to read the schema was concluded. Unlock it here so that this doesn't
   202         -  ** cause problems for the call to BtreeSetPageSize() below.  */
   203         -  sqlite3BtreeCommit(pTemp);
   204         -
          220  +  if( pOut ){
          221  +    sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp));
          222  +    i64 sz = 0;
          223  +    if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){
          224  +      rc = SQLITE_ERROR;
          225  +      sqlite3SetString(pzErrMsg, db, "output file already exists");
          226  +      goto end_of_vacuum;
          227  +    }
          228  +  }
   205    229     nRes = sqlite3BtreeGetOptimalReserve(pMain);
   206    230   
   207    231     /* A VACUUM cannot change the pagesize of an encrypted database. */
   208    232   #ifdef SQLITE_HAS_CODEC
   209    233     if( db->nextPagesize ){
   210    234       extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*);
   211    235       int nKey;
................................................................................
   221    245   
   222    246     /* Begin a transaction and take an exclusive lock on the main database
   223    247     ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
   224    248     ** to ensure that we do not try to change the page-size on a WAL database.
   225    249     */
   226    250     rc = execSql(db, pzErrMsg, "BEGIN");
   227    251     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   228         -  rc = sqlite3BtreeBeginTrans(pMain, 2, 0);
          252  +  rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0);
   229    253     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   230    254   
   231    255     /* Do not attempt to change the page size for a WAL database */
   232    256     if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain))
   233    257                                                  ==PAGER_JOURNALMODE_WAL ){
   234    258       db->nextPagesize = 0;
   235    259     }
................................................................................
   316    340          BTREE_DEFAULT_CACHE_SIZE, 0,  /* Preserve the default page cache size */
   317    341          BTREE_TEXT_ENCODING,      0,  /* Preserve the text encoding */
   318    342          BTREE_USER_VERSION,       0,  /* Preserve the user version */
   319    343          BTREE_APPLICATION_ID,     0,  /* Preserve the application id */
   320    344       };
   321    345   
   322    346       assert( 1==sqlite3BtreeIsInTrans(pTemp) );
   323         -    assert( 1==sqlite3BtreeIsInTrans(pMain) );
          347  +    assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) );
   324    348   
   325    349       /* Copy Btree meta values */
   326    350       for(i=0; i<ArraySize(aCopy); i+=2){
   327    351         /* GetMeta() and UpdateMeta() cannot fail in this context because
   328    352         ** we already have page 1 loaded into cache and marked dirty. */
   329    353         sqlite3BtreeGetMeta(pMain, aCopy[i], &meta);
   330    354         rc = sqlite3BtreeUpdateMeta(pTemp, aCopy[i], meta+aCopy[i+1]);
   331    355         if( NEVER(rc!=SQLITE_OK) ) goto end_of_vacuum;
   332    356       }
   333    357   
   334         -    rc = sqlite3BtreeCopyFile(pMain, pTemp);
          358  +    if( pOut==0 ){
          359  +      rc = sqlite3BtreeCopyFile(pMain, pTemp);
          360  +    }
   335    361       if( rc!=SQLITE_OK ) goto end_of_vacuum;
   336    362       rc = sqlite3BtreeCommit(pTemp);
   337    363       if( rc!=SQLITE_OK ) goto end_of_vacuum;
   338    364   #ifndef SQLITE_OMIT_AUTOVACUUM
   339         -    sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
          365  +    if( pOut==0 ){
          366  +      sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
          367  +    }
   340    368   #endif
   341    369     }
   342    370   
   343    371     assert( rc==SQLITE_OK );
   344         -  rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
          372  +  if( pOut==0 ){
          373  +    rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
          374  +  }
   345    375   
   346    376   end_of_vacuum:
   347    377     /* Restore the original value of db->flags */
   348    378     db->init.iDb = 0;
   349    379     db->mDbFlags = saved_mDbFlags;
   350    380     db->flags = saved_flags;
   351    381     db->nChange = saved_nChange;

Changes to src/vdbe.c.

  6723   6723     sqlite3VdbeChangeEncoding(pOut, encoding);
  6724   6724     if( rc ) goto abort_due_to_error;
  6725   6725     break;
  6726   6726   };
  6727   6727   #endif /* SQLITE_OMIT_PRAGMA */
  6728   6728   
  6729   6729   #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
  6730         -/* Opcode: Vacuum P1 * * * *
         6730  +/* Opcode: Vacuum P1 P2 * * *
  6731   6731   **
  6732   6732   ** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
  6733   6733   ** for an attached database.  The "temp" database may not be vacuumed.
         6734  +**
         6735  +** If P2 is not zero, then it is a register holding a string which is
         6736  +** the file into which the result of vacuum should be written.  When
         6737  +** P2 is zero, the vacuum overwrites the original database.
  6734   6738   */
  6735   6739   case OP_Vacuum: {
  6736   6740     assert( p->readOnly==0 );
  6737         -  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1);
         6741  +  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1,
         6742  +                        pOp->p2 ? &aMem[pOp->p2] : 0);
  6738   6743     if( rc ) goto abort_due_to_error;
  6739   6744     break;
  6740   6745   }
  6741   6746   #endif
  6742   6747   
  6743   6748   #if !defined(SQLITE_OMIT_AUTOVACUUM)
  6744   6749   /* Opcode: IncrVacuum P1 P2 * * *
................................................................................
  7133   7138     assert( pName->flags & MEM_Str );
  7134   7139     testcase( pName->enc==SQLITE_UTF8 );
  7135   7140     testcase( pName->enc==SQLITE_UTF16BE );
  7136   7141     testcase( pName->enc==SQLITE_UTF16LE );
  7137   7142     rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
  7138   7143     if( rc ) goto abort_due_to_error;
  7139   7144     rc = pVtab->pModule->xRename(pVtab, pName->z);
  7140         -  if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter;
         7145  +  if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter;
  7141   7146     sqlite3VtabImportErrmsg(p, pVtab);
  7142   7147     p->expired = 0;
  7143   7148     if( rc ) goto abort_due_to_error;
  7144   7149     break;
  7145   7150   }
  7146   7151   #endif
  7147   7152   

Changes to src/vdbe.h.

   247    247   int sqlite3VdbeReset(Vdbe*);
   248    248   void sqlite3VdbeSetNumCols(Vdbe*,int);
   249    249   int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*));
   250    250   void sqlite3VdbeCountChanges(Vdbe*);
   251    251   sqlite3 *sqlite3VdbeDb(Vdbe*);
   252    252   u8 sqlite3VdbePrepareFlags(Vdbe*);
   253    253   void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8);
          254  +#ifdef SQLITE_ENABLE_NORMALIZE
          255  +void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*);
          256  +int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*);
          257  +#endif
   254    258   void sqlite3VdbeSwap(Vdbe*,Vdbe*);
   255    259   VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*);
   256    260   sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8);
   257    261   void sqlite3VdbeSetVarmask(Vdbe*, int);
   258    262   #ifndef SQLITE_OMIT_TRACE
   259    263     char *sqlite3VdbeExpandSql(Vdbe*, const char*);
   260    264   #endif

Changes to src/vdbeInt.h.

   331    331   };
   332    332   
   333    333   /* A bitfield type for use inside of structures.  Always follow with :N where
   334    334   ** N is the number of bits.
   335    335   */
   336    336   typedef unsigned bft;  /* Bit Field Type */
   337    337   
          338  +/* The ScanStatus object holds a single value for the
          339  +** sqlite3_stmt_scanstatus() interface.
          340  +*/
   338    341   typedef struct ScanStatus ScanStatus;
   339    342   struct ScanStatus {
   340    343     int addrExplain;                /* OP_Explain for loop */
   341    344     int addrLoop;                   /* Address of "loops" counter */
   342    345     int addrVisit;                  /* Address of "rows visited" counter */
   343    346     int iSelectID;                  /* The "Select-ID" for this loop */
   344    347     LogEst nEst;                    /* Estimated output rows per loop */
   345    348     char *zName;                    /* Name of table or index */
   346    349   };
          350  +
          351  +/* The DblquoteStr object holds the text of a double-quoted
          352  +** string for a prepared statement.  A linked list of these objects
          353  +** is constructed during statement parsing and is held on Vdbe.pDblStr.
          354  +** When computing a normalized SQL statement for an SQL statement, that
          355  +** list is consulted for each double-quoted identifier to see if the
          356  +** identifier should really be a string literal.
          357  +*/
          358  +typedef struct DblquoteStr DblquoteStr;
          359  +struct DblquoteStr {
          360  +  DblquoteStr *pNextStr;   /* Next string literal in the list */
          361  +  char z[8];               /* Dequoted value for the string */
          362  +};
   347    363   
   348    364   /*
   349    365   ** An instance of the virtual machine.  This structure contains the complete
   350    366   ** state of the virtual machine.
   351    367   **
   352    368   ** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare()
   353    369   ** is really a pointer to an instance of this structure.
................................................................................
   360    376     u32 magic;              /* Magic number for sanity checking */
   361    377     int nMem;               /* Number of memory locations currently allocated */
   362    378     int nCursor;            /* Number of slots in apCsr[] */
   363    379     u32 cacheCtr;           /* VdbeCursor row cache generation counter */
   364    380     int pc;                 /* The program counter */
   365    381     int rc;                 /* Value to return */
   366    382     int nChange;            /* Number of db changes made since last reset */
   367         -  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
          383  +  int iStatement;         /* Statement number (or 0 if has no opened stmt) */
   368    384     i64 iCurrentTime;       /* Value of julianday('now') for this statement */
   369    385     i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
   370    386     i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
   371    387     i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
   372    388   
   373    389     /* When allocating a new Vdbe object, all of the fields below should be
   374    390     ** initialized to zero or NULL */
................................................................................
   404    420     bft bIsReader:1;        /* True for statements that read */
   405    421     yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   406    422     yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   407    423     u32 aCounter[7];        /* Counters used by sqlite3_stmt_status() */
   408    424     char *zSql;             /* Text of the SQL statement that generated this */
   409    425   #ifdef SQLITE_ENABLE_NORMALIZE
   410    426     char *zNormSql;         /* Normalization of the associated SQL statement */
          427  +  DblquoteStr *pDblStr;   /* List of double-quoted string literals */
   411    428   #endif
   412    429     void *pFree;            /* Free this when deleting the vdbe */
   413    430     VdbeFrame *pFrame;      /* Parent frame */
   414    431     VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
   415    432     int nFrame;             /* Number of frames in pFrame list */
   416    433     u32 expmask;            /* Binding to these vars invalidates VM */
   417    434     SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */

Changes to src/vdbeapi.c.

    58     58   ** Invoke the profile callback.  This routine is only called if we already
    59     59   ** know that the profile callback is defined and needs to be invoked.
    60     60   */
    61     61   static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
    62     62     sqlite3_int64 iNow;
    63     63     sqlite3_int64 iElapse;
    64     64     assert( p->startTime>0 );
    65         -  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
           65  +  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
    66     66     assert( db->init.busy==0 );
    67     67     assert( p->zSql!=0 );
    68     68     sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
    69     69     iElapse = (iNow - p->startTime)*1000000;
           70  +#ifndef SQLITE_OMIT_DEPRECATED  	
    70     71     if( db->xProfile ){
    71     72       db->xProfile(db->pProfileArg, p->zSql, iElapse);
    72     73     }
           74  +#endif
    73     75     if( db->mTrace & SQLITE_TRACE_PROFILE ){
    74     76       db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
    75     77     }
    76     78     p->startTime = 0;
    77     79   }
    78     80   /*
    79     81   ** The checkProfileCallback(DB,P) macro checks to see if a profile callback
................................................................................
   579    581     /* Check that malloc() has not failed. If it has, return early. */
   580    582     db = p->db;
   581    583     if( db->mallocFailed ){
   582    584       p->rc = SQLITE_NOMEM;
   583    585       return SQLITE_NOMEM_BKPT;
   584    586     }
   585    587   
   586         -  if( p->pc<=0 && p->expired ){
          588  +  if( p->pc<0 && p->expired ){
   587    589       p->rc = SQLITE_SCHEMA;
   588    590       rc = SQLITE_ERROR;
   589    591       goto end_of_step;
   590    592     }
   591    593     if( p->pc<0 ){
   592    594       /* If there are no other statements currently running, then
   593    595       ** reset the interrupt flag.  This prevents a call to sqlite3_interrupt
................................................................................
   598    600       }
   599    601   
   600    602       assert( db->nVdbeWrite>0 || db->autoCommit==0 
   601    603           || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
   602    604       );
   603    605   
   604    606   #ifndef SQLITE_OMIT_TRACE
   605         -    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
          607  +    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
   606    608           && !db->init.busy && p->zSql ){
   607    609         sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
   608    610       }else{
   609    611         assert( p->startTime==0 );
   610    612       }
   611    613   #endif
   612    614   
................................................................................
   625    627   #endif /* SQLITE_OMIT_EXPLAIN */
   626    628     {
   627    629       db->nVdbeExec++;
   628    630       rc = sqlite3VdbeExec(p);
   629    631       db->nVdbeExec--;
   630    632     }
   631    633   
          634  +  if( rc!=SQLITE_ROW ){
   632    635   #ifndef SQLITE_OMIT_TRACE
   633         -  /* If the statement completed successfully, invoke the profile callback */
   634         -  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
          636  +    /* If the statement completed successfully, invoke the profile callback */
          637  +    checkProfileCallback(db, p);
   635    638   #endif
   636    639   
   637         -  if( rc==SQLITE_DONE && db->autoCommit ){
   638         -    assert( p->rc==SQLITE_OK );
   639         -    p->rc = doWalCallbacks(db);
   640         -    if( p->rc!=SQLITE_OK ){
   641         -      rc = SQLITE_ERROR;
          640  +    if( rc==SQLITE_DONE && db->autoCommit ){
          641  +      assert( p->rc==SQLITE_OK );
          642  +      p->rc = doWalCallbacks(db);
          643  +      if( p->rc!=SQLITE_OK ){
          644  +        rc = SQLITE_ERROR;
          645  +      }
   642    646       }
   643    647     }
   644    648   
   645    649     db->errCode = rc;
   646    650     if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
   647    651       p->rc = SQLITE_NOMEM_BKPT;
   648    652     }
................................................................................
   654    658     ** contains the value that would be returned if sqlite3_finalize() 
   655    659     ** were called on statement p.
   656    660     */
   657    661     assert( rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR 
   658    662          || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE
   659    663     );
   660    664     assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp );
   661         -  if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 
   662         -   && rc!=SQLITE_ROW 
   663         -   && rc!=SQLITE_DONE 
          665  +  if( rc!=SQLITE_ROW 
          666  +   && rc!=SQLITE_DONE
          667  +   && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0
   664    668     ){
   665    669       /* If this statement was prepared using saved SQL and an 
   666    670       ** error has occurred, then return the error code in p->rc to the
   667    671       ** caller. Set the error code in the database handle to the same value.
   668    672       */ 
   669    673       rc = sqlite3VdbeTransferError(p);
   670    674     }
................................................................................
  1278   1282       sqlite3_mutex_leave(p->db->mutex);
  1279   1283       return SQLITE_RANGE;
  1280   1284     }
  1281   1285     i--;
  1282   1286     pVar = &p->aVar[i];
  1283   1287     sqlite3VdbeMemRelease(pVar);
  1284   1288     pVar->flags = MEM_Null;
  1285         -  sqlite3Error(p->db, SQLITE_OK);
         1289  +  p->db->errCode = SQLITE_OK;
  1286   1290   
  1287   1291     /* If the bit corresponding to this variable in Vdbe.expmask is set, then 
  1288   1292     ** binding a new value to this variable invalidates the current query plan.
  1289   1293     **
  1290   1294     ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host
  1291   1295     ** parameter in the WHERE clause might influence the choice of query plan
  1292   1296     ** for a statement, then the statement will be automatically recompiled,
................................................................................
  1704   1708   
  1705   1709   #ifdef SQLITE_ENABLE_NORMALIZE
  1706   1710   /*
  1707   1711   ** Return the normalized SQL associated with a prepared statement.
  1708   1712   */
  1709   1713   const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
  1710   1714     Vdbe *p = (Vdbe *)pStmt;
  1711         -  return p ? p->zNormSql : 0;
         1715  +  if( p==0 ) return 0;
         1716  +  if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){
         1717  +    sqlite3_mutex_enter(p->db->mutex);
         1718  +    p->zNormSql = sqlite3Normalize(p, p->zSql);
         1719  +    sqlite3_mutex_leave(p->db->mutex);
         1720  +  }
         1721  +  return p->zNormSql;
  1712   1722   }
  1713   1723   #endif /* SQLITE_ENABLE_NORMALIZE */
  1714   1724   
  1715   1725   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
  1716   1726   /*
  1717   1727   ** Allocate and populate an UnpackedRecord structure based on the serialized
  1718   1728   ** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure

Changes to src/vdbeaux.c.

    60     60     if( p==0 ) return;
    61     61     p->prepFlags = prepFlags;
    62     62     if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
    63     63       p->expmask = 0;
    64     64     }
    65     65     assert( p->zSql==0 );
    66     66     p->zSql = sqlite3DbStrNDup(p->db, z, n);
           67  +}
           68  +
           69  +#ifdef SQLITE_ENABLE_NORMALIZE
           70  +/*
           71  +** Add a new element to the Vdbe->pDblStr list.
           72  +*/
           73  +void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){
           74  +  if( p ){
           75  +    int n = sqlite3Strlen30(z);
           76  +    DblquoteStr *pStr = sqlite3DbMallocRawNN(db,
           77  +                            sizeof(*pStr)+n+1-sizeof(pStr->z));
           78  +    if( pStr ){
           79  +      pStr->pNextStr = p->pDblStr;
           80  +      p->pDblStr = pStr;
           81  +      memcpy(pStr->z, z, n+1);
           82  +    }
           83  +  }
           84  +}
           85  +#endif
           86  +
    67     87   #ifdef SQLITE_ENABLE_NORMALIZE
    68         -  assert( p->zNormSql==0 );
    69         -  if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
    70         -    sqlite3Normalize(p, p->zSql, n, prepFlags);
    71         -    assert( p->zNormSql!=0 || p->db->mallocFailed );
           88  +/*
           89  +** zId of length nId is a double-quoted identifier.  Check to see if
           90  +** that identifier is really used as a string literal.
           91  +*/
           92  +int sqlite3VdbeUsesDoubleQuotedString(
           93  +  Vdbe *pVdbe,            /* The prepared statement */
           94  +  const char *zId         /* The double-quoted identifier, already dequoted */
           95  +){
           96  +  DblquoteStr *pStr;
           97  +  assert( zId!=0 );
           98  +  if( pVdbe->pDblStr==0 ) return 0;
           99  +  for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){
          100  +    if( strcmp(zId, pStr->z)==0 ) return 1;
    72    101     }
          102  +  return 0;
          103  +}
    73    104   #endif
    74         -}
    75    105   
    76    106   /*
    77    107   ** Swap all content between two VDBE structures.
    78    108   */
    79    109   void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
    80    110     Vdbe tmp, *pTmp;
    81    111     char *zTmp;
................................................................................
    88    118     pB->pNext = pTmp;
    89    119     pTmp = pA->pPrev;
    90    120     pA->pPrev = pB->pPrev;
    91    121     pB->pPrev = pTmp;
    92    122     zTmp = pA->zSql;
    93    123     pA->zSql = pB->zSql;
    94    124     pB->zSql = zTmp;
    95         -#ifdef SQLITE_ENABLE_NORMALIZE
          125  +#if 0
    96    126     zTmp = pA->zNormSql;
    97    127     pA->zNormSql = pB->zNormSql;
    98    128     pB->zNormSql = zTmp;
    99    129   #endif
   100    130     pB->expmask = pA->expmask;
   101    131     pB->prepFlags = pA->prepFlags;
   102    132     memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));
................................................................................
  2869   2899           }else if( rc!=SQLITE_OK ){
  2870   2900             p->rc = rc;
  2871   2901             sqlite3RollbackAll(db, SQLITE_OK);
  2872   2902             p->nChange = 0;
  2873   2903           }else{
  2874   2904             db->nDeferredCons = 0;
  2875   2905             db->nDeferredImmCons = 0;
  2876         -          db->flags &= ~SQLITE_DeferFKs;
         2906  +          db->flags &= ~(u64)SQLITE_DeferFKs;
  2877   2907             sqlite3CommitInternalChanges(db);
  2878   2908           }
  2879   2909         }else{
  2880   2910           sqlite3RollbackAll(db, SQLITE_OK);
  2881   2911           p->nChange = 0;
  2882   2912         }
  2883   2913         db->nStatement = 0;
................................................................................
  3186   3216       sqlite3DbFree(db, p->pFree);
  3187   3217     }
  3188   3218     vdbeFreeOpArray(db, p->aOp, p->nOp);
  3189   3219     sqlite3DbFree(db, p->aColName);
  3190   3220     sqlite3DbFree(db, p->zSql);
  3191   3221   #ifdef SQLITE_ENABLE_NORMALIZE
  3192   3222     sqlite3DbFree(db, p->zNormSql);
         3223  +  {
         3224  +    DblquoteStr *pThis, *pNext;
         3225  +    for(pThis=p->pDblStr; pThis; pThis=pNext){
         3226  +      pNext = pThis->pNextStr;
         3227  +      sqlite3DbFree(db, pThis);
         3228  +    }
         3229  +  }
  3193   3230   #endif
  3194   3231   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  3195   3232     {
  3196   3233       int i;
  3197   3234       for(i=0; i<p->nScan; i++){
  3198   3235         sqlite3DbFree(db, p->aScan[i].zName);
  3199   3236       }

Changes to src/wal.c.

   358    358   ** it is held, but does not prevent a checkpointer from checkpointing 
   359    359   ** it.
   360    360   **
   361    361   ** There is still a single WRITER and a single CHECKPOINTER lock. The
   362    362   ** recovery procedure still takes the same exclusive lock on the entire
   363    363   ** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks
   364    364   ** above use four of the six read-locking slots used by legacy wal mode.
   365         -** See the header comment for function walLockReader() for details.
   366    365   **
   367    366   ** STARTUP/RECOVERY
   368    367   **
   369    368   ** The read and write version fields of the database header in a wal2
   370    369   ** database are set to 0x03, instead of 0x02 as in legacy wal mode.
   371    370   **
   372    371   ** The wal file format used in wal2 mode is the same as the format used
................................................................................
   469    468   /*
   470    469   ** Values that may be stored in Wal.readLock in wal2 mode.
   471    470   **
   472    471   ** In wal mode, the Wal.readLock member is set to -1 when no read-lock
   473    472   ** is held, or else is the index of the read-mark on which a lock is
   474    473   ** held.
   475    474   **
   476         -** In wal2 mode, Wal.readLock must be set to one of the following values.
   477         -** A value of -1 still indicates that no read-lock is held, but the other
   478         -** values are symbolic. See the implementation of walLockReader() for
   479         -** details of how the symbols map to OS level locks.
          475  +** In wal2 mode, a value of -1 still indicates that no read-lock is held.
          476  +** And a non-zero value still represents the index of the read-mark on
          477  +** which a lock is held. There are two differences:
          478  +**
          479  +**   1. wal2 mode never uses read-mark 0.
          480  +**
          481  +**   2. locks on each read-mark have a different interpretation, as 
          482  +**      indicated by the symbolic names below.
   480    483   */
   481    484   #define WAL_LOCK_NONE        -1
   482    485   #define WAL_LOCK_PART1        1
   483    486   #define WAL_LOCK_PART1_FULL2  2
   484         -#define WAL_LOCK_PART2        3
   485         -#define WAL_LOCK_PART2_FULL1  4
          487  +#define WAL_LOCK_PART2_FULL1  3
          488  +#define WAL_LOCK_PART2        4
   486    489   
   487    490   /* 
   488    491   ** This constant is used in wal2 mode only.
   489    492   **
   490    493   ** In wal2 mode, when committing a transaction, if the current wal file 
   491    494   ** is sufficiently large and there are no conflicting locks held, the
   492    495   ** writer writes the new transaction into the start of the other wal
................................................................................
  1108   1111     if( pWal->exclusiveMode ) return;
  1109   1112     (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n,
  1110   1113                            SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE);
  1111   1114     WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal,
  1112   1115                walLockName(lockIdx), n));
  1113   1116   }
  1114   1117   
  1115         -/*
  1116         -** This function is used to take and release read-locks in wal2 mode.
  1117         -**
  1118         -** Use of WAL_READ_LOCK(x) slots for (1<=x<=4).
  1119         -**
  1120         -** 1) Partial read of *-wal-1   (blocks checkpointer from checkpointing)
  1121         -** 2) Full read of *-wal-2      (blocks writer from writing)
  1122         -** 3) Partial read of *-wal-2   (blocks checkpointer from checkpointing)
  1123         -** 4) Full read of *-wal-1      (blocks writer from writing)
  1124         -*/
  1125         -static int walLockReader(Wal *pWal, int eLock, int bLock){
  1126         -  int i;                          /* Index of first readmark to lock */
  1127         -  int n;                          /* Number of readmarks to lock */
  1128         -
  1129         -  assert( pWal->hdr.iVersion==WAL_VERSION2 );
  1130         -  if( pWal->exclusiveMode ) return SQLITE_OK;
  1131         -
  1132         -  switch( eLock ){
  1133         -    case WAL_LOCK_PART1      : i = 1; n = 1; break; 
  1134         -    case WAL_LOCK_PART1_FULL2: i = 1; n = 2; break; 
  1135         -    case WAL_LOCK_PART2      : i = 3; n = 1; break; 
  1136         -    case WAL_LOCK_PART2_FULL1: i = 3; n = 2; break; 
  1137         -    default: assert( !"cannot happen" );
  1138         -  }
  1139         -
  1140         -  return sqlite3OsShmLock(pWal->pDbFd, WAL_READ_LOCK(i), n,
  1141         -      SQLITE_SHM_SHARED | (bLock ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) 
  1142         -  );
  1143         -}
  1144         -
  1145   1118   /*
  1146   1119   ** Compute a hash on a page number.  The resulting hash value must land
  1147   1120   ** between 0 and (HASHTABLE_NSLOT-1).  The walHashNext() function advances
  1148   1121   ** the hash to the next value in the event of a collision.
  1149   1122   */
  1150   1123   static int walHash(u32 iPage){
  1151   1124     assert( iPage>0 );
................................................................................
  1228   1201   
  1229   1202     if( 0==(iHash & 0x01) ){
  1230   1203       /* A frame in wal file 0 */
  1231   1204       *piRead = (iExternal <= HASHTABLE_NPAGE_ONE) ? iExternal :
  1232   1205         iExternal - (iHash/2) * HASHTABLE_NPAGE;
  1233   1206       return 0;
  1234   1207     }
  1235         -  if( iHash==0 ){
  1236         -    *piRead = iExternal;
  1237         -    return 0;
  1238         -  }else{
  1239         -    *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE;
  1240         -  }
  1241   1208   
  1242         -  return (iHash % 2);
         1209  +  *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE;
         1210  +  return 1;
  1243   1211   }
  1244   1212   
  1245   1213   /*
  1246   1214   ** Return the number of the wal-index page that contains the hash-table
  1247   1215   ** and page-number array that contain entries corresponding to WAL frame
  1248   1216   ** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages 
  1249   1217   ** are numbered starting from 0.
................................................................................
  1561   1529             testcase( szPage>=65536 );
  1562   1530             aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
  1563   1531             aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
  1564   1532           }
  1565   1533         }
  1566   1534     
  1567   1535         sqlite3_free(aFrame);
  1568         -    }else if( pbZero && nSize==0 ){
         1536  +    }else if( pbZero ){
  1569   1537         *pbZero = 1;
  1570   1538       }
  1571   1539     }
  1572   1540   
  1573   1541     pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
  1574   1542     pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
  1575   1543   
................................................................................
  1685   1653     if( rc==SQLITE_OK ){
  1686   1654       volatile WalCkptInfo *pInfo;
  1687   1655   
  1688   1656       if( isOpen(pWal->apWalFd[1]) ){
  1689   1657         /* The case where *-wal2 may follow *-wal */
  1690   1658         if( nCkpt2<=0x0F && nCkpt2==nCkpt1+1 ){
  1691   1659           if( sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[0]))==hdr.aFrameCksum[0]
  1692         -            && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1]
  1693         -          ){
         1660  +         && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1]
         1661  +        ){
  1694   1662             walidxSetFile(&pWal->hdr, 1);
  1695   1663             walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame);
  1696   1664             walidxSetMxFrame(&pWal->hdr, 0, hdr.mxFrame);
  1697   1665           }else{
  1698   1666             pWal->hdr = hdr;
  1699   1667           }
  1700   1668         }else
................................................................................
  1744   1712       /* If more than one frame was recovered from the log file, report an
  1745   1713       ** event via sqlite3_log(). This is to help with identifying performance
  1746   1714       ** problems caused by applications routinely shutting down without
  1747   1715       ** checkpointing the log file.  */
  1748   1716       if( pWal->hdr.nPage ){
  1749   1717         if( isWalMode2(pWal) ){
  1750   1718           sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
  1751         -            "recovered (%d,%d) frames from WAL files %s[2] (%s mode)",
         1719  +            "recovered (%d,%d) frames from WAL files %s[2] (wal2 mode)",
  1752   1720               walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), 
  1753         -            pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal"
         1721  +            pWal->zWalName
  1754   1722           );
  1755   1723         }else{
  1756   1724           sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
  1757   1725               "recovered %d frames from WAL file %s",
  1758   1726               pWal->hdr.mxFrame, pWal->zWalName
  1759   1727           );
  1760   1728         }
................................................................................
  2247   2215     walIndexWriteHdr(pWal);
  2248   2216     pInfo->nBackfill = 0;
  2249   2217     pInfo->nBackfillAttempted = 0;
  2250   2218     pInfo->aReadMark[1] = 0;
  2251   2219     for(i=2; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
  2252   2220     assert( pInfo->aReadMark[0]==0 );
  2253   2221   }
         2222  +
         2223  +/*
         2224  +** This function is used in wal2 mode.
         2225  +**
         2226  +** This function is called when writer pWal is just about to start 
         2227  +** writing out frames. Parameter iApp is the current wal file. The "other" wal
         2228  +** file (wal file !iApp) has been fully checkpointed. This function returns
         2229  +** SQLITE_OK if there are no readers preventing the writer from switching to
         2230  +** the other wal file. Or SQLITE_BUSY if there are.
         2231  +*/
         2232  +static int wal2RestartOk(Wal *pWal, int iApp){
         2233  +  /* The other wal file (wal file !iApp) can be overwritten if there
         2234  +  ** are no readers reading from it - no "full" or "partial" locks.
         2235  +  ** Technically speaking it is not possible for any reader to hold
         2236  +  ** a "part" lock, as this would have prevented the file from being
         2237  +  ** checkpointed. But checking anyway doesn't hurt. The following
         2238  +  ** is equivalent to:
         2239  +  **
         2240  +  **   if( iApp==0 ) eLock = WAL_LOCK_PART1_FULL2;
         2241  +  **   if( iApp==1 ) eLock = WAL_LOCK_PART1;
         2242  +  */
         2243  +  int eLock = 1 + (iApp==0);
         2244  +
         2245  +  assert( WAL_LOCK_PART1==1 );
         2246  +  assert( WAL_LOCK_PART1_FULL2==2 );
         2247  +  assert( WAL_LOCK_PART2_FULL1==3 );
         2248  +  assert( WAL_LOCK_PART2==4 );
         2249  +
         2250  +  assert( iApp!=0 || eLock==WAL_LOCK_PART1_FULL2 );
         2251  +  assert( iApp!=1 || eLock==WAL_LOCK_PART1 );
         2252  +
         2253  +  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 3);
         2254  +}
         2255  +static void wal2RestartFinished(Wal *pWal, int iApp){
         2256  +  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iApp==0)), 3);
         2257  +}
         2258  +
         2259  +/*
         2260  +** This function is used in wal2 mode.
         2261  +**
         2262  +** This function is called when a checkpointer wishes to checkpoint wal
         2263  +** file iCkpt. It takes the required lock and, if successful, returns
         2264  +** SQLITE_OK. Otherwise, an SQLite error code (e.g. SQLITE_BUSY). If this
         2265  +** function returns SQLITE_OK, it is the responsibility of the caller
         2266  +** to invoke wal2CheckpointFinished() to release the lock.
         2267  +*/
         2268  +static int wal2CheckpointOk(Wal *pWal, int iCkpt){
         2269  +  int eLock = 1 + (iCkpt*2);
         2270  +
         2271  +  assert( WAL_LOCK_PART1==1 );
         2272  +  assert( WAL_LOCK_PART1_FULL2==2 );
         2273  +  assert( WAL_LOCK_PART2_FULL1==3 );
         2274  +  assert( WAL_LOCK_PART2==4 );
         2275  +
         2276  +  assert( iCkpt!=0 || eLock==WAL_LOCK_PART1 );
         2277  +  assert( iCkpt!=1 || eLock==WAL_LOCK_PART2_FULL1 );
         2278  +
         2279  +  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 2);
         2280  +}
         2281  +static void wal2CheckpointFinished(Wal *pWal, int iCkpt){
         2282  +  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iCkpt*2)), 2);
         2283  +}
  2254   2284   
  2255   2285   /*
  2256   2286   ** Copy as much content as we can from the WAL back into the database file
  2257   2287   ** in response to an sqlite3_wal_checkpoint() request or the equivalent.
  2258   2288   **
  2259   2289   ** The amount of information copies from WAL to database might be limited
  2260   2290   ** by active readers.  This routine will never overwrite a database page
................................................................................
  2315   2345       sqlite3_file *pWalFd = pWal->apWalFd[iCkpt];
  2316   2346       mxPage = pWal->hdr.nPage;
  2317   2347   
  2318   2348       /* If this is a wal2 system, check for a reader holding a lock 
  2319   2349       ** preventing this checkpoint operation. If one is found, return
  2320   2350       ** early.  */
  2321   2351       if( bWal2 ){
  2322         -      rc = walLockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1);
         2352  +      rc = wal2CheckpointOk(pWal, iCkpt);
  2323   2353         if( rc!=SQLITE_OK ) return rc;
  2324   2354       }
  2325   2355   
  2326   2356       /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
  2327   2357       ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
  2328   2358       assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
  2329   2359   
................................................................................
  2364   2394       /* Allocate the iterator */
  2365   2395       if( bWal2 || pInfo->nBackfill<mxSafeFrame ){
  2366   2396         assert( bWal2==0 || pInfo->nBackfill==0 );
  2367   2397         rc = walIteratorInit(pWal, iCkpt, pInfo->nBackfill, &pIter);
  2368   2398         assert( rc==SQLITE_OK || pIter==0 );
  2369   2399       }
  2370   2400   
  2371         -    if( pIter
  2372         -     && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK
  2373         -    ){
         2401  +    if( pIter && (bWal2 
         2402  +     || (rc = walBusyLock(pWal, xBusy, pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
         2403  +    )){
  2374   2404         u32 nBackfill = pInfo->nBackfill;
  2375   2405   
  2376   2406         assert( bWal2==0 || nBackfill==0 );
  2377   2407         pInfo->nBackfillAttempted = mxSafeFrame;
  2378   2408   
  2379   2409         /* Sync the wal file being checkpointed to disk */
  2380   2410         rc = sqlite3OsSync(pWalFd, CKPT_SYNC_FLAGS(sync_flags));
................................................................................
  2431   2461           }
  2432   2462         }
  2433   2463         if( rc==SQLITE_OK ){
  2434   2464           pInfo->nBackfill = bWal2 ? 1 : mxSafeFrame;
  2435   2465         }
  2436   2466   
  2437   2467         /* Release the reader lock held while backfilling */
  2438         -      if( bWal2 ){
  2439         -        walUnlockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1);
         2468  +      if( bWal2==0 ){
         2469  +        walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
  2440   2470         }
  2441         -      walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
  2442   2471       }
  2443   2472   
  2444   2473       if( rc==SQLITE_BUSY ){
  2445   2474         /* Reset the return code so as not to report a checkpoint failure
  2446   2475         ** just because there are active readers.  */
  2447   2476         rc = SQLITE_OK;
  2448   2477       }
         2478  +    if( bWal2 ) wal2CheckpointFinished(pWal, iCkpt);
  2449   2479     }
  2450   2480   
  2451   2481     /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
  2452   2482     ** entire wal file has been copied into the database file, then block 
  2453   2483     ** until all readers have finished using the wal file. This ensures that 
  2454   2484     ** the next process to write to the database restarts the wal file.
  2455   2485     */
................................................................................
  3076   3106       }
  3077   3107     }
  3078   3108   
  3079   3109     assert( pWal->nWiData>0 );
  3080   3110     assert( pWal->apWiData[0]!=0 );
  3081   3111     pInfo = walCkptInfo(pWal);
  3082   3112     if( isWalMode2(pWal) ){
  3083         -    int eLock = 1 + (walidxGetFile(&pWal->hdr)*2);
  3084         -    if( pInfo->nBackfill==0 ){
  3085         -      eLock += walidxGetMxFrame(&pWal->hdr, !walidxGetFile(&pWal->hdr))>0;
  3086         -    }
  3087         -    rc = walLockReader(pWal, eLock, 1);
         3113  +    /* This connection needs a "part" lock on the current wal file and, 
         3114  +    ** unless pInfo->nBackfill is set to indicate that it has already been
         3115  +    ** checkpointed, a "full" lock on the other wal file.  */
         3116  +    int iWal = walidxGetFile(&pWal->hdr);
         3117  +    int nBackfill = pInfo->nBackfill || walidxGetMxFrame(&pWal->hdr, !iWal)==0;
         3118  +    int eLock = 1 + (iWal*2) + (nBackfill==iWal);
         3119  +
         3120  +    assert( nBackfill==0 || nBackfill==1 );
         3121  +    assert( iWal==0 || iWal==1 );
         3122  +    assert( iWal!=0 || nBackfill!=1 || eLock==WAL_LOCK_PART1 );
         3123  +    assert( iWal!=0 || nBackfill!=0 || eLock==WAL_LOCK_PART1_FULL2 );
         3124  +    assert( iWal!=1 || nBackfill!=1 || eLock==WAL_LOCK_PART2 );
         3125  +    assert( iWal!=1 || nBackfill!=0 || eLock==WAL_LOCK_PART2_FULL1 );
         3126  +
         3127  +    rc = walLockShared(pWal, WAL_READ_LOCK(eLock));
  3088   3128       if( rc!=SQLITE_OK ){
  3089         -      return rc;
         3129  +      return (rc==SQLITE_BUSY ? WAL_RETRY : rc);
  3090   3130       }
  3091         -
  3092   3131       walShmBarrier(pWal);
  3093   3132       if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
  3094         -      walLockReader(pWal, eLock, 0);
         3133  +      walUnlockShared(pWal, WAL_READ_LOCK(eLock));
  3095   3134         return WAL_RETRY;
  3096   3135       }else{
  3097   3136         pWal->readLock = eLock;
  3098   3137       }
  3099   3138       assert( pWal->minFrame==0 && walFramePage(pWal->minFrame)==0 );
  3100   3139     }else{
  3101   3140       u32 mxReadMark;               /* Largest aReadMark[] value */
................................................................................
  3420   3459   /*
  3421   3460   ** Finish with a read transaction.  All this does is release the
  3422   3461   ** read-lock.
  3423   3462   */
  3424   3463   void sqlite3WalEndReadTransaction(Wal *pWal){
  3425   3464     sqlite3WalEndWriteTransaction(pWal);
  3426   3465     if( pWal->readLock!=WAL_LOCK_NONE ){
  3427         -    if( isWalMode2(pWal) ){
  3428         -      (void)walLockReader(pWal, pWal->readLock, 0);
  3429         -    }else{
  3430         -      walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  3431         -    }
         3466  +    walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  3432   3467       pWal->readLock = WAL_LOCK_NONE;
  3433   3468     }
  3434   3469   }
  3435   3470   
  3436   3471   /* Search hash table iHash for an entry matching page number
  3437   3472   ** pgno. Each call to this function searches a single hash table
  3438   3473   ** (each hash table indexes up to HASHTABLE_NPAGE frames).
................................................................................
  4059   4094       pWal->hdr.aFrameCksum[1] = aWalData[2];
  4060   4095       walCleanupHash(pWal);
  4061   4096     }
  4062   4097   
  4063   4098     return rc;
  4064   4099   }
  4065   4100   
  4066         -/*
  4067         -** This function is used in wal2 mode.
  4068         -**
  4069         -** This function is called when writer pWal is just about to start 
  4070         -** writing out frames. The "other" wal file (wal file !pWal->hdr.iAppend)
  4071         -** has been fully checkpointed. This function returns SQLITE_OK if there
  4072         -** are no readers preventing the writer from switching to the other wal
  4073         -** file. Or SQLITE_BUSY if there are.
  4074         -*/
  4075         -static int walRestartOk(Wal *pWal){
  4076         -  int rc;                                        /* Return code */
  4077         -  int iApp = walidxGetFile(&pWal->hdr);          /* Current WAL file */
  4078         -
  4079         -  /* No reader can be doing a "partial" read of wal file !iApp - in that
  4080         -  ** case it would not have been possible to checkpoint the file. So
  4081         -  ** it is only necessary to test for "full" readers. See the comment
  4082         -  ** above walLockReader() function for exactly what this means in terms
  4083         -  ** of locks.  */
  4084         -  int i = (iApp==0) ? 2 : 4;
  4085         -
  4086         -  rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
  4087         -  if( rc==SQLITE_OK ){
  4088         -    walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
  4089         -  }
  4090         -  return rc;
  4091         -}
  4092         -
  4093   4101   /*
  4094   4102   ** This function is called just before writing a set of frames to the log
  4095   4103   ** file (see sqlite3WalFrames()). It checks to see if, instead of appending
  4096   4104   ** to the current log file, it is possible and desirable to switch to the
  4097   4105   ** other log file and write the new transaction to the start of it.
  4098   4106   ** If so, the wal-index header is updated accordingly - both in heap memory
  4099   4107   ** and in the *-shm file.
................................................................................
  4113   4121           / (pWal->szPage+WAL_FRAME_HDRSIZE);
  4114   4122         nWalSize = MAX(nWalSize, 1);
  4115   4123       }
  4116   4124   
  4117   4125       if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){
  4118   4126         volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
  4119   4127         if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){
  4120         -        rc = walRestartOk(pWal);
         4128  +        rc = wal2RestartOk(pWal, iApp);
  4121   4129           if( rc==SQLITE_OK ){
  4122         -          iApp = !iApp;
         4130  +          int iNew = !iApp;
  4123   4131             pWal->nCkpt++;
  4124         -          walidxSetFile(&pWal->hdr, iApp);
  4125         -          walidxSetMxFrame(&pWal->hdr, iApp, 0);
         4132  +          walidxSetFile(&pWal->hdr, iNew);
         4133  +          walidxSetMxFrame(&pWal->hdr, iNew, 0);
  4126   4134             sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]);
  4127   4135             sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]);
  4128   4136             walIndexWriteHdr(pWal);
  4129   4137             pInfo->nBackfill = 0;
  4130         -          walLockReader(pWal, pWal->readLock, 0);
  4131         -          pWal->readLock = iApp ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2;
  4132         -          rc = walLockReader(pWal, pWal->readLock, 1);
         4138  +          wal2RestartFinished(pWal, iApp);
         4139  +          walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
         4140  +          pWal->readLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2;
         4141  +          rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  4133   4142           }else if( rc==SQLITE_BUSY ){
  4134   4143             rc = SQLITE_OK;
  4135   4144           }
  4136   4145         }
  4137   4146       }
  4138   4147     }else if( pWal->readLock==0 ){
  4139   4148       volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
................................................................................
  4754   4763     */
  4755   4764     assert( pWal->readLock!=WAL_LOCK_NONE || pWal->lockError );
  4756   4765     assert( pWal->readLock!=WAL_LOCK_NONE || (op<=0 && pWal->exclusiveMode==0) );
  4757   4766   
  4758   4767     if( op==0 ){
  4759   4768       if( pWal->exclusiveMode ){
  4760   4769         pWal->exclusiveMode = WAL_NORMAL_MODE;
  4761         -      if( isWalMode2(pWal) ){
  4762         -        rc = walLockReader(pWal, pWal->readLock, 1);
  4763         -      }else{
  4764         -        rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  4765         -      }
         4770  +      rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  4766   4771         if( rc!=SQLITE_OK ){
  4767   4772           pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
  4768   4773         }
  4769   4774         rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
  4770   4775       }else{
  4771   4776         /* Already in locking_mode=NORMAL */
  4772   4777         rc = 0;
  4773   4778       }
  4774   4779     }else if( op>0 ){
  4775   4780       assert( pWal->exclusiveMode==WAL_NORMAL_MODE );
  4776   4781       assert( pWal->readLock>=0 );
  4777         -    if( isWalMode2(pWal) ){
  4778         -      walLockReader(pWal, pWal->readLock, 0);
  4779         -    }else{
  4780         -      walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  4781         -    }
         4782  +    walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
  4782   4783       pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
  4783   4784       rc = 1;
  4784   4785     }else{
  4785   4786       rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
  4786   4787     }
  4787   4788     return rc;
  4788   4789   }

Changes to src/window.c.

  2129   2129   Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){
  2130   2130     Window *pNew = 0;
  2131   2131     if( ALWAYS(p) ){
  2132   2132       pNew = sqlite3DbMallocZero(db, sizeof(Window));
  2133   2133       if( pNew ){
  2134   2134         pNew->zName = sqlite3DbStrDup(db, p->zName);
  2135   2135         pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0);
         2136  +      pNew->pFunc = p->pFunc;
  2136   2137         pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0);
  2137   2138         pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0);
  2138   2139         pNew->eType = p->eType;
  2139   2140         pNew->eEnd = p->eEnd;
  2140   2141         pNew->eStart = p->eStart;
  2141   2142         pNew->pStart = sqlite3ExprDup(db, p->pStart, 0);
  2142   2143         pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0);

Changes to test/altertab2.test.

     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #*************************************************************************
    11     11   #
    12     12   
    13     13   set testdir [file dirname $argv0]
    14     14   source $testdir/tester.tcl
    15         -set testprefix altertab
           15  +set testprefix altertab2
    16     16   
    17     17   # If SQLITE_OMIT_ALTERTABLE is defined, omit this file.
    18     18   ifcapable !altertable {
    19     19     finish_test
    20     20     return
    21     21   }
    22     22   
................................................................................
    81     81     SELECT sql FROM sqlite_master WHERE name LIKE 'c%';
    82     82   } {
    83     83     {CREATE TABLE c1(x REFERENCES "p3")}
    84     84     {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")}
    85     85     {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))}
    86     86   }
    87     87   
           88  +#-------------------------------------------------------------------------
           89  +# Table name in WITH clauses that are part of views or triggers.
           90  +#
           91  +foreach {tn schema} {
           92  +  1 {
           93  +    CREATE TABLE log_entry(col1, y);
           94  +    CREATE INDEX i1 ON log_entry(col1);
           95  +  }
           96  +
           97  +  2 {
           98  +    CREATE TABLE t1(a, b, c);
           99  +    CREATE TABLE t2(x);
          100  +    CREATE TABLE log_entry(col1);
          101  +    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
          102  +      INSERT INTO t2 SELECT col1 FROM log_entry;
          103  +    END;
          104  +  }
          105  +
          106  +  3 {
          107  +    CREATE TABLE t1(a, b, c);
          108  +    CREATE TABLE t2(x);
          109  +    CREATE TABLE log_entry(col1);
          110  +    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
          111  +      INSERT INTO t2
          112  +        WITH xyz(x) AS (SELECT col1 FROM log_entry)
          113  +        SELECT x FROM xyz;
          114  +    END;
          115  +  }
          116  +
          117  +  4 {
          118  +    CREATE TABLE log_entry(col1);
          119  +    CREATE VIEW ttt AS
          120  +        WITH xyz(x) AS (SELECT col1 FROM log_entry)
          121  +        SELECT x FROM xyz;
          122  +  }
          123  +} {
          124  +  reset_db
          125  +  do_execsql_test 3.$tn.1 $schema
          126  +  set expect [db eval "SELECT sql FROM sqlite_master"]
          127  +  set expect [string map {log_entry {"newname"}} $expect]
          128  +
          129  +  do_execsql_test 3.$tn.2 {
          130  +    ALTER TABLE log_entry RENAME TO newname;
          131  +    SELECT sql FROM sqlite_master;
          132  +  } $expect
          133  +
          134  +  reset_db
          135  +  do_execsql_test 3.$tn.3 $schema
          136  +  set expect [db eval "SELECT sql FROM sqlite_master"]
          137  +  set expect [string map {col1 newname} $expect]
          138  +
          139  +  do_execsql_test 3.$tn.4 {
          140  +    ALTER TABLE log_entry RENAME col1 TO newname;
          141  +    SELECT sql FROM sqlite_master;
          142  +  } $expect
          143  +}
    88    144   
    89    145   finish_test

Changes to test/dbfuzz2.c.

    74     74     int rc;
    75     75     int i;
    76     76   
    77     77     if( eVerbosity>=1 ){
    78     78       printf("************** nByte=%d ***************\n", (int)nByte);
    79     79       fflush(stdout);
    80     80     }
           81  +  if( sqlite3_initialize() ) return 0;
    81     82     rc = sqlite3_open(0, &db);
    82     83     if( rc ) return 1;
    83     84     a = sqlite3_malloc64(nByte+1);
    84     85     if( a==0 ) return 1;
    85     86     memcpy(a, aData, nByte);
    86     87     sqlite3_deserialize(db, "main", a, nByte, nByte,
    87     88           SQLITE_DESERIALIZE_RESIZEABLE |

Added test/fts4umlaut.test.

            1  +# 2018 December 3
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix fts4umlaut
           18  +
           19  +ifcapable !fts3 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts4(
           27  +      x, 
           28  +      tokenize=unicode61 "remove_diacritics=2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1 WHERE t1 MATCH $q
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2 WHERE t2 MATCH $q
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to test/index6.test.

   385    385     CREATE TABLE t11(a,b,c);
   386    386     CREATE INDEX t11x ON t11(a) WHERE b<>99;
   387    387     EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
   388    388   } {/USING INDEX t11x/}
   389    389   do_execsql_test index6-11.2 {
   390    390     EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
   391    391   } {/USING INDEX t11x/}
   392         -  
   393    392   
          393  +# 2018-12-08
          394  +# Ticket https://www.sqlite.org/src/info/1d958d90596593a7
          395  +# NOT IN operator fails when using a partial index.
          396  +#
          397  +do_execsql_test index6-12.1 {
          398  +  DROP TABLE IF EXISTS t1;
          399  +  DROP TABLE IF EXISTS t2;
          400  +  CREATE TABLE t1(a,b);
          401  +  INSERT INTO t1 VALUES(1,1);
          402  +  INSERT INTO t1 VALUES(2,2);
          403  +  CREATE TABLE t2(x);
          404  +  INSERT INTO t2 VALUES(1);
          405  +  INSERT INTO t2 VALUES(2);
          406  +  SELECT 'one', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
          407  +  CREATE INDEX t1a ON t1(a) WHERE b=1;
          408  +  SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
          409  +} {}
          410  +do_execsql_test index6-12.2 {
          411  +  SELECT x FROM t2 WHERE x IN (SELECT a FROM t1) ORDER BY +x;
          412  +} {1 2}
   394    413   finish_test

Changes to test/json101.test.

   809    809   do_execsql_test json-14.160 {
   810    810     SELECT fullkey FROM json_tree('"hello"');
   811    811   } {$}
   812    812   do_execsql_test json-14.170 {
   813    813     SELECT fullkey FROM json_tree('null');
   814    814   } {$}
   815    815   
   816         -
          816  +# 2018-12-03
          817  +# Make sure the table-valued functions contained within parentheses
          818  +# work correctly.
          819  +#
          820  +# Bug reported via private email. See TH3 for more information.
          821  +#
          822  +do_execsql_test json-15.100 {
          823  +  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
          824  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          825  +do_execsql_test json-15.110 {
          826  +  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
          827  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          828  +do_execsql_test json-15.120 {
          829  +  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
          830  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          831  +do_execsql_test json-15.130 {
          832  +  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
          833  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
   817    834   
   818    835   finish_test

Changes to test/normalize.test.

    82     82     set STMT [sqlite3_prepare_v3 $DB \
    83     83         "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 0 TAIL]
    84     84   
    85     85     sqlite3_bind_null $STMT 1
    86     86   } {}
    87     87   do_test 202 {
    88     88     sqlite3_normalized_sql $STMT
    89         -} {}
           89  +} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;}
    90     90   do_test 203 {
    91     91     sqlite3_finalize $STMT
    92     92   } {SQLITE_OK}
    93     93   
    94     94   do_test 210 {
    95     95     set STMT [sqlite3_prepare_v3 $DB \
    96     96         "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 2 TAIL]
................................................................................
   203    203     {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
   204    204     0x2
   205    205     {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}
   206    206   
   207    207     430
   208    208     {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
   209    209     0x2
   210         -  {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}}
          210  +  {0 {SELECT a FROM t1 WHERE x IN(?,?,?);}}
   211    211   
   212    212     440
   213    213     {SELECT 'a' FROM t1 WHERE 'x';}
   214    214     0x2
   215    215     {0 {SELECT?FROM t1 WHERE?;}}
   216    216   
   217    217     450
   218    218     {SELECT [a] FROM t1 WHERE [x];}
   219    219     0x2
   220         -  {0 {SELECT"a"FROM t1 WHERE"x";}}
          220  +  {0 {SELECT a FROM t1 WHERE x;}}
   221    221   
   222    222     460
   223    223     {SELECT * FROM t1 WHERE x IN (x);}
   224    224     0x2
   225    225     {0 {SELECT*FROM t1 WHERE x IN(x);}}
   226    226   
   227    227     470
................................................................................
   228    228     {SELECT * FROM t1 WHERE x IN (x,a);}
   229    229     0x2
   230    230     {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
   231    231   
   232    232     480
   233    233     {SELECT * FROM t1 WHERE x IN ([x],"a");}
   234    234     0x2
   235         -  {0 {SELECT*FROM t1 WHERE x IN("x","a");}}
          235  +  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
   236    236   
   237    237     500
   238    238     {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());}
   239    239     0x2
   240         -  {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}}
          240  +  {0 {SELECT*FROM t1 WHERE x IN(x,a,?,sqlite_version());}}
   241    241   
   242    242     520
   243    243     {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
   244    244     0x2
   245    245     {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}
   246    246   
   247    247     540
   248    248     {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));}
   249    249     0x2
   250         -  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
          250  +  {0 {SELECT*FROM t1 WHERE x IN((SELECT x FROM t1));}}
   251    251   
   252    252     550
   253    253     {SELECT a, a+1, a||'b', a+"b" FROM t1;}
   254    254     0x2
   255         -  {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}}
          255  +  {0 {SELECT a,a+?,a||?,a+b FROM t1;}}
   256    256   
   257    257     570
   258    258     {SELECT * FROM t1 WHERE x IN (1);}
   259    259     0x2
   260    260     {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
   261    261   
   262    262     580
................................................................................
   312    312     {SELECT "col f", [col f] FROM t1;}
   313    313     0x2
   314    314     {0 {SELECT"col f","col f"FROM t1;}}
   315    315   
   316    316     680
   317    317     {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];}
   318    318     0x2
   319         -  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}}
          319  +  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON t1."col f"==t2."col y";}}
   320    320   
   321    321     690
   322    322     {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);}
   323    323     0x2
   324    324     {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}}
   325    325   
   326    326     700
................................................................................
   342    342     {SELECT x FROM t1 WHERE x = NULL;}
   343    343     0x2
   344    344     {0 {SELECT x FROM t1 WHERE x=?;}}
   345    345   
   346    346     760
   347    347     {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');}
   348    348     0x2
   349         -  {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}}
          349  +  {0 {SELECT x FROM t1 WHERE x IN(x IS NOT NULL,?,?,?,b,?);}}
   350    350   } {
   351    351     do_test $tnum {
   352    352       set code [catch {
   353    353         set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL]
   354    354         sqlite3_normalized_sql $STMT
   355    355       } res]
   356    356       if {[info exists STMT]} {

Changes to test/permutations.test.

   426    426   # Define the coverage related test suites:
   427    427   #
   428    428   #   coverage-wal
   429    429   #
   430    430   test_suite "coverage-wal" -description {
   431    431     Coverage tests for file wal.c.
   432    432   } -files {
   433         -  wal.test       wal2.test     wal3.test       walmode.test    
   434         -  walbak.test    walhook.test  walcrash2.test  walcksum.test
   435         -  walfault.test  walbig.test   walnoshm.test
   436         -  wal5.test
          433  +wal2big.test wal2recover.test wal2rewrite.test
          434  +wal2simple.test wal2snapshot.test wal2.test
          435  +wal3.test wal4.test wal5.test
          436  +wal64k.test wal6.test wal7.test wal8.test wal9.test
          437  +walbak.test walbig.test walblock.test walcksum.test 
          438  +walfault.test walhook.test walmode.test walnoshm.test
          439  +waloverwrite.test walpersist.test walprotocol2.test
          440  +walprotocol.test walro2.test walrofault.test walro.test
          441  +walshared.test walslow.test wal.test
          442  +wal2savepoint.test wal2lock.test wal2recover2.test
   437    443   } 
   438    444   
   439    445   test_suite "coverage-pager" -description {
   440    446     Coverage tests for file pager.c.
   441    447   } -files {
   442    448     pager1.test    pager2.test  pagerfault.test  pagerfault2.test
   443    449     walfault.test  walbak.test  journal2.test    tkt-9d68c883.test

Changes to test/shell1.test.

   252    252   } {0 {}}
   253    253   do_test shell1-3.1.3 {
   254    254     catchcmd "test.db" ".backup FOO BAR"
   255    255   } {1 {Error: unknown database FOO}}
   256    256   do_test shell1-3.1.4 {
   257    257     # too many arguments
   258    258     catchcmd "test.db" ".backup FOO BAR BAD"
   259         -} {1 {Usage: .backup ?DB? ?--append? FILENAME}}
          259  +} {1 {Usage: .backup ?DB? ?OPTIONS? FILENAME}}
   260    260   
   261    261   # .bail ON|OFF           Stop after hitting an error.  Default OFF
   262    262   do_test shell1-3.2.1 {
   263    263     catchcmd "test.db" ".bail"
   264    264   } {1 {Usage: .bail on|off}}
   265    265   do_test shell1-3.2.2 {
   266    266     catchcmd "test.db" ".bail ON"

Changes to test/shell4.test.

   103    103   }]
   104    104     list [regexp {Memory Used} $res] \
   105    105          [regexp {Heap Usage} $res] \
   106    106          [regexp {Autoindex Inserts} $res]
   107    107   } {1 1 1}
   108    108   
   109    109   do_test shell4-2.1 {
   110         -  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
   111         -} {1 {Usage: .trace FILE|off}}
          110  +  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
          111  +} {1 {Unknown option "--unknown" on ".trace"}}
   112    112   do_test shell4-2.2 {
   113    113     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
   114    114   } {0 {}}
   115    115   do_test shell4-2.3 {
   116         -  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
   117         -} {/^1 {PRAGMA.*Usage:.*}$/}
          116  +  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
          117  +} {/^0 {PRAGMA.*}$/}
   118    118   ifcapable trace {
   119    119   do_test shell4-2.4 {
   120    120     catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
   121    121   } {0 {CREATE TABLE t1(x);
   122    122   SELECT * FROM t1;}}
   123    123   do_test shell4-2.5 {
   124    124     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;"

Changes to test/tabfunc01.test.

   120    120     SELECT * FROM temp.generate_series(1,4)
   121    121   } {1 2 3 4}
   122    122   do_execsql_test tabfunc01-4.3 {
   123    123     ATTACH ':memory:' AS aux1;
   124    124     CREATE TABLE aux1.t1(a,b,c);
   125    125     SELECT * FROM aux1.generate_series(1,4)
   126    126   } {1 2 3 4}
          127  +
          128  +# 2018-12-03: Fix bug reported by by private email.
          129  +do_execsql_test tabfunc01-4.4 {
          130  +  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
          131  +} {1 3 5}
   127    132   
   128    133   # The next series of tests is verifying that virtual table are able
   129    134   # to optimize the IN operator, even on terms that are not marked "omit".
   130    135   # When the generate_series virtual table is compiled for the testfixture,
   131    136   # the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
   132    137   # causes the xBestIndex method of generate_series to leave the
   133    138   # sqlite3_index_constraint_usage.omit flag set to 0, which should cause

Added test/vacuum-into.test.

            1  +# 2018-12-07
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the VACUUM INTO statement.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +
           18  +# If the VACUUM statement is disabled in the current build, skip all
           19  +# the tests in this file.
           20  +#
           21  +ifcapable {!vacuum} {
           22  +  omit_test vacuum.test {Compiled with SQLITE_OMIT_VACUUM}
           23  +  finish_test
           24  +  return
           25  +}
           26  +
           27  +forcedelete out.db
           28  +do_execsql_test vacuum-into-100 {
           29  +  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
           30  +  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
           31  +  INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c;
           32  +  CREATE INDEX t1b ON t1(b);
           33  +  DELETE FROM t1 WHERE a%2;
           34  +  SELECT count(*), sum(a), sum(length(b)) FROM t1;
           35  +} {50 2550 30000}
           36  +do_execsql_test vacuum-into-110 {
           37  +  VACUUM main INTO 'out.db';
           38  +} {}
           39  +sqlite3 db2 out.db
           40  +do_test vacuum-into-120 {
           41  +  db2 eval {SELECT count(*), sum(a), sum(length(b)) FROM t1}
           42  +} {50 2550 30000}
           43  +do_catchsql_test vacuum-into-130 {
           44  +  VACUUM INTO 'out.db';
           45  +} {1 {output file already exists}}
           46  +forcedelete out2.db
           47  +do_catchsql_test vacuum-into-140 {
           48  +  VACUUM INTO 'out2.db';
           49  +} {0 {}}
           50  +do_catchsql_test vacuum-into-150 {
           51  +  VACUUM INTO 'out2.db';
           52  +} {1 {output file already exists}}
           53  +
           54  +do_catchsql_test vacuum-into-200 {
           55  +  VACUUM main INTO ':memory:';
           56  +} {0 {}}
           57  +
           58  +# The INTO argument can be an arbitrary expression.
           59  +#
           60  +do_execsql_test vacuum-into-300 {
           61  +  CREATE TABLE t2(name TEXT);
           62  +  INSERT INTO t2 VALUES(':memory:');
           63  +  VACUUM main INTO (SELECT name FROM t2);
           64  +} {}
           65  +do_catchsql_test vacuum-into-310 {
           66  +  VACUUM INTO null;
           67  +} {1 {non-text filename}}
           68  +
           69  +finish_test

Added test/wal2big.test.

            1  +# 2017 September 19
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the operation of the library in
           13  +# "PRAGMA journal_mode=WAL2" mode.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +source $testdir/lock_common.tcl
           19  +source $testdir/malloc_common.tcl
           20  +source $testdir/wal_common.tcl
           21  +
           22  +set testprefix wal2big
           23  +ifcapable !wal {finish_test ; return }
           24  +
           25  +do_execsql_test 1.0 {
           26  +  CREATE TABLE t1(a, b, c);
           27  +  CREATE INDEX t1a ON t1(a);
           28  +  CREATE INDEX t1b ON t1(b);
           29  +  CREATE INDEX t1c ON t1(c);
           30  +  PRAGMA journal_mode = wal2;
           31  +  PRAGMA journal_size_limit = 10000000;
           32  +
           33  +  WITH s(i) AS (
           34  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
           35  +  )
           36  +  INSERT INTO t1 SELECT random(), random(), random() FROM s;
           37  +} {wal2 10000000}
           38  +
           39  +do_execsql_test 1.1 {
           40  +  WITH s(i) AS (
           41  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
           42  +  )
           43  +  INSERT INTO t1 SELECT random(), random(), random() FROM s;
           44  +}
           45  +
           46  +do_test 1.2 {
           47  +  list [expr [file size test.db-wal]>10000000] \
           48  +       [expr [file size test.db-wal2]>10000000]
           49  +} {1 1}
           50  +
           51  +do_test 1.3 {
           52  +  sqlite3 db2 test.db
           53  +  execsql {
           54  +    SELECT count(*) FROM t1;
           55  +    PRAGMA integrity_check;
           56  +  } db2
           57  +} {400000 ok}
           58  +
           59  +do_test 1.4 {
           60  +  db2 close
           61  +  forcecopy test.db test.db2
           62  +  forcecopy test.db-wal test.db2-wal
           63  +  forcecopy test.db-wal2 test.db2-wal2
           64  +  sqlite3 db2 test.db2
           65  +  execsql {
           66  +    SELECT count(*) FROM t1;
           67  +    PRAGMA integrity_check;
           68  +  }
           69  +} {400000 ok}
           70  +
           71  +finish_test

Added test/wal2lock.test.

            1  +# 2018 December 15
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the operation of the library in
           13  +# "PRAGMA journal_mode=WAL2" mode.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +source $testdir/lock_common.tcl
           19  +source $testdir/malloc_common.tcl
           20  +source $testdir/wal_common.tcl
           21  +
           22  +set testprefix wal2lock
           23  +ifcapable !wal {finish_test ; return }
           24  +
           25  +db close
           26  +testvfs tvfs 
           27  +sqlite3 db test.db -vfs tvfs
           28  +
           29  +do_execsql_test 1.0 {
           30  +  PRAGMA journal_mode = wal2;
           31  +  CREATE TABLE y1(y, yy);
           32  +  CREATE INDEX y1y ON y1(y);
           33  +  CREATE INDEX y1yy ON y1(yy);
           34  +  INSERT INTO y1 VALUES(1, 2), (3, 4), (5, 6);
           35  +} {wal2}
           36  +
           37  +tvfs script vfs_callback
           38  +tvfs filter xShmLock
           39  +
           40  +set ::lock [list]
           41  +proc vfs_callback {func file name lock} {
           42  +  lappend ::lock $lock
           43  +  return SQLITE_OK
           44  +}
           45  +
           46  +do_execsql_test 1.1.1 {
           47  +  SELECT * FROM y1
           48  +} {1 2 3 4 5 6}
           49  +do_test 1.1.2 {
           50  +  set ::lock
           51  +} {{4 1 lock shared} {4 1 unlock shared}}
           52  +
           53  +set ::bFirst 1
           54  +proc vfs_callback {func file name lock} {
           55  +  if {$::bFirst} {
           56  +    set ::bFirst 0
           57  +    return SQLITE_BUSY
           58  +  }
           59  +  return SQLITE_OK
           60  +}
           61  +do_execsql_test 1.2 {
           62  +  SELECT * FROM y1
           63  +} {1 2 3 4 5 6}
           64  +
           65  +set ::bFirst 1
           66  +proc vfs_callback {func file name lock} {
           67  +  if {$::bFirst} {
           68  +    set ::bFirst 0
           69  +    return SQLITE_IOERR
           70  +  }
           71  +  return SQLITE_OK
           72  +}
           73  +do_catchsql_test 1.3 {
           74  +  SELECT * FROM y1
           75  +} {1 {disk I/O error}}
           76  +
           77  +puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
           78  +puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite"
           79  +puts "# is built on unix without HAVE_USLEEP defined, it may be much longer."
           80  +proc vfs_callback {func file name lock} { return SQLITE_BUSY }
           81  +do_catchsql_test 1.4 {
           82  +  SELECT * FROM y1
           83  +} {1 {locking protocol}}
           84  +proc vfs_callback {func file name lock} { return SQLITE_OK }
           85  +
           86  +sqlite3 db2 test.db -vfs tvfs
           87  +set ::bFirst 1
           88  +
           89  +proc vfs_callback {func file name lock} {
           90  +  if {$::bFirst} {
           91  +    set ::bFirst 0
           92  +    db2 eval { INSERT INTO y1 VALUES(7, 8) }
           93  +  }
           94  +}
           95  +
           96  +do_execsql_test 1.5.1 {
           97  +  SELECT * FROM y1
           98  +} {1 2 3 4 5 6 7 8}
           99  +do_execsql_test 1.5.2 {
          100  +  SELECT * FROM y1
          101  +} {1 2 3 4 5 6 7 8}
          102  +
          103  +db close
          104  +db2 close
          105  +tvfs delete
          106  +finish_test

Added test/wal2recover.test.

            1  +# 2018 December 13
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the operation of the library in
           13  +# "PRAGMA journal_mode=WAL2" mode.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +source $testdir/lock_common.tcl
           19  +source $testdir/malloc_common.tcl
           20  +source $testdir/wal_common.tcl
           21  +
           22  +set testprefix wal2recover
           23  +ifcapable !wal {finish_test ; return }
           24  +
           25  +proc db_copy {from to} {
           26  +  forcecopy $from $to
           27  +  forcecopy ${from}-wal ${to}-wal
           28  +  forcecopy ${from}-wal2 ${to}-wal2
           29  +}
           30  +
           31  +do_execsql_test 1.0 {
           32  +  CREATE TABLE t1(a, b, c);
           33  +  CREATE INDEX t1a ON t1(a);
           34  +  CREATE INDEX t1b ON t1(b);
           35  +  CREATE INDEX t1c ON t1(c);
           36  +  PRAGMA journal_mode = wal2;
           37  +  PRAGMA journal_size_limit = 15000;
           38  +  PRAGMA wal_autocheckpoint = 0;
           39  +} {wal2 15000 0}
           40  +
           41  +do_test 1.1 {
           42  +  for {set i 1} {$i <= 1000} {incr i} {
           43  +    execsql { INSERT INTO t1 VALUES(random(), random(), random()) }
           44  +    db_copy test.db test.db2
           45  +    sqlite3 db2 test.db
           46  +    set res [execsql {
           47  +      SELECT count(*) FROM t1;
           48  +      PRAGMA integrity_check;
           49  +    } db2]
           50  +    db2 close
           51  +    if {$res != [list $i ok]} {
           52  +      error "failure on iteration $i"
           53  +    }
           54  +  }
           55  +  set {} {}
           56  +} {}
           57  +
           58  +#--------------------------------------------------------------------------
           59  +reset_db
           60  +do_execsql_test 2.0 {
           61  +  CREATE TABLE t1(x UNIQUE);
           62  +  CREATE TABLE t2(x UNIQUE);
           63  +  PRAGMA journal_mode = wal2;
           64  +  PRAGMA journal_size_limit = 10000;
           65  +  PRAGMA wal_autocheckpoint = 0;
           66  +  BEGIN;
           67  +    INSERT INTO t1 VALUES(randomblob(4000));
           68  +    INSERT INTO t1 VALUES(randomblob(4000));
           69  +    INSERT INTO t1 VALUES(randomblob(4000));
           70  +  COMMIT;
           71  +  BEGIN;
           72  +    INSERT INTO t2 VALUES(randomblob(4000));
           73  +    INSERT INTO t2 VALUES(randomblob(4000));
           74  +    INSERT INTO t2 VALUES(randomblob(4000));
           75  +  COMMIT;
           76  +} {wal2 10000 0}
           77  +do_test 2.0.1 {
           78  +  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
           79  +} {5120 28328 28328}
           80  +
           81  +# Test recovery with both wal files intact.
           82  +#
           83  +do_test 2.1 {
           84  +  db_copy test.db test.db2
           85  +  sqlite3 db2 test.db2
           86  +  execsql {
           87  +    SELECT count(*) FROM t1;
           88  +    SELECT count(*) FROM t2;
           89  +    PRAGMA integrity_check;
           90  +  } db2
           91  +} {3 3 ok}
           92  +
           93  +do_test 2.2 {
           94  +  db2 close
           95  +  db_copy test.db test.db2
           96  +  hexio_write test.db2-wal 16 12345678
           97  +  sqlite3 db2 test.db2
           98  +  execsql {
           99  +    SELECT count(*) FROM t1;
          100  +    SELECT count(*) FROM t2;
          101  +  } db2
          102  +} {0 3}
          103  +
          104  +do_test 2.3 {
          105  +  db2 close
          106  +  db_copy test.db test.db2
          107  +  hexio_write test.db2-wal2 16 12345678
          108  +  sqlite3 db2 test.db2
          109  +  execsql {
          110  +    SELECT count(*) FROM t1;
          111  +    SELECT count(*) FROM t2;
          112  +    PRAGMA integrity_check;
          113  +  } db2
          114  +} {3 0 ok}
          115  +
          116  +do_test 2.4 {
          117  +  db2 close
          118  +  db_copy test.db test.db2
          119  +  forcecopy test.db-wal test.db2-wal2
          120  +  sqlite3 db2 test.db2
          121  +  execsql {
          122  +    SELECT count(*) FROM t1;
          123  +    SELECT count(*) FROM t2;
          124  +    PRAGMA integrity_check;
          125  +  } db2
          126  +} {3 0 ok}
          127  +
          128  +do_test 2.5 {
          129  +  db2 close
          130  +  db_copy test.db test.db2
          131  +  forcecopy test.db-wal  test.db2-wal2
          132  +  forcecopy test.db-wal2  test.db2-wal
          133  +  sqlite3 db2 test.db2
          134  +  execsql {
          135  +    SELECT count(*) FROM t1;
          136  +    SELECT count(*) FROM t2;
          137  +    PRAGMA integrity_check;
          138  +  } db2
          139  +} {3 3 ok}
          140  +
          141  +do_test 2.6 {
          142  +  db2 close
          143  +  db_copy test.db test.db2
          144  +  forcecopy test.db-wal test.db2-wal2
          145  +  close [open test.db-wal w]
          146  +  sqlite3 db2 test.db2
          147  +  execsql {
          148  +    SELECT count(*) FROM t1;
          149  +    SELECT count(*) FROM t2;
          150  +    PRAGMA integrity_check;
          151  +  } db2
          152  +} {3 0 ok}
          153  +
          154  +do_test 2.7 {
          155  +  db2 close
          156  +  db_copy test.db test.db2
          157  +  forcedelete test.db2-wal
          158  +  sqlite3 db2 test.db2
          159  +  execsql {
          160  +    SELECT count(*) FROM t1;
          161  +    SELECT count(*) FROM t2;
          162  +    PRAGMA integrity_check;
          163  +  } db2
          164  +} {0 0 ok}
          165  +
          166  +#-------------------------------------------------------------------------
          167  +#
          168  +reset_db
          169  +do_execsql_test 3.0 {
          170  +  CREATE TABLE t1(a TEXT, b TEXT, c TEXT);
          171  +  CREATE INDEX t1a ON t1(a);
          172  +  CREATE INDEX t1b ON t1(b);
          173  +  CREATE INDEX t1c ON t1(c);
          174  +  PRAGMA journal_mode = wal2;
          175  +  PRAGMA journal_size_limit = 10000;
          176  +  PRAGMA wal_autocheckpoint = 0;
          177  +  PRAGMA cache_size = 5;
          178  +} {wal2 10000 0}
          179  +
          180  +do_execsql_test 3.1 {
          181  +  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
          182  +  INSERT INTO t1 SELECT i, i, i FROM s;
          183  +
          184  +  INSERT INTO t1 VALUES(201, 201, 201);
          185  +} {}
          186  +
          187  +do_test 3.2 {
          188  +  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
          189  +} {5120 15752 4224}
          190  +
          191  +do_test 3.3 {
          192  +  forcecopy test.db test.db2
          193  +  forcecopy test.db-wal test.db2-wal
          194  +  forcecopy test.db-wal2 test.db2-wal2
          195  +  sqlite3 db2 test.db2
          196  +  execsql {
          197  +    PRAGMA journal_size_limit = 10000;
          198  +    PRAGMA wal_autocheckpoint = 0;
          199  +    PRAGMA cache_size = 5;
          200  +    BEGIN;
          201  +      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
          202  +      INSERT INTO t1 SELECT i, i, i FROM s;
          203  +  } db2
          204  +  list [file size test.db2] [file size test.db2-wal] [file size test.db2-wal2]
          205  +} {5120 15752 23088}
          206  +
          207  +do_test 3.4 {
          208  +  set fd [open test.db2-shm]
          209  +  fconfigure $fd -encoding binary -translation binary
          210  +  set data [read $fd]
          211  +  close $fd
          212  +
          213  +  set fd [open test.db-shm w]
          214  +  fconfigure $fd -encoding binary -translation binary
          215  +  puts -nonewline $fd $data
          216  +  close $fd
          217  +
          218  +  execsql {
          219  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
          220  +      INSERT INTO t1 SELECT i, i, i FROM s;
          221  +    SELECT count(*) FROM t1;
          222  +    PRAGMA integrity_check;
          223  +  }
          224  +} {211 ok}
          225  +
          226  +do_test 3.5 {
          227  +  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
          228  +} {5120 15752 18896}
          229  +
          230  +
          231  +finish_test
          232  +

Added test/wal2recover2.test.

            1  +# 2018 December 13
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the operation of the library in
           13  +# "PRAGMA journal_mode=WAL2" mode.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +source $testdir/lock_common.tcl
           19  +source $testdir/malloc_common.tcl
           20  +source $testdir/wal_common.tcl
           21  +
           22  +set testprefix wal2recover2
           23  +ifcapable !wal {finish_test ; return }
           24  +
           25  +do_execsql_test 1.0 {
           26  +  CREATE TABLE t1(x);
           27  +  CREATE TABLE t2(x);
           28  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
           29  +    INSERT INTO t1 SELECT i FROM s;
           30  +  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
           31  +    INSERT INTO t2 SELECT i FROM s;
           32  +
           33  +  PRAGMA journal_mode = wal2;
           34  +  PRAGMA journal_size_limit = 10000;
           35  +} {wal2 10000}
           36  +
           37  +set ::L 1125750
           38  +set ::M 1126500
           39  +set ::H 1127250
           40  +
           41  +do_execsql_test 1.1 {
           42  +  UPDATE t1 SET x=x+1;
           43  +  UPDATE t2 SET x=x+1 WHERE rowid<=750;
           44  +
           45  +  SELECT sum(x) FROM t1;
           46  +  SELECT sum(x) FROM t2;
           47  +} [list $H $M]
           48  +
           49  +do_test 1.2 {
           50  +  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
           51  +} {31744 14704 7368}
           52  +
           53  +proc cksum {zIn data} {
           54  +  if {[string length $zIn]==0} {
           55  +    set s0 0
           56  +    set s1 0
           57  +  } else {
           58  +    set s0 [hexio_get_int [string range $zIn 0 7]]
           59  +    set s1 [hexio_get_int [string range $zIn 8 15]]
           60  +  }
           61  +  set n [expr [string length $data] / 8]
           62  +
           63  +  for {set i 0} {$i < $n} {incr i 2} {
           64  +    set x0 [hexio_get_int -l [string range $data [expr $i*8]   [expr $i*8+7]]]
           65  +    set x1 [hexio_get_int -l [string range $data [expr $i*8+8] [expr $i*8+8+7]]]
           66  +
           67  +    set s0 [expr ($s0 + $x0 + $s1) & 0xFFFFFFFF]
           68  +    set s1 [expr ($s1 + $x1 + $s0) & 0xFFFFFFFF]
           69  +  }
           70  +
           71  +  return "[hexio_render_int32 $s0][hexio_render_int32 $s1]"
           72  +}
           73  +
           74  +proc fix_wal_cksums {file} {
           75  +  # Fix the checksum on the wal header.
           76  +  set data [hexio_read $file 0 32]
           77  +  set cksum [cksum {} [string range $data 0 47]]
           78  +  set salt [hexio_read $file 16 8]
           79  +  hexio_write $file 24 $cksum
           80  +
           81  +  # Fix the checksums for all pages in the wal file.
           82  +  set pgsz [hexio_get_int [hexio_read $file 8 4]]
           83  +  set sz [file size $file]
           84  +  for {set off 32} {$off < $sz} {incr off [expr $pgsz+24]} {
           85  +    set e [hexio_read $file $off 8]
           86  +    set cksum [cksum $cksum $e]
           87  +
           88  +    set p [hexio_read $file [expr $off+24] $pgsz]
           89  +    set cksum [cksum $cksum $p]
           90  +
           91  +    hexio_write $file [expr $off+8] $salt
           92  +    hexio_write $file [expr $off+16] $cksum
           93  +  }
           94  +}
           95  +
           96  +proc wal_incr_hdrfield {file field} {
           97  +  switch -- $field {
           98  +    nCkpt { set offset 12 }
           99  +    salt0 { set offset 16 }
          100  +    salt1 { set offset 20 }
          101  +    default {
          102  +      error "unknown field $field - should be \"nCkpt\", \"salt0\" or \"salt1\""
          103  +    }
          104  +  }
          105  +
          106  +  # Increment the value in the wal header.
          107  +  set v [hexio_get_int [hexio_read $file $offset 4]]
          108  +  incr v
          109  +  hexio_write $file $offset [hexio_render_int32 $v]
          110  +  
          111  +  # Fix various checksums
          112  +  fix_wal_cksums $file
          113  +}
          114  +
          115  +proc wal_set_nckpt {file val} {
          116  +  # Increment the value in the wal header.
          117  +  hexio_write $file 12 [hexio_render_int32 $val]
          118  +  
          119  +  # Fix various checksums
          120  +  fix_wal_cksums $file
          121  +}
          122  +
          123  +proc wal_set_follow {file prevfile} {
          124  +  set pgsz [hexio_get_int [hexio_read $prevfile 8 4]]
          125  +  set sz [file size $prevfile]
          126  +  set cksum [hexio_read $prevfile [expr $sz-$pgsz-8] 8]
          127  +
          128  +  hexio_write $file 16 $cksum
          129  +  fix_wal_cksums $file
          130  +}
          131  +
          132  +foreach {tn file field} {
          133  +  1 test.db2-wal    salt0
          134  +  2 test.db2-wal    salt1
          135  +  3 test.db2-wal    nCkpt
          136  +  4 test.db2-wal2   salt0
          137  +  5 test.db2-wal2   salt1
          138  +  6 test.db2-wal2   nCkpt
          139  +} {
          140  +  do_test 1.3.$tn {
          141  +    forcecopy test.db test.db2
          142  +    forcecopy test.db-wal test.db2-wal
          143  +    forcecopy test.db-wal2 test.db2-wal2
          144  +    wal_incr_hdrfield $file $field
          145  +    sqlite3 db2 test.db2
          146  +    execsql {
          147  +      SELECT sum(x) FROM t1;
          148  +      SELECT sum(x) FROM t2;
          149  +    } db2
          150  +  } [list $H $L]
          151  +  db2 close
          152  +}
          153  +
          154  +do_test 1.4 {
          155  +  forcecopy test.db test.db2
          156  +  forcecopy test.db-wal2 test.db2-wal
          157  +  forcedelete test.db2-wal2
          158  +  sqlite3 db2 test.db2
          159  +  execsql {
          160  +    SELECT sum(x) FROM t1;
          161  +    SELECT sum(x) FROM t2;
          162  +  } db2
          163  +} [list $L $M]
          164  +
          165  +do_test 1.5 {
          166  +  forcecopy test.db test.db2
          167  +  forcecopy test.db-wal2 test.db2-wal
          168  +  forcecopy test.db-wal test.db2-wal2
          169  +  sqlite3 db2 test.db2
          170  +  execsql {
          171  +    SELECT sum(x) FROM t1;
          172  +    SELECT sum(x) FROM t2;
          173  +  } db2
          174  +} [list $H $M]
          175  +
          176  +foreach {tn file field} {
          177  +  1 test.db2-wal    salt0
          178  +  2 test.db2-wal    salt1
          179  +  3 test.db2-wal2   salt0
          180  +  4 test.db2-wal2   salt1
          181  +} {
          182  +  do_test 1.6.$tn {
          183  +    forcecopy test.db test.db2
          184  +    forcecopy test.db-wal2 test.db2-wal
          185  +    forcecopy test.db-wal test.db2-wal2
          186  +    wal_incr_hdrfield $file $field
          187  +    breakpoint
          188  +    sqlite3 db2 test.db2
          189  +    execsql {
          190  +      SELECT sum(x) FROM t1;
          191  +      SELECT sum(x) FROM t2;
          192  +    } db2
          193  +  } [list $H $L]
          194  +  db2 close
          195  +}
          196  +
          197  +foreach {tn nCkpt1 nCkpt2 res} [list \
          198  +  1   2 1   "$H $M"                  \
          199  +  2   2 2   "$L $M"                  \
          200  +  3   3 1   "$H $L"                  \
          201  +  4   15 14 "$H $M"                  \
          202  +  5   0 15  "$H $M"                  \
          203  +  6   1 15  "$L $M"                  \
          204  +] {
          205  +  do_test 1.7.$tn {
          206  +    forcecopy test.db test.db2
          207  +    forcecopy test.db-wal2 test.db2-wal
          208  +    forcecopy test.db-wal test.db2-wal2
          209  +
          210  +    wal_set_nckpt test.db2-wal2 $nCkpt2
          211  +    wal_set_nckpt test.db2-wal  $nCkpt1
          212  +    wal_set_follow test.db2-wal test.db2-wal2
          213  +
          214  +    if {$tn==1} breakpoint
          215  +
          216  +    sqlite3 db2 test.db2
          217  +    execsql {
          218  +      SELECT sum(x) FROM t1;
          219  +      SELECT sum(x) FROM t2;
          220  +    } db2
          221  +  } $res
          222  +  db2 close
          223  +}
          224  +
          225  +
          226  +finish_test
          227  +

Added test/wal2savepoint.test.

            1  +# 2018 December 13
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the operation of the library in
           13  +# "PRAGMA journal_mode=WAL2" mode.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +source $testdir/lock_common.tcl
           19  +source $testdir/malloc_common.tcl
           20  +source $testdir/wal_common.tcl
           21  +
           22  +set testprefix wal2savepoint
           23  +ifcapable !wal {finish_test ; return }
           24  +
           25  +do_execsql_test 1.0 {
           26  +  CREATE TABLE t1(a, b, c);
           27  +  CREATE INDEX t1a ON t1(a);
           28  +  CREATE INDEX t1b ON t1(b);
           29  +  CREATE INDEX t1c ON t1(c);
           30  +  PRAGMA journal_mode = wal2;
           31  +  PRAGMA journal_size_limit = 15000;
           32  +  PRAGMA wal_autocheckpoint = 0;
           33  +  PRAGMA cache_size = 5;
           34  +} {wal2 15000 0}
           35  +
           36  +do_execsql_test 1.1 {
           37  +  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
           38  +  INSERT INTO t1 SELECT random(), random(), random() FROM s;
           39  +} {}
           40  +
           41  +do_test 1.2 {
           42  +  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
           43  +} {5120 23088 0}
           44  +
           45  +do_execsql_test 1.3 {
           46  +  BEGIN;
           47  +    SAVEPOINT abc;
           48  +      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100)
           49  +      INSERT INTO t1 SELECT random(), random(), random() FROM s;
           50  +    ROLLBACK TO abc;
           51  +    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
           52  +    INSERT INTO t1 SELECT random(), random(), random() FROM s;
           53  +  COMMIT;
           54  +  SELECT count(*) FROM t1;
           55  +  PRAGMA integrity_check;
           56  +} {210 ok}
           57  +
           58  +
           59  +finish_test
           60  +

Changes to test/wal2simple.test.

   110    110   do_execsql_test 3.0 {
   111    111     CREATE TABLE t1(x BLOB, y INTEGER PRIMARY KEY);
   112    112     CREATE INDEX i1 ON t1(x);
   113    113     PRAGMA cache_size = 5;
   114    114     PRAGMA journal_mode = wal2;
   115    115   } {wal2}
   116    116   
   117         -breakpoint
   118    117   do_test 3.1 {
   119    118     execsql BEGIN
   120    119     for {set i 1} {$i < 1000} {incr i} {
   121    120       execsql { INSERT INTO t1 VALUES(randomblob(800), $i) }
   122    121     }
   123    122     execsql COMMIT
   124    123   } {}
................................................................................
   239    238   
   240    239   do_test 6.1 {
   241    240     for {set i 0} {$i < 10} {incr i} {
   242    241       execsql "CREATE TABLE t$i (x);"
   243    242     }
   244    243   } {}
   245    244   
   246         -puts "[file size test.db-wal] [file size test.db-wal2]"
   247         -
   248    245   do_test 6.2.1 {
   249    246     foreach f [glob -nocomplain test.db2*] { forcedelete $f }
   250    247     forcecopy test.db-wal2 test.db2-wal2
   251    248     sqlite3 db2 test.db2
   252    249     db2 eval { SELECT * FROM sqlite_master }
   253    250   } {}
   254    251   do_test 6.2.2 {
................................................................................
   284    281   do_test 6.4.2 {
   285    282     db2 eval {
   286    283       PRAGMA journal_mode = wal2;
   287    284       SELECT * FROM sqlite_master;
   288    285     }
   289    286   } {wal2}
   290    287   db2 close
          288  +
          289  +#-------------------------------------------------------------------------
          290  +reset_db
          291  +sqlite3 db2 test.db
          292  +do_execsql_test 7.0 {
          293  +  PRAGMA journal_size_limit = 10000;
          294  +  PRAGMA journal_mode = wal2;
          295  +  PRAGMA wal_autocheckpoint = 0;
          296  +  BEGIN;
          297  +    CREATE TABLE t1(a);
          298  +    INSERT INTO t1 VALUES( randomblob(8000) );
          299  +  COMMIT;
          300  +} {10000 wal2 0}
          301  +
          302  +do_test 7.1 {
          303  +  list [file size test.db-wal] [file size test.db-wal2]
          304  +} {9464 0}
          305  +
          306  +# Connection db2 is holding a PART1 lock. 
          307  +#
          308  +#   7.2.2: Test that the PART1 does not prevent db from switching to the
          309  +#          other wal file.
          310  +#
          311  +#   7.2.3: Test that the PART1 does prevent a checkpoint of test.db-wal.
          312  +#
          313  +#   7.2.4: Test that after the PART1 is released the checkpoint is possible.
          314  +#
          315  +do_test 7.2.1 {
          316  +  execsql {
          317  +    BEGIN;
          318  +      SELECT count(*) FROM t1;
          319  +  } db2
          320  +} {1}
          321  +do_test 7.2.2 {
          322  +  execsql {
          323  +    INSERT INTO t1 VALUES( randomblob(800) );
          324  +    INSERT INTO t1 VALUES( randomblob(800) );
          325  +  }
          326  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          327  +} {13656 3176 1024}
          328  +do_test 7.2.3 {
          329  +  execsql { PRAGMA wal_checkpoint }
          330  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          331  +} {13656 3176 1024}
          332  +do_test 7.2.4 {
          333  +  execsql { END } db2
          334  +  execsql { PRAGMA wal_checkpoint }
          335  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          336  +} {13656 3176 11264}
          337  +
          338  +# Connection db2 is holding a PART2_FULL1 lock. 
          339  +#
          340  +#   7.3.2: Test that the lock does not prevent checkpointing.
          341  +#
          342  +#   7.3.3: Test that the lock does prevent the writer from overwriting 
          343  +#          test.db-wal.
          344  +#
          345  +#   7.3.4: Test that after the PART2_FULL1 is released the writer can
          346  +#          switch wal files and overwrite test.db-wal
          347  +#
          348  +db close
          349  +db2 close
          350  +sqlite3 db test.db
          351  +sqlite3 db2 test.db
          352  +do_test 7.3.1 {
          353  +  execsql {
          354  +    PRAGMA wal_autocheckpoint = 0;
          355  +    PRAGMA journal_size_limit = 10000;
          356  +    INSERT INTO t1 VALUES(randomblob(10000));
          357  +    INSERT INTO t1 VALUES(randomblob(500));
          358  +  }
          359  +  execsql {
          360  +    BEGIN;
          361  +      SELECT count(*) FROM t1;
          362  +  } db2
          363  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          364  +} {12608 3176 11264}
          365  +do_test 7.3.2 {
          366  +  execsql { PRAGMA wal_checkpoint }
          367  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          368  +} {12608 3176 21504}
          369  +do_test 7.3.3 {
          370  +  execsql { 
          371  +    INSERT INTO t1 VALUES(randomblob(10000));
          372  +    INSERT INTO t1 VALUES(randomblob(500));
          373  +  }
          374  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          375  +} {12608 18896 21504}
          376  +do_test 7.3.4 {
          377  +  execsql END db2
          378  +  execsql { INSERT INTO t1 VALUES(randomblob(5000)); }
          379  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          380  +} {12608 18896 21504}
          381  +
          382  +# Connection db2 is holding a PART2 lock. 
          383  +#
          384  +#   7.4.2: Test that the lock does not prevent writer switching to test.db-wal.
          385  +#
          386  +#   7.3.3: Test that the lock does prevent checkpointing of test.db-wal2.
          387  +#
          388  +#   7.3.4: Test that after the PART2 is released test.db-wal2 can be
          389  +#          checkpointed.
          390  +#
          391  +db close
          392  +db2 close
          393  +sqlite3 db test.db
          394  +sqlite3 db2 test.db
          395  +do_test 7.4.1 {
          396  +  execsql {
          397  +    PRAGMA wal_autocheckpoint = 0;
          398  +    PRAGMA journal_size_limit = 10000;
          399  +    INSERT INTO t1 VALUES(randomblob(10000));
          400  +    INSERT INTO t1 VALUES(randomblob(10000));
          401  +    PRAGMA wal_checkpoint;
          402  +  }
          403  +  execsql {
          404  +    BEGIN;
          405  +      SELECT count(*) FROM t1;
          406  +  } db2
          407  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          408  +} {12608 12608 44032}
          409  +do_test 7.4.2 {
          410  +  execsql { 
          411  +    INSERT INTO t1 VALUES(randomblob(5000));
          412  +  }
          413  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          414  +} {12608 12608 44032}
          415  +do_test 7.4.3 {
          416  +  execsql { PRAGMA wal_checkpoint }
          417  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          418  +} {12608 12608 44032}
          419  +do_test 7.4.4 {
          420  +  execsql END db2
          421  +  execsql { PRAGMA wal_checkpoint }
          422  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          423  +} {12608 12608 54272}
          424  +
          425  +# Connection db2 is holding a PART1_FULL2 lock. 
          426  +#
          427  +#   7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2.
          428  +#
          429  +#   7.5.3: Test that the lock does prevent the writer from overwriting
          430  +#          test.db-wal2.
          431  +#
          432  +#   7.5.4: Test that after the PART1_FULL2 lock is released, the writer
          433  +#          can switch to test.db-wal2.
          434  +#
          435  +db close
          436  +db2 close
          437  +sqlite3 db test.db
          438  +sqlite3 db2 test.db
          439  +do_test 7.5.1 {
          440  +  execsql {
          441  +    PRAGMA wal_autocheckpoint = 0;
          442  +    PRAGMA journal_size_limit = 10000;
          443  +    INSERT INTO t1 VALUES(randomblob(10000));
          444  +    INSERT INTO t1 VALUES(randomblob(10000));
          445  +    PRAGMA wal_checkpoint;
          446  +    INSERT INTO t1 VALUES(randomblob(5000));
          447  +  }
          448  +  execsql {
          449  +    BEGIN;
          450  +      SELECT count(*) FROM t1;
          451  +  } db2
          452  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          453  +} {12608 12608 64512}
          454  +do_test 7.5.2 {
          455  +  execsql { PRAGMA wal_checkpoint }
          456  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          457  +} {12608 12608 75776}
          458  +do_test 7.5.3.1 {
          459  +  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
          460  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          461  +} {14704 12608 75776}
          462  +do_test 7.5.3.2 {
          463  +  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
          464  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          465  +} {22040 12608 75776}
          466  +do_test 7.5.4 {
          467  +  execsql END db2
          468  +  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
          469  +  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
          470  +} {22040 12608 75776}
          471  +
   291    472   
   292    473   finish_test
          474  +

Added test/wal2snapshot.test.

            1  +# 2018 December 5
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the operation of the library in
           13  +# "PRAGMA journal_mode=WAL2" mode.
           14  +#
           15  +
           16  +set testdir [file dirname $argv0]
           17  +source $testdir/tester.tcl
           18  +
           19  +set testprefix wal2snapshot
           20  +ifcapable !wal {finish_test ; return }
           21  +ifcapable !snapshot {finish_test; return}
           22  +
           23  +foreach {tn mode} {1 wal 2 wal2} {
           24  +  reset_db
           25  +  do_execsql_test $tn.1 "PRAGMA journal_mode = $mode" $mode
           26  +
           27  +  do_execsql_test $tn.2 {
           28  +    CREATE TABLE t1(a, b);
           29  +    INSERT INTO t1 VALUES(1, 2);
           30  +    INSERT INTO t1 VALUES(3, 4);
           31  +    BEGIN;
           32  +  }
           33  +
           34  +  # Check that sqlite3_snapshot_get() is an error for a wal2 db.
           35  +  #
           36  +  if {$tn==1} {
           37  +    do_test 1.3 {
           38  +      set S [sqlite3_snapshot_get db main]
           39  +      sqlite3_snapshot_free $S
           40  +    } {}
           41  +  } else {
           42  +    do_test 2.3 {
           43  +      list [catch { sqlite3_snapshot_get db main } msg] $msg
           44  +    } {1 SQLITE_ERROR}
           45  +  }
           46  +  
           47  +  # Check that sqlite3_snapshot_recover() is an error for a wal2 db.
           48  +  #
           49  +  do_execsql_test $tn.4 COMMIT
           50  +  if {$tn==1} {
           51  +    do_test 1.5 {
           52  +      sqlite3_snapshot_recover db main
           53  +    } {}
           54  +  } else {
           55  +    do_test 2.5 {
           56  +      list [catch { sqlite3_snapshot_recover db main } msg] $msg
           57  +    } {1 SQLITE_ERROR}
           58  +  }
           59  + 
           60  +  # Check that sqlite3_snapshot_open() is an error for a wal2 db.
           61  +  #
           62  +  if {$tn==1} {
           63  +    do_test 1.6 {
           64  +      execsql BEGIN
           65  +      set SNAPSHOT [sqlite3_snapshot_get_blob db main]
           66  +      sqlite3_snapshot_open_blob db main $SNAPSHOT
           67  +      execsql COMMIT
           68  +    } {}
           69  +  } else {
           70  +    do_test 2.6 {
           71  +      execsql BEGIN
           72  +      set res [
           73  +        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
           74  +      ]
           75  +      execsql COMMIT
           76  +      set res
           77  +    } {1 SQLITE_ERROR}
           78  +  }
           79  +}
           80  +
           81  +
           82  +finish_test
           83  +
           84  +

Changes to test/window1.test.

   589    589   
   590    590   do_execsql_test 13.5 {
   591    591     SELECT a, rank() OVER(ORDER BY b) FROM t1
   592    592       INTERSECT 
   593    593     SELECT a, rank() OVER(ORDER BY b DESC) FROM t1;
   594    594   } {
   595    595   }
          596  +
          597  +# 2018-12-06
          598  +# https://www.sqlite.org/src/info/f09fcd17810f65f7
          599  +# Assertion fault when window functions are used.
          600  +#
          601  +# Root cause is the query flattener invoking sqlite3ExprDup() on
          602  +# expressions that contain subqueries with window functions.  The
          603  +# sqlite3ExprDup() routine is not making correctly initializing
          604  +# Select.pWin field of the subqueries.
          605  +#
          606  +sqlite3 db :memory:
          607  +do_execsql_test 14.0 {
          608  +  SELECT * FROM(
          609  +    SELECT * FROM (SELECT 1 AS c) WHERE c IN (
          610  +        SELECT (row_number() OVER()) FROM (VALUES (0))
          611  +    )
          612  +  );
          613  +} {1}
          614  +do_execsql_test 14.1 {
          615  +  CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(12345);
          616  +  CREATE TABLE t2(c); INSERT INTO t2(c) VALUES(1);
          617  +  SELECT y, y+1, y+2 FROM (
          618  +    SELECT c IN (
          619  +      SELECT (row_number() OVER()) FROM t1
          620  +    ) AS y FROM t2
          621  +  );
          622  +} {1 2 3}
   596    623   
   597    624   finish_test

Added tool/index_usage.c.

            1  +/*
            2  +** 2018-12-04
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** 
           13  +** This file implements a utility program used to help determine which
           14  +** indexes in a database schema are used and unused, and how often specific
           15  +** indexes are used.
           16  +*/
           17  +#include "sqlite3.h"
           18  +#include <stdio.h>
           19  +#include <stdlib.h>
           20  +#include <assert.h>
           21  +#include <string.h>
           22  +
           23  +static void usage(const char *argv0){
           24  +  printf("Usage: %s DATABASE LOG\n\n", argv0);
           25  +  printf(
           26  +    "DATABASE is an SQLite database against which various statements\n"
           27  +    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
           28  +    "database with this schema:\n"
           29  +    "\n"
           30  +    "    CREATE TABLE sqllog(sql TEXT);\n"
           31  +    "\n"
           32  +    "This utility program analyzes statements contained in LOG and prints\n"
           33  +    "a report showing how many times each index in DATABASE is used by the\n"
           34  +    "statements in LOG.\n"
           35  +    "\n"
           36  +    "DATABASE only needs to contain the schema used by the statements in\n"
           37  +    "LOG. The content can be removed from DATABASE.\n"
           38  +  );
           39  +  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
           40  +         "checkin number %.40s. Different versions\n"
           41  +         "of SQLite might use different indexes.\n",
           42  +         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
           43  +  exit(1);
           44  +}
           45  +
           46  +int main(int argc, char **argv){
           47  +  sqlite3 *db = 0;          /* The main database */
           48  +  sqlite3_stmt *pStmt = 0;  /* a query */
           49  +  char *zSql;
           50  +  int nErr = 0;
           51  +  int rc;
           52  +
           53  +  if( argc!=3 ) usage(argv[0]);
           54  +  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
           55  +  if( rc ){
           56  +    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
           57  +    goto errorOut;
           58  +  }
           59  +  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
           60  +  if( rc ){
           61  +    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
           62  +           sqlite3_errmsg(db));
           63  +    goto errorOut;
           64  +  }
           65  +  sqlite3_finalize(pStmt);
           66  +  pStmt = 0;
           67  +  rc = sqlite3_exec(db, 
           68  +     "CREATE TABLE temp.idxu(\n"
           69  +     "  tbl TEXT,\n"
           70  +     "  idx TEXT,\n"
           71  +     "  cnt INT,\n"
           72  +     "  PRIMARY KEY(idx)\n"
           73  +     ") WITHOUT ROWID;", 0, 0, 0);
           74  +  if( rc ){
           75  +    printf("Cannot create the result table - %s\n",
           76  +           sqlite3_errmsg(db));
           77  +    goto errorOut;
           78  +  }
           79  +  rc = sqlite3_exec(db,
           80  +     "INSERT INTO temp.idxu(tbl,idx,cnt)"
           81  +     " SELECT tbl_name, name, 0 FROM sqlite_master"
           82  +     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);
           83  +
           84  +  /* Open the LOG database */
           85  +  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
           86  +  rc = sqlite3_exec(db, zSql, 0, 0, 0);
           87  +  sqlite3_free(zSql);
           88  +  if( rc ){
           89  +    printf("Cannot open the LOG database \"%s\" - %s\n",
           90  +           argv[2], sqlite3_errmsg(db));
           91  +    goto errorOut;
           92  +  }
           93  +  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
           94  +                          -1, &pStmt, 0);
           95  +  if( rc ){
           96  +    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
           97  +           argv[2], sqlite3_errmsg(db));
           98  +    goto errorOut;
           99  +  }
          100  +
          101  +  /* Update the counts based on LOG */
          102  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          103  +    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
          104  +    sqlite3_stmt *pS2;
          105  +    if( zLog==0 ) continue;
          106  +    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
          107  +    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
          108  +    sqlite3_free(zSql);
          109  +    if( rc ){
          110  +      printf("Cannot compile LOG entry %d (%s): %s\n",
          111  +             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
          112  +      nErr++;
          113  +    }else{
          114  +      while( sqlite3_step(pS2)==SQLITE_ROW ){
          115  +        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
          116  +        const char *z1, *z2;
          117  +        int n;
          118  +        /* printf("EXPLAIN: %s\n", zExplain); */
          119  +        z1 = strstr(zExplain, " USING INDEX ");
          120  +        if( z1==0 ) continue;
          121  +        z1 += 13;
          122  +        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
          123  +        n = z2 - z1;
          124  +        zSql = sqlite3_mprintf(
          125  +          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
          126  +        );
          127  +        /* printf("sql: %s\n", zSql); */
          128  +        sqlite3_exec(db, zSql, 0, 0, 0);
          129  +        sqlite3_free(zSql);
          130  +      }
          131  +    }
          132  +    sqlite3_finalize(pS2);
          133  +  }
          134  +  sqlite3_finalize(pStmt);
          135  +
          136  +  /* Generate the report */
          137  +  rc = sqlite3_prepare_v2(db,
          138  +     "SELECT tbl, idx, cnt, "
          139  +     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
          140  +     " FROM temp.idxu, main.sqlite_master"
          141  +     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
          142  +     "   AND temp.idxu.idx=main.sqlite_master.name"
          143  +     " ORDER BY cnt DESC, tbl, idx",
          144  +     -1, &pStmt, 0);
          145  +  if( rc ){
          146  +    printf("Cannot query the result table - %s\n",
          147  +           sqlite3_errmsg(db));
          148  +    goto errorOut;
          149  +  }
          150  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          151  +    printf("%10d %s on %s(%s)\n", 
          152  +       sqlite3_column_int(pStmt, 2),
          153  +       sqlite3_column_text(pStmt, 1),
          154  +       sqlite3_column_text(pStmt, 0),
          155  +       sqlite3_column_text(pStmt, 3));
          156  +  }
          157  +  sqlite3_finalize(pStmt);
          158  +  pStmt = 0;
          159  +
          160  +errorOut:
          161  +  sqlite3_finalize(pStmt);
          162  +  sqlite3_close(db);
          163  +  return nErr;
          164  +}

Changes to tool/lemon.c.

  4586   4586     }
  4587   4587     tplt_xfer(lemp->name,in,out,&lineno);
  4588   4588   
  4589   4589     /* Generate code which executes whenever the parser stack overflows */
  4590   4590     tplt_print(out,lemp,lemp->overflow,&lineno);
  4591   4591     tplt_xfer(lemp->name,in,out,&lineno);
  4592   4592   
  4593         -  /* Generate the table of rule information
         4593  +  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
         4594  +  ** yyRuleInfoNRhs[].
  4594   4595     **
  4595   4596     ** Note: This code depends on the fact that rules are number
  4596   4597     ** sequentually beginning with 0.
  4597   4598     */
  4598   4599     for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
  4599         -    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
         4600  +    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
         4601  +     rule_print(out, rp);
         4602  +    fprintf(out," */\n"); lineno++;
         4603  +  }
         4604  +  tplt_xfer(lemp->name,in,out,&lineno);
         4605  +  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
         4606  +    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
  4600   4607       rule_print(out, rp);
  4601   4608       fprintf(out," */\n"); lineno++;
  4602   4609     }
  4603   4610     tplt_xfer(lemp->name,in,out,&lineno);
  4604   4611   
  4605   4612     /* Generate code which execution during each REDUCE action */
  4606   4613     i = 0;

Changes to tool/lempar.c.

   682    682     yytos = yypParser->yytos;
   683    683     yytos->stateno = yyNewState;
   684    684     yytos->major = yyMajor;
   685    685     yytos->minor.yy0 = yyMinor;
   686    686     yyTraceShift(yypParser, yyNewState, "Shift");
   687    687   }
   688    688   
   689         -/* The following table contains information about every rule that
   690         -** is used during the reduce.
   691         -*/
   692         -static const struct {
   693         -  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
   694         -  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
   695         -} yyRuleInfo[] = {
          689  +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
          690  +** of that rule */
          691  +static const YYCODETYPE yyRuleInfoLhs[] = {
          692  +%%
          693  +};
          694  +
          695  +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
          696  +** of symbols on the right-hand side of that rule. */
          697  +static const signed char yyRuleInfoNRhs[] = {
   696    698   %%
   697    699   };
   698    700   
   699    701   static void yy_accept(yyParser*);  /* Forward Declaration */
   700    702   
   701    703   /*
   702    704   ** Perform a reduce action and the shift that must immediately
................................................................................
   721    723     int yysize;                     /* Amount to pop the stack */
   722    724     ParseARG_FETCH
   723    725     (void)yyLookahead;
   724    726     (void)yyLookaheadToken;
   725    727     yymsp = yypParser->yytos;
   726    728   #ifndef NDEBUG
   727    729     if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
   728         -    yysize = yyRuleInfo[yyruleno].nrhs;
          730  +    yysize = yyRuleInfoNRhs[yyruleno];
   729    731       if( yysize ){
   730    732         fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
   731    733           yyTracePrompt,
   732    734           yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
   733    735       }else{
   734    736         fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
   735    737           yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
................................................................................
   736    738       }
   737    739     }
   738    740   #endif /* NDEBUG */
   739    741   
   740    742     /* Check that the stack is large enough to grow by a single entry
   741    743     ** if the RHS of the rule is empty.  This ensures that there is room
   742    744     ** enough on the stack to push the LHS value */
   743         -  if( yyRuleInfo[yyruleno].nrhs==0 ){
          745  +  if( yyRuleInfoNRhs[yyruleno]==0 ){
   744    746   #ifdef YYTRACKMAXSTACKDEPTH
   745    747       if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
   746    748         yypParser->yyhwm++;
   747    749         assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
   748    750       }
   749    751   #endif
   750    752   #if YYSTACKDEPTH>0 
................................................................................
   778    780     **  #line <lineno> <thisfile>
   779    781     **     break;
   780    782     */
   781    783   /********** Begin reduce actions **********************************************/
   782    784   %%
   783    785   /********** End reduce actions ************************************************/
   784    786     };
   785         -  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
   786         -  yygoto = yyRuleInfo[yyruleno].lhs;
   787         -  yysize = yyRuleInfo[yyruleno].nrhs;
          787  +  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
          788  +  yygoto = yyRuleInfoLhs[yyruleno];
          789  +  yysize = yyRuleInfoNRhs[yyruleno];
   788    790     yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
   789    791   
   790    792     /* There are no SHIFTREDUCE actions on nonterminals because the table
   791    793     ** generator has simplified them to pure REDUCE actions. */
   792    794     assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );
   793    795   
   794    796     /* It is not possible for a REDUCE to be followed by an error */