/ Check-in [d8dd98a3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest trunk changes into this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | wal2
Files: files | file ages | folders
SHA3-256: d8dd98a39ea061dea264a7d81153f7b1be2f81b554b30d0ce289897c802209bd
User & Date: dan 2018-12-11 13:44:59
Wiki:wal2
Context
2018-12-11
17:56
Change the way wal2 locks work to ensure a reader only ever has to lock a single slot. check-in: 18b2c23a user: dan tags: wal2
13:44
Merge latest trunk changes into this branch. check-in: d8dd98a3 user: dan tags: wal2
12:51
Small performance improvement in sqlite3_step(). check-in: d1db8d58 user: drh tags: trunk
2018-12-06
16:54
Add test file wal2snapshot.test that should have been added as part of an earlier commit. check-in: f6baa7e1 user: dan tags: wal2
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

  1302   1302   
  1303   1303   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1304   1304   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1305   1305   
  1306   1306   showshm$(TEXE):	$(TOP)/tool/showshm.c
  1307   1307   	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
  1308   1308   
         1309  +index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
         1310  +	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
         1311  +
  1309   1312   changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
  1310   1313   	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
  1311   1314   
  1312   1315   changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
  1313   1316   	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
  1314   1317   
  1315   1318   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo

Changes to Makefile.msc.

  2440   2440   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  2441   2441   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2442   2442   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2443   2443   
  2444   2444   showshm.exe:	$(TOP)\tool\showshm.c
  2445   2445   	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
  2446   2446   
         2447  +index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
         2448  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2449  +		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2450  +
  2447   2451   changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
  2448   2452   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2449   2453   		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
  2450   2454   		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2451   2455   
  2452   2456   changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
  2453   2457   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \

Changes to VERSION.

     1         -3.26.0
            1  +3.27.0

Changes to autoconf/Makefile.msc.

   279    279   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
   280    280   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
   281    281   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
   282    282   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
   283    283   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
   284    284   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
   285    285   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
          286  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
   286    287   !ENDIF
   287    288   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   288    289   !ENDIF
   289    290   
   290    291   # Should the session extension be enabled?  If so, add compilation options
   291    292   # to enable it.
   292    293   #
................................................................................
   933    934   # Additional compiler options for the shell.  These are only effective
   934    935   # when the shell is not being dynamically linked.
   935    936   #
   936    937   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   937    938   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
   938    939   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
   939    940   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
          941  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
   940    942   !ENDIF
   941    943   
   942    944   
   943    945   # This is the default Makefile target.  The objects listed here
   944    946   # are what get build when you type just "make" with no arguments.
   945    947   #
   946    948   core:	dll shell

Changes to configure.

     1      1   #! /bin/sh
     2      2   # Guess values for system-dependent variables and create Makefiles.
     3         -# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
            3  +# Generated by GNU Autoconf 2.69 for sqlite 3.27.0.
     4      4   #
     5      5   #
     6      6   # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
     7      7   #
     8      8   #
     9      9   # This configure script is free software; the Free Software Foundation
    10     10   # gives unlimited permission to copy, distribute and modify it.
................................................................................
   722    722   subdirs=
   723    723   MFLAGS=
   724    724   MAKEFLAGS=
   725    725   
   726    726   # Identity of this package.
   727    727   PACKAGE_NAME='sqlite'
   728    728   PACKAGE_TARNAME='sqlite'
   729         -PACKAGE_VERSION='3.26.0'
   730         -PACKAGE_STRING='sqlite 3.26.0'
          729  +PACKAGE_VERSION='3.27.0'
          730  +PACKAGE_STRING='sqlite 3.27.0'
   731    731   PACKAGE_BUGREPORT=''
   732    732   PACKAGE_URL=''
   733    733   
   734    734   # Factoring default headers for most tests.
   735    735   ac_includes_default="\
   736    736   #include <stdio.h>
   737    737   #ifdef HAVE_SYS_TYPES_H
................................................................................
  1462   1462   #
  1463   1463   # Report the --help message.
  1464   1464   #
  1465   1465   if test "$ac_init_help" = "long"; then
  1466   1466     # Omit some internal or obsolete options to make the list less imposing.
  1467   1467     # This message is too long to be a string in the A/UX 3.1 sh.
  1468   1468     cat <<_ACEOF
  1469         -\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.
         1469  +\`configure' configures sqlite 3.27.0 to adapt to many kinds of systems.
  1470   1470   
  1471   1471   Usage: $0 [OPTION]... [VAR=VALUE]...
  1472   1472   
  1473   1473   To assign environment variables (e.g., CC, CFLAGS...), specify them as
  1474   1474   VAR=VALUE.  See below for descriptions of some of the useful variables.
  1475   1475   
  1476   1476   Defaults for the options are specified in brackets.
................................................................................
  1527   1527     --build=BUILD     configure for building on BUILD [guessed]
  1528   1528     --host=HOST       cross-compile to build programs to run on HOST [BUILD]
  1529   1529   _ACEOF
  1530   1530   fi
  1531   1531   
  1532   1532   if test -n "$ac_init_help"; then
  1533   1533     case $ac_init_help in
  1534         -     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
         1534  +     short | recursive ) echo "Configuration of sqlite 3.27.0:";;
  1535   1535      esac
  1536   1536     cat <<\_ACEOF
  1537   1537   
  1538   1538   Optional Features:
  1539   1539     --disable-option-checking  ignore unrecognized --enable/--with options
  1540   1540     --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  1541   1541     --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
  1653   1653       cd "$ac_pwd" || { ac_status=$?; break; }
  1654   1654     done
  1655   1655   fi
  1656   1656   
  1657   1657   test -n "$ac_init_help" && exit $ac_status
  1658   1658   if $ac_init_version; then
  1659   1659     cat <<\_ACEOF
  1660         -sqlite configure 3.26.0
         1660  +sqlite configure 3.27.0
  1661   1661   generated by GNU Autoconf 2.69
  1662   1662   
  1663   1663   Copyright (C) 2012 Free Software Foundation, Inc.
  1664   1664   This configure script is free software; the Free Software Foundation
  1665   1665   gives unlimited permission to copy, distribute and modify it.
  1666   1666   _ACEOF
  1667   1667     exit
................................................................................
  2072   2072     eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
  2073   2073   
  2074   2074   } # ac_fn_c_check_header_mongrel
  2075   2075   cat >config.log <<_ACEOF
  2076   2076   This file contains any messages produced by compilers while
  2077   2077   running configure, to aid debugging if configure makes a mistake.
  2078   2078   
  2079         -It was created by sqlite $as_me 3.26.0, which was
         2079  +It was created by sqlite $as_me 3.27.0, which was
  2080   2080   generated by GNU Autoconf 2.69.  Invocation command line was
  2081   2081   
  2082   2082     $ $0 $@
  2083   2083   
  2084   2084   _ACEOF
  2085   2085   exec 5>>config.log
  2086   2086   {
................................................................................
 12228  12228   test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1
 12229  12229   
 12230  12230   cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 12231  12231   # Save the log message, to keep $0 and so on meaningful, and to
 12232  12232   # report actual input values of CONFIG_FILES etc. instead of their
 12233  12233   # values after options handling.
 12234  12234   ac_log="
 12235         -This file was extended by sqlite $as_me 3.26.0, which was
        12235  +This file was extended by sqlite $as_me 3.27.0, which was
 12236  12236   generated by GNU Autoconf 2.69.  Invocation command line was
 12237  12237   
 12238  12238     CONFIG_FILES    = $CONFIG_FILES
 12239  12239     CONFIG_HEADERS  = $CONFIG_HEADERS
 12240  12240     CONFIG_LINKS    = $CONFIG_LINKS
 12241  12241     CONFIG_COMMANDS = $CONFIG_COMMANDS
 12242  12242     $ $0 $@
................................................................................
 12294  12294   
 12295  12295   Report bugs to the package provider."
 12296  12296   
 12297  12297   _ACEOF
 12298  12298   cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 12299  12299   ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 12300  12300   ac_cs_version="\\
 12301         -sqlite config.status 3.26.0
        12301  +sqlite config.status 3.27.0
 12302  12302   configured by $0, generated by GNU Autoconf 2.69,
 12303  12303     with options \\"\$ac_cs_config\\"
 12304  12304   
 12305  12305   Copyright (C) 2012 Free Software Foundation, Inc.
 12306  12306   This config.status script is free software; the Free Software Foundation
 12307  12307   gives unlimited permission to copy, distribute and modify it."
 12308  12308   

Changes to ext/fts3/fts3_unicode.c.

    78     78   #endif /* ifndef SQLITE_AMALGAMATION */
    79     79   
    80     80   typedef struct unicode_tokenizer unicode_tokenizer;
    81     81   typedef struct unicode_cursor unicode_cursor;
    82     82   
    83     83   struct unicode_tokenizer {
    84     84     sqlite3_tokenizer base;
    85         -  int bRemoveDiacritic;
           85  +  int eRemoveDiacritic;
    86     86     int nException;
    87     87     int *aiException;
    88     88   };
    89     89   
    90     90   struct unicode_cursor {
    91     91     sqlite3_tokenizer_cursor base;
    92     92     const unsigned char *aInput;    /* Input text being tokenized */
................................................................................
   223    223     unicode_tokenizer *pNew;        /* New tokenizer object */
   224    224     int i;
   225    225     int rc = SQLITE_OK;
   226    226   
   227    227     pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   228    228     if( pNew==NULL ) return SQLITE_NOMEM;
   229    229     memset(pNew, 0, sizeof(unicode_tokenizer));
   230         -  pNew->bRemoveDiacritic = 1;
          230  +  pNew->eRemoveDiacritic = 1;
   231    231   
   232    232     for(i=0; rc==SQLITE_OK && i<nArg; i++){
   233    233       const char *z = azArg[i];
   234    234       int n = (int)strlen(z);
   235    235   
   236    236       if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
   237         -      pNew->bRemoveDiacritic = 1;
          237  +      pNew->eRemoveDiacritic = 1;
   238    238       }
   239    239       else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
   240         -      pNew->bRemoveDiacritic = 0;
          240  +      pNew->eRemoveDiacritic = 0;
          241  +    }
          242  +    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
          243  +      pNew->eRemoveDiacritic = 2;
   241    244       }
   242    245       else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
   243    246         rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
   244    247       }
   245    248       else if( n>=11 && memcmp("separators=", z, 11)==0 ){
   246    249         rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
   247    250       }
................................................................................
   346    349         zOut = &zNew[zOut - pCsr->zToken];
   347    350         pCsr->zToken = zNew;
   348    351         pCsr->nAlloc += 64;
   349    352       }
   350    353   
   351    354       /* Write the folded case of the last character read to the output */
   352    355       zEnd = z;
   353         -    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
          356  +    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
   354    357       if( iOut ){
   355    358         WRITE_UTF8(zOut, iOut);
   356    359       }
   357    360   
   358    361       /* If the cursor is not at EOF, read the next character */
   359    362       if( z>=zTerm ) break;
   360    363       READ_UTF8(z, zTerm, iCode);

Changes to ext/fts3/fts3_unicode2.c.

   155    155   ** If the argument is a codepoint corresponding to a lowercase letter
   156    156   ** in the ASCII range with a diacritic added, return the codepoint
   157    157   ** of the ASCII letter only. For example, if passed 235 - "LATIN
   158    158   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
   159    159   ** E"). The resuls of passing a codepoint that corresponds to an
   160    160   ** uppercase letter are undefined.
   161    161   */
   162         -static int remove_diacritic(int c){
          162  +static int remove_diacritic(int c, int bComplex){
   163    163     unsigned short aDia[] = {
   164    164           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
   165    165        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
   166    166        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
   167    167        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
   168         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
   169         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
   170         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
   171         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
   172         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
   173         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
   174         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
   175         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
   176         -    62924, 63050, 63082, 63274, 63390, 
          168  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
          169  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
          170  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
          171  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
          172  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
          173  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
          174  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
          175  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
          176  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
          177  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
          178  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
          179  +    63182, 63242, 63274, 63310, 63368, 63390, 
   177    180     };
   178    181     char aChar[] = {
   179         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
   180         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
   181         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
   182         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
   183         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
   184         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
   185         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
   186         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
   187         -    'e',  'i',  'o',  'u',  'y',  
          182  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
          183  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
          184  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
          185  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
          186  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
          187  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
          188  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
          189  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
          190  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
          191  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
          192  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
          193  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
          194  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
          195  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
          196  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
          197  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
          198  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
          199  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
          200  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
          201  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
          202  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   188    203     };
   189    204   
   190    205     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   191    206     int iRes = 0;
   192    207     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
   193    208     int iLo = 0;
   194    209     while( iHi>=iLo ){
................................................................................
   197    212         iRes = iTest;
   198    213         iLo = iTest+1;
   199    214       }else{
   200    215         iHi = iTest-1;
   201    216       }
   202    217     }
   203    218     assert( key>=aDia[iRes] );
   204         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
          219  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
          220  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
   205    221   }
   206    222   
   207    223   
   208    224   /*
   209    225   ** Return true if the argument interpreted as a unicode codepoint
   210    226   ** is a diacritical modifier character.
   211    227   */
................................................................................
   224    240   ** is an upper case character that has a lower case equivalent,
   225    241   ** return the codepoint corresponding to the lower case version.
   226    242   ** Otherwise, return a copy of the argument.
   227    243   **
   228    244   ** The results are undefined if the value passed to this function
   229    245   ** is less than zero.
   230    246   */
   231         -int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
          247  +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
   232    248     /* Each entry in the following array defines a rule for folding a range
   233    249     ** of codepoints to lower case. The rule applies to a range of nRange
   234    250     ** codepoints starting at codepoint iCode.
   235    251     **
   236    252     ** If the least significant bit in flags is clear, then the rule applies
   237    253     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   238    254     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   347    363       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   348    364       p = &aEntry[iRes];
   349    365       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   350    366         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   351    367         assert( ret>0 );
   352    368       }
   353    369   
   354         -    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
          370  +    if( eRemoveDiacritic ){
          371  +      ret = remove_diacritic(ret, eRemoveDiacritic==2);
          372  +    }
   355    373     }
   356    374     
   357    375     else if( c>=66560 && c<66600 ){
   358    376       ret = c + 40;
   359    377     }
   360    378   
   361    379     return ret;
   362    380   }
   363    381   #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
   364    382   #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/unicode/mkunicode.tcl.

     5      5     global tl_lookup_table
     6      6     set aChar [list]
     7      7     set lRange [list]
     8      8   
     9      9     set nRange 1
    10     10     set iFirst  [lindex $map 0 0]
    11     11     set cPrev   [lindex $map 0 1]
           12  +  set fPrev   [lindex $map 0 2]
    12     13   
    13     14     foreach m [lrange $map 1 end] {
    14         -    foreach {i c} $m {}
           15  +    foreach {i c f} $m {}
    15     16   
    16         -    if {$cPrev == $c} {
           17  +    if {$cPrev == $c && $fPrev==$f} {
    17     18         for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
    18     19           if {[info exists tl_lookup_table($j)]==0} break
    19     20         }
    20     21   
    21     22         if {$j==$i} {
    22     23           set nNew [expr {(1 + $i - $iFirst)}]
    23     24           if {$nNew<=8} {
................................................................................
    25     26             continue
    26     27           }
    27     28         }
    28     29       }
    29     30   
    30     31       lappend lRange [list $iFirst $nRange]
    31     32       lappend aChar  $cPrev
           33  +    lappend aFlag  $fPrev
    32     34   
    33     35       set iFirst $i
    34     36       set cPrev  $c
           37  +    set fPrev  $f
    35     38       set nRange 1
    36     39     }
    37     40     lappend lRange [list $iFirst $nRange]
    38     41     lappend aChar $cPrev
           42  +  lappend aFlag $fPrev
    39     43   
    40     44     puts "/*"
    41     45     puts "** If the argument is a codepoint corresponding to a lowercase letter"
    42     46     puts "** in the ASCII range with a diacritic added, return the codepoint"
    43     47     puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
    44     48     puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
    45     49     puts "** E\"). The resuls of passing a codepoint that corresponds to an"
    46     50     puts "** uppercase letter are undefined."
    47     51     puts "*/"
    48         -  puts "static int ${::remove_diacritic}(int c)\{"
           52  +  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
    49     53     puts "  unsigned short aDia\[\] = \{"
    50     54     puts -nonewline "        0, "
    51     55     set i 1
    52     56     foreach r $lRange {
    53     57       foreach {iCode nRange} $r {}
    54     58       if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
    55     59       incr i
................................................................................
    56     60   
    57     61       puts -nonewline [format "%5d" [expr ($iCode<<3) + $nRange-1]]
    58     62       puts -nonewline ", "
    59     63     }
    60     64     puts ""
    61     65     puts "  \};"
    62     66     puts "  char aChar\[\] = \{"
    63         -  puts -nonewline "    '\\0', "
           67  +  puts -nonewline "    '\\0',      "
    64     68     set i 1
    65         -  foreach c $aChar {
    66         -    set str "'$c',  "
    67         -    if {$c == ""} { set str "'\\0', " }
           69  +  foreach c $aChar f $aFlag {
           70  +    if { $f } {
           71  +      set str "'$c'|0x80,  "
           72  +    } else {
           73  +      set str "'$c'|0x00,  "
           74  +    }
           75  +    if {$c == ""} { set str "'\\0',      " }
    68     76   
    69         -    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
           77  +    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    70     78       incr i
    71     79       puts -nonewline "$str"
    72     80     }
    73     81     puts ""
    74     82     puts "  \};"
    75     83     puts {
    76     84     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
................................................................................
    83     91         iRes = iTest;
    84     92         iLo = iTest+1;
    85     93       }else{
    86     94         iHi = iTest-1;
    87     95       }
    88     96     }
    89     97     assert( key>=aDia[iRes] );
    90         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
           98  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           99  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
    91    100     puts "\}"
    92    101   }
    93    102   
    94    103   proc print_isdiacritic {zFunc map} {
    95    104   
    96    105     set lCode [list]
    97    106     foreach m $map {
    98         -    foreach {code char} $m {}
          107  +    foreach {code char flag} $m {}
          108  +    if {$flag} continue
    99    109       if {$code && $char == ""} { lappend lCode $code }
   100    110     }
   101    111     set lCode [lsort -integer $lCode]
   102    112     set iFirst [lindex $lCode 0]
   103    113     set iLast [lindex $lCode end]
   104    114   
   105    115     set i1 0
................................................................................
   468    478     puts "** is an upper case character that has a lower case equivalent,"
   469    479     puts "** return the codepoint corresponding to the lower case version."
   470    480     puts "** Otherwise, return a copy of the argument."
   471    481     puts "**"
   472    482     puts "** The results are undefined if the value passed to this function"
   473    483     puts "** is less than zero."
   474    484     puts "*/"
   475         -  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
          485  +  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
   476    486   
   477    487     set liOff [tl_generate_ioff_table $lRecord]
   478    488     tl_print_table_header
   479    489     foreach entry $lRecord { 
   480    490       if {[tl_print_table_entry toggle $entry $liOff]} { 
   481    491         lappend lHigh $entry 
   482    492       } 
................................................................................
   512    522       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   513    523       p = &aEntry[iRes];
   514    524       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   515    525         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   516    526         assert( ret>0 );
   517    527       }
   518    528   
   519         -    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
          529  +    if( eRemoveDiacritic ){
          530  +      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
          531  +    }
   520    532     }
   521    533     }]
   522    534   
   523    535     foreach entry $lHigh {
   524    536       tl_print_if_entry $entry
   525    537     }
   526    538   
................................................................................
   601    613     set caseN [categories_switch C N {d l o}]
   602    614     set caseP [categories_switch C P {c d e f i o s}]
   603    615     set caseS [categories_switch C S {c k m o}]
   604    616     set caseZ [categories_switch C Z {l p s}]
   605    617   
   606    618     set nCat [expr [llength [array names C]] + 1]
   607    619     puts [code {
   608         -    int sqlite3Fts5UnicodeNCat(void) { 
   609         -      return $nCat;
   610         -    }
   611         -
   612    620       int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   613    621         aArray[0] = 1;
   614    622         switch( zCat[0] ){
   615    623           $caseC
   616    624           $caseL
   617    625           $caseM
   618    626           $caseN

Changes to ext/fts3/unicode/parseunicode.tcl.

     3      3   # Parameter $zName must be a path to the file UnicodeData.txt. This command
     4      4   # reads the file and returns a list of mappings required to remove all
     5      5   # diacritical marks from a unicode string. Each mapping is itself a list
     6      6   # consisting of two elements - the unicode codepoint and the single ASCII
     7      7   # character that it should be replaced with, or an empty string if the 
     8      8   # codepoint should simply be removed from the input. Examples:
     9      9   #
    10         -#   { 224 a  }     (replace codepoint 224 to "a")
    11         -#   { 769 "" }     (remove codepoint 769 from input)
           10  +#   { 224 a  0 }     (replace codepoint 224 to "a")
           11  +#   { 769 "" 0 }     (remove codepoint 769 from input)
    12     12   #
    13     13   # Mappings are only returned for non-upper case codepoints. It is assumed
    14     14   # that the input has already been folded to lower case.
           15  +#
           16  +# The third value in the list is always either 0 or 1. 0 if the 
           17  +# UnicodeData.txt file maps the codepoint to a single ASCII character and
           18  +# a diacritic, or 1 if the mapping is indirect. For example, consider the 
           19  +# two entries:
           20  +#
           21  +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
           22  +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
           23  +#
           24  +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
           25  +# diacritic). The second is an indirect mapping, as it maps to the
           26  +# first codepoint plus 0302 (a diacritic).
    15     27   #
    16     28   proc rd_load_unicodedata_text {zName} {
    17     29     global tl_lookup_table
    18     30   
    19     31     set fd [open $zName]
    20     32     set lField {
    21     33       code
................................................................................
    49     61         continue
    50     62       }
    51     63   
    52     64       set iCode  [expr "0x$code"]
    53     65       set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    54     66       set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
    55     67   
           68  +    # Filter out upper-case characters, as they will be mapped to their
           69  +    # lower-case equivalents before this data is used.
    56     70       if {[info exists tl_lookup_table($iCode)]} continue
           71  +
           72  +    # Check if this is an indirect mapping. If so, set bIndirect to true
           73  +    # and change $iAscii to the indirectly mappped ASCII character.
           74  +    set bIndirect 0
           75  +    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
           76  +      set iAscii $mapping($iAscii)
           77  +      set bIndirect 1
           78  +    }
    57     79   
    58     80       if { ($iAscii >= 97 && $iAscii <= 122)
    59     81         || ($iAscii >= 65 && $iAscii <= 90)
    60     82       } {
    61         -      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
           83  +      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
           84  +      set mapping($iCode) $iAscii
    62     85         set dia($iDia) 1
    63     86       }
    64     87     }
    65     88   
    66     89     foreach d [array names dia] {
    67         -    lappend lRet [list $d ""]
           90  +    lappend lRet [list $d "" 0]
    68     91     }
    69     92     set lRet [lsort -integer -index 0 $lRet]
    70     93   
    71     94     close $fd
    72     95     set lRet
    73     96   }
    74     97   

Changes to ext/fts5/fts5_tokenize.c.

   230    230   #endif /* ifndef SQLITE_AMALGAMATION */
   231    231   
   232    232   typedef struct Unicode61Tokenizer Unicode61Tokenizer;
   233    233   struct Unicode61Tokenizer {
   234    234     unsigned char aTokenChar[128];  /* ASCII range token characters */
   235    235     char *aFold;                    /* Buffer to fold text into */
   236    236     int nFold;                      /* Size of aFold[] in bytes */
   237         -  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
          237  +  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   238    238     int nException;
   239    239     int *aiException;
   240    240   
   241    241     unsigned char aCategory[32];    /* True for token char categories */
   242    242   };
          243  +
          244  +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
          245  +#define FTS5_REMOVE_DIACRITICS_NONE    0
          246  +#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
          247  +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
   243    248   
   244    249   static int fts5UnicodeAddExceptions(
   245    250     Unicode61Tokenizer *p,          /* Tokenizer object */
   246    251     const char *z,                  /* Characters to treat as exceptions */
   247    252     int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
   248    253   ){
   249    254     int rc = SQLITE_OK;
................................................................................
   357    362     }else{
   358    363       p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
   359    364       if( p ){
   360    365         const char *zCat = "L* N* Co";
   361    366         int i;
   362    367         memset(p, 0, sizeof(Unicode61Tokenizer));
   363    368   
   364         -      p->bRemoveDiacritic = 1;
          369  +      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
   365    370         p->nFold = 64;
   366    371         p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
   367    372         if( p->aFold==0 ){
   368    373           rc = SQLITE_NOMEM;
   369    374         }
   370    375   
   371    376         /* Search for a "categories" argument */
................................................................................
   378    383         if( rc==SQLITE_OK ){
   379    384           rc = unicodeSetCategories(p, zCat);
   380    385         }
   381    386   
   382    387         for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
   383    388           const char *zArg = azArg[i+1];
   384    389           if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
   385         -          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
          390  +          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
   386    391               rc = SQLITE_ERROR;
          392  +          }else{
          393  +            p->eRemoveDiacritic = (zArg[0] - '0');
          394  +            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
          395  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
          396  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
          397  +            );
   387    398             }
   388         -          p->bRemoveDiacritic = (zArg[0]=='1');
   389    399           }else
   390    400           if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
   391    401             rc = fts5UnicodeAddExceptions(p, zArg, 1);
   392    402           }else
   393    403           if( 0==sqlite3_stricmp(azArg[i], "separators") ){
   394    404             rc = fts5UnicodeAddExceptions(p, zArg, 0);
   395    405           }else
................................................................................
   495    505   
   496    506         if( *zCsr & 0x80 ){
   497    507           /* An non-ascii-range character. Fold it into the output buffer if
   498    508           ** it is a token character, or break out of the loop if it is not. */
   499    509           READ_UTF8(zCsr, zTerm, iCode);
   500    510           if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
   501    511    non_ascii_tokenchar:
   502         -          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
          512  +          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
   503    513             if( iCode ) WRITE_UTF8(zOut, iCode);
   504    514           }else{
   505    515             break;
   506    516           }
   507    517         }else if( a[*zCsr]==0 ){
   508    518           /* An ascii-range separator character. End of token. */
   509    519           break; 

Changes to ext/fts5/fts5_unicode2.c.

    24     24   ** If the argument is a codepoint corresponding to a lowercase letter
    25     25   ** in the ASCII range with a diacritic added, return the codepoint
    26     26   ** of the ASCII letter only. For example, if passed 235 - "LATIN
    27     27   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
    28     28   ** E"). The resuls of passing a codepoint that corresponds to an
    29     29   ** uppercase letter are undefined.
    30     30   */
    31         -static int fts5_remove_diacritic(int c){
           31  +static int fts5_remove_diacritic(int c, int bComplex){
    32     32     unsigned short aDia[] = {
    33     33           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
    34     34        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
    35     35        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
    36     36        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
    37         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
    38         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
    39         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
    40         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
    41         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
    42         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
    43         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
    44         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
    45         -    62924, 63050, 63082, 63274, 63390, 
           37  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
           38  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
           39  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
           40  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
           41  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
           42  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
           43  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
           44  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
           45  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
           46  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
           47  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
           48  +    63182, 63242, 63274, 63310, 63368, 63390, 
    46     49     };
    47     50     char aChar[] = {
    48         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
    49         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
    50         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
    51         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
    52         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
    53         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
    54         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
    55         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
    56         -    'e',  'i',  'o',  'u',  'y',  
           51  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
           52  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
           53  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
           54  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
           55  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
           56  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
           57  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
           58  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
           59  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
           60  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
           61  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
           62  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
           63  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
           64  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
           65  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
           66  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
           67  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
           68  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
           69  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
           70  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
           71  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
    57     72     };
    58     73   
    59     74     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
    60     75     int iRes = 0;
    61     76     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
    62     77     int iLo = 0;
    63     78     while( iHi>=iLo ){
................................................................................
    66     81         iRes = iTest;
    67     82         iLo = iTest+1;
    68     83       }else{
    69     84         iHi = iTest-1;
    70     85       }
    71     86     }
    72     87     assert( key>=aDia[iRes] );
    73         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
           88  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           89  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
    74     90   }
    75     91   
    76     92   
    77     93   /*
    78     94   ** Return true if the argument interpreted as a unicode codepoint
    79     95   ** is a diacritical modifier character.
    80     96   */
................................................................................
    93    109   ** is an upper case character that has a lower case equivalent,
    94    110   ** return the codepoint corresponding to the lower case version.
    95    111   ** Otherwise, return a copy of the argument.
    96    112   **
    97    113   ** The results are undefined if the value passed to this function
    98    114   ** is less than zero.
    99    115   */
   100         -int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
          116  +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   101    117     /* Each entry in the following array defines a rule for folding a range
   102    118     ** of codepoints to lower case. The rule applies to a range of nRange
   103    119     ** codepoints starting at codepoint iCode.
   104    120     **
   105    121     ** If the least significant bit in flags is clear, then the rule applies
   106    122     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   107    123     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   216    232       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   217    233       p = &aEntry[iRes];
   218    234       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   219    235         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   220    236         assert( ret>0 );
   221    237       }
   222    238   
   223         -    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
          239  +    if( eRemoveDiacritic ){
          240  +      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
          241  +    }
   224    242     }
   225    243     
   226    244     else if( c>=66560 && c<66600 ){
   227    245       ret = c + 40;
   228    246     }
   229    247   
   230    248     return ret;
   231    249   }
   232    250   
   233         -
   234         -#if 0
   235         -int sqlite3Fts5UnicodeNCat(void) { 
   236         -  return 32;
   237         -}
   238         -#endif
   239         -
   240    251   int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   241    252     aArray[0] = 1;
   242    253     switch( zCat[0] ){
   243    254       case 'C':
   244    255             switch( zCat[1] ){
   245    256               case 'c': aArray[1] = 1; break;
   246    257               case 'f': aArray[2] = 1; break;
................................................................................
   752    763   void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
   753    764     int i = 0;
   754    765     int iTbl = 0;
   755    766     while( i<128 ){
   756    767       int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
   757    768       int n = (aFts5UnicodeData[iTbl] >> 5) + i;
   758    769       for(; i<128 && i<n; i++){
   759         -      aAscii[i] = (u8)bToken;
          770  +      aAscii[i] = bToken;
   760    771       }
   761    772       iTbl++;
   762    773     }
   763    774   }
   764         -

Changes to ext/fts5/test/fts5tokenizer.test.

   185    185     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 tokenchars');
   186    186   } {1 {error in tokenizer constructor}}
   187    187   do_catchsql_test 6.2 {
   188    188     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
   189    189   } {1 {error in tokenizer constructor}}
   190    190   do_catchsql_test 6.3 {
   191    191     CREATE VIRTUAL TABLE a3 USING fts5(
   192         -    x, y, tokenize = 'unicode61 remove_diacritics 2'
          192  +    x, y, tokenize = 'unicode61 remove_diacritics 3'
   193    193     );
   194    194   } {1 {error in tokenizer constructor}}
   195    195   do_catchsql_test 6.4 {
   196    196     CREATE VIRTUAL TABLE a3 USING fts5(
   197    197       x, y, tokenize = 'unicode61 remove_diacritics 10'
   198    198     );
   199    199   } {1 {error in tokenizer constructor}}

Added ext/fts5/test/fts5umlaut.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5umlaut
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts5(
           27  +      x, 
           28  +      tokenize="unicode61 remove_diacritics 2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1($q)
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1('Ha Noi')
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2($q)
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2('Ha Noi')
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to ext/fts5/test/fts5unicode3.test.

    32     32   
    33     33   tl_load_casefolding_txt $CF
    34     34   foreach x [an_load_unicodedata_text $UD] {
    35     35     set aNotAlnum($x) 1
    36     36   }
    37     37   
    38     38   foreach {y} [rd_load_unicodedata_text $UD] {
    39         -  foreach {code ascii} $y {}
           39  +  foreach {code ascii f} $y {}
    40     40     if {$ascii==""} {
    41     41       set int 0
    42     42     } else {
    43     43       binary scan $ascii c int
    44     44     }
    45         -  set aDiacritic($code) $int
           45  +  set aDiacritic($code,$f) $int
           46  +  if {$f==0} { set aDiacritic($code,1) $int }
    46     47   }
    47     48   
    48     49   proc tcl_fold {i {bRemoveDiacritic 0}} {
    49     50     global tl_lookup_table
    50     51     global aDiacritic
           52  +  set f [expr $bRemoveDiacritic==2]
    51     53   
    52     54     if {[info exists tl_lookup_table($i)]} {
    53     55       set i $tl_lookup_table($i)
    54     56     }
    55         -  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
    56         -    set i $aDiacritic($i)
           57  +  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
           58  +    set i $aDiacritic($i,$f)
    57     59     }
    58     60     expr $i
    59     61   }
    60     62   db func tcl_fold tcl_fold
    61     63   
    62     64   proc tcl_isalnum {i} {
    63     65     global aNotAlnum
................................................................................
    81     83       SELECT -1
    82     84       UNION ALL
    83     85       SELECT i+1 FROM ii WHERE i<100000
    84     86     )
    85     87     SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
    86     88   } {0 {}}
    87     89   
    88         -do_execsql_test 1.2 {
           90  +do_execsql_test 1.2.1 {
    89     91     WITH ii(i) AS (
    90     92       SELECT -1
    91     93       UNION ALL
    92     94       SELECT i+1 FROM ii WHERE i<100000
    93     95     )
    94     96     SELECT count(*), min(i) FROM ii 
    95     97     WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
    96     98   } {0 {}}
           99  +
          100  +do_execsql_test 1.2.2 {
          101  +  WITH ii(i) AS (
          102  +    SELECT -1
          103  +    UNION ALL
          104  +    SELECT i+1 FROM ii WHERE i<100000
          105  +  )
          106  +  SELECT count(*), min(i) FROM ii 
          107  +  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
          108  +} {0 {}}
    97    109   
    98    110   do_execsql_test 1.3 {
    99    111     WITH ii(i) AS (
   100    112       SELECT -1
   101    113       UNION ALL
   102    114       SELECT i+1 FROM ii WHERE i<100000
   103    115     )

Changes to ext/rbu/rbu_common.tcl.

    82     82       sqlite3 tmpdb $rbu
    83     83       tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
    84     84       tmpdb close
    85     85     }
    86     86     set rc
    87     87   }
    88     88   
    89         -proc do_rbu_vacuum_test {tn step} {
    90         -  forcedelete state.db
    91         -  uplevel [list do_test $tn.1 {
    92         -    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
           89  +proc do_rbu_vacuum_test {tn step {statedb state.db}} {
           90  +  forcedelete $statedb
           91  +  if {$statedb=="" && $step==1} breakpoint
           92  +  uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
           93  +    if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
    93     94       while 1 {
    94         -      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
           95  +      if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
    95     96         set state [rbu state]
    96     97         check_prestep_state test.db $state
    97     98         set rc [rbu step]
    98     99         check_poststep_state $rc test.db $state
    99    100         if {$rc!="SQLITE_OK"} break
   100    101         if {$step==1} { rbu close }
   101    102       }
   102    103       rbu close
   103         -  } {SQLITE_DONE}]
          104  +  }] {SQLITE_DONE}]
   104    105   
   105    106     uplevel [list do_execsql_test $tn.2 {
   106    107       PRAGMA integrity_check
   107    108     } ok]
   108    109   }
   109    110   

Changes to ext/rbu/rbuvacuum2.test.

    12     12   # This file contains tests for the RBU module. More specifically, it
    13     13   # contains tests to ensure that the sqlite3rbu_vacuum() API works as
    14     14   # expected.
    15     15   #
    16     16   
    17     17   source [file join [file dirname [info script]] rbu_common.tcl]
    18     18   
    19         -foreach step {0 1} {
    20         -  set ::testprefix rbuvacuum2-$step
           19  +foreach {step} {0 1} {
           20  +foreach {ttt state} {
           21  +  s state.db t test.db-vacuum n {}
           22  +} {
           23  +  set ::testprefix rbuvacuum2-$step$ttt
    21     24     
    22     25     #-------------------------------------------------------------------------
    23     26     # Test that a database that contains fts3 tables can be vacuumed.
    24     27     #
    25     28     ifcapable fts3 {
    26     29       reset_db
    27     30       do_execsql_test 1.1 {
    28     31         CREATE VIRTUAL TABLE t1 USING fts3(z, y);
    29     32         INSERT INTO t1 VALUES('fix this issue', 'at some point');
    30     33       }
    31     34     
    32         -    do_rbu_vacuum_test 1.2 $step
           35  +    do_rbu_vacuum_test 1.2 $step $state
    33     36     
    34     37       do_execsql_test 1.3 {
    35     38         SELECT * FROM t1;
    36     39       } {{fix this issue} {at some point}}
    37     40     
    38     41       do_execsql_test 1.4 {
    39     42         SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
................................................................................
    42     45       do_execsql_test 1.5 {
    43     46         INSERT INTO t1 VALUES('a b c', 'd e f');
    44     47         INSERT INTO t1 VALUES('l h i', 'd e f');
    45     48         DELETE FROM t1 WHERE docid = 2;
    46     49         INSERT INTO t1 VALUES('a b c', 'x y z');
    47     50       }
    48     51   
    49         -    do_rbu_vacuum_test 1.6 $step
           52  +    do_rbu_vacuum_test 1.6 $step $state
    50     53       do_execsql_test 1.7 {
    51     54         INSERT INTO t1(t1) VALUES('integrity-check');
    52     55         SELECT * FROM t1;
    53     56       } {
    54     57         {fix this issue} {at some point}
    55     58         {l h i} {d e f}
    56     59         {a b c} {x y z}
................................................................................
    63     66     ifcapable fts5 {
    64     67       reset_db
    65     68       do_execsql_test 2.1 {
    66     69         CREATE VIRTUAL TABLE t1 USING fts5(z, y);
    67     70         INSERT INTO t1 VALUES('fix this issue', 'at some point');
    68     71       }
    69     72     
    70         -    do_rbu_vacuum_test 2.2 $step
           73  +    do_rbu_vacuum_test 2.2 $step $state
    71     74     
    72     75       do_execsql_test 2.3 {
    73     76         SELECT * FROM t1;
    74     77       } {{fix this issue} {at some point}}
    75     78     
    76     79       do_execsql_test 2.4 {
    77     80         SELECT rowid FROM t1 ('fix');
................................................................................
    80     83       do_execsql_test 2.5 {
    81     84         INSERT INTO t1 VALUES('a b c', 'd e f');
    82     85         INSERT INTO t1 VALUES('l h i', 'd e f');
    83     86         DELETE FROM t1 WHERE rowid = 2;
    84     87         INSERT INTO t1 VALUES('a b c', 'x y z');
    85     88       }
    86     89   
    87         -    do_rbu_vacuum_test 2.6 $step
           90  +    do_rbu_vacuum_test 2.6 $step $state
    88     91       do_execsql_test 2.7 {
    89     92         INSERT INTO t1(t1) VALUES('integrity-check');
    90     93         SELECT * FROM t1;
    91     94       } {
    92     95         {fix this issue} {at some point}
    93     96         {l h i} {d e f}
    94     97         {a b c} {x y z}
................................................................................
   103    106       do_execsql_test 3.1 {
   104    107         CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
   105    108         INSERT INTO rt VALUES(1, 45, 55);
   106    109         INSERT INTO rt VALUES(2, 50, 60);
   107    110         INSERT INTO rt VALUES(3, 55, 65);
   108    111       }
   109    112     
   110         -    do_rbu_vacuum_test 3.2 $step
          113  +    do_rbu_vacuum_test 3.2 $step $state
   111    114     
   112    115       do_execsql_test 3.3 {
   113    116         SELECT * FROM rt;
   114    117       } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
   115    118     
   116    119       do_execsql_test 3.4.1 {
   117    120         SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
   118    121       } {1 2}
   119    122       do_execsql_test 3.4.2 {
   120    123         SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
   121    124       } {2 3}
   122    125   
   123         -    do_rbu_vacuum_test 3.5 $step
          126  +    do_rbu_vacuum_test 3.5 $step $state
   124    127   
   125    128       do_execsql_test 3.6.1 {
   126    129         SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
   127    130       } {1 2}
   128    131       do_execsql_test 3.6.2 {
   129    132         SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
   130    133       } {2 3}
................................................................................
   143    146         SELECT * FROM sqlite_master;
   144    147       } {
   145    148       table t1 t1 2 {CREATE TABLE t1(a, b, c)}
   146    149       view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
   147    150       trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
   148    151       }
   149    152   
   150         -    do_rbu_vacuum_test 4.3 $step
          153  +    do_rbu_vacuum_test 4.3 $step $state
   151    154       do_execsql_test 4.4 {
   152    155         SELECT * FROM sqlite_master;
   153    156       } {
   154    157       table t1 t1 2 {CREATE TABLE t1(a, b, c)}
   155    158       view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
   156    159       trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
   157    160       }
   158    161     }
          162  +}
   159    163   }
   160    164     
   161    165   #-------------------------------------------------------------------------
   162    166   # Test that passing a NULL value as the second argument to 
   163    167   # sqlite3rbu_vacuum() causes it to:
   164    168   #
   165    169   #   * Use <database>-vacuum as the state db, and
................................................................................
   227    231   do_test 6.3 {
   228    232     sqlite3rbu_vacuum rbu test.db test.db2
   229    233     while {[rbu step]!="SQLITE_DONE"} { rbu step }
   230    234     rbu close
   231    235     execsql { PRAGMA integrity_check }
   232    236   } {ok}
   233    237   
          238  +do_test 6.4 {
          239  +  sqlite3rbu_vacuum rbu test.db test.db-vactmp
          240  +  list [catch { rbu close } msg] $msg
          241  +} {1 SQLITE_MISUSE}
          242  +
   234    243   finish_test

Changes to ext/rbu/sqlite3rbu.c.

  2473   2473           zExtra = &p->zRbu[5];
  2474   2474           while( *zExtra ){
  2475   2475             if( *zExtra++=='?' ) break;
  2476   2476           }
  2477   2477           if( *zExtra=='\0' ) zExtra = 0;
  2478   2478         }
  2479   2479   
  2480         -      zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", 
         2480  +      zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", 
  2481   2481             sqlite3_db_filename(p->dbRbu, "main"),
  2482   2482             (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
  2483   2483         );
  2484   2484   
  2485   2485         if( zTarget==0 ){
  2486   2486           p->rc = SQLITE_NOMEM;
  2487   2487           return;
................................................................................
  3739   3739   ** Open a handle to begin or resume an RBU VACUUM operation.
  3740   3740   */
  3741   3741   sqlite3rbu *sqlite3rbu_vacuum(
  3742   3742     const char *zTarget, 
  3743   3743     const char *zState
  3744   3744   ){
  3745   3745     if( zTarget==0 ){ return rbuMisuseError(); }
         3746  +  if( zState ){
         3747  +    int n = strlen(zState);
         3748  +    if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
         3749  +      return rbuMisuseError();
         3750  +    }
         3751  +  }
  3746   3752     /* TODO: Check that both arguments are non-NULL */
  3747   3753     return openRbuHandle(0, zTarget, zState);
  3748   3754   }
  3749   3755   
  3750   3756   /*
  3751   3757   ** Return the database handle used by pRbu.
  3752   3758   */

Changes to ext/rbu/sqlite3rbu.h.

   329    329   ** The vacuum can be resumed by calling this function to open a new RBU
   330    330   ** handle specifying the same target and state databases.
   331    331   **
   332    332   ** If the second argument passed to this function is NULL, then the
   333    333   ** name of the state database is "<database>-vacuum", where <database>
   334    334   ** is the name of the target database file. In this case, on UNIX, if the
   335    335   ** state database is not already present in the file-system, it is created
   336         -** with the same permissions as the target db is made.
          336  +** with the same permissions as the target db is made. 
          337  +**
          338  +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the 
          339  +** state database ends with "-vactmp". This name is reserved for internal 
          340  +** use.
   337    341   **
   338    342   ** This function does not delete the state database after an RBU vacuum
   339    343   ** is completed, even if it created it. However, if the call to
   340    344   ** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
   341    345   ** of the state tables within the state database are zeroed. This way,
   342    346   ** the next call to sqlite3rbu_vacuum() opens a handle that starts a 
   343    347   ** new RBU vacuum operation.

Changes to ext/rbu/test_rbu.c.

   269    269     if( objc!=3 && objc!=4 ){
   270    270       Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?");
   271    271       return TCL_ERROR;
   272    272     }
   273    273     zCmd = Tcl_GetString(objv[1]);
   274    274     zTarget = Tcl_GetString(objv[2]);
   275    275     if( objc==4 ) zStateDb = Tcl_GetString(objv[3]);
          276  +  if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0;
   276    277   
   277    278     pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
   278    279     Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
   279    280     Tcl_SetObjResult(interp, objv[1]);
   280    281     return TCL_OK;
   281    282   }
   282    283   

Changes to main.mk.

   991    991   
   992    992   showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
   993    993   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
   994    994   		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
   995    995   
   996    996   showshm$(EXE):	$(TOP)/tool/showshm.c
   997    997   	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
          998  +
          999  +index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
         1000  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
         1001  +		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)
   998   1002   
   999   1003   changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
  1000   1004   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
  1001   1005   		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
  1002   1006   
  1003   1007   changesetfuzz$(EXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.o
  1004   1008   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changesetfuzz$(EXE) \

Changes to src/alter.c.

   775    775         pToken->pNext = pCtx->pList;
   776    776         pCtx->pList = pToken;
   777    777         pCtx->nList++;
   778    778         break;
   779    779       }
   780    780     }
   781    781   }
          782  +
          783  +/*
          784  +** Iterate through the Select objects that are part of WITH clauses attached
          785  +** to select statement pSelect.
          786  +*/
          787  +static void renameWalkWith(Walker *pWalker, Select *pSelect){
          788  +  if( pSelect->pWith ){
          789  +    int i;
          790  +    for(i=0; i<pSelect->pWith->nCte; i++){
          791  +      Select *p = pSelect->pWith->a[i].pSelect;
          792  +      NameContext sNC;
          793  +      memset(&sNC, 0, sizeof(sNC));
          794  +      sNC.pParse = pWalker->pParse;
          795  +      sqlite3SelectPrep(sNC.pParse, p, &sNC);
          796  +      sqlite3WalkSelect(pWalker, p);
          797  +    }
          798  +  }
          799  +}
   782    800   
   783    801   /*
   784    802   ** This is a Walker select callback. It does nothing. It is only required
   785    803   ** because without a dummy callback, sqlite3WalkExpr() and similar do not
   786    804   ** descend into sub-select statements.
   787    805   */
   788    806   static int renameColumnSelectCb(Walker *pWalker, Select *p){
   789         -  UNUSED_PARAMETER(pWalker);
   790         -  UNUSED_PARAMETER(p);
          807  +  renameWalkWith(pWalker, p);
   791    808     return WRC_Continue;
   792    809   }
   793    810   
   794    811   /*
   795    812   ** This is a Walker expression callback.
   796    813   **
   797    814   ** For every TK_COLUMN node in the expression tree, search to see
................................................................................
  1354   1371   /*
  1355   1372   ** Walker select callback used by "RENAME TABLE". 
  1356   1373   */
  1357   1374   static int renameTableSelectCb(Walker *pWalker, Select *pSelect){
  1358   1375     int i;
  1359   1376     RenameCtx *p = pWalker->u.pRename;
  1360   1377     SrcList *pSrc = pSelect->pSrc;
         1378  +  if( pSrc==0 ){
         1379  +    assert( pWalker->pParse->db->mallocFailed );
         1380  +    return WRC_Abort;
         1381  +  }
  1361   1382     for(i=0; i<pSrc->nSrc; i++){
  1362   1383       struct SrcList_item *pItem = &pSrc->a[i];
  1363   1384       if( pItem->pTab==p->pTab ){
  1364   1385         renameTokenFind(pWalker->pParse, p, pItem->zName);
  1365   1386       }
  1366   1387     }
         1388  +  renameWalkWith(pWalker, pSelect);
  1367   1389   
  1368   1390     return WRC_Continue;
  1369   1391   }
  1370   1392   
  1371   1393   
  1372   1394   /*
  1373   1395   ** This C function implements an SQL user function that is used by SQL code

Changes to src/btree.c.

  9852   9852     }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){
  9853   9853       checkAppendMsg(&sCheck,
  9854   9854         "incremental_vacuum enabled with a max rootpage of zero"
  9855   9855       );
  9856   9856     }
  9857   9857   #endif
  9858   9858     testcase( pBt->db->flags & SQLITE_CellSizeCk );
  9859         -  pBt->db->flags &= ~SQLITE_CellSizeCk;
         9859  +  pBt->db->flags &= ~(u64)SQLITE_CellSizeCk;
  9860   9860     for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
  9861   9861       i64 notUsed;
  9862   9862       if( aRoot[i]==0 ) continue;
  9863   9863   #ifndef SQLITE_OMIT_AUTOVACUUM
  9864   9864       if( pBt->autoVacuum && aRoot[i]>1 ){
  9865   9865         checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
  9866   9866       }

Changes to src/build.c.

   223    223   
   224    224   
   225    225     /* Get the VDBE program ready for execution
   226    226     */
   227    227     if( v && pParse->nErr==0 && !db->mallocFailed ){
   228    228       /* A minimum of one cursor is required if autoincrement is used
   229    229       *  See ticket [a696379c1f08866] */
   230         -    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
          230  +    assert( pParse->pAinc==0 || pParse->nTab>0 );
   231    231       sqlite3VdbeMakeReady(v, pParse);
   232    232       pParse->rc = SQLITE_DONE;
   233    233     }else{
   234    234       pParse->rc = SQLITE_ERROR;
   235    235     }
   236    236   }
   237    237   
................................................................................
   632    632     }
   633    633   
   634    634     /* Delete any foreign keys attached to this table. */
   635    635     sqlite3FkDelete(db, pTable);
   636    636   
   637    637     /* Delete the Table structure itself.
   638    638     */
   639         -#ifdef SQLITE_ENABLE_NORMALIZE
   640         -  if( pTable->pColHash ){
   641         -    sqlite3HashClear(pTable->pColHash);
   642         -    sqlite3_free(pTable->pColHash);
   643         -  }
   644         -#endif
   645    639     sqlite3DeleteColumnNames(db, pTable);
   646    640     sqlite3DbFree(db, pTable->zName);
   647    641     sqlite3DbFree(db, pTable->zColAff);
   648    642     sqlite3SelectDelete(db, pTable->pSelect);
   649    643     sqlite3ExprListDelete(db, pTable->pCheck);
   650    644   #ifndef SQLITE_OMIT_VIRTUALTABLE
   651    645     sqlite3VtabClear(db, pTable);

Changes to src/callback.c.

   279    279     return match;
   280    280   }
   281    281   
   282    282   /*
   283    283   ** Search a FuncDefHash for a function with the given name.  Return
   284    284   ** a pointer to the matching FuncDef if found, or 0 if there is no match.
   285    285   */
   286         -static FuncDef *functionSearch(
          286  +FuncDef *sqlite3FunctionSearch(
   287    287     int h,               /* Hash of the name */
   288    288     const char *zFunc    /* Name of function */
   289    289   ){
   290    290     FuncDef *p;
   291    291     for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
   292    292       if( sqlite3StrICmp(p->zName, zFunc)==0 ){
   293    293         return p;
   294    294       }
   295    295     }
   296    296     return 0;
   297    297   }
   298         -#ifdef SQLITE_ENABLE_NORMALIZE
   299         -FuncDef *sqlite3FunctionSearchN(
   300         -  int h,               /* Hash of the name */
   301         -  const char *zFunc,   /* Name of function */
   302         -  int nFunc            /* Length of the name */
   303         -){
   304         -  FuncDef *p;
   305         -  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
   306         -    if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
   307         -      return p;
   308         -    }
   309         -  }
   310         -  return 0;
   311         -}
   312         -#endif /* SQLITE_ENABLE_NORMALIZE */
   313    298   
   314    299   /*
   315    300   ** Insert a new FuncDef into a FuncDefHash hash table.
   316    301   */
   317    302   void sqlite3InsertBuiltinFuncs(
   318    303     FuncDef *aDef,      /* List of global functions to be inserted */
   319    304     int nDef            /* Length of the apDef[] list */
................................................................................
   321    306     int i;
   322    307     for(i=0; i<nDef; i++){
   323    308       FuncDef *pOther;
   324    309       const char *zName = aDef[i].zName;
   325    310       int nName = sqlite3Strlen30(zName);
   326    311       int h = SQLITE_FUNC_HASH(zName[0], nName);
   327    312       assert( zName[0]>='a' && zName[0]<='z' );
   328         -    pOther = functionSearch(h, zName);
          313  +    pOther = sqlite3FunctionSearch(h, zName);
   329    314       if( pOther ){
   330    315         assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] );
   331    316         aDef[i].pNext = pOther->pNext;
   332    317         pOther->pNext = &aDef[i];
   333    318       }else{
   334    319         aDef[i].pNext = 0;
   335    320         aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h];
................................................................................
   399    384     ** have fields overwritten with new information appropriate for the
   400    385     ** new function.  But the FuncDefs for built-in functions are read-only.
   401    386     ** So we must not search for built-ins when creating a new function.
   402    387     */ 
   403    388     if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
   404    389       bestScore = 0;
   405    390       h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
   406         -    p = functionSearch(h, zName);
          391  +    p = sqlite3FunctionSearch(h, zName);
   407    392       while( p ){
   408    393         int score = matchQuality(p, nArg, enc);
   409    394         if( score>bestScore ){
   410    395           pBest = p;
   411    396           bestScore = score;
   412    397         }
   413    398         p = p->pNext;

Changes to src/expr.c.

  1325   1325       }
  1326   1326     }
  1327   1327     return pRet;
  1328   1328   }
  1329   1329   #else
  1330   1330   # define withDup(x,y) 0
  1331   1331   #endif
         1332  +
         1333  +#ifndef SQLITE_OMIT_WINDOWFUNC
         1334  +/*
         1335  +** The gatherSelectWindows() procedure and its helper routine
         1336  +** gatherSelectWindowsCallback() are used to scan all the expressions
         1337  +** an a newly duplicated SELECT statement and gather all of the Window
         1338  +** objects found there, assembling them onto the linked list at Select->pWin.
         1339  +*/
         1340  +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){
         1341  +  if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){
         1342  +    assert( ExprHasProperty(pExpr, EP_WinFunc) );
         1343  +    pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin;
         1344  +    pWalker->u.pSelect->pWin = pExpr->y.pWin;
         1345  +  }
         1346  +  return WRC_Continue;
         1347  +}
         1348  +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){
         1349  +  return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune;
         1350  +}
         1351  +static void gatherSelectWindows(Select *p){
         1352  +  Walker w;
         1353  +  w.xExprCallback = gatherSelectWindowsCallback;
         1354  +  w.xSelectCallback = gatherSelectWindowsSelectCallback;
         1355  +  w.xSelectCallback2 = 0;
         1356  +  w.u.pSelect = p;
         1357  +  sqlite3WalkSelect(&w, p);
         1358  +}
         1359  +#endif
         1360  +
  1332   1361   
  1333   1362   /*
  1334   1363   ** The following group of routines make deep copies of expressions,
  1335   1364   ** expression lists, ID lists, and select statements.  The copies can
  1336   1365   ** be deleted (by being passed to their respective ...Delete() routines)
  1337   1366   ** without effecting the originals.
  1338   1367   **
................................................................................
  1493   1522       pNew->addrOpenEphm[0] = -1;
  1494   1523       pNew->addrOpenEphm[1] = -1;
  1495   1524       pNew->nSelectRow = p->nSelectRow;
  1496   1525       pNew->pWith = withDup(db, p->pWith);
  1497   1526   #ifndef SQLITE_OMIT_WINDOWFUNC
  1498   1527       pNew->pWin = 0;
  1499   1528       pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn);
         1529  +    if( p->pWin ) gatherSelectWindows(pNew);
  1500   1530   #endif
  1501   1531       pNew->selId = p->selId;
  1502   1532       *pp = pNew;
  1503   1533       pp = &pNew->pPrior;
  1504   1534       pNext = pNew;
  1505   1535     }
  1506   1536   
................................................................................
  2145   2175   */
  2146   2176   int sqlite3IsRowid(const char *z){
  2147   2177     if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
  2148   2178     if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
  2149   2179     if( sqlite3StrICmp(z, "OID")==0 ) return 1;
  2150   2180     return 0;
  2151   2181   }
  2152         -#ifdef SQLITE_ENABLE_NORMALIZE
  2153         -int sqlite3IsRowidN(const char *z, int n){
  2154         -  if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1;
  2155         -  if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1;
  2156         -  if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1;
  2157         -  return 0;
  2158         -}
  2159         -#endif
  2160   2182   
  2161   2183   /*
  2162   2184   ** pX is the RHS of an IN operator.  If pX is a SELECT statement 
  2163   2185   ** that can be simplified to a direct table access, then return
  2164   2186   ** a pointer to the SELECT statement.  If pX is not a SELECT statement,
  2165   2187   ** or if the SELECT statement needs to be manifested into a transient
  2166   2188   ** table, then return NULL.
................................................................................
  2417   2439   
  2418   2440         if( affinity_ok ){
  2419   2441           /* Search for an existing index that will work for this IN operator */
  2420   2442           for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){
  2421   2443             Bitmask colUsed;      /* Columns of the index used */
  2422   2444             Bitmask mCol;         /* Mask for the current column */
  2423   2445             if( pIdx->nColumn<nExpr ) continue;
         2446  +          if( pIdx->pPartIdxWhere!=0 ) continue;
  2424   2447             /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute
  2425   2448             ** BITMASK(nExpr) without overflowing */
  2426   2449             testcase( pIdx->nColumn==BMS-2 );
  2427   2450             testcase( pIdx->nColumn==BMS-1 );
  2428   2451             if( pIdx->nColumn>=BMS-1 ) continue;
  2429   2452             if( mustBeUnique ){
  2430   2453               if( pIdx->nKeyCol>nExpr

Changes to src/hash.c.

    60     60       ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    61     61       ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    62     62       h += sqlite3UpperToLower[c];
    63     63       h *= 0x9e3779b1;
    64     64     }
    65     65     return h;
    66     66   }
    67         -#ifdef SQLITE_ENABLE_NORMALIZE
    68         -static unsigned int strHashN(const char *z, int n){
    69         -  unsigned int h = 0;
    70         -  int i;
    71         -  for(i=0; i<n; i++){
    72         -    /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
    73         -    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    74         -    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    75         -    h += sqlite3UpperToLower[z[i]];
    76         -    h *= 0x9e3779b1;
    77         -  }
    78         -  return h;
    79         -}
    80         -#endif /* SQLITE_ENABLE_NORMALIZE */
    81     67   
    82     68   
    83     69   /* Link pNew element into the hash table pH.  If pEntry!=0 then also
    84     70   ** insert pNew into the pEntry hash bucket.
    85     71   */
    86     72   static void insertElement(
    87     73     Hash *pH,              /* The complete hash table */
................................................................................
   185    171       if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ 
   186    172         return elem;
   187    173       }
   188    174       elem = elem->next;
   189    175     }
   190    176     return &nullElement;
   191    177   }
   192         -#ifdef SQLITE_ENABLE_NORMALIZE
   193         -static HashElem *findElementWithHashN(
   194         -  const Hash *pH,     /* The pH to be searched */
   195         -  const char *pKey,   /* The key we are searching for */
   196         -  int nKey,           /* Number of key bytes to use */
   197         -  unsigned int *pHash /* Write the hash value here */
   198         -){
   199         -  HashElem *elem;                /* Used to loop thru the element list */
   200         -  int count;                     /* Number of elements left to test */
   201         -  unsigned int h;                /* The computed hash */
   202         -  static HashElem nullElement = { 0, 0, 0, 0 };
   203         -
   204         -  if( pH->ht ){   /*OPTIMIZATION-IF-TRUE*/
   205         -    struct _ht *pEntry;
   206         -    h = strHashN(pKey, nKey) % pH->htsize;
   207         -    pEntry = &pH->ht[h];
   208         -    elem = pEntry->chain;
   209         -    count = pEntry->count;
   210         -  }else{
   211         -    h = 0;
   212         -    elem = pH->first;
   213         -    count = pH->count;
   214         -  }
   215         -  if( pHash ) *pHash = h;
   216         -  while( count-- ){
   217         -    assert( elem!=0 );
   218         -    if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ 
   219         -      return elem;
   220         -    }
   221         -    elem = elem->next;
   222         -  }
   223         -  return &nullElement;
   224         -}
   225         -#endif /* SQLITE_ENABLE_NORMALIZE */
   226    178   
   227    179   /* Remove a single entry from the hash table given a pointer to that
   228    180   ** element and a hash on the element's key.
   229    181   */
   230    182   static void removeElementGivenHash(
   231    183     Hash *pH,         /* The pH containing "elem" */
   232    184     HashElem* elem,   /* The element to be removed from the pH */
................................................................................
   263    215   ** found, or NULL if there is no match.
   264    216   */
   265    217   void *sqlite3HashFind(const Hash *pH, const char *pKey){
   266    218     assert( pH!=0 );
   267    219     assert( pKey!=0 );
   268    220     return findElementWithHash(pH, pKey, 0)->data;
   269    221   }
   270         -#ifdef SQLITE_ENABLE_NORMALIZE
   271         -void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
   272         -  assert( pH!=0 );
   273         -  assert( pKey!=0 );
   274         -  assert( nKey>=0 );
   275         -  return findElementWithHashN(pH, pKey, nKey, 0)->data;
   276         -}
   277         -#endif /* SQLITE_ENABLE_NORMALIZE */
   278    222   
   279    223   /* Insert an element into the hash table pH.  The key is pKey
   280    224   ** and the data is "data".
   281    225   **
   282    226   ** If no element exists with a matching key, then a new
   283    227   ** element is created and NULL is returned.
   284    228   **

Changes to src/insert.c.

   315    315       aOp[3].p5 = SQLITE_JUMPIFNULL;
   316    316       aOp[4].p2 = memId+1;
   317    317       aOp[5].p3 = memId;
   318    318       aOp[6].p1 = memId;
   319    319       aOp[7].p2 = memId+2;
   320    320       aOp[7].p1 = memId;
   321    321       aOp[10].p2 = memId;
          322  +    if( pParse->nTab==0 ) pParse->nTab = 1;
   322    323     }
   323    324   }
   324    325   
   325    326   /*
   326    327   ** Update the maximum rowid for an autoincrement calculation.
   327    328   **
   328    329   ** This routine should be called when the regRowid register holds a

Changes to src/loadext.c.

   646    646   ** default so as not to open security holes in older applications.
   647    647   */
   648    648   int sqlite3_enable_load_extension(sqlite3 *db, int onoff){
   649    649     sqlite3_mutex_enter(db->mutex);
   650    650     if( onoff ){
   651    651       db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc;
   652    652     }else{
   653         -    db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
          653  +    db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
   654    654     }
   655    655     sqlite3_mutex_leave(db->mutex);
   656    656     return SQLITE_OK;
   657    657   }
   658    658   
   659    659   #endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */
   660    660   

Changes to src/main.c.

   839    839         };
   840    840         unsigned int i;
   841    841         rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
   842    842         for(i=0; i<ArraySize(aFlagOp); i++){
   843    843           if( aFlagOp[i].op==op ){
   844    844             int onoff = va_arg(ap, int);
   845    845             int *pRes = va_arg(ap, int*);
   846         -          u32 oldFlags = db->flags;
          846  +          u64 oldFlags = db->flags;
   847    847             if( onoff>0 ){
   848    848               db->flags |= aFlagOp[i].mask;
   849    849             }else if( onoff==0 ){
   850         -            db->flags &= ~aFlagOp[i].mask;
          850  +            db->flags &= ~(u64)aFlagOp[i].mask;
   851    851             }
   852    852             if( oldFlags!=db->flags ){
   853    853               sqlite3ExpirePreparedStatements(db, 0);
   854    854             }
   855    855             if( pRes ){
   856    856               *pRes = (db->flags & aFlagOp[i].mask)!=0;
   857    857             }
................................................................................
  1306   1306       sqlite3ResetAllSchemasOfConnection(db);
  1307   1307     }
  1308   1308     sqlite3BtreeLeaveAll(db);
  1309   1309   
  1310   1310     /* Any deferred constraint violations have now been resolved. */
  1311   1311     db->nDeferredCons = 0;
  1312   1312     db->nDeferredImmCons = 0;
  1313         -  db->flags &= ~SQLITE_DeferFKs;
         1313  +  db->flags &= ~(u64)SQLITE_DeferFKs;
  1314   1314   
  1315   1315     /* If one has been configured, invoke the rollback-hook callback */
  1316   1316     if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
  1317   1317       db->xRollbackCallback(db->pRollbackArg);
  1318   1318     }
  1319   1319   }
  1320   1320   
................................................................................
  2048   2048       return 0;
  2049   2049     }
  2050   2050   #endif
  2051   2051     sqlite3_mutex_enter(db->mutex);
  2052   2052     pOld = db->pProfileArg;
  2053   2053     db->xProfile = xProfile;
  2054   2054     db->pProfileArg = pArg;
         2055  +  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
         2056  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  2055   2057     sqlite3_mutex_leave(db->mutex);
  2056   2058     return pOld;
  2057   2059   }
  2058   2060   #endif /* SQLITE_OMIT_DEPRECATED */
  2059   2061   #endif /* SQLITE_OMIT_TRACE */
  2060   2062   
  2061   2063   /*
................................................................................
  2399   2401       return sqlite3ErrStr(SQLITE_MISUSE_BKPT);
  2400   2402     }
  2401   2403     sqlite3_mutex_enter(db->mutex);
  2402   2404     if( db->mallocFailed ){
  2403   2405       z = sqlite3ErrStr(SQLITE_NOMEM_BKPT);
  2404   2406     }else{
  2405   2407       testcase( db->pErr==0 );
  2406         -    z = (char*)sqlite3_value_text(db->pErr);
         2408  +    z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0;
  2407   2409       assert( !db->mallocFailed );
  2408   2410       if( z==0 ){
  2409   2411         z = sqlite3ErrStr(db->errCode);
  2410   2412       }
  2411   2413     }
  2412   2414     sqlite3_mutex_leave(db->mutex);
  2413   2415     return z;

Changes to src/parse.y.

   660    660         A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,N,U);
   661    661         if( A ){
   662    662           struct SrcList_item *pNew = &A->a[A->nSrc-1];
   663    663           struct SrcList_item *pOld = F->a;
   664    664           pNew->zName = pOld->zName;
   665    665           pNew->zDatabase = pOld->zDatabase;
   666    666           pNew->pSelect = pOld->pSelect;
          667  +        if( pOld->fg.isTabFunc ){
          668  +          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
          669  +          pOld->u1.pFuncArg = 0;
          670  +          pOld->fg.isTabFunc = 0;
          671  +          pNew->fg.isTabFunc = 1;
          672  +        }
   667    673           pOld->zName = pOld->zDatabase = 0;
   668    674           pOld->pSelect = 0;
   669    675         }
   670    676         sqlite3SrcListDelete(pParse->db, F);
   671    677       }else{
   672    678         Select *pSubquery;
   673    679         sqlite3SrcListShiftJoinType(F);
................................................................................
  1357   1363   //
  1358   1364   cmd ::= DROP INDEX ifexists(E) fullname(X).   {sqlite3DropIndex(pParse, X, E);}
  1359   1365   
  1360   1366   ///////////////////////////// The VACUUM command /////////////////////////////
  1361   1367   //
  1362   1368   %ifndef SQLITE_OMIT_VACUUM
  1363   1369   %ifndef SQLITE_OMIT_ATTACH
  1364         -cmd ::= VACUUM.                {sqlite3Vacuum(pParse,0);}
  1365         -cmd ::= VACUUM nm(X).          {sqlite3Vacuum(pParse,&X);}
         1370  +%type vinto {Expr*}
         1371  +%destructor vinto {sqlite3ExprDelete(pParse->db, $$);}
         1372  +cmd ::= VACUUM vinto(Y).                {sqlite3Vacuum(pParse,0,Y);}
         1373  +cmd ::= VACUUM nm(X) vinto(Y).          {sqlite3Vacuum(pParse,&X,Y);}
         1374  +vinto(A) ::= INTO expr(X).              {A = X;}
         1375  +vinto(A) ::= .                          {A = 0;}
  1366   1376   %endif  SQLITE_OMIT_ATTACH
  1367   1377   %endif  SQLITE_OMIT_VACUUM
  1368   1378   
  1369   1379   ///////////////////////////// The PRAGMA command /////////////////////////////
  1370   1380   //
  1371   1381   %ifndef SQLITE_OMIT_PRAGMA
  1372   1382   cmd ::= PRAGMA nm(X) dbnm(Z).                {sqlite3Pragma(pParse,&X,&Z,0,0);}

Changes to src/pcache1.c.

    98     98     unsigned int iKey;             /* Key value (page number) */
    99     99     u8 isBulkLocal;                /* This page from bulk local storage */
   100    100     u8 isAnchor;                   /* This is the PGroup.lru element */
   101    101     PgHdr1 *pNext;                 /* Next in hash table chain */
   102    102     PCache1 *pCache;               /* Cache that currently owns this page */
   103    103     PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
   104    104     PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
          105  +                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
   105    106   };
   106    107   
   107    108   /*
   108    109   ** A page is pinned if it is not on the LRU list.  To be "pinned" means
   109    110   ** that the page is in active use and must not be deallocated.
   110    111   */
   111    112   #define PAGE_IS_PINNED(p)    ((p)->pLruNext==0)
................................................................................
   566    567     assert( PAGE_IS_UNPINNED(pPage) );
   567    568     assert( pPage->pLruNext );
   568    569     assert( pPage->pLruPrev );
   569    570     assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
   570    571     pPage->pLruPrev->pLruNext = pPage->pLruNext;
   571    572     pPage->pLruNext->pLruPrev = pPage->pLruPrev;
   572    573     pPage->pLruNext = 0;
   573         -  pPage->pLruPrev = 0;
          574  +  /* pPage->pLruPrev = 0;
          575  +  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
   574    576     assert( pPage->isAnchor==0 );
   575    577     assert( pPage->pCache->pGroup->lru.isAnchor==1 );
   576    578     pPage->pCache->nRecyclable--;
   577    579     return pPage;
   578    580   }
   579    581   
   580    582   
................................................................................
   904    906   
   905    907     if( pPage ){
   906    908       unsigned int h = iKey % pCache->nHash;
   907    909       pCache->nPage++;
   908    910       pPage->iKey = iKey;
   909    911       pPage->pNext = pCache->apHash[h];
   910    912       pPage->pCache = pCache;
   911         -    pPage->pLruPrev = 0;
   912    913       pPage->pLruNext = 0;
          914  +    /* pPage->pLruPrev = 0;
          915  +    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
   913    916       *(void **)pPage->page.pExtra = 0;
   914    917       pCache->apHash[h] = pPage;
   915    918       if( iKey>pCache->iMaxKey ){
   916    919         pCache->iMaxKey = iKey;
   917    920       }
   918    921     }
   919    922     return pPage;
................................................................................
  1065   1068    
  1066   1069     assert( pPage->pCache==pCache );
  1067   1070     pcache1EnterMutex(pGroup);
  1068   1071   
  1069   1072     /* It is an error to call this function if the page is already 
  1070   1073     ** part of the PGroup LRU list.
  1071   1074     */
  1072         -  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
         1075  +  assert( pPage->pLruNext==0 );
  1073   1076     assert( PAGE_IS_PINNED(pPage) );
  1074   1077   
  1075   1078     if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
  1076   1079       pcache1RemoveFromHash(pPage, 1);
  1077   1080     }else{
  1078   1081       /* Add the page to the PGroup LRU list. */
  1079   1082       PgHdr1 **ppFirst = &pGroup->lru.pLruNext;

Changes to src/pragma.c.

   813    813         int size = 1;
   814    814         if( sqlite3GetInt32(zRight, &size) ){
   815    815           sqlite3BtreeSetSpillSize(pDb->pBt, size);
   816    816         }
   817    817         if( sqlite3GetBoolean(zRight, size!=0) ){
   818    818           db->flags |= SQLITE_CacheSpill;
   819    819         }else{
   820         -        db->flags &= ~SQLITE_CacheSpill;
          820  +        db->flags &= ~(u64)SQLITE_CacheSpill;
   821    821         }
   822    822         setAllPagerFlags(db);
   823    823       }
   824    824       break;
   825    825     }
   826    826   
   827    827     /*

Changes to src/prepare.c.

   289    289   
   290    290     /* Ticket #2804:  When we open a database in the newer file format,
   291    291     ** clear the legacy_file_format pragma flag so that a VACUUM will
   292    292     ** not downgrade the database and thus invalidate any descending
   293    293     ** indices that the user might have created.
   294    294     */
   295    295     if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){
   296         -    db->flags &= ~SQLITE_LegacyFileFmt;
          296  +    db->flags &= ~(u64)SQLITE_LegacyFileFmt;
   297    297     }
   298    298   
   299    299     /* Read the schema information out of the schema tables
   300    300     */
   301    301     assert( db->init.busy );
   302    302     {
   303    303       char *zSql;
................................................................................
   705    705     sqlite3BtreeLeaveAll(db);
   706    706     rc = sqlite3ApiExit(db, rc);
   707    707     assert( (rc&db->errMask)==rc );
   708    708     sqlite3_mutex_leave(db->mutex);
   709    709     return rc;
   710    710   }
   711    711   
   712         -#ifdef SQLITE_ENABLE_NORMALIZE
   713         -/*
   714         -** Checks if the specified token is a table, column, or function name,
   715         -** based on the databases associated with the statement being prepared.
   716         -** If the function fails, zero is returned and pRc is filled with the
   717         -** error code.
   718         -*/
   719         -static int shouldTreatAsIdentifier(
   720         -  sqlite3 *db,        /* Database handle. */
   721         -  const char *zToken, /* Pointer to start of token to be checked */
   722         -  int nToken,         /* Length of token to be checked */
   723         -  int *pRc            /* Pointer to error code upon failure */
   724         -){
   725         -  int bFound = 0;     /* Non-zero if token is an identifier name. */
   726         -  int i, j;           /* Database and column loop indexes. */
   727         -  Schema *pSchema;    /* Schema for current database. */
   728         -  Hash *pHash;        /* Hash table of tables for current database. */
   729         -  HashElem *e;        /* Hash element for hash table iteration. */
   730         -  Table *pTab;        /* Database table for columns being checked. */
   731         -
   732         -  if( sqlite3IsRowidN(zToken, nToken) ){
   733         -    return 1;
   734         -  }
   735         -  if( nToken>0 ){
   736         -    int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
   737         -    if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
   738         -  }
   739         -  assert( db!=0 );
   740         -  sqlite3_mutex_enter(db->mutex);
   741         -  sqlite3BtreeEnterAll(db);
   742         -  for(i=0; i<db->nDb; i++){
   743         -    pHash = &db->aFunc;
   744         -    if( sqlite3HashFindN(pHash, zToken, nToken) ){
   745         -      bFound = 1;
   746         -      break;
   747         -    }
   748         -    pSchema = db->aDb[i].pSchema;
   749         -    if( pSchema==0 ) continue;
   750         -    pHash = &pSchema->tblHash;
   751         -    if( sqlite3HashFindN(pHash, zToken, nToken) ){
   752         -      bFound = 1;
   753         -      break;
   754         -    }
   755         -    for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
   756         -      pTab = sqliteHashData(e);
   757         -      if( pTab==0 ) continue;
   758         -      pHash = pTab->pColHash;
   759         -      if( pHash==0 ){
   760         -        pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
   761         -        if( pHash ){
   762         -          sqlite3HashInit(pHash);
   763         -          for(j=0; j<pTab->nCol; j++){
   764         -            Column *pCol = &pTab->aCol[j];
   765         -            sqlite3HashInsert(pHash, pCol->zName, pCol);
   766         -          }
   767         -        }else{
   768         -          *pRc = SQLITE_NOMEM_BKPT;
   769         -          bFound = 0;
   770         -          goto done;
   771         -        }
   772         -      }
   773         -      if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
   774         -        bFound = 1;
   775         -        goto done;
   776         -      }
   777         -    }
   778         -  }
   779         -done:
   780         -  sqlite3BtreeLeaveAll(db);
   781         -  sqlite3_mutex_leave(db->mutex);
   782         -  return bFound;
   783         -}
   784         -
   785         -/*
   786         -** Attempt to estimate the final output buffer size needed for the fully
   787         -** normalized version of the specified SQL string.  This should take into
   788         -** account any potential expansion that could occur (e.g. via IN clauses
   789         -** being expanded, etc).  This size returned is the total number of bytes
   790         -** including the NUL terminator.
   791         -*/
   792         -static int estimateNormalizedSize(
   793         -  const char *zSql, /* The original SQL string */
   794         -  int nSql,         /* Length of original SQL string */
   795         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
   796         -){
   797         -  int nOut = nSql + 4;
   798         -  const char *z = zSql;
   799         -  while( nOut<nSql*5 ){
   800         -    while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
   801         -    if( z[0]==0 ) break;
   802         -    z++;
   803         -    if( z[0]!='N' && z[0]!='n' ) break;
   804         -    z++;
   805         -    while( sqlite3Isspace(z[0]) ){ z++; }
   806         -    if( z[0]!='(' ) break;
   807         -    z++;
   808         -    nOut += 5; /* ?,?,? */
   809         -  }
   810         -  return nOut;
   811         -}
   812         -
   813         -/*
   814         -** Copy the current token into the output buffer while dealing with quoted
   815         -** identifiers.  By default, all letters will be converted into lowercase.
   816         -** If the bUpper flag is set, uppercase will be used.  The piOut argument
   817         -** will be used to update the target index into the output string.
   818         -*/
   819         -static void copyNormalizedToken(
   820         -  const char *zSql, /* The original SQL string */
   821         -  int iIn,          /* Current index into the original SQL string */
   822         -  int nToken,       /* Number of bytes in the current token */
   823         -  int tokenFlags,   /* Flags returned by the tokenizer */
   824         -  char *zOut,       /* The output string */
   825         -  int *piOut        /* Pointer to target index into the output string */
   826         -){
   827         -  int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
   828         -  int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
   829         -  int j = *piOut, k = 0;
   830         -  for(; k<nToken; k++){
   831         -    if( bQuoted ){
   832         -      if( k==0 && iIn>0 ){
   833         -        zOut[j++] = '"';
   834         -        continue;
   835         -      }else if( k==nToken-1 ){
   836         -        zOut[j++] = '"';
   837         -        continue;
   838         -      }
   839         -    }
   840         -    if( bKeyword ){
   841         -      zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
   842         -    }else{
   843         -      zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
   844         -    }
   845         -  }
   846         -  *piOut = j;
   847         -}
   848         -
   849         -/*
   850         -** Perform normalization of the SQL contained in the prepared statement and
   851         -** store the result in the zNormSql field.  The schema for the associated
   852         -** databases are consulted while performing the normalization in order to
   853         -** determine if a token appears to be an identifier.  All identifiers are
   854         -** left intact in the normalized SQL and all literals are replaced with a
   855         -** single '?'.
   856         -*/
   857         -void sqlite3Normalize(
   858         -  Vdbe *pVdbe,      /* VM being reprepared */
   859         -  const char *zSql, /* The original SQL string */
   860         -  int nSql,         /* Size of the input string in bytes */
   861         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
   862         -){
   863         -  sqlite3 *db;           /* Database handle. */
   864         -  char *z;               /* The output string */
   865         -  int nZ;                /* Size of the output string in bytes */
   866         -  int i;                 /* Next character to read from zSql[] */
   867         -  int j;                 /* Next character to fill in on z[] */
   868         -  int tokenType = 0;     /* Type of the next token */
   869         -  int prevTokenType = 0; /* Type of the previous token, except spaces */
   870         -  int n;                 /* Size of the next token */
   871         -  int nParen = 0;        /* Nesting level of parenthesis */
   872         -  Hash inHash;           /* Table of parenthesis levels to output index. */
   873         -
   874         -  db = sqlite3VdbeDb(pVdbe);
   875         -  assert( db!=0 );
   876         -  assert( pVdbe->zNormSql==0 );
   877         -  if( zSql==0 ) return;
   878         -  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
   879         -  z = sqlite3DbMallocRawNN(db, nZ);
   880         -  if( z==0 ) return;
   881         -  sqlite3HashInit(&inHash);
   882         -  for(i=j=0; i<nSql && zSql[i]; i+=n){
   883         -    int flags = 0;
   884         -    if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
   885         -    n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
   886         -    switch( tokenType ){
   887         -      case TK_SPACE: {
   888         -        break;
   889         -      }
   890         -      case TK_ILLEGAL: {
   891         -        sqlite3DbFree(db, z);
   892         -        sqlite3HashClear(&inHash);
   893         -        return;
   894         -      }
   895         -      case TK_STRING:
   896         -      case TK_INTEGER:
   897         -      case TK_FLOAT:
   898         -      case TK_VARIABLE:
   899         -      case TK_BLOB: {
   900         -        z[j++] = '?';
   901         -        break;
   902         -      }
   903         -      case TK_LP:
   904         -      case TK_RP: {
   905         -        if( tokenType==TK_LP ){
   906         -          nParen++;
   907         -          if( prevTokenType==TK_IN ){
   908         -            assert( nParen<nSql );
   909         -            sqlite3HashInsert(&inHash, zSql+nParen, SQLITE_INT_TO_PTR(j));
   910         -          }
   911         -        }else{
   912         -          int jj;
   913         -          assert( nParen<nSql );
   914         -          jj = SQLITE_PTR_TO_INT(sqlite3HashFind(&inHash, zSql+nParen));
   915         -          if( jj>0 ){
   916         -            sqlite3HashInsert(&inHash, zSql+nParen, 0);
   917         -            assert( jj+6<nZ );
   918         -            memcpy(z+jj+1, "?,?,?", 5);
   919         -            j = jj+6;
   920         -            assert( nZ-1-j>=0 );
   921         -            assert( nZ-1-j<nZ );
   922         -            memset(z+j, 0, nZ-1-j);
   923         -          }
   924         -          nParen--;
   925         -        }
   926         -        assert( nParen>=0 );
   927         -        /* Fall through */
   928         -      }
   929         -      case TK_MINUS:
   930         -      case TK_SEMI:
   931         -      case TK_PLUS:
   932         -      case TK_STAR:
   933         -      case TK_SLASH:
   934         -      case TK_REM:
   935         -      case TK_EQ:
   936         -      case TK_LE:
   937         -      case TK_NE:
   938         -      case TK_LSHIFT:
   939         -      case TK_LT:
   940         -      case TK_RSHIFT:
   941         -      case TK_GT:
   942         -      case TK_GE:
   943         -      case TK_BITOR:
   944         -      case TK_CONCAT:
   945         -      case TK_COMMA:
   946         -      case TK_BITAND:
   947         -      case TK_BITNOT:
   948         -      case TK_DOT:
   949         -      case TK_IN:
   950         -      case TK_IS:
   951         -      case TK_NOT:
   952         -      case TK_NULL:
   953         -      case TK_ID: {
   954         -        if( tokenType==TK_NULL ){
   955         -          if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
   956         -            /* NULL is a keyword in this case, not a literal value */
   957         -          }else{
   958         -            /* Here the NULL is a literal value */
   959         -            z[j++] = '?';
   960         -            break;
   961         -          }
   962         -        }
   963         -        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
   964         -          z[j++] = ' ';
   965         -        }
   966         -        if( tokenType==TK_ID ){
   967         -          int i2 = i, n2 = n, rc = SQLITE_OK;
   968         -          if( nParen>0 ){
   969         -            assert( nParen<nSql );
   970         -            sqlite3HashInsert(&inHash, zSql+nParen, 0);
   971         -          }
   972         -          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
   973         -          if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
   974         -            if( rc!=SQLITE_OK ){
   975         -              sqlite3DbFree(db, z);
   976         -              sqlite3HashClear(&inHash);
   977         -              return;
   978         -            }
   979         -            if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
   980         -              z[j++] = '?';
   981         -              break;
   982         -            }
   983         -          }
   984         -        }
   985         -        copyNormalizedToken(zSql, i, n, flags, z, &j);
   986         -        break;
   987         -      }
   988         -    }
   989         -  }
   990         -  assert( j<nZ && "one" );
   991         -  while( j>0 && z[j-1]==' ' ){ j--; }
   992         -  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   993         -  z[j] = 0;
   994         -  assert( j<nZ && "two" );
   995         -  pVdbe->zNormSql = z;
   996         -  sqlite3HashClear(&inHash);
   997         -}
   998         -#endif /* SQLITE_ENABLE_NORMALIZE */
   999    712   
  1000    713   /*
  1001    714   ** Rerun the compilation of a statement after a schema change.
  1002    715   **
  1003    716   ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,
  1004    717   ** if the statement cannot be recompiled because another connection has
  1005    718   ** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error

Changes to src/resolve.c.

    76     76     db = pParse->db;
    77     77     pDup = sqlite3ExprDup(db, pOrig, 0);
    78     78     if( pDup!=0 ){
    79     79       if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
    80     80       if( pExpr->op==TK_COLLATE ){
    81     81         pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
    82     82       }
    83         -    ExprSetProperty(pDup, EP_Alias);
           83  +//    ExprSetProperty(pDup, EP_Alias);
    84     84   
    85     85       /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
    86     86       ** prevents ExprDelete() from deleting the Expr structure itself,
    87     87       ** allowing it to be repopulated by the memcpy() on the following line.
    88     88       ** The pExpr->u.zToken might point into memory that will be freed by the
    89     89       ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
    90     90       ** make a copy of the token before doing the sqlite3DbFree().
................................................................................
   470    470     **
   471    471     ** Because no reference was made to outer contexts, the pNC->nRef
   472    472     ** fields are not changed in any context.
   473    473     */
   474    474     if( cnt==0 && zTab==0 ){
   475    475       assert( pExpr->op==TK_ID );
   476    476       if( ExprHasProperty(pExpr,EP_DblQuoted) ){
          477  +      /* If a double-quoted identifier does not match any known column name,
          478  +      ** then treat it as a string.
          479  +      **
          480  +      ** This hack was added in the early days of SQLite in a misguided attempt
          481  +      ** to be compatible with MySQL 3.x, which used double-quotes for strings.
          482  +      ** I now sorely regret putting in this hack. The effect of this hack is
          483  +      ** that misspelled identifier names are silently converted into strings
          484  +      ** rather than causing an error, to the frustration of countless
          485  +      ** programmers. To all those frustrated programmers, my apologies.
          486  +      **
          487  +      ** Someday, I hope to get rid of this hack. Unfortunately there is
          488  +      ** a huge amount of legacy SQL that uses it. So for now, we just
          489  +      ** issue a warning.
          490  +      */
          491  +      sqlite3_log(SQLITE_WARNING,
          492  +        "double-quoted string literal: \"%w\"", zCol);
          493  +#ifdef SQLITE_ENABLE_NORMALIZE
          494  +      sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol);
          495  +#endif
   477    496         pExpr->op = TK_STRING;
   478    497         pExpr->y.pTab = 0;
   479    498         return WRC_Prune;
   480    499       }
   481    500       if( sqlite3ExprIdToTrueFalse(pExpr) ){
   482    501         return WRC_Prune;
   483    502       }

Changes to src/select.c.

  2073   2073   /*
  2074   2074   ** Given a SELECT statement, generate a Table structure that describes
  2075   2075   ** the result set of that SELECT.
  2076   2076   */
  2077   2077   Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){
  2078   2078     Table *pTab;
  2079   2079     sqlite3 *db = pParse->db;
  2080         -  int savedFlags;
         2080  +  u64 savedFlags;
  2081   2081   
  2082   2082     savedFlags = db->flags;
  2083         -  db->flags &= ~SQLITE_FullColNames;
         2083  +  db->flags &= ~(u64)SQLITE_FullColNames;
  2084   2084     db->flags |= SQLITE_ShortColNames;
  2085   2085     sqlite3SelectPrep(pParse, pSelect, 0);
  2086   2086     if( pParse->nErr ) return 0;
  2087   2087     while( pSelect->pPrior ) pSelect = pSelect->pPrior;
  2088   2088     db->flags = savedFlags;
  2089   2089     pTab = sqlite3DbMallocZero(db, sizeof(Table) );
  2090   2090     if( pTab==0 ){
................................................................................
  3457   3457           if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
  3458   3458             memset(&ifNullRow, 0, sizeof(ifNullRow));
  3459   3459             ifNullRow.op = TK_IF_NULL_ROW;
  3460   3460             ifNullRow.pLeft = pCopy;
  3461   3461             ifNullRow.iTable = pSubst->iNewTable;
  3462   3462             pCopy = &ifNullRow;
  3463   3463           }
         3464  +        testcase( ExprHasProperty(pCopy, EP_Subquery) );
  3464   3465           pNew = sqlite3ExprDup(db, pCopy, 0);
  3465   3466           if( pNew && pSubst->isLeftJoin ){
  3466   3467             ExprSetProperty(pNew, EP_CanBeNull);
  3467   3468           }
  3468   3469           if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){
  3469   3470             pNew->iRightJoinTable = pExpr->iRightJoinTable;
  3470   3471             ExprSetProperty(pNew, EP_FromJoin);
................................................................................
  4021   4022         for(i=0; i<pOrderBy->nExpr; i++){
  4022   4023           pOrderBy->a[i].u.x.iOrderByCol = 0;
  4023   4024         }
  4024   4025         assert( pParent->pOrderBy==0 );
  4025   4026         pParent->pOrderBy = pOrderBy;
  4026   4027         pSub->pOrderBy = 0;
  4027   4028       }
  4028         -    pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
         4029  +    pWhere = pSub->pWhere;
         4030  +    pSub->pWhere = 0;
  4029   4031       if( isLeftJoin>0 ){
  4030   4032         setJoinExpr(pWhere, iNewParent);
  4031   4033       }
  4032   4034       pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
  4033   4035       if( db->mallocFailed==0 ){
  4034   4036         SubstContext x;
  4035   4037         x.pParse = pParse;

Changes to src/shell.c.in.

  1003   1003     u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  1004   1004     u8 autoEQPtest;        /* autoEQP is in test mode */
  1005   1005     u8 statsOn;            /* True to display memory stats before each finalize */
  1006   1006     u8 scanstatsOn;        /* True to display scan stats before each finalize */
  1007   1007     u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  1008   1008     u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
  1009   1009     u8 nEqpLevel;          /* Depth of the EQP output graph */
         1010  +  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
  1010   1011     unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  1011   1012     int outCount;          /* Revert to stdout when reaching zero */
  1012   1013     int cnt;               /* Number of records displayed so far */
  1013   1014     FILE *out;             /* Write results here */
  1014   1015     FILE *traceOut;        /* Output for sqlite3_trace() */
  1015   1016     int nErr;              /* Number of errors seen */
  1016   1017     int mode;              /* An output mode setting */
................................................................................
  1062   1063   #define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
  1063   1064   #define SHELL_OPEN_NORMAL      1      /* Normal database file */
  1064   1065   #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
  1065   1066   #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
  1066   1067   #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
  1067   1068   #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
  1068   1069   
         1070  +/* Allowed values for ShellState.eTraceType
         1071  +*/
         1072  +#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
         1073  +#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
         1074  +#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
         1075  +
  1069   1076   /*
  1070   1077   ** These are the allowed shellFlgs values
  1071   1078   */
  1072   1079   #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
  1073   1080   #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
  1074   1081   #define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
  1075   1082   #define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
................................................................................
  3366   3373     "      http://sqlite.org/cli.html#sqlar_archive_support",
  3367   3374   #endif
  3368   3375   #ifndef SQLITE_OMIT_AUTHORIZATION
  3369   3376     ".auth ON|OFF             Show authorizer callbacks",
  3370   3377   #endif
  3371   3378     ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
  3372   3379     "       --append            Use the appendvfs",
         3380  +  "       --async             Write to FILE without a journal and without fsync()",
  3373   3381     ".bail on|off             Stop after hitting an error.  Default OFF",
  3374   3382     ".binary on|off           Turn binary output on or off.  Default OFF",
  3375   3383     ".cd DIRECTORY            Change the working directory to DIRECTORY",
  3376   3384     ".changes on|off          Show number of rows changed by SQL",
  3377   3385     ".check GLOB              Fail if output since .testcase does not match",
  3378   3386     ".clone NEWDB             Clone data into NEWDB from the existing database",
  3379   3387     ".databases               List names and files of attached databases",
................................................................................
  3488   3496   #ifndef SQLITE_NOHAVE_SYSTEM
  3489   3497     ".system CMD ARGS...      Run CMD ARGS... in a system shell",
  3490   3498   #endif
  3491   3499     ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
  3492   3500     ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
  3493   3501     ".timeout MS              Try opening locked tables for MS milliseconds",
  3494   3502     ".timer on|off            Turn SQL timer on or off",
  3495         -  ".trace FILE|off          Output each SQL statement as it is run",
         3503  +#ifndef SQLITE_OMIT_TRACE
         3504  +  ".trace ?OPTIONS?         Output each SQL statement as it is run",
         3505  +  "    FILE                    Send output to FILE",
         3506  +  "    stdout                  Send output to stdout",
         3507  +  "    stderr                  Send output to stderr",
         3508  +  "    off                     Disable tracing",
         3509  +  "    --expanded              Expand query parameters",
         3510  +#ifdef SQLITE_ENABLE_NORMALIZE
         3511  +  "    --normalized            Normal the SQL statements",
         3512  +#endif
         3513  +  "    --plain                 Show SQL as it is input",
         3514  +  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
         3515  +  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
         3516  +  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
         3517  +  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
         3518  +#endif /* SQLITE_OMIT_TRACE */
  3496   3519     ".vfsinfo ?AUX?           Information about the top-level VFS",
  3497   3520     ".vfslist                 List all available VFSes",
  3498   3521     ".vfsname ?AUX?           Print the name of the VFS stack",
  3499   3522     ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
  3500   3523     "     Negative values right-justify",
  3501   3524   };
  3502   3525   
................................................................................
  3995   4018       if( f==0 ){
  3996   4019         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
  3997   4020       }
  3998   4021     }
  3999   4022     return f;
  4000   4023   }
  4001   4024   
  4002         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
         4025  +#ifndef SQLITE_OMIT_TRACE
  4003   4026   /*
  4004   4027   ** A routine for handling output from sqlite3_trace().
  4005   4028   */
  4006   4029   static int sql_trace_callback(
  4007         -  unsigned mType,
  4008         -  void *pArg,
  4009         -  void *pP,
  4010         -  void *pX
  4011         -){
  4012         -  FILE *f = (FILE*)pArg;
  4013         -  UNUSED_PARAMETER(mType);
  4014         -  UNUSED_PARAMETER(pP);
  4015         -  if( f ){
  4016         -    const char *z = (const char*)pX;
  4017         -    int i = strlen30(z);
  4018         -    while( i>0 && z[i-1]==';' ){ i--; }
  4019         -    utf8_printf(f, "%.*s;\n", i, z);
         4030  +  unsigned mType,         /* The trace type */
         4031  +  void *pArg,             /* The ShellState pointer */
         4032  +  void *pP,               /* Usually a pointer to sqlite_stmt */
         4033  +  void *pX                /* Auxiliary output */
         4034  +){
         4035  +  ShellState *p = (ShellState*)pArg;
         4036  +  sqlite3_stmt *pStmt;
         4037  +  const char *zSql;
         4038  +  int nSql;
         4039  +  if( p->traceOut==0 ) return 0;
         4040  +  if( mType==SQLITE_TRACE_CLOSE ){
         4041  +    utf8_printf(p->traceOut, "-- closing database connection\n");
         4042  +    return 0;
         4043  +  }
         4044  +  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
         4045  +    zSql = (const char*)pX;
         4046  +  }else{
         4047  +    pStmt = (sqlite3_stmt*)pP;
         4048  +    switch( p->eTraceType ){
         4049  +      case SHELL_TRACE_EXPANDED: {
         4050  +        zSql = sqlite3_expanded_sql(pStmt);
         4051  +        break;
         4052  +      }
         4053  +#ifdef SQLITE_ENABLE_NORMALIZE
         4054  +      case SHELL_TRACE_NORMALIZED: {
         4055  +        zSql = sqlite3_normalized_sql(pStmt);
         4056  +        break;
         4057  +      }
         4058  +#endif
         4059  +      default: {
         4060  +        zSql = sqlite3_sql(pStmt);
         4061  +        break;
         4062  +      }
         4063  +    }
         4064  +  }
         4065  +  if( zSql==0 ) return 0;
         4066  +  nSql = strlen30(zSql);
         4067  +  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
         4068  +  switch( mType ){
         4069  +    case SQLITE_TRACE_ROW:
         4070  +    case SQLITE_TRACE_STMT: {
         4071  +      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
         4072  +      break;
         4073  +    }
         4074  +    case SQLITE_TRACE_PROFILE: {
         4075  +      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
         4076  +      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
         4077  +      break;
         4078  +    }
  4020   4079     }
  4021   4080     return 0;
  4022   4081   }
  4023   4082   #endif
  4024   4083   
  4025   4084   /*
  4026   4085   ** A no-op routine that runs with the ".breakpoint" doc-command.  This is
................................................................................
  5778   5837      || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
  5779   5838     ){
  5780   5839       const char *zDestFile = 0;
  5781   5840       const char *zDb = 0;
  5782   5841       sqlite3 *pDest;
  5783   5842       sqlite3_backup *pBackup;
  5784   5843       int j;
         5844  +    int bAsync = 0;
  5785   5845       const char *zVfs = 0;
  5786   5846       for(j=1; j<nArg; j++){
  5787   5847         const char *z = azArg[j];
  5788   5848         if( z[0]=='-' ){
  5789   5849           if( z[1]=='-' ) z++;
  5790   5850           if( strcmp(z, "-append")==0 ){
  5791   5851             zVfs = "apndvfs";
         5852  +        }else
         5853  +        if( strcmp(z, "-async")==0 ){
         5854  +          bAsync = 1;
  5792   5855           }else
  5793   5856           {
  5794   5857             utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
  5795   5858             return 1;
  5796   5859           }
  5797   5860         }else if( zDestFile==0 ){
  5798   5861           zDestFile = azArg[j];
  5799   5862         }else if( zDb==0 ){
  5800   5863           zDb = zDestFile;
  5801   5864           zDestFile = azArg[j];
  5802   5865         }else{
  5803         -        raw_printf(stderr, "Usage: .backup ?DB? ?--append? FILENAME\n");
         5866  +        raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
  5804   5867           return 1;
  5805   5868         }
  5806   5869       }
  5807   5870       if( zDestFile==0 ){
  5808   5871         raw_printf(stderr, "missing FILENAME argument on .backup\n");
  5809   5872         return 1;
  5810   5873       }
................................................................................
  5812   5875       rc = sqlite3_open_v2(zDestFile, &pDest, 
  5813   5876                     SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs);
  5814   5877       if( rc!=SQLITE_OK ){
  5815   5878         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
  5816   5879         close_db(pDest);
  5817   5880         return 1;
  5818   5881       }
         5882  +    if( bAsync ){
         5883  +      sqlite3_exec(pDest, "PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;",
         5884  +                   0, 0, 0);
         5885  +    }
  5819   5886       open_db(p, 0);
  5820   5887       pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
  5821   5888       if( pBackup==0 ){
  5822   5889         utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
  5823   5890         close_db(pDest);
  5824   5891         return 1;
  5825   5892       }
................................................................................
  7834   7901         }
  7835   7902       }else{
  7836   7903         raw_printf(stderr, "Usage: .timer on|off\n");
  7837   7904         rc = 1;
  7838   7905       }
  7839   7906     }else
  7840   7907   
         7908  +#ifndef SQLITE_OMIT_TRACE
  7841   7909     if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
         7910  +    int mType = 0;
         7911  +    int jj;
  7842   7912       open_db(p, 0);
  7843         -    if( nArg!=2 ){
  7844         -      raw_printf(stderr, "Usage: .trace FILE|off\n");
  7845         -      rc = 1;
  7846         -      goto meta_command_exit;
         7913  +    for(jj=1; jj<nArg; jj++){
         7914  +      const char *z = azArg[jj];
         7915  +      if( z[0]=='-' ){
         7916  +        if( optionMatch(z, "expanded") ){
         7917  +          p->eTraceType = SHELL_TRACE_EXPANDED;
         7918  +        }
         7919  +#ifdef SQLITE_ENABLE_NORMALIZE
         7920  +        else if( optionMatch(z, "normalized") ){
         7921  +          p->eTraceType = SHELL_TRACE_NORMALIZED;
         7922  +        }
         7923  +#endif
         7924  +        else if( optionMatch(z, "plain") ){
         7925  +          p->eTraceType = SHELL_TRACE_PLAIN;
         7926  +        }
         7927  +        else if( optionMatch(z, "profile") ){
         7928  +          mType |= SQLITE_TRACE_PROFILE;
         7929  +        }
         7930  +        else if( optionMatch(z, "row") ){
         7931  +          mType |= SQLITE_TRACE_ROW;
         7932  +        }
         7933  +        else if( optionMatch(z, "stmt") ){
         7934  +          mType |= SQLITE_TRACE_STMT;
         7935  +        }
         7936  +        else if( optionMatch(z, "close") ){
         7937  +          mType |= SQLITE_TRACE_CLOSE;
         7938  +        }
         7939  +        else {
         7940  +          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
         7941  +          rc = 1;
         7942  +          goto meta_command_exit;
         7943  +        }
         7944  +      }else{
         7945  +        output_file_close(p->traceOut);
         7946  +        p->traceOut = output_file_open(azArg[1], 0);
         7947  +      }
  7847   7948       }
  7848         -    output_file_close(p->traceOut);
  7849         -    p->traceOut = output_file_open(azArg[1], 0);
  7850         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
  7851   7949       if( p->traceOut==0 ){
  7852   7950         sqlite3_trace_v2(p->db, 0, 0, 0);
  7853   7951       }else{
  7854         -      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
         7952  +      if( mType==0 ) mType = SQLITE_TRACE_STMT;
         7953  +      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
  7855   7954       }
  7856         -#endif
  7857   7955     }else
         7956  +#endif /* !defined(SQLITE_OMIT_TRACE) */
  7858   7957   
  7859   7958   #if SQLITE_USER_AUTHENTICATION
  7860   7959     if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
  7861   7960       if( nArg<2 ){
  7862   7961         raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
  7863   7962         rc = 1;
  7864   7963         goto meta_command_exit;

Changes to src/sqlite.h.in.

  2987   2987   ** ^The callback function registered by sqlite3_profile() is invoked
  2988   2988   ** as each SQL statement finishes.  ^The profile callback contains
  2989   2989   ** the original statement text and an estimate of wall-clock time
  2990   2990   ** of how long that statement took to run.  ^The profile callback
  2991   2991   ** time is in units of nanoseconds, however the current implementation
  2992   2992   ** is only capable of millisecond resolution so the six least significant
  2993   2993   ** digits in the time are meaningless.  Future versions of SQLite
  2994         -** might provide greater resolution on the profiler callback.  The
  2995         -** sqlite3_profile() function is considered experimental and is
  2996         -** subject to change in future versions of SQLite.
         2994  +** might provide greater resolution on the profiler callback.  Invoking
         2995  +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the
         2996  +** profile callback.
  2997   2997   */
  2998   2998   SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
  2999   2999      void(*xTrace)(void*,const char*), void*);
  3000   3000   SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
  3001   3001      void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
  3002   3002   
  3003   3003   /*
................................................................................
  3625   3625   ** and [sqlite3_prepare16_v3()] assume that the prepared statement will 
  3626   3626   ** be used just once or at most a few times and then destroyed using
  3627   3627   ** [sqlite3_finalize()] relatively soon. The current implementation acts
  3628   3628   ** on this hint by avoiding the use of [lookaside memory] so as not to
  3629   3629   ** deplete the limited store of lookaside memory. Future versions of
  3630   3630   ** SQLite may act on this hint differently.
  3631   3631   **
  3632         -** [[SQLITE_PREPARE_NORMALIZE]] ^(<dt>SQLITE_PREPARE_NORMALIZE</dt>
  3633         -** <dd>The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized
  3634         -** representation of the SQL statement should be calculated and then
  3635         -** associated with the prepared statement, which can be obtained via
  3636         -** the [sqlite3_normalized_sql()] interface.)^  The semantics used to
  3637         -** normalize a SQL statement are unspecified and subject to change.
  3638         -** At a minimum, literal values will be replaced with suitable
  3639         -** placeholders.
         3632  +** [[SQLITE_PREPARE_NORMALIZE]] <dt>SQLITE_PREPARE_NORMALIZE</dt>
         3633  +** <dd>The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used
         3634  +** to be required for any prepared statement that wanted to use the
         3635  +** [sqlite3_normalized_sql()] interface.  However, the
         3636  +** [sqlite3_normalized_sql()] interface is now available to all
         3637  +** prepared statements, regardless of whether or not they use this
         3638  +** flag.
  3640   3639   ** </dl>
  3641   3640   */
  3642   3641   #define SQLITE_PREPARE_PERSISTENT              0x01
  3643   3642   #define SQLITE_PREPARE_NORMALIZE               0x02
  3644   3643   
  3645   3644   /*
  3646   3645   ** CAPI3REF: Compiling An SQL Statement

Changes to src/sqliteInt.h.

  1352   1352                                  const char*);
  1353   1353   #endif
  1354   1354   
  1355   1355   #ifndef SQLITE_OMIT_DEPRECATED
  1356   1356   /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
  1357   1357   ** in the style of sqlite3_trace()
  1358   1358   */
  1359         -#define SQLITE_TRACE_LEGACY  0x80
         1359  +#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
         1360  +#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
  1360   1361   #else
  1361         -#define SQLITE_TRACE_LEGACY  0
         1362  +#define SQLITE_TRACE_LEGACY          0
         1363  +#define SQLITE_TRACE_XPROFILE        0
  1362   1364   #endif /* SQLITE_OMIT_DEPRECATED */
         1365  +#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */
  1363   1366   
  1364   1367   
  1365   1368   /*
  1366   1369   ** Each database connection is an instance of the following structure.
  1367   1370   */
  1368   1371   struct sqlite3 {
  1369   1372     sqlite3_vfs *pVfs;            /* OS Interface */
................................................................................
  1414   1417     int nVdbeWrite;               /* Number of active VDBEs that read and write */
  1415   1418     int nVdbeExec;                /* Number of nested calls to VdbeExec() */
  1416   1419     int nVDestroy;                /* Number of active OP_VDestroy operations */
  1417   1420     int nExtension;               /* Number of loaded extensions */
  1418   1421     void **aExtension;            /* Array of shared library handles */
  1419   1422     int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
  1420   1423     void *pTraceArg;                          /* Argument to the trace function */
         1424  +#ifndef SQLITE_OMIT_DEPRECATED
  1421   1425     void (*xProfile)(void*,const char*,u64);  /* Profiling function */
  1422   1426     void *pProfileArg;                        /* Argument to profile function */
         1427  +#endif
  1423   1428     void *pCommitArg;                 /* Argument to xCommitCallback() */
  1424   1429     int (*xCommitCallback)(void*);    /* Invoked at every commit. */
  1425   1430     void *pRollbackArg;               /* Argument to xRollbackCallback() */
  1426   1431     void (*xRollbackCallback)(void*); /* Invoked at every commit. */
  1427   1432     void *pUpdateArg;
  1428   1433     void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
  1429   1434   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
................................................................................
  1948   1953   /*
  1949   1954   ** The schema for each SQL table and view is represented in memory
  1950   1955   ** by an instance of the following structure.
  1951   1956   */
  1952   1957   struct Table {
  1953   1958     char *zName;         /* Name of the table or view */
  1954   1959     Column *aCol;        /* Information about each column */
  1955         -#ifdef SQLITE_ENABLE_NORMALIZE
  1956         -  Hash *pColHash;      /* All columns indexed by name */
  1957         -#endif
  1958   1960     Index *pIndex;       /* List of SQL indexes on this table. */
  1959   1961     Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
  1960   1962     FKey *pFKey;         /* Linked list of all foreign keys in this table */
  1961   1963     char *zColAff;       /* String defining the affinity of each column */
  1962   1964     ExprList *pCheck;    /* All CHECK constraints */
  1963   1965                          /*   ... also used as column name list in a VIEW */
  1964   1966     int tnum;            /* Root BTree page for this table */
................................................................................
  3979   3981   #define LOCATE_VIEW    0x01
  3980   3982   #define LOCATE_NOERR   0x02
  3981   3983   Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
  3982   3984   Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
  3983   3985   Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
  3984   3986   void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
  3985   3987   void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
  3986         -void sqlite3Vacuum(Parse*,Token*);
  3987         -int sqlite3RunVacuum(char**, sqlite3*, int);
         3988  +void sqlite3Vacuum(Parse*,Token*,Expr*);
         3989  +int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*);
  3988   3990   char *sqlite3NameFromToken(sqlite3*, Token*);
  3989   3991   int sqlite3ExprCompare(Parse*,Expr*, Expr*, int);
  3990   3992   int sqlite3ExprCompareSkip(Expr*, Expr*, int);
  3991   3993   int sqlite3ExprListCompare(ExprList*, ExprList*, int);
  3992   3994   int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int);
  3993   3995   int sqlite3ExprImpliesNonNullRow(Expr*,int);
  3994   3996   void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
................................................................................
  4018   4020   #ifdef SQLITE_ENABLE_CURSOR_HINTS
  4019   4021   int sqlite3ExprContainsSubquery(Expr*);
  4020   4022   #endif
  4021   4023   int sqlite3ExprIsInteger(Expr*, int*);
  4022   4024   int sqlite3ExprCanBeNull(const Expr*);
  4023   4025   int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
  4024   4026   int sqlite3IsRowid(const char*);
  4025         -#ifdef SQLITE_ENABLE_NORMALIZE
  4026         -int sqlite3IsRowidN(const char*, int);
  4027         -#endif
  4028   4027   void sqlite3GenerateRowDelete(
  4029   4028       Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
  4030   4029   void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
  4031   4030   int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
  4032   4031   void sqlite3ResolvePartIdxLabel(Parse*,int);
  4033   4032   int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int);
  4034   4033   void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
................................................................................
  4047   4046   void sqlite3UniqueConstraint(Parse*, int, Index*);
  4048   4047   void sqlite3RowidConstraint(Parse*, int, Table*);
  4049   4048   Expr *sqlite3ExprDup(sqlite3*,Expr*,int);
  4050   4049   ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
  4051   4050   SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
  4052   4051   IdList *sqlite3IdListDup(sqlite3*,IdList*);
  4053   4052   Select *sqlite3SelectDup(sqlite3*,Select*,int);
  4054         -#ifdef SQLITE_ENABLE_NORMALIZE
  4055         -FuncDef *sqlite3FunctionSearchN(int,const char*,int);
  4056         -#endif
         4053  +FuncDef *sqlite3FunctionSearch(int,const char*);
  4057   4054   void sqlite3InsertBuiltinFuncs(FuncDef*,int);
  4058   4055   FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
  4059   4056   void sqlite3RegisterBuiltinFunctions(void);
  4060   4057   void sqlite3RegisterDateTimeFunctions(void);
  4061   4058   void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
  4062   4059   int sqlite3SafetyCheckOk(sqlite3*);
  4063   4060   int sqlite3SafetyCheckSickOrOk(sqlite3*);
................................................................................
  4254   4251   #endif
  4255   4252   void sqlite3RootPageMoved(sqlite3*, int, int, int);
  4256   4253   void sqlite3Reindex(Parse*, Token*, Token*);
  4257   4254   void sqlite3AlterFunctions(void);
  4258   4255   void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
  4259   4256   void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
  4260   4257   int sqlite3GetToken(const unsigned char *, int *);
  4261         -#ifdef SQLITE_ENABLE_NORMALIZE
  4262         -int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
  4263         -#endif
  4264   4258   void sqlite3NestedParse(Parse*, const char*, ...);
  4265   4259   void sqlite3ExpirePreparedStatements(sqlite3*, int);
  4266   4260   int sqlite3CodeSubselect(Parse*, Expr *, int, int);
  4267   4261   void sqlite3SelectPrep(Parse*, Select*, NameContext*);
  4268   4262   void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
  4269   4263   int sqlite3MatchSpanName(const char*, const char*, const char*, const char*);
  4270   4264   int sqlite3ResolveExprNames(NameContext*, Expr*);
................................................................................
  4415   4409   int sqlite3VtabBegin(sqlite3 *, VTable *);
  4416   4410   FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
  4417   4411   sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
  4418   4412   int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
  4419   4413   int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
  4420   4414   void sqlite3ParserReset(Parse*);
  4421   4415   #ifdef SQLITE_ENABLE_NORMALIZE
  4422         -void sqlite3Normalize(Vdbe*, const char*, int, u8);
         4416  +char *sqlite3Normalize(Vdbe*, const char*);
  4423   4417   #endif
  4424   4418   int sqlite3Reprepare(Vdbe*);
  4425   4419   void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
  4426   4420   CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
  4427   4421   int sqlite3TempInMemory(const sqlite3*);
  4428   4422   const char *sqlite3JournalModename(int);
  4429   4423   #ifndef SQLITE_OMIT_WAL

Changes to src/tokenize.c.

   541    541       }
   542    542     }
   543    543     while( IdChar(z[i]) ){ i++; }
   544    544     *tokenType = TK_ID;
   545    545     return i;
   546    546   }
   547    547   
   548         -#ifdef SQLITE_ENABLE_NORMALIZE
   549         -/*
   550         -** Return the length (in bytes) of the token that begins at z[0].
   551         -** Store the token type in *tokenType before returning.  If flags has
   552         -** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
   553         -** for keywords.  Add SQLITE_TOKEN_QUOTED to flags if the token was
   554         -** actually a quoted identifier.  Add SQLITE_TOKEN_KEYWORD to flags
   555         -** if the token was recognized as a keyword; this is useful when the
   556         -** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
   557         -** to differentiate between a keyword being treated as an identifier
   558         -** (for normalization purposes) and an actual identifier.
   559         -*/
   560         -int sqlite3GetTokenNormalized(
   561         -  const unsigned char *z,
   562         -  int *tokenType,
   563         -  int *flags
   564         -){
   565         -  int n;
   566         -  unsigned char iClass = aiClass[*z];
   567         -  if( iClass==CC_KYWD ){
   568         -    int i;
   569         -    for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
   570         -    if( IdChar(z[i]) ){
   571         -      /* This token started out using characters that can appear in keywords,
   572         -      ** but z[i] is a character not allowed within keywords, so this must
   573         -      ** be an identifier instead */
   574         -      i++;
   575         -      while( IdChar(z[i]) ){ i++; }
   576         -      *tokenType = TK_ID;
   577         -      return i;
   578         -    }
   579         -    *tokenType = TK_ID;
   580         -    n = keywordCode((char*)z, i, tokenType);
   581         -    /* If the token is no longer considered to be an identifier, then it is a
   582         -    ** keyword of some kind.  Make the token back into an identifier and then
   583         -    ** set the SQLITE_TOKEN_KEYWORD flag.  Several non-identifier tokens are
   584         -    ** used verbatim, including IN, IS, NOT, and NULL. */
   585         -    switch( *tokenType ){
   586         -      case TK_ID: {
   587         -        /* do nothing, handled by caller */
   588         -        break;
   589         -      }
   590         -      case TK_IN:
   591         -      case TK_IS:
   592         -      case TK_NOT:
   593         -      case TK_NULL: {
   594         -        *flags |= SQLITE_TOKEN_KEYWORD;
   595         -        break;
   596         -      }
   597         -      default: {
   598         -        *tokenType = TK_ID;
   599         -        *flags |= SQLITE_TOKEN_KEYWORD;
   600         -        break;
   601         -      }
   602         -    }
   603         -  }else{
   604         -    n = sqlite3GetToken(z, tokenType);
   605         -    /* If the token is considered to be an identifier and the character class
   606         -    ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
   607         -    if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
   608         -      *flags |= SQLITE_TOKEN_QUOTED;
   609         -    }
   610         -  }
   611         -  return n;
   612         -}
   613         -#endif /* SQLITE_ENABLE_NORMALIZE */
   614         -
   615    548   /*
   616    549   ** Run the parser on the given SQL string.  The parser structure is
   617    550   ** passed in.  An SQLITE_ status code is returned.  If an error occurs
   618    551   ** then an and attempt is made to write an error message into 
   619    552   ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
   620    553   ** error message.
   621    554   */
................................................................................
   777    710       Table *p = pParse->pZombieTab;
   778    711       pParse->pZombieTab = p->pNextZombie;
   779    712       sqlite3DeleteTable(db, p);
   780    713     }
   781    714     assert( nErr==0 || pParse->rc!=SQLITE_OK );
   782    715     return nErr;
   783    716   }
          717  +
          718  +
          719  +#ifdef SQLITE_ENABLE_NORMALIZE
          720  +/*
          721  +** Insert a single space character into pStr if the current string
          722  +** ends with an identifier
          723  +*/
          724  +static void addSpaceSeparator(sqlite3_str *pStr){
          725  +  if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){
          726  +    sqlite3_str_append(pStr, " ", 1);
          727  +  }
          728  +}
          729  +
          730  +/*
          731  +** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
          732  +** the normalization in space obtained from sqlite3DbMalloc().  Or return
          733  +** NULL if anything goes wrong or if zSql is NULL.
          734  +*/
          735  +char *sqlite3Normalize(
          736  +  Vdbe *pVdbe,       /* VM being reprepared */
          737  +  const char *zSql   /* The original SQL string */
          738  +){
          739  +  sqlite3 *db;       /* The database connection */
          740  +  int i;             /* Next unread byte of zSql[] */
          741  +  int n;             /* length of current token */
          742  +  int tokenType;     /* type of current token */
          743  +  int prevType;      /* Previous non-whitespace token */
          744  +  int nParen;        /* Number of nested levels of parentheses */
          745  +  int iStartIN;      /* Start of RHS of IN operator in z[] */
          746  +  int nParenAtIN;    /* Value of nParent at start of RHS of IN operator */
          747  +  int j;             /* Bytes of normalized SQL generated so far */
          748  +  sqlite3_str *pStr; /* The normalized SQL string under construction */
          749  +
          750  +  db = sqlite3VdbeDb(pVdbe);
          751  +  tokenType = -1;
          752  +  nParen = iStartIN = nParenAtIN = 0;
          753  +  pStr = sqlite3_str_new(db);
          754  +  assert( pStr!=0 );  /* sqlite3_str_new() never returns NULL */
          755  +  for(i=0; zSql[i] && pStr->accError==0; i+=n){
          756  +    if( tokenType!=TK_SPACE ){
          757  +      prevType = tokenType;
          758  +    }
          759  +    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
          760  +    if( NEVER(n<=0) ) break;
          761  +    switch( tokenType ){
          762  +      case TK_SPACE: {
          763  +        break;
          764  +      }
          765  +      case TK_NULL: {
          766  +        if( prevType==TK_IS || prevType==TK_NOT ){
          767  +          sqlite3_str_append(pStr, " NULL", 5);
          768  +          break;
          769  +        }
          770  +        /* Fall through */
          771  +      }
          772  +      case TK_STRING:
          773  +      case TK_INTEGER:
          774  +      case TK_FLOAT:
          775  +      case TK_VARIABLE:
          776  +      case TK_BLOB: {
          777  +        sqlite3_str_append(pStr, "?", 1);
          778  +        break;
          779  +      }
          780  +      case TK_LP: {
          781  +        nParen++;
          782  +        if( prevType==TK_IN ){
          783  +          iStartIN = pStr->nChar;
          784  +          nParenAtIN = nParen;
          785  +        }
          786  +        sqlite3_str_append(pStr, "(", 1);
          787  +        break;
          788  +      }
          789  +      case TK_RP: {
          790  +        if( iStartIN>0 && nParen==nParenAtIN ){
          791  +          assert( pStr->nChar>=iStartIN );
          792  +          pStr->nChar = iStartIN+1;
          793  +          sqlite3_str_append(pStr, "?,?,?", 5);
          794  +          iStartIN = 0;
          795  +        }
          796  +        nParen--;
          797  +        sqlite3_str_append(pStr, ")", 1);
          798  +        break;
          799  +      }
          800  +      case TK_ID: {
          801  +        iStartIN = 0;
          802  +        j = pStr->nChar;
          803  +        if( sqlite3Isquote(zSql[i]) ){
          804  +          char *zId = sqlite3DbStrNDup(db, zSql+i, n);
          805  +          int nId;
          806  +          int eType = 0;
          807  +          if( zId==0 ) break;
          808  +          sqlite3Dequote(zId);
          809  +          if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){
          810  +            sqlite3_str_append(pStr, "?", 1);
          811  +            sqlite3DbFree(db, zId);
          812  +            break;
          813  +          }
          814  +          nId = sqlite3Strlen30(zId);
          815  +          if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){
          816  +            addSpaceSeparator(pStr);
          817  +            sqlite3_str_append(pStr, zId, nId);
          818  +          }else{
          819  +            sqlite3_str_appendf(pStr, "\"%w\"", zId);
          820  +          }
          821  +          sqlite3DbFree(db, zId);
          822  +        }else{
          823  +          addSpaceSeparator(pStr);
          824  +          sqlite3_str_append(pStr, zSql+i, n);
          825  +        }
          826  +        while( j<pStr->nChar ){
          827  +          pStr->zText[j] = sqlite3Tolower(pStr->zText[j]);
          828  +          j++;
          829  +        }
          830  +        break;
          831  +      }
          832  +      case TK_SELECT: {
          833  +        iStartIN = 0;
          834  +        /* fall through */
          835  +      }
          836  +      default: {
          837  +        if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr);
          838  +        j = pStr->nChar;
          839  +        sqlite3_str_append(pStr, zSql+i, n);
          840  +        while( j<pStr->nChar ){
          841  +          pStr->zText[j] = sqlite3Toupper(pStr->zText[j]);
          842  +          j++;
          843  +        }
          844  +        break;
          845  +      }
          846  +    }
          847  +  }
          848  +  if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1);
          849  +  return sqlite3_str_finish(pStr);
          850  +}
          851  +#endif /* SQLITE_ENABLE_NORMALIZE */

Changes to src/vacuum.c.

    98     98   ** the copy of step (3) were replaced by deleting the original database
    99     99   ** and renaming the transient database as the original.  But that will
   100    100   ** not work if other processes are attached to the original database.
   101    101   ** And a power loss in between deleting the original and renaming the
   102    102   ** transient would cause the database file to appear to be deleted
   103    103   ** following reboot.
   104    104   */
   105         -void sqlite3Vacuum(Parse *pParse, Token *pNm){
          105  +void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){
   106    106     Vdbe *v = sqlite3GetVdbe(pParse);
   107    107     int iDb = 0;
   108         -  if( v==0 ) return;
          108  +  if( v==0 ) goto build_vacuum_end;
   109    109     if( pNm ){
   110    110   #ifndef SQLITE_BUG_COMPATIBLE_20160819
   111    111       /* Default behavior:  Report an error if the argument to VACUUM is
   112    112       ** not recognized */
   113    113       iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm);
   114         -    if( iDb<0 ) return;
          114  +    if( iDb<0 ) goto build_vacuum_end;
   115    115   #else
   116    116       /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
   117    117       ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
   118    118       ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
   119    119       ** The buggy behavior is required for binary compatibility with some
   120    120       ** legacy applications. */
   121    121       iDb = sqlite3FindDb(pParse->db, pNm);
   122    122       if( iDb<0 ) iDb = 0;
   123    123   #endif
   124    124     }
   125    125     if( iDb!=1 ){
   126         -    sqlite3VdbeAddOp1(v, OP_Vacuum, iDb);
          126  +    int iIntoReg = 0;
          127  +    if( pInto ){
          128  +      iIntoReg = ++pParse->nMem;
          129  +      sqlite3ExprCode(pParse, pInto, iIntoReg);
          130  +    }
          131  +    sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg);
   127    132       sqlite3VdbeUsesBtree(v, iDb);
   128    133     }
          134  +build_vacuum_end:
          135  +  sqlite3ExprDelete(pParse->db, pInto);
   129    136     return;
   130    137   }
   131    138   
   132    139   /*
   133    140   ** This routine implements the OP_Vacuum opcode of the VDBE.
   134    141   */
   135         -int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){
          142  +int sqlite3RunVacuum(
          143  +  char **pzErrMsg,        /* Write error message here */
          144  +  sqlite3 *db,            /* Database connection */
          145  +  int iDb,                /* Which attached DB to vacuum */
          146  +  sqlite3_value *pOut     /* Write results here, if not NULL */
          147  +){
   136    148     int rc = SQLITE_OK;     /* Return code from service routines */
   137    149     Btree *pMain;           /* The database being vacuumed */
   138    150     Btree *pTemp;           /* The temporary database we vacuum into */
   139         -  u16 saved_mDbFlags;     /* Saved value of db->mDbFlags */
   140         -  u32 saved_flags;        /* Saved value of db->flags */
          151  +  u32 saved_mDbFlags;     /* Saved value of db->mDbFlags */
          152  +  u64 saved_flags;        /* Saved value of db->flags */
   141    153     int saved_nChange;      /* Saved value of db->nChange */
   142    154     int saved_nTotalChange; /* Saved value of db->nTotalChange */
   143    155     u8 saved_mTrace;        /* Saved trace settings */
   144    156     Db *pDb = 0;            /* Database to detach at end of vacuum */
   145    157     int isMemDb;            /* True if vacuuming a :memory: database */
   146    158     int nRes;               /* Bytes of reserved space at the end of each page */
   147    159     int nDb;                /* Number of attached databases */
   148    160     const char *zDbMain;    /* Schema name of database to vacuum */
          161  +  const char *zOut;       /* Name of output file */
   149    162   
   150    163     if( !db->autoCommit ){
   151    164       sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
   152    165       return SQLITE_ERROR;
   153    166     }
   154    167     if( db->nVdbeActive>1 ){
   155    168       sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress");
   156    169       return SQLITE_ERROR;
   157    170     }
          171  +  if( pOut ){
          172  +    if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){
          173  +      sqlite3SetString(pzErrMsg, db, "non-text filename");
          174  +      return SQLITE_ERROR;
          175  +    }
          176  +    zOut = (const char*)sqlite3_value_text(pOut);
          177  +  }else{
          178  +    zOut = "";
          179  +  }
   158    180   
   159    181     /* Save the current value of the database flags so that it can be 
   160    182     ** restored before returning. Then set the writable-schema flag, and
   161    183     ** disable CHECK and foreign key constraints.  */
   162    184     saved_flags = db->flags;
   163    185     saved_mDbFlags = db->mDbFlags;
   164    186     saved_nChange = db->nChange;
   165    187     saved_nTotalChange = db->nTotalChange;
   166    188     saved_mTrace = db->mTrace;
   167    189     db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
   168    190     db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
   169         -  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder
          191  +  db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder
   170    192                      | SQLITE_Defensive | SQLITE_CountRows);
   171    193     db->mTrace = 0;
   172    194   
   173    195     zDbMain = db->aDb[iDb].zDbSName;
   174    196     pMain = db->aDb[iDb].pBt;
   175    197     isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));
   176    198   
................................................................................
   185    207     ** that actually made the VACUUM run slower.  Very little journalling
   186    208     ** actually occurs when doing a vacuum since the vacuum_db is initially
   187    209     ** empty.  Only the journal header is written.  Apparently it takes more
   188    210     ** time to parse and run the PRAGMA to turn journalling off than it does
   189    211     ** to write the journal header file.
   190    212     */
   191    213     nDb = db->nDb;
   192         -  rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db");
          214  +  rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut);
   193    215     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   194    216     assert( (db->nDb-1)==nDb );
   195    217     pDb = &db->aDb[nDb];
   196    218     assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
   197    219     pTemp = pDb->pBt;
   198         -
   199         -  /* The call to execSql() to attach the temp database has left the file
   200         -  ** locked (as there was more than one active statement when the transaction
   201         -  ** to read the schema was concluded. Unlock it here so that this doesn't
   202         -  ** cause problems for the call to BtreeSetPageSize() below.  */
   203         -  sqlite3BtreeCommit(pTemp);
   204         -
          220  +  if( pOut ){
          221  +    sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp));
          222  +    i64 sz = 0;
          223  +    if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){
          224  +      rc = SQLITE_ERROR;
          225  +      sqlite3SetString(pzErrMsg, db, "output file already exists");
          226  +      goto end_of_vacuum;
          227  +    }
          228  +  }
   205    229     nRes = sqlite3BtreeGetOptimalReserve(pMain);
   206    230   
   207    231     /* A VACUUM cannot change the pagesize of an encrypted database. */
   208    232   #ifdef SQLITE_HAS_CODEC
   209    233     if( db->nextPagesize ){
   210    234       extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*);
   211    235       int nKey;
................................................................................
   221    245   
   222    246     /* Begin a transaction and take an exclusive lock on the main database
   223    247     ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
   224    248     ** to ensure that we do not try to change the page-size on a WAL database.
   225    249     */
   226    250     rc = execSql(db, pzErrMsg, "BEGIN");
   227    251     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   228         -  rc = sqlite3BtreeBeginTrans(pMain, 2, 0);
          252  +  rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0);
   229    253     if( rc!=SQLITE_OK ) goto end_of_vacuum;
   230    254   
   231    255     /* Do not attempt to change the page size for a WAL database */
   232    256     if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain))
   233    257                                                  ==PAGER_JOURNALMODE_WAL ){
   234    258       db->nextPagesize = 0;
   235    259     }
................................................................................
   316    340          BTREE_DEFAULT_CACHE_SIZE, 0,  /* Preserve the default page cache size */
   317    341          BTREE_TEXT_ENCODING,      0,  /* Preserve the text encoding */
   318    342          BTREE_USER_VERSION,       0,  /* Preserve the user version */
   319    343          BTREE_APPLICATION_ID,     0,  /* Preserve the application id */
   320    344       };
   321    345   
   322    346       assert( 1==sqlite3BtreeIsInTrans(pTemp) );
   323         -    assert( 1==sqlite3BtreeIsInTrans(pMain) );
          347  +    assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) );
   324    348   
   325    349       /* Copy Btree meta values */
   326    350       for(i=0; i<ArraySize(aCopy); i+=2){
   327    351         /* GetMeta() and UpdateMeta() cannot fail in this context because
   328    352         ** we already have page 1 loaded into cache and marked dirty. */
   329    353         sqlite3BtreeGetMeta(pMain, aCopy[i], &meta);
   330    354         rc = sqlite3BtreeUpdateMeta(pTemp, aCopy[i], meta+aCopy[i+1]);
   331    355         if( NEVER(rc!=SQLITE_OK) ) goto end_of_vacuum;
   332    356       }
   333    357   
   334         -    rc = sqlite3BtreeCopyFile(pMain, pTemp);
          358  +    if( pOut==0 ){
          359  +      rc = sqlite3BtreeCopyFile(pMain, pTemp);
          360  +    }
   335    361       if( rc!=SQLITE_OK ) goto end_of_vacuum;
   336    362       rc = sqlite3BtreeCommit(pTemp);
   337    363       if( rc!=SQLITE_OK ) goto end_of_vacuum;
   338    364   #ifndef SQLITE_OMIT_AUTOVACUUM
   339         -    sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
          365  +    if( pOut==0 ){
          366  +      sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
          367  +    }
   340    368   #endif
   341    369     }
   342    370   
   343    371     assert( rc==SQLITE_OK );
   344         -  rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
          372  +  if( pOut==0 ){
          373  +    rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
          374  +  }
   345    375   
   346    376   end_of_vacuum:
   347    377     /* Restore the original value of db->flags */
   348    378     db->init.iDb = 0;
   349    379     db->mDbFlags = saved_mDbFlags;
   350    380     db->flags = saved_flags;
   351    381     db->nChange = saved_nChange;

Changes to src/vdbe.c.

  6692   6692     sqlite3VdbeChangeEncoding(pOut, encoding);
  6693   6693     if( rc ) goto abort_due_to_error;
  6694   6694     break;
  6695   6695   };
  6696   6696   #endif /* SQLITE_OMIT_PRAGMA */
  6697   6697   
  6698   6698   #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
  6699         -/* Opcode: Vacuum P1 * * * *
         6699  +/* Opcode: Vacuum P1 P2 * * *
  6700   6700   **
  6701   6701   ** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
  6702   6702   ** for an attached database.  The "temp" database may not be vacuumed.
         6703  +**
         6704  +** If P2 is not zero, then it is a register holding a string which is
         6705  +** the file into which the result of vacuum should be written.  When
         6706  +** P2 is zero, the vacuum overwrites the original database.
  6703   6707   */
  6704   6708   case OP_Vacuum: {
  6705   6709     assert( p->readOnly==0 );
  6706         -  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1);
         6710  +  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1,
         6711  +                        pOp->p2 ? &aMem[pOp->p2] : 0);
  6707   6712     if( rc ) goto abort_due_to_error;
  6708   6713     break;
  6709   6714   }
  6710   6715   #endif
  6711   6716   
  6712   6717   #if !defined(SQLITE_OMIT_AUTOVACUUM)
  6713   6718   /* Opcode: IncrVacuum P1 P2 * * *
................................................................................
  7094   7099     assert( pName->flags & MEM_Str );
  7095   7100     testcase( pName->enc==SQLITE_UTF8 );
  7096   7101     testcase( pName->enc==SQLITE_UTF16BE );
  7097   7102     testcase( pName->enc==SQLITE_UTF16LE );
  7098   7103     rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
  7099   7104     if( rc ) goto abort_due_to_error;
  7100   7105     rc = pVtab->pModule->xRename(pVtab, pName->z);
  7101         -  if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter;
         7106  +  if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter;
  7102   7107     sqlite3VtabImportErrmsg(p, pVtab);
  7103   7108     p->expired = 0;
  7104   7109     if( rc ) goto abort_due_to_error;
  7105   7110     break;
  7106   7111   }
  7107   7112   #endif
  7108   7113   

Changes to src/vdbe.h.

   247    247   int sqlite3VdbeReset(Vdbe*);
   248    248   void sqlite3VdbeSetNumCols(Vdbe*,int);
   249    249   int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*));
   250    250   void sqlite3VdbeCountChanges(Vdbe*);
   251    251   sqlite3 *sqlite3VdbeDb(Vdbe*);
   252    252   u8 sqlite3VdbePrepareFlags(Vdbe*);
   253    253   void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8);
          254  +#ifdef SQLITE_ENABLE_NORMALIZE
          255  +void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*);
          256  +int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*);
          257  +#endif
   254    258   void sqlite3VdbeSwap(Vdbe*,Vdbe*);
   255    259   VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*);
   256    260   sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8);
   257    261   void sqlite3VdbeSetVarmask(Vdbe*, int);
   258    262   #ifndef SQLITE_OMIT_TRACE
   259    263     char *sqlite3VdbeExpandSql(Vdbe*, const char*);
   260    264   #endif

Changes to src/vdbeInt.h.

   331    331   };
   332    332   
   333    333   /* A bitfield type for use inside of structures.  Always follow with :N where
   334    334   ** N is the number of bits.
   335    335   */
   336    336   typedef unsigned bft;  /* Bit Field Type */
   337    337   
          338  +/* The ScanStatus object holds a single value for the
          339  +** sqlite3_stmt_scanstatus() interface.
          340  +*/
   338    341   typedef struct ScanStatus ScanStatus;
   339    342   struct ScanStatus {
   340    343     int addrExplain;                /* OP_Explain for loop */
   341    344     int addrLoop;                   /* Address of "loops" counter */
   342    345     int addrVisit;                  /* Address of "rows visited" counter */
   343    346     int iSelectID;                  /* The "Select-ID" for this loop */
   344    347     LogEst nEst;                    /* Estimated output rows per loop */
   345    348     char *zName;                    /* Name of table or index */
   346    349   };
          350  +
          351  +/* The DblquoteStr object holds the text of a double-quoted
          352  +** string for a prepared statement.  A linked list of these objects
          353  +** is constructed during statement parsing and is held on Vdbe.pDblStr.
          354  +** When computing a normalized SQL statement for an SQL statement, that
          355  +** list is consulted for each double-quoted identifier to see if the
          356  +** identifier should really be a string literal.
          357  +*/
          358  +typedef struct DblquoteStr DblquoteStr;
          359  +struct DblquoteStr {
          360  +  DblquoteStr *pNextStr;   /* Next string literal in the list */
          361  +  char z[8];               /* Dequoted value for the string */
          362  +};
   347    363   
   348    364   /*
   349    365   ** An instance of the virtual machine.  This structure contains the complete
   350    366   ** state of the virtual machine.
   351    367   **
   352    368   ** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare()
   353    369   ** is really a pointer to an instance of this structure.
................................................................................
   360    376     u32 magic;              /* Magic number for sanity checking */
   361    377     int nMem;               /* Number of memory locations currently allocated */
   362    378     int nCursor;            /* Number of slots in apCsr[] */
   363    379     u32 cacheCtr;           /* VdbeCursor row cache generation counter */
   364    380     int pc;                 /* The program counter */
   365    381     int rc;                 /* Value to return */
   366    382     int nChange;            /* Number of db changes made since last reset */
   367         -  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
          383  +  int iStatement;         /* Statement number (or 0 if has no opened stmt) */
   368    384     i64 iCurrentTime;       /* Value of julianday('now') for this statement */
   369    385     i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
   370    386     i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
   371    387     i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
   372    388   
   373    389     /* When allocating a new Vdbe object, all of the fields below should be
   374    390     ** initialized to zero or NULL */
................................................................................
   404    420     bft bIsReader:1;        /* True for statements that read */
   405    421     yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   406    422     yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   407    423     u32 aCounter[7];        /* Counters used by sqlite3_stmt_status() */
   408    424     char *zSql;             /* Text of the SQL statement that generated this */
   409    425   #ifdef SQLITE_ENABLE_NORMALIZE
   410    426     char *zNormSql;         /* Normalization of the associated SQL statement */
          427  +  DblquoteStr *pDblStr;   /* List of double-quoted string literals */
   411    428   #endif
   412    429     void *pFree;            /* Free this when deleting the vdbe */
   413    430     VdbeFrame *pFrame;      /* Parent frame */
   414    431     VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
   415    432     int nFrame;             /* Number of frames in pFrame list */
   416    433     u32 expmask;            /* Binding to these vars invalidates VM */
   417    434     SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */

Changes to src/vdbeapi.c.

    58     58   ** Invoke the profile callback.  This routine is only called if we already
    59     59   ** know that the profile callback is defined and needs to be invoked.
    60     60   */
    61     61   static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
    62     62     sqlite3_int64 iNow;
    63     63     sqlite3_int64 iElapse;
    64     64     assert( p->startTime>0 );
    65         -  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
           65  +  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
    66     66     assert( db->init.busy==0 );
    67     67     assert( p->zSql!=0 );
    68     68     sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
    69     69     iElapse = (iNow - p->startTime)*1000000;
           70  +#ifndef SQLITE_OMIT_DEPRECATED  	
    70     71     if( db->xProfile ){
    71     72       db->xProfile(db->pProfileArg, p->zSql, iElapse);
    72     73     }
           74  +#endif
    73     75     if( db->mTrace & SQLITE_TRACE_PROFILE ){
    74     76       db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
    75     77     }
    76     78     p->startTime = 0;
    77     79   }
    78     80   /*
    79     81   ** The checkProfileCallback(DB,P) macro checks to see if a profile callback
................................................................................
   579    581     /* Check that malloc() has not failed. If it has, return early. */
   580    582     db = p->db;
   581    583     if( db->mallocFailed ){
   582    584       p->rc = SQLITE_NOMEM;
   583    585       return SQLITE_NOMEM_BKPT;
   584    586     }
   585    587   
   586         -  if( p->pc<=0 && p->expired ){
          588  +  if( p->pc<0 && p->expired ){
   587    589       p->rc = SQLITE_SCHEMA;
   588    590       rc = SQLITE_ERROR;
   589    591       goto end_of_step;
   590    592     }
   591    593     if( p->pc<0 ){
   592    594       /* If there are no other statements currently running, then
   593    595       ** reset the interrupt flag.  This prevents a call to sqlite3_interrupt
................................................................................
   598    600       }
   599    601   
   600    602       assert( db->nVdbeWrite>0 || db->autoCommit==0 
   601    603           || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
   602    604       );
   603    605   
   604    606   #ifndef SQLITE_OMIT_TRACE
   605         -    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
          607  +    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
   606    608           && !db->init.busy && p->zSql ){
   607    609         sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
   608    610       }else{
   609    611         assert( p->startTime==0 );
   610    612       }
   611    613   #endif
   612    614   
................................................................................
   625    627   #endif /* SQLITE_OMIT_EXPLAIN */
   626    628     {
   627    629       db->nVdbeExec++;
   628    630       rc = sqlite3VdbeExec(p);
   629    631       db->nVdbeExec--;
   630    632     }
   631    633   
          634  +  if( rc!=SQLITE_ROW ){
   632    635   #ifndef SQLITE_OMIT_TRACE
   633         -  /* If the statement completed successfully, invoke the profile callback */
   634         -  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
          636  +    /* If the statement completed successfully, invoke the profile callback */
          637  +    checkProfileCallback(db, p);
   635    638   #endif
   636    639   
   637         -  if( rc==SQLITE_DONE && db->autoCommit ){
   638         -    assert( p->rc==SQLITE_OK );
   639         -    p->rc = doWalCallbacks(db);
   640         -    if( p->rc!=SQLITE_OK ){
   641         -      rc = SQLITE_ERROR;
          640  +    if( rc==SQLITE_DONE && db->autoCommit ){
          641  +      assert( p->rc==SQLITE_OK );
          642  +      p->rc = doWalCallbacks(db);
          643  +      if( p->rc!=SQLITE_OK ){
          644  +        rc = SQLITE_ERROR;
          645  +      }
   642    646       }
   643    647     }
   644    648   
   645    649     db->errCode = rc;
   646    650     if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
   647    651       p->rc = SQLITE_NOMEM_BKPT;
   648    652     }
................................................................................
   654    658     ** contains the value that would be returned if sqlite3_finalize() 
   655    659     ** were called on statement p.
   656    660     */
   657    661     assert( rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR 
   658    662          || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE
   659    663     );
   660    664     assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp );
   661         -  if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 
   662         -   && rc!=SQLITE_ROW 
   663         -   && rc!=SQLITE_DONE 
          665  +  if( rc!=SQLITE_ROW 
          666  +   && rc!=SQLITE_DONE
          667  +   && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0
   664    668     ){
   665    669       /* If this statement was prepared using saved SQL and an 
   666    670       ** error has occurred, then return the error code in p->rc to the
   667    671       ** caller. Set the error code in the database handle to the same value.
   668    672       */ 
   669    673       rc = sqlite3VdbeTransferError(p);
   670    674     }
................................................................................
  1278   1282       sqlite3_mutex_leave(p->db->mutex);
  1279   1283       return SQLITE_RANGE;
  1280   1284     }
  1281   1285     i--;
  1282   1286     pVar = &p->aVar[i];
  1283   1287     sqlite3VdbeMemRelease(pVar);
  1284   1288     pVar->flags = MEM_Null;
  1285         -  sqlite3Error(p->db, SQLITE_OK);
         1289  +  p->db->errCode = SQLITE_OK;
  1286   1290   
  1287   1291     /* If the bit corresponding to this variable in Vdbe.expmask is set, then 
  1288   1292     ** binding a new value to this variable invalidates the current query plan.
  1289   1293     **
  1290   1294     ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host
  1291   1295     ** parameter in the WHERE clause might influence the choice of query plan
  1292   1296     ** for a statement, then the statement will be automatically recompiled,
................................................................................
  1704   1708   
  1705   1709   #ifdef SQLITE_ENABLE_NORMALIZE
  1706   1710   /*
  1707   1711   ** Return the normalized SQL associated with a prepared statement.
  1708   1712   */
  1709   1713   const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
  1710   1714     Vdbe *p = (Vdbe *)pStmt;
  1711         -  return p ? p->zNormSql : 0;
         1715  +  if( p==0 ) return 0;
         1716  +  if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){
         1717  +    sqlite3_mutex_enter(p->db->mutex);
         1718  +    p->zNormSql = sqlite3Normalize(p, p->zSql);
         1719  +    sqlite3_mutex_leave(p->db->mutex);
         1720  +  }
         1721  +  return p->zNormSql;
  1712   1722   }
  1713   1723   #endif /* SQLITE_ENABLE_NORMALIZE */
  1714   1724   
  1715   1725   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
  1716   1726   /*
  1717   1727   ** Allocate and populate an UnpackedRecord structure based on the serialized
  1718   1728   ** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure

Changes to src/vdbeaux.c.

    60     60     if( p==0 ) return;
    61     61     p->prepFlags = prepFlags;
    62     62     if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
    63     63       p->expmask = 0;
    64     64     }
    65     65     assert( p->zSql==0 );
    66     66     p->zSql = sqlite3DbStrNDup(p->db, z, n);
           67  +}
           68  +
           69  +#ifdef SQLITE_ENABLE_NORMALIZE
           70  +/*
           71  +** Add a new element to the Vdbe->pDblStr list.
           72  +*/
           73  +void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){
           74  +  if( p ){
           75  +    int n = sqlite3Strlen30(z);
           76  +    DblquoteStr *pStr = sqlite3DbMallocRawNN(db,
           77  +                            sizeof(*pStr)+n+1-sizeof(pStr->z));
           78  +    if( pStr ){
           79  +      pStr->pNextStr = p->pDblStr;
           80  +      p->pDblStr = pStr;
           81  +      memcpy(pStr->z, z, n+1);
           82  +    }
           83  +  }
           84  +}
           85  +#endif
           86  +
    67     87   #ifdef SQLITE_ENABLE_NORMALIZE
    68         -  assert( p->zNormSql==0 );
    69         -  if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
    70         -    sqlite3Normalize(p, p->zSql, n, prepFlags);
    71         -    assert( p->zNormSql!=0 || p->db->mallocFailed );
           88  +/*
           89  +** zId of length nId is a double-quoted identifier.  Check to see if
           90  +** that identifier is really used as a string literal.
           91  +*/
           92  +int sqlite3VdbeUsesDoubleQuotedString(
           93  +  Vdbe *pVdbe,            /* The prepared statement */
           94  +  const char *zId         /* The double-quoted identifier, already dequoted */
           95  +){
           96  +  DblquoteStr *pStr;
           97  +  assert( zId!=0 );
           98  +  if( pVdbe->pDblStr==0 ) return 0;
           99  +  for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){
          100  +    if( strcmp(zId, pStr->z)==0 ) return 1;
    72    101     }
          102  +  return 0;
          103  +}
    73    104   #endif
    74         -}
    75    105   
    76    106   /*
    77    107   ** Swap all content between two VDBE structures.
    78    108   */
    79    109   void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
    80    110     Vdbe tmp, *pTmp;
    81    111     char *zTmp;
................................................................................
    88    118     pB->pNext = pTmp;
    89    119     pTmp = pA->pPrev;
    90    120     pA->pPrev = pB->pPrev;
    91    121     pB->pPrev = pTmp;
    92    122     zTmp = pA->zSql;
    93    123     pA->zSql = pB->zSql;
    94    124     pB->zSql = zTmp;
    95         -#ifdef SQLITE_ENABLE_NORMALIZE
          125  +#if 0
    96    126     zTmp = pA->zNormSql;
    97    127     pA->zNormSql = pB->zNormSql;
    98    128     pB->zNormSql = zTmp;
    99    129   #endif
   100    130     pB->expmask = pA->expmask;
   101    131     pB->prepFlags = pA->prepFlags;
   102    132     memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));
................................................................................
  2851   2881           }else if( rc!=SQLITE_OK ){
  2852   2882             p->rc = rc;
  2853   2883             sqlite3RollbackAll(db, SQLITE_OK);
  2854   2884             p->nChange = 0;
  2855   2885           }else{
  2856   2886             db->nDeferredCons = 0;
  2857   2887             db->nDeferredImmCons = 0;
  2858         -          db->flags &= ~SQLITE_DeferFKs;
         2888  +          db->flags &= ~(u64)SQLITE_DeferFKs;
  2859   2889             sqlite3CommitInternalChanges(db);
  2860   2890           }
  2861   2891         }else{
  2862   2892           sqlite3RollbackAll(db, SQLITE_OK);
  2863   2893           p->nChange = 0;
  2864   2894         }
  2865   2895         db->nStatement = 0;
................................................................................
  3166   3196       sqlite3DbFree(db, p->pFree);
  3167   3197     }
  3168   3198     vdbeFreeOpArray(db, p->aOp, p->nOp);
  3169   3199     sqlite3DbFree(db, p->aColName);
  3170   3200     sqlite3DbFree(db, p->zSql);
  3171   3201   #ifdef SQLITE_ENABLE_NORMALIZE
  3172   3202     sqlite3DbFree(db, p->zNormSql);
         3203  +  {
         3204  +    DblquoteStr *pThis, *pNext;
         3205  +    for(pThis=p->pDblStr; pThis; pThis=pNext){
         3206  +      pNext = pThis->pNextStr;
         3207  +      sqlite3DbFree(db, pThis);
         3208  +    }
         3209  +  }
  3173   3210   #endif
  3174   3211   #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  3175   3212     {
  3176   3213       int i;
  3177   3214       for(i=0; i<p->nScan; i++){
  3178   3215         sqlite3DbFree(db, p->aScan[i].zName);
  3179   3216       }

Changes to src/window.c.

  2129   2129   Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){
  2130   2130     Window *pNew = 0;
  2131   2131     if( ALWAYS(p) ){
  2132   2132       pNew = sqlite3DbMallocZero(db, sizeof(Window));
  2133   2133       if( pNew ){
  2134   2134         pNew->zName = sqlite3DbStrDup(db, p->zName);
  2135   2135         pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0);
         2136  +      pNew->pFunc = p->pFunc;
  2136   2137         pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0);
  2137   2138         pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0);
  2138   2139         pNew->eType = p->eType;
  2139   2140         pNew->eEnd = p->eEnd;
  2140   2141         pNew->eStart = p->eStart;
  2141   2142         pNew->pStart = sqlite3ExprDup(db, p->pStart, 0);
  2142   2143         pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0);

Changes to test/altertab2.test.

     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #*************************************************************************
    11     11   #
    12     12   
    13     13   set testdir [file dirname $argv0]
    14     14   source $testdir/tester.tcl
    15         -set testprefix altertab
           15  +set testprefix altertab2
    16     16   
    17     17   # If SQLITE_OMIT_ALTERTABLE is defined, omit this file.
    18     18   ifcapable !altertable {
    19     19     finish_test
    20     20     return
    21     21   }
    22     22   
................................................................................
    81     81     SELECT sql FROM sqlite_master WHERE name LIKE 'c%';
    82     82   } {
    83     83     {CREATE TABLE c1(x REFERENCES "p3")}
    84     84     {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")}
    85     85     {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))}
    86     86   }
    87     87   
           88  +#-------------------------------------------------------------------------
           89  +# Table name in WITH clauses that are part of views or triggers.
           90  +#
           91  +foreach {tn schema} {
           92  +  1 {
           93  +    CREATE TABLE log_entry(col1, y);
           94  +    CREATE INDEX i1 ON log_entry(col1);
           95  +  }
           96  +
           97  +  2 {
           98  +    CREATE TABLE t1(a, b, c);
           99  +    CREATE TABLE t2(x);
          100  +    CREATE TABLE log_entry(col1);
          101  +    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
          102  +      INSERT INTO t2 SELECT col1 FROM log_entry;
          103  +    END;
          104  +  }
          105  +
          106  +  3 {
          107  +    CREATE TABLE t1(a, b, c);
          108  +    CREATE TABLE t2(x);
          109  +    CREATE TABLE log_entry(col1);
          110  +    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
          111  +      INSERT INTO t2
          112  +        WITH xyz(x) AS (SELECT col1 FROM log_entry)
          113  +        SELECT x FROM xyz;
          114  +    END;
          115  +  }
          116  +
          117  +  4 {
          118  +    CREATE TABLE log_entry(col1);
          119  +    CREATE VIEW ttt AS
          120  +        WITH xyz(x) AS (SELECT col1 FROM log_entry)
          121  +        SELECT x FROM xyz;
          122  +  }
          123  +} {
          124  +  reset_db
          125  +  do_execsql_test 3.$tn.1 $schema
          126  +  set expect [db eval "SELECT sql FROM sqlite_master"]
          127  +  set expect [string map {log_entry {"newname"}} $expect]
          128  +
          129  +  do_execsql_test 3.$tn.2 {
          130  +    ALTER TABLE log_entry RENAME TO newname;
          131  +    SELECT sql FROM sqlite_master;
          132  +  } $expect
          133  +
          134  +  reset_db
          135  +  do_execsql_test 3.$tn.3 $schema
          136  +  set expect [db eval "SELECT sql FROM sqlite_master"]
          137  +  set expect [string map {col1 newname} $expect]
          138  +
          139  +  do_execsql_test 3.$tn.4 {
          140  +    ALTER TABLE log_entry RENAME col1 TO newname;
          141  +    SELECT sql FROM sqlite_master;
          142  +  } $expect
          143  +}
    88    144   
    89    145   finish_test

Changes to test/dbfuzz2.c.

    74     74     int rc;
    75     75     int i;
    76     76   
    77     77     if( eVerbosity>=1 ){
    78     78       printf("************** nByte=%d ***************\n", (int)nByte);
    79     79       fflush(stdout);
    80     80     }
           81  +  if( sqlite3_initialize() ) return 0;
    81     82     rc = sqlite3_open(0, &db);
    82     83     if( rc ) return 1;
    83     84     a = sqlite3_malloc64(nByte+1);
    84     85     if( a==0 ) return 1;
    85     86     memcpy(a, aData, nByte);
    86     87     sqlite3_deserialize(db, "main", a, nByte, nByte,
    87     88           SQLITE_DESERIALIZE_RESIZEABLE |

Added test/fts4umlaut.test.

            1  +# 2018 December 3
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix fts4umlaut
           18  +
           19  +ifcapable !fts3 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts4(
           27  +      x, 
           28  +      tokenize=unicode61 "remove_diacritics=2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1 WHERE t1 MATCH $q
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2 WHERE t2 MATCH $q
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to test/index6.test.

   385    385     CREATE TABLE t11(a,b,c);
   386    386     CREATE INDEX t11x ON t11(a) WHERE b<>99;
   387    387     EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
   388    388   } {/USING INDEX t11x/}
   389    389   do_execsql_test index6-11.2 {
   390    390     EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
   391    391   } {/USING INDEX t11x/}
   392         -  
   393    392   
          393  +# 2018-12-08
          394  +# Ticket https://www.sqlite.org/src/info/1d958d90596593a7
          395  +# NOT IN operator fails when using a partial index.
          396  +#
          397  +do_execsql_test index6-12.1 {
          398  +  DROP TABLE IF EXISTS t1;
          399  +  DROP TABLE IF EXISTS t2;
          400  +  CREATE TABLE t1(a,b);
          401  +  INSERT INTO t1 VALUES(1,1);
          402  +  INSERT INTO t1 VALUES(2,2);
          403  +  CREATE TABLE t2(x);
          404  +  INSERT INTO t2 VALUES(1);
          405  +  INSERT INTO t2 VALUES(2);
          406  +  SELECT 'one', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
          407  +  CREATE INDEX t1a ON t1(a) WHERE b=1;
          408  +  SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
          409  +} {}
          410  +do_execsql_test index6-12.2 {
          411  +  SELECT x FROM t2 WHERE x IN (SELECT a FROM t1) ORDER BY +x;
          412  +} {1 2}
   394    413   finish_test

Changes to test/json101.test.

   809    809   do_execsql_test json-14.160 {
   810    810     SELECT fullkey FROM json_tree('"hello"');
   811    811   } {$}
   812    812   do_execsql_test json-14.170 {
   813    813     SELECT fullkey FROM json_tree('null');
   814    814   } {$}
   815    815   
   816         -
          816  +# 2018-12-03
          817  +# Make sure the table-valued functions contained within parentheses
          818  +# work correctly.
          819  +#
          820  +# Bug reported via private email. See TH3 for more information.
          821  +#
          822  +do_execsql_test json-15.100 {
          823  +  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
          824  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          825  +do_execsql_test json-15.110 {
          826  +  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
          827  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          828  +do_execsql_test json-15.120 {
          829  +  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
          830  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          831  +do_execsql_test json-15.130 {
          832  +  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
          833  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
   817    834   
   818    835   finish_test

Changes to test/normalize.test.

    82     82     set STMT [sqlite3_prepare_v3 $DB \
    83     83         "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 0 TAIL]
    84     84   
    85     85     sqlite3_bind_null $STMT 1
    86     86   } {}
    87     87   do_test 202 {
    88     88     sqlite3_normalized_sql $STMT
    89         -} {}
           89  +} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;}
    90     90   do_test 203 {
    91     91     sqlite3_finalize $STMT
    92     92   } {SQLITE_OK}
    93     93   
    94     94   do_test 210 {
    95     95     set STMT [sqlite3_prepare_v3 $DB \
    96     96         "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 2 TAIL]
................................................................................
   203    203     {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
   204    204     0x2
   205    205     {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}
   206    206   
   207    207     430
   208    208     {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
   209    209     0x2
   210         -  {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}}
          210  +  {0 {SELECT a FROM t1 WHERE x IN(?,?,?);}}
   211    211   
   212    212     440
   213    213     {SELECT 'a' FROM t1 WHERE 'x';}
   214    214     0x2
   215    215     {0 {SELECT?FROM t1 WHERE?;}}
   216    216   
   217    217     450
   218    218     {SELECT [a] FROM t1 WHERE [x];}
   219    219     0x2
   220         -  {0 {SELECT"a"FROM t1 WHERE"x";}}
          220  +  {0 {SELECT a FROM t1 WHERE x;}}
   221    221   
   222    222     460
   223    223     {SELECT * FROM t1 WHERE x IN (x);}
   224    224     0x2
   225    225     {0 {SELECT*FROM t1 WHERE x IN(x);}}
   226    226   
   227    227     470
................................................................................
   228    228     {SELECT * FROM t1 WHERE x IN (x,a);}
   229    229     0x2
   230    230     {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
   231    231   
   232    232     480
   233    233     {SELECT * FROM t1 WHERE x IN ([x],"a");}
   234    234     0x2
   235         -  {0 {SELECT*FROM t1 WHERE x IN("x","a");}}
          235  +  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
   236    236   
   237    237     500
   238    238     {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());}
   239    239     0x2
   240         -  {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}}
          240  +  {0 {SELECT*FROM t1 WHERE x IN(x,a,?,sqlite_version());}}
   241    241   
   242    242     520
   243    243     {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
   244    244     0x2
   245    245     {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}
   246    246   
   247    247     540
   248    248     {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));}
   249    249     0x2
   250         -  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
          250  +  {0 {SELECT*FROM t1 WHERE x IN((SELECT x FROM t1));}}
   251    251   
   252    252     550
   253    253     {SELECT a, a+1, a||'b', a+"b" FROM t1;}
   254    254     0x2
   255         -  {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}}
          255  +  {0 {SELECT a,a+?,a||?,a+b FROM t1;}}
   256    256   
   257    257     570
   258    258     {SELECT * FROM t1 WHERE x IN (1);}
   259    259     0x2
   260    260     {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
   261    261   
   262    262     580
................................................................................
   312    312     {SELECT "col f", [col f] FROM t1;}
   313    313     0x2
   314    314     {0 {SELECT"col f","col f"FROM t1;}}
   315    315   
   316    316     680
   317    317     {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];}
   318    318     0x2
   319         -  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}}
          319  +  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON t1."col f"==t2."col y";}}
   320    320   
   321    321     690
   322    322     {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);}
   323    323     0x2
   324    324     {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}}
   325    325   
   326    326     700
................................................................................
   342    342     {SELECT x FROM t1 WHERE x = NULL;}
   343    343     0x2
   344    344     {0 {SELECT x FROM t1 WHERE x=?;}}
   345    345   
   346    346     760
   347    347     {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');}
   348    348     0x2
   349         -  {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}}
          349  +  {0 {SELECT x FROM t1 WHERE x IN(x IS NOT NULL,?,?,?,b,?);}}
   350    350   } {
   351    351     do_test $tnum {
   352    352       set code [catch {
   353    353         set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL]
   354    354         sqlite3_normalized_sql $STMT
   355    355       } res]
   356    356       if {[info exists STMT]} {

Changes to test/shell1.test.

   252    252   } {0 {}}
   253    253   do_test shell1-3.1.3 {
   254    254     catchcmd "test.db" ".backup FOO BAR"
   255    255   } {1 {Error: unknown database FOO}}
   256    256   do_test shell1-3.1.4 {
   257    257     # too many arguments
   258    258     catchcmd "test.db" ".backup FOO BAR BAD"
   259         -} {1 {Usage: .backup ?DB? ?--append? FILENAME}}
          259  +} {1 {Usage: .backup ?DB? ?OPTIONS? FILENAME}}
   260    260   
   261    261   # .bail ON|OFF           Stop after hitting an error.  Default OFF
   262    262   do_test shell1-3.2.1 {
   263    263     catchcmd "test.db" ".bail"
   264    264   } {1 {Usage: .bail on|off}}
   265    265   do_test shell1-3.2.2 {
   266    266     catchcmd "test.db" ".bail ON"

Changes to test/shell4.test.

   103    103   }]
   104    104     list [regexp {Memory Used} $res] \
   105    105          [regexp {Heap Usage} $res] \
   106    106          [regexp {Autoindex Inserts} $res]
   107    107   } {1 1 1}
   108    108   
   109    109   do_test shell4-2.1 {
   110         -  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
   111         -} {1 {Usage: .trace FILE|off}}
          110  +  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
          111  +} {1 {Unknown option "--unknown" on ".trace"}}
   112    112   do_test shell4-2.2 {
   113    113     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
   114    114   } {0 {}}
   115    115   do_test shell4-2.3 {
   116         -  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
   117         -} {/^1 {PRAGMA.*Usage:.*}$/}
          116  +  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
          117  +} {/^0 {PRAGMA.*}$/}
   118    118   ifcapable trace {
   119    119   do_test shell4-2.4 {
   120    120     catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
   121    121   } {0 {CREATE TABLE t1(x);
   122    122   SELECT * FROM t1;}}
   123    123   do_test shell4-2.5 {
   124    124     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;"

Changes to test/tabfunc01.test.

   120    120     SELECT * FROM temp.generate_series(1,4)
   121    121   } {1 2 3 4}
   122    122   do_execsql_test tabfunc01-4.3 {
   123    123     ATTACH ':memory:' AS aux1;
   124    124     CREATE TABLE aux1.t1(a,b,c);
   125    125     SELECT * FROM aux1.generate_series(1,4)
   126    126   } {1 2 3 4}
          127  +
          128  +# 2018-12-03: Fix bug reported by by private email.
          129  +do_execsql_test tabfunc01-4.4 {
          130  +  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
          131  +} {1 3 5}
   127    132   
   128    133   # The next series of tests is verifying that virtual table are able
   129    134   # to optimize the IN operator, even on terms that are not marked "omit".
   130    135   # When the generate_series virtual table is compiled for the testfixture,
   131    136   # the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
   132    137   # causes the xBestIndex method of generate_series to leave the
   133    138   # sqlite3_index_constraint_usage.omit flag set to 0, which should cause

Added test/vacuum-into.test.

            1  +# 2018-12-07
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this file is testing the VACUUM INTO statement.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +
           18  +# If the VACUUM statement is disabled in the current build, skip all
           19  +# the tests in this file.
           20  +#
           21  +ifcapable {!vacuum} {
           22  +  omit_test vacuum.test {Compiled with SQLITE_OMIT_VACUUM}
           23  +  finish_test
           24  +  return
           25  +}
           26  +
           27  +forcedelete out.db
           28  +do_execsql_test vacuum-into-100 {
           29  +  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
           30  +  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
           31  +  INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c;
           32  +  CREATE INDEX t1b ON t1(b);
           33  +  DELETE FROM t1 WHERE a%2;
           34  +  SELECT count(*), sum(a), sum(length(b)) FROM t1;
           35  +} {50 2550 30000}
           36  +do_execsql_test vacuum-into-110 {
           37  +  VACUUM main INTO 'out.db';
           38  +} {}
           39  +sqlite3 db2 out.db
           40  +do_test vacuum-into-120 {
           41  +  db2 eval {SELECT count(*), sum(a), sum(length(b)) FROM t1}
           42  +} {50 2550 30000}
           43  +do_catchsql_test vacuum-into-130 {
           44  +  VACUUM INTO 'out.db';
           45  +} {1 {output file already exists}}
           46  +forcedelete out2.db
           47  +do_catchsql_test vacuum-into-140 {
           48  +  VACUUM INTO 'out2.db';
           49  +} {0 {}}
           50  +do_catchsql_test vacuum-into-150 {
           51  +  VACUUM INTO 'out2.db';
           52  +} {1 {output file already exists}}
           53  +
           54  +do_catchsql_test vacuum-into-200 {
           55  +  VACUUM main INTO ':memory:';
           56  +} {0 {}}
           57  +
           58  +# The INTO argument can be an arbitrary expression.
           59  +#
           60  +do_execsql_test vacuum-into-300 {
           61  +  CREATE TABLE t2(name TEXT);
           62  +  INSERT INTO t2 VALUES(':memory:');
           63  +  VACUUM main INTO (SELECT name FROM t2);
           64  +} {}
           65  +do_catchsql_test vacuum-into-310 {
           66  +  VACUUM INTO null;
           67  +} {1 {non-text filename}}
           68  +
           69  +finish_test

Changes to test/window1.test.

   589    589   
   590    590   do_execsql_test 13.5 {
   591    591     SELECT a, rank() OVER(ORDER BY b) FROM t1
   592    592       INTERSECT 
   593    593     SELECT a, rank() OVER(ORDER BY b DESC) FROM t1;
   594    594   } {
   595    595   }
          596  +
          597  +# 2018-12-06
          598  +# https://www.sqlite.org/src/info/f09fcd17810f65f7
          599  +# Assertion fault when window functions are used.
          600  +#
          601  +# Root cause is the query flattener invoking sqlite3ExprDup() on
          602  +# expressions that contain subqueries with window functions.  The
          603  +# sqlite3ExprDup() routine is not making correctly initializing
          604  +# Select.pWin field of the subqueries.
          605  +#
          606  +sqlite3 db :memory:
          607  +do_execsql_test 14.0 {
          608  +  SELECT * FROM(
          609  +    SELECT * FROM (SELECT 1 AS c) WHERE c IN (
          610  +        SELECT (row_number() OVER()) FROM (VALUES (0))
          611  +    )
          612  +  );
          613  +} {1}
          614  +do_execsql_test 14.1 {
          615  +  CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(12345);
          616  +  CREATE TABLE t2(c); INSERT INTO t2(c) VALUES(1);
          617  +  SELECT y, y+1, y+2 FROM (
          618  +    SELECT c IN (
          619  +      SELECT (row_number() OVER()) FROM t1
          620  +    ) AS y FROM t2
          621  +  );
          622  +} {1 2 3}
   596    623   
   597    624   finish_test

Added tool/index_usage.c.

            1  +/*
            2  +** 2018-12-04
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** 
           13  +** This file implements a utility program used to help determine which
           14  +** indexes in a database schema are used and unused, and how often specific
           15  +** indexes are used.
           16  +*/
           17  +#include "sqlite3.h"
           18  +#include <stdio.h>
           19  +#include <stdlib.h>
           20  +#include <assert.h>
           21  +#include <string.h>
           22  +
           23  +static void usage(const char *argv0){
           24  +  printf("Usage: %s DATABASE LOG\n\n", argv0);
           25  +  printf(
           26  +    "DATABASE is an SQLite database against which various statements\n"
           27  +    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
           28  +    "database with this schema:\n"
           29  +    "\n"
           30  +    "    CREATE TABLE sqllog(sql TEXT);\n"
           31  +    "\n"
           32  +    "This utility program analyzes statements contained in LOG and prints\n"
           33  +    "a report showing how many times each index in DATABASE is used by the\n"
           34  +    "statements in LOG.\n"
           35  +    "\n"
           36  +    "DATABASE only needs to contain the schema used by the statements in\n"
           37  +    "LOG. The content can be removed from DATABASE.\n"
           38  +  );
           39  +  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
           40  +         "checkin number %.40s. Different versions\n"
           41  +         "of SQLite might use different indexes.\n",
           42  +         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
           43  +  exit(1);
           44  +}
           45  +
           46  +int main(int argc, char **argv){
           47  +  sqlite3 *db = 0;          /* The main database */
           48  +  sqlite3_stmt *pStmt = 0;  /* a query */
           49  +  char *zSql;
           50  +  int nErr = 0;
           51  +  int rc;
           52  +
           53  +  if( argc!=3 ) usage(argv[0]);
           54  +  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
           55  +  if( rc ){
           56  +    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
           57  +    goto errorOut;
           58  +  }
           59  +  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
           60  +  if( rc ){
           61  +    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
           62  +           sqlite3_errmsg(db));
           63  +    goto errorOut;
           64  +  }
           65  +  sqlite3_finalize(pStmt);
           66  +  pStmt = 0;
           67  +  rc = sqlite3_exec(db, 
           68  +     "CREATE TABLE temp.idxu(\n"
           69  +     "  tbl TEXT,\n"
           70  +     "  idx TEXT,\n"
           71  +     "  cnt INT,\n"
           72  +     "  PRIMARY KEY(idx)\n"
           73  +     ") WITHOUT ROWID;", 0, 0, 0);
           74  +  if( rc ){
           75  +    printf("Cannot create the result table - %s\n",
           76  +           sqlite3_errmsg(db));
           77  +    goto errorOut;
           78  +  }
           79  +  rc = sqlite3_exec(db,
           80  +     "INSERT INTO temp.idxu(tbl,idx,cnt)"
           81  +     " SELECT tbl_name, name, 0 FROM sqlite_master"
           82  +     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);
           83  +
           84  +  /* Open the LOG database */
           85  +  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
           86  +  rc = sqlite3_exec(db, zSql, 0, 0, 0);
           87  +  sqlite3_free(zSql);
           88  +  if( rc ){
           89  +    printf("Cannot open the LOG database \"%s\" - %s\n",
           90  +           argv[2], sqlite3_errmsg(db));
           91  +    goto errorOut;
           92  +  }
           93  +  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
           94  +                          -1, &pStmt, 0);
           95  +  if( rc ){
           96  +    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
           97  +           argv[2], sqlite3_errmsg(db));
           98  +    goto errorOut;
           99  +  }
          100  +
          101  +  /* Update the counts based on LOG */
          102  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          103  +    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
          104  +    sqlite3_stmt *pS2;
          105  +    if( zLog==0 ) continue;
          106  +    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
          107  +    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
          108  +    sqlite3_free(zSql);
          109  +    if( rc ){
          110  +      printf("Cannot compile LOG entry %d (%s): %s\n",
          111  +             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
          112  +      nErr++;
          113  +    }else{
          114  +      while( sqlite3_step(pS2)==SQLITE_ROW ){
          115  +        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
          116  +        const char *z1, *z2;
          117  +        int n;
          118  +        /* printf("EXPLAIN: %s\n", zExplain); */
          119  +        z1 = strstr(zExplain, " USING INDEX ");
          120  +        if( z1==0 ) continue;
          121  +        z1 += 13;
          122  +        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
          123  +        n = z2 - z1;
          124  +        zSql = sqlite3_mprintf(
          125  +          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
          126  +        );
          127  +        /* printf("sql: %s\n", zSql); */
          128  +        sqlite3_exec(db, zSql, 0, 0, 0);
          129  +        sqlite3_free(zSql);
          130  +      }
          131  +    }
          132  +    sqlite3_finalize(pS2);
          133  +  }
          134  +  sqlite3_finalize(pStmt);
          135  +
          136  +  /* Generate the report */
          137  +  rc = sqlite3_prepare_v2(db,
          138  +     "SELECT tbl, idx, cnt, "
          139  +     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
          140  +     " FROM temp.idxu, main.sqlite_master"
          141  +     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
          142  +     "   AND temp.idxu.idx=main.sqlite_master.name"
          143  +     " ORDER BY cnt DESC, tbl, idx",
          144  +     -1, &pStmt, 0);
          145  +  if( rc ){
          146  +    printf("Cannot query the result table - %s\n",
          147  +           sqlite3_errmsg(db));
          148  +    goto errorOut;
          149  +  }
          150  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          151  +    printf("%10d %s on %s(%s)\n", 
          152  +       sqlite3_column_int(pStmt, 2),
          153  +       sqlite3_column_text(pStmt, 1),
          154  +       sqlite3_column_text(pStmt, 0),
          155  +       sqlite3_column_text(pStmt, 3));
          156  +  }
          157  +  sqlite3_finalize(pStmt);
          158  +  pStmt = 0;
          159  +
          160  +errorOut:
          161  +  sqlite3_finalize(pStmt);
          162  +  sqlite3_close(db);
          163  +  return nErr;
          164  +}

Changes to tool/lemon.c.

  4586   4586     }
  4587   4587     tplt_xfer(lemp->name,in,out,&lineno);
  4588   4588   
  4589   4589     /* Generate code which executes whenever the parser stack overflows */
  4590   4590     tplt_print(out,lemp,lemp->overflow,&lineno);
  4591   4591     tplt_xfer(lemp->name,in,out,&lineno);
  4592   4592   
  4593         -  /* Generate the table of rule information
         4593  +  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
         4594  +  ** yyRuleInfoNRhs[].
  4594   4595     **
  4595   4596     ** Note: This code depends on the fact that rules are number
  4596   4597     ** sequentually beginning with 0.
  4597   4598     */
  4598   4599     for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
  4599         -    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
         4600  +    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
         4601  +     rule_print(out, rp);
         4602  +    fprintf(out," */\n"); lineno++;
         4603  +  }
         4604  +  tplt_xfer(lemp->name,in,out,&lineno);
         4605  +  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
         4606  +    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
  4600   4607       rule_print(out, rp);
  4601   4608       fprintf(out," */\n"); lineno++;
  4602   4609     }
  4603   4610     tplt_xfer(lemp->name,in,out,&lineno);
  4604   4611   
  4605   4612     /* Generate code which execution during each REDUCE action */
  4606   4613     i = 0;

Changes to tool/lempar.c.

   682    682     yytos = yypParser->yytos;
   683    683     yytos->stateno = yyNewState;
   684    684     yytos->major = yyMajor;
   685    685     yytos->minor.yy0 = yyMinor;
   686    686     yyTraceShift(yypParser, yyNewState, "Shift");
   687    687   }
   688    688   
   689         -/* The following table contains information about every rule that
   690         -** is used during the reduce.
   691         -*/
   692         -static const struct {
   693         -  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
   694         -  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
   695         -} yyRuleInfo[] = {
          689  +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
          690  +** of that rule */
          691  +static const YYCODETYPE yyRuleInfoLhs[] = {
          692  +%%
          693  +};
          694  +
          695  +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
          696  +** of symbols on the right-hand side of that rule. */
          697  +static const signed char yyRuleInfoNRhs[] = {
   696    698   %%
   697    699   };
   698    700   
   699    701   static void yy_accept(yyParser*);  /* Forward Declaration */
   700    702   
   701    703   /*
   702    704   ** Perform a reduce action and the shift that must immediately
................................................................................
   721    723     int yysize;                     /* Amount to pop the stack */
   722    724     ParseARG_FETCH
   723    725     (void)yyLookahead;
   724    726     (void)yyLookaheadToken;
   725    727     yymsp = yypParser->yytos;
   726    728   #ifndef NDEBUG
   727    729     if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
   728         -    yysize = yyRuleInfo[yyruleno].nrhs;
          730  +    yysize = yyRuleInfoNRhs[yyruleno];
   729    731       if( yysize ){
   730    732         fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
   731    733           yyTracePrompt,
   732    734           yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
   733    735       }else{
   734    736         fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
   735    737           yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
................................................................................
   736    738       }
   737    739     }
   738    740   #endif /* NDEBUG */
   739    741   
   740    742     /* Check that the stack is large enough to grow by a single entry
   741    743     ** if the RHS of the rule is empty.  This ensures that there is room
   742    744     ** enough on the stack to push the LHS value */
   743         -  if( yyRuleInfo[yyruleno].nrhs==0 ){
          745  +  if( yyRuleInfoNRhs[yyruleno]==0 ){
   744    746   #ifdef YYTRACKMAXSTACKDEPTH
   745    747       if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
   746    748         yypParser->yyhwm++;
   747    749         assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
   748    750       }
   749    751   #endif
   750    752   #if YYSTACKDEPTH>0 
................................................................................
   778    780     **  #line <lineno> <thisfile>
   779    781     **     break;
   780    782     */
   781    783   /********** Begin reduce actions **********************************************/
   782    784   %%
   783    785   /********** End reduce actions ************************************************/
   784    786     };
   785         -  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
   786         -  yygoto = yyRuleInfo[yyruleno].lhs;
   787         -  yysize = yyRuleInfo[yyruleno].nrhs;
          787  +  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
          788  +  yygoto = yyRuleInfoLhs[yyruleno];
          789  +  yysize = yyRuleInfoNRhs[yyruleno];
   788    790     yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
   789    791   
   790    792     /* There are no SHIFTREDUCE actions on nonterminals because the table
   791    793     ** generator has simplified them to pure REDUCE actions. */
   792    794     assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );
   793    795   
   794    796     /* It is not possible for a REDUCE to be followed by an error */