/ Check-in [06bb80ee]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest wal2 changes into this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | begin-concurrent-wal2
Files: files | file ages | folders
SHA3-256: 06bb80eeb84f57b1e8109a35f5b14992a2e23f2bf5a8921fa29087f96cb44d10
User & Date: dan 2018-12-15 20:59:14
Wiki:begin-concurrent-wal2
Context
2018-12-17
15:22
Merge changes from wal2 branch. check-in: 85f49f0d user: dan tags: begin-concurrent-wal2
2018-12-15
20:59
Merge latest wal2 changes into this branch. check-in: 06bb80ee user: dan tags: begin-concurrent-wal2
20:20
Further test cases for wal2 mode. check-in: 9cb5f8da user: dan tags: wal2
2018-12-07
20:25
Add multi-threaded performance test program "tserver" to this branch. Fix bugs in the begin-concurrent/wal2 integration revealed by the same. check-in: 7bd3b356 user: dan tags: begin-concurrent-wal2
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to Makefile.in.

1302
1303
1304
1305
1306
1307
1308



1309
1310
1311
1312
1313
1314
1315

showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)

showshm$(TEXE):	$(TOP)/tool/showshm.c
	$(LTLINK) -o $@ $(TOP)/tool/showshm.c




changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)

changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)

rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo







>
>
>







1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318

showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)

showshm$(TEXE):	$(TOP)/tool/showshm.c
	$(LTLINK) -o $@ $(TOP)/tool/showshm.c

index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)

changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)

changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)

rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo

Changes to Makefile.msc.

2440
2441
2442
2443
2444
2445
2446




2447
2448
2449
2450
2451
2452
2453
showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

showshm.exe:	$(TOP)\tool\showshm.c
	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)





changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \







>
>
>
>







2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

showshm.exe:	$(TOP)\tool\showshm.c
	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)

index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \

Changes to VERSION.

1
3.26.0
|
1
3.27.0

Changes to autoconf/Makefile.msc.

279
280
281
282
283
284
285

286
287
288
289
290
291
292
...
933
934
935
936
937
938
939

940
941
942
943
944
945
946
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1

!ENDIF
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
!ENDIF

# Should the session extension be enabled?  If so, add compilation options
# to enable it.
#
................................................................................
# Additional compiler options for the shell.  These are only effective
# when the shell is not being dynamically linked.
#
!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1

!ENDIF


# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
core:	dll shell







>







 







>







279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
...
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
!ENDIF
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
!ENDIF

# Should the session extension be enabled?  If so, add compilation options
# to enable it.
#
................................................................................
# Additional compiler options for the shell.  These are only effective
# when the shell is not being dynamically linked.
#
!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
!ENDIF


# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
core:	dll shell

Changes to configure.

1
2
3
4
5
6
7
8
9
10
...
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
....
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
....
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
....
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
....
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
.....
12228
12229
12230
12231
12232
12233
12234
12235
12236
12237
12238
12239
12240
12241
12242
.....
12294
12295
12296
12297
12298
12299
12300
12301
12302
12303
12304
12305
12306
12307
12308
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for sqlite 3.26.0.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
#
#
# This configure script is free software; the Free Software Foundation
# gives unlimited permission to copy, distribute and modify it.
................................................................................
subdirs=
MFLAGS=
MAKEFLAGS=

# Identity of this package.
PACKAGE_NAME='sqlite'
PACKAGE_TARNAME='sqlite'
PACKAGE_VERSION='3.26.0'
PACKAGE_STRING='sqlite 3.26.0'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''

# Factoring default headers for most tests.
ac_includes_default="\
#include <stdio.h>
#ifdef HAVE_SYS_TYPES_H
................................................................................
#
# Report the --help message.
#
if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems.

Usage: $0 [OPTION]... [VAR=VALUE]...

To assign environment variables (e.g., CC, CFLAGS...), specify them as
VAR=VALUE.  See below for descriptions of some of the useful variables.

Defaults for the options are specified in brackets.
................................................................................
  --build=BUILD     configure for building on BUILD [guessed]
  --host=HOST       cross-compile to build programs to run on HOST [BUILD]
_ACEOF
fi

if test -n "$ac_init_help"; then
  case $ac_init_help in
     short | recursive ) echo "Configuration of sqlite 3.26.0:";;
   esac
  cat <<\_ACEOF

Optional Features:
  --disable-option-checking  ignore unrecognized --enable/--with options
  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
    cd "$ac_pwd" || { ac_status=$?; break; }
  done
fi

test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
  cat <<\_ACEOF
sqlite configure 3.26.0
generated by GNU Autoconf 2.69

Copyright (C) 2012 Free Software Foundation, Inc.
This configure script is free software; the Free Software Foundation
gives unlimited permission to copy, distribute and modify it.
_ACEOF
  exit
................................................................................
  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno

} # ac_fn_c_check_header_mongrel
cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.

It was created by sqlite $as_me 3.26.0, which was
generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@

_ACEOF
exec 5>>config.log
{
................................................................................
test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1

cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# Save the log message, to keep $0 and so on meaningful, and to
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by sqlite $as_me 3.26.0, which was
generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
  CONFIG_HEADERS  = $CONFIG_HEADERS
  CONFIG_LINKS    = $CONFIG_LINKS
  CONFIG_COMMANDS = $CONFIG_COMMANDS
  $ $0 $@
................................................................................

Report bugs to the package provider."

_ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
sqlite config.status 3.26.0
configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

Copyright (C) 2012 Free Software Foundation, Inc.
This config.status script is free software; the Free Software Foundation
gives unlimited permission to copy, distribute and modify it."



|







 







|
|







 







|







 







|







 







|







 







|







 







|







 







|







1
2
3
4
5
6
7
8
9
10
...
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
....
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
....
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
....
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
....
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
.....
12228
12229
12230
12231
12232
12233
12234
12235
12236
12237
12238
12239
12240
12241
12242
.....
12294
12295
12296
12297
12298
12299
12300
12301
12302
12303
12304
12305
12306
12307
12308
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for sqlite 3.27.0.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
#
#
# This configure script is free software; the Free Software Foundation
# gives unlimited permission to copy, distribute and modify it.
................................................................................
subdirs=
MFLAGS=
MAKEFLAGS=

# Identity of this package.
PACKAGE_NAME='sqlite'
PACKAGE_TARNAME='sqlite'
PACKAGE_VERSION='3.27.0'
PACKAGE_STRING='sqlite 3.27.0'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''

# Factoring default headers for most tests.
ac_includes_default="\
#include <stdio.h>
#ifdef HAVE_SYS_TYPES_H
................................................................................
#
# Report the --help message.
#
if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
\`configure' configures sqlite 3.27.0 to adapt to many kinds of systems.

Usage: $0 [OPTION]... [VAR=VALUE]...

To assign environment variables (e.g., CC, CFLAGS...), specify them as
VAR=VALUE.  See below for descriptions of some of the useful variables.

Defaults for the options are specified in brackets.
................................................................................
  --build=BUILD     configure for building on BUILD [guessed]
  --host=HOST       cross-compile to build programs to run on HOST [BUILD]
_ACEOF
fi

if test -n "$ac_init_help"; then
  case $ac_init_help in
     short | recursive ) echo "Configuration of sqlite 3.27.0:";;
   esac
  cat <<\_ACEOF

Optional Features:
  --disable-option-checking  ignore unrecognized --enable/--with options
  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
................................................................................
    cd "$ac_pwd" || { ac_status=$?; break; }
  done
fi

test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
  cat <<\_ACEOF
sqlite configure 3.27.0
generated by GNU Autoconf 2.69

Copyright (C) 2012 Free Software Foundation, Inc.
This configure script is free software; the Free Software Foundation
gives unlimited permission to copy, distribute and modify it.
_ACEOF
  exit
................................................................................
  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno

} # ac_fn_c_check_header_mongrel
cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.

It was created by sqlite $as_me 3.27.0, which was
generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@

_ACEOF
exec 5>>config.log
{
................................................................................
test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1

cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# Save the log message, to keep $0 and so on meaningful, and to
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by sqlite $as_me 3.27.0, which was
generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
  CONFIG_HEADERS  = $CONFIG_HEADERS
  CONFIG_LINKS    = $CONFIG_LINKS
  CONFIG_COMMANDS = $CONFIG_COMMANDS
  $ $0 $@
................................................................................

Report bugs to the package provider."

_ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
sqlite config.status 3.27.0
configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

Copyright (C) 2012 Free Software Foundation, Inc.
This config.status script is free software; the Free Software Foundation
gives unlimited permission to copy, distribute and modify it."

Changes to ext/fts3/fts3_unicode.c.

78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
...
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240



241
242
243
244
245
246
247
...
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
#endif /* ifndef SQLITE_AMALGAMATION */

typedef struct unicode_tokenizer unicode_tokenizer;
typedef struct unicode_cursor unicode_cursor;

struct unicode_tokenizer {
  sqlite3_tokenizer base;
  int bRemoveDiacritic;
  int nException;
  int *aiException;
};

struct unicode_cursor {
  sqlite3_tokenizer_cursor base;
  const unsigned char *aInput;    /* Input text being tokenized */
................................................................................
  unicode_tokenizer *pNew;        /* New tokenizer object */
  int i;
  int rc = SQLITE_OK;

  pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
  if( pNew==NULL ) return SQLITE_NOMEM;
  memset(pNew, 0, sizeof(unicode_tokenizer));
  pNew->bRemoveDiacritic = 1;

  for(i=0; rc==SQLITE_OK && i<nArg; i++){
    const char *z = azArg[i];
    int n = (int)strlen(z);

    if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
      pNew->bRemoveDiacritic = 1;
    }
    else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
      pNew->bRemoveDiacritic = 0;



    }
    else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
    }
    else if( n>=11 && memcmp("separators=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
    }
................................................................................
      zOut = &zNew[zOut - pCsr->zToken];
      pCsr->zToken = zNew;
      pCsr->nAlloc += 64;
    }

    /* Write the folded case of the last character read to the output */
    zEnd = z;
    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }

    /* If the cursor is not at EOF, read the next character */
    if( z>=zTerm ) break;
    READ_UTF8(z, zTerm, iCode);







|







 







|






|


|
>
>
>







 







|







78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
...
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
...
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
#endif /* ifndef SQLITE_AMALGAMATION */

typedef struct unicode_tokenizer unicode_tokenizer;
typedef struct unicode_cursor unicode_cursor;

struct unicode_tokenizer {
  sqlite3_tokenizer base;
  int eRemoveDiacritic;
  int nException;
  int *aiException;
};

struct unicode_cursor {
  sqlite3_tokenizer_cursor base;
  const unsigned char *aInput;    /* Input text being tokenized */
................................................................................
  unicode_tokenizer *pNew;        /* New tokenizer object */
  int i;
  int rc = SQLITE_OK;

  pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
  if( pNew==NULL ) return SQLITE_NOMEM;
  memset(pNew, 0, sizeof(unicode_tokenizer));
  pNew->eRemoveDiacritic = 1;

  for(i=0; rc==SQLITE_OK && i<nArg; i++){
    const char *z = azArg[i];
    int n = (int)strlen(z);

    if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
      pNew->eRemoveDiacritic = 1;
    }
    else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
      pNew->eRemoveDiacritic = 0;
    }
    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
      pNew->eRemoveDiacritic = 2;
    }
    else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
    }
    else if( n>=11 && memcmp("separators=", z, 11)==0 ){
      rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
    }
................................................................................
      zOut = &zNew[zOut - pCsr->zToken];
      pCsr->zToken = zNew;
      pCsr->nAlloc += 64;
    }

    /* Write the folded case of the last character read to the output */
    zEnd = z;
    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
    if( iOut ){
      WRITE_UTF8(zOut, iOut);
    }

    /* If the cursor is not at EOF, read the next character */
    if( z>=zTerm ) break;
    READ_UTF8(z, zTerm, iCode);

Changes to ext/fts3/fts3_unicode2.c.

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170

171
172
173
174


175
176


177
178
179
180
181
182
183
184
185
186
187















188
189
190
191
192
193
194
...
197
198
199
200
201
202
203

204
205
206
207
208
209
210
211
...
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
...
347
348
349
350
351
352
353

354

355
356
357
358
359
360
361
362
363
364
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int remove_diacritic(int c){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 

     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 


    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
    62924, 63050, 63082, 63274, 63390, 


  };
  char aChar[] = {
    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
    'e',  'i',  'o',  'u',  'y',  















  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
  int iRes = 0;
  int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
  int iLo = 0;
  while( iHi>=iLo ){
................................................................................
      iRes = iTest;
      iLo = iTest+1;
    }else{
      iHi = iTest-1;
    }
  }
  assert( key>=aDia[iRes] );

  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}


/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
................................................................................
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
  **
  ** If the least significant bit in flags is clear, then the rule applies
  ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
  ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
    assert( iRes>=0 && c>=aEntry[iRes].iCode );
    p = &aEntry[iRes];
    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
      assert( ret>0 );
    }


    if( bRemoveDiacritic ) ret = remove_diacritic(ret);

  }
  
  else if( c>=66560 && c<66600 ){
    ret = c + 40;
  }

  return ret;
}
#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */







|





|
|
|
>
|
<
|
|
>
>
|
<
>
>


<
<
<
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
|







 







|







 







>
|
>










155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172

173
174
175
176
177

178
179
180
181



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
...
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
...
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
...
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int remove_diacritic(int c, int bComplex){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 

    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 

    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
    63182, 63242, 63274, 63310, 63368, 63390, 
  };
  char aChar[] = {



    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
  int iRes = 0;
  int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
  int iLo = 0;
  while( iHi>=iLo ){
................................................................................
      iRes = iTest;
      iLo = iTest+1;
    }else{
      iHi = iTest-1;
    }
  }
  assert( key>=aDia[iRes] );
  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
}


/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
................................................................................
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
  **
  ** If the least significant bit in flags is clear, then the rule applies
  ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
  ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
    assert( iRes>=0 && c>=aEntry[iRes].iCode );
    p = &aEntry[iRes];
    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
      assert( ret>0 );
    }

    if( eRemoveDiacritic ){
      ret = remove_diacritic(ret, eRemoveDiacritic==2);
    }
  }
  
  else if( c>=66560 && c<66600 ){
    ret = c + 40;
  }

  return ret;
}
#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/unicode/mkunicode.tcl.

5
6
7
8
9
10
11

12
13
14
15
16
17
18
19
20
21
22
23
..
25
26
27
28
29
30
31

32
33
34

35
36
37
38

39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
..
58
59
60
61
62
63
64
65

66



67
68
69
70
71
72
73
74
75
76
..
83
84
85
86
87
88
89

90
91
92
93
94
95
96
97
98

99
100
101
102
103
104
105
...
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
...
512
513
514
515
516
517
518

519

520
521
522
523
524
525
526
...
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
  global tl_lookup_table
  set aChar [list]
  set lRange [list]

  set nRange 1
  set iFirst  [lindex $map 0 0]
  set cPrev   [lindex $map 0 1]


  foreach m [lrange $map 1 end] {
    foreach {i c} $m {}

    if {$cPrev == $c} {
      for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
        if {[info exists tl_lookup_table($j)]==0} break
      }

      if {$j==$i} {
        set nNew [expr {(1 + $i - $iFirst)}]
        if {$nNew<=8} {
................................................................................
          continue
        }
      }
    }

    lappend lRange [list $iFirst $nRange]
    lappend aChar  $cPrev


    set iFirst $i
    set cPrev  $c

    set nRange 1
  }
  lappend lRange [list $iFirst $nRange]
  lappend aChar $cPrev


  puts "/*"
  puts "** If the argument is a codepoint corresponding to a lowercase letter"
  puts "** in the ASCII range with a diacritic added, return the codepoint"
  puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
  puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
  puts "** E\"). The resuls of passing a codepoint that corresponds to an"
  puts "** uppercase letter are undefined."
  puts "*/"
  puts "static int ${::remove_diacritic}(int c)\{"
  puts "  unsigned short aDia\[\] = \{"
  puts -nonewline "        0, "
  set i 1
  foreach r $lRange {
    foreach {iCode nRange} $r {}
    if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
    incr i
................................................................................
    puts -nonewline ", "
  }
  puts ""
  puts "  \};"
  puts "  char aChar\[\] = \{"
  puts -nonewline "    '\\0', "
  set i 1
  foreach c $aChar {

    set str "'$c',  "



    if {$c == ""} { set str "'\\0', " }

    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
    incr i
    puts -nonewline "$str"
  }
  puts ""
  puts "  \};"
  puts {
  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
................................................................................
      iRes = iTest;
      iLo = iTest+1;
    }else{
      iHi = iTest-1;
    }
  }
  assert( key>=aDia[iRes] );

  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
  puts "\}"
}

proc print_isdiacritic {zFunc map} {

  set lCode [list]
  foreach m $map {
    foreach {code char} $m {}

    if {$code && $char == ""} { lappend lCode $code }
  }
  set lCode [lsort -integer $lCode]
  set iFirst [lindex $lCode 0]
  set iLast [lindex $lCode end]

  set i1 0
................................................................................
  puts "** is an upper case character that has a lower case equivalent,"
  puts "** return the codepoint corresponding to the lower case version."
  puts "** Otherwise, return a copy of the argument."
  puts "**"
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"

  set liOff [tl_generate_ioff_table $lRecord]
  tl_print_table_header
  foreach entry $lRecord { 
    if {[tl_print_table_entry toggle $entry $liOff]} { 
      lappend lHigh $entry 
    } 
................................................................................
    assert( iRes>=0 && c>=aEntry[iRes].iCode );
    p = &aEntry[iRes];
    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
      assert( ret>0 );
    }


    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);

  }
  }]

  foreach entry $lHigh {
    tl_print_if_entry $entry
  }

................................................................................
  set caseN [categories_switch C N {d l o}]
  set caseP [categories_switch C P {c d e f i o s}]
  set caseS [categories_switch C S {c k m o}]
  set caseZ [categories_switch C Z {l p s}]

  set nCat [expr [llength [array names C]] + 1]
  puts [code {
    int sqlite3Fts5UnicodeNCat(void) { 
      return $nCat;
    }

    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
      aArray[0] = 1;
      switch( zCat[0] ){
        $caseC
        $caseL
        $caseM
        $caseN







>


|

|







 







>



>




>









|







 







|
>
|
>
>
>
|

|







 







>
|







|
>







 







|







 







>
|
>







 







<
<
<
<







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
..
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
..
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
..
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
...
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
...
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
...
613
614
615
616
617
618
619




620
621
622
623
624
625
626
  global tl_lookup_table
  set aChar [list]
  set lRange [list]

  set nRange 1
  set iFirst  [lindex $map 0 0]
  set cPrev   [lindex $map 0 1]
  set fPrev   [lindex $map 0 2]

  foreach m [lrange $map 1 end] {
    foreach {i c f} $m {}

    if {$cPrev == $c && $fPrev==$f} {
      for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
        if {[info exists tl_lookup_table($j)]==0} break
      }

      if {$j==$i} {
        set nNew [expr {(1 + $i - $iFirst)}]
        if {$nNew<=8} {
................................................................................
          continue
        }
      }
    }

    lappend lRange [list $iFirst $nRange]
    lappend aChar  $cPrev
    lappend aFlag  $fPrev

    set iFirst $i
    set cPrev  $c
    set fPrev  $f
    set nRange 1
  }
  lappend lRange [list $iFirst $nRange]
  lappend aChar $cPrev
  lappend aFlag $fPrev

  puts "/*"
  puts "** If the argument is a codepoint corresponding to a lowercase letter"
  puts "** in the ASCII range with a diacritic added, return the codepoint"
  puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
  puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
  puts "** E\"). The resuls of passing a codepoint that corresponds to an"
  puts "** uppercase letter are undefined."
  puts "*/"
  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
  puts "  unsigned short aDia\[\] = \{"
  puts -nonewline "        0, "
  set i 1
  foreach r $lRange {
    foreach {iCode nRange} $r {}
    if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
    incr i
................................................................................
    puts -nonewline ", "
  }
  puts ""
  puts "  \};"
  puts "  char aChar\[\] = \{"
  puts -nonewline "    '\\0',      "
  set i 1
  foreach c $aChar f $aFlag {
    if { $f } {
      set str "'$c'|0x80,  "
    } else {
      set str "'$c'|0x00,  "
    }
    if {$c == ""} { set str "'\\0',      " }

    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    incr i
    puts -nonewline "$str"
  }
  puts ""
  puts "  \};"
  puts {
  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
................................................................................
      iRes = iTest;
      iLo = iTest+1;
    }else{
      iHi = iTest-1;
    }
  }
  assert( key>=aDia[iRes] );
  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
  puts "\}"
}

proc print_isdiacritic {zFunc map} {

  set lCode [list]
  foreach m $map {
    foreach {code char flag} $m {}
    if {$flag} continue
    if {$code && $char == ""} { lappend lCode $code }
  }
  set lCode [lsort -integer $lCode]
  set iFirst [lindex $lCode 0]
  set iLast [lindex $lCode end]

  set i1 0
................................................................................
  puts "** is an upper case character that has a lower case equivalent,"
  puts "** return the codepoint corresponding to the lower case version."
  puts "** Otherwise, return a copy of the argument."
  puts "**"
  puts "** The results are undefined if the value passed to this function"
  puts "** is less than zero."
  puts "*/"
  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"

  set liOff [tl_generate_ioff_table $lRecord]
  tl_print_table_header
  foreach entry $lRecord { 
    if {[tl_print_table_entry toggle $entry $liOff]} { 
      lappend lHigh $entry 
    } 
................................................................................
    assert( iRes>=0 && c>=aEntry[iRes].iCode );
    p = &aEntry[iRes];
    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
      assert( ret>0 );
    }

    if( eRemoveDiacritic ){
      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
    }
  }
  }]

  foreach entry $lHigh {
    tl_print_if_entry $entry
  }

................................................................................
  set caseN [categories_switch C N {d l o}]
  set caseP [categories_switch C P {c d e f i o s}]
  set caseS [categories_switch C S {c k m o}]
  set caseZ [categories_switch C Z {l p s}]

  set nCat [expr [llength [array names C]] + 1]
  puts [code {




    int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
      aArray[0] = 1;
      switch( zCat[0] ){
        $caseC
        $caseL
        $caseM
        $caseN

Changes to ext/fts3/unicode/parseunicode.tcl.

3
4
5
6
7
8
9
10
11
12
13
14












15
16
17
18
19
20
21
..
49
50
51
52
53
54
55


56








57
58
59
60
61

62
63
64
65
66
67
68
69
70
71
72
73
74
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of mappings required to remove all
# diacritical marks from a unicode string. Each mapping is itself a list
# consisting of two elements - the unicode codepoint and the single ASCII
# character that it should be replaced with, or an empty string if the 
# codepoint should simply be removed from the input. Examples:
#
#   { 224 a  }     (replace codepoint 224 to "a")
#   { 769 "" }     (remove codepoint 769 from input)
#
# Mappings are only returned for non-upper case codepoints. It is assumed
# that the input has already been folded to lower case.












#
proc rd_load_unicodedata_text {zName} {
  global tl_lookup_table

  set fd [open $zName]
  set lField {
    code
................................................................................
      continue
    }

    set iCode  [expr "0x$code"]
    set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]



    if {[info exists tl_lookup_table($iCode)]} continue









    if { ($iAscii >= 97 && $iAscii <= 122)
      || ($iAscii >= 65 && $iAscii <= 90)
    } {
      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]

      set dia($iDia) 1
    }
  }

  foreach d [array names dia] {
    lappend lRet [list $d ""]
  }
  set lRet [lsort -integer -index 0 $lRet]

  close $fd
  set lRet
}








|
|



>
>
>
>
>
>
>
>
>
>
>
>







 







>
>

>
>
>
>
>
>
>
>




|
>





|







3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
..
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of mappings required to remove all
# diacritical marks from a unicode string. Each mapping is itself a list
# consisting of two elements - the unicode codepoint and the single ASCII
# character that it should be replaced with, or an empty string if the 
# codepoint should simply be removed from the input. Examples:
#
#   { 224 a  0 }     (replace codepoint 224 to "a")
#   { 769 "" 0 }     (remove codepoint 769 from input)
#
# Mappings are only returned for non-upper case codepoints. It is assumed
# that the input has already been folded to lower case.
#
# The third value in the list is always either 0 or 1. 0 if the 
# UnicodeData.txt file maps the codepoint to a single ASCII character and
# a diacritic, or 1 if the mapping is indirect. For example, consider the 
# two entries:
#
# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
#
# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
# diacritic). The second is an indirect mapping, as it maps to the
# first codepoint plus 0302 (a diacritic).
#
proc rd_load_unicodedata_text {zName} {
  global tl_lookup_table

  set fd [open $zName]
  set lField {
    code
................................................................................
      continue
    }

    set iCode  [expr "0x$code"]
    set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]

    # Filter out upper-case characters, as they will be mapped to their
    # lower-case equivalents before this data is used.
    if {[info exists tl_lookup_table($iCode)]} continue

    # Check if this is an indirect mapping. If so, set bIndirect to true
    # and change $iAscii to the indirectly mappped ASCII character.
    set bIndirect 0
    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
      set iAscii $mapping($iAscii)
      set bIndirect 1
    }

    if { ($iAscii >= 97 && $iAscii <= 122)
      || ($iAscii >= 65 && $iAscii <= 90)
    } {
      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
      set mapping($iCode) $iAscii
      set dia($iDia) 1
    }
  }

  foreach d [array names dia] {
    lappend lRet [list $d "" 0]
  }
  set lRet [lsort -integer -index 0 $lRet]

  close $fd
  set lRet
}

Changes to ext/fts5/fts5_tokenize.c.

230
231
232
233
234
235
236
237
238
239
240
241
242





243
244
245
246
247
248
249
...
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
...
378
379
380
381
382
383
384
385
386
387
388





389
390
391
392
393
394
395
...
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
#endif /* ifndef SQLITE_AMALGAMATION */

typedef struct Unicode61Tokenizer Unicode61Tokenizer;
struct Unicode61Tokenizer {
  unsigned char aTokenChar[128];  /* ASCII range token characters */
  char *aFold;                    /* Buffer to fold text into */
  int nFold;                      /* Size of aFold[] in bytes */
  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
  int nException;
  int *aiException;

  unsigned char aCategory[32];    /* True for token char categories */
};






static int fts5UnicodeAddExceptions(
  Unicode61Tokenizer *p,          /* Tokenizer object */
  const char *z,                  /* Characters to treat as exceptions */
  int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
){
  int rc = SQLITE_OK;
................................................................................
  }else{
    p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
    if( p ){
      const char *zCat = "L* N* Co";
      int i;
      memset(p, 0, sizeof(Unicode61Tokenizer));

      p->bRemoveDiacritic = 1;
      p->nFold = 64;
      p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
      if( p->aFold==0 ){
        rc = SQLITE_NOMEM;
      }

      /* Search for a "categories" argument */
................................................................................
      if( rc==SQLITE_OK ){
        rc = unicodeSetCategories(p, zCat);
      }

      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        const char *zArg = azArg[i+1];
        if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
            rc = SQLITE_ERROR;
          }
          p->bRemoveDiacritic = (zArg[0]=='1');





        }else
        if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 1);
        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 0);
        }else
................................................................................

      if( *zCsr & 0x80 ){
        /* An non-ascii-range character. Fold it into the output buffer if
        ** it is a token character, or break out of the loop if it is not. */
        READ_UTF8(zCsr, zTerm, iCode);
        if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
 non_ascii_tokenchar:
          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
          if( iCode ) WRITE_UTF8(zOut, iCode);
        }else{
          break;
        }
      }else if( a[*zCsr]==0 ){
        /* An ascii-range separator character. End of token. */
        break; 







|





>
>
>
>
>







 







|







 







|

|
|
>
>
>
>
>







 







|







230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
...
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
...
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
...
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
#endif /* ifndef SQLITE_AMALGAMATION */

typedef struct Unicode61Tokenizer Unicode61Tokenizer;
struct Unicode61Tokenizer {
  unsigned char aTokenChar[128];  /* ASCII range token characters */
  char *aFold;                    /* Buffer to fold text into */
  int nFold;                      /* Size of aFold[] in bytes */
  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
  int nException;
  int *aiException;

  unsigned char aCategory[32];    /* True for token char categories */
};

/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
#define FTS5_REMOVE_DIACRITICS_NONE    0
#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
#define FTS5_REMOVE_DIACRITICS_COMPLEX 2

static int fts5UnicodeAddExceptions(
  Unicode61Tokenizer *p,          /* Tokenizer object */
  const char *z,                  /* Characters to treat as exceptions */
  int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
){
  int rc = SQLITE_OK;
................................................................................
  }else{
    p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
    if( p ){
      const char *zCat = "L* N* Co";
      int i;
      memset(p, 0, sizeof(Unicode61Tokenizer));

      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
      p->nFold = 64;
      p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
      if( p->aFold==0 ){
        rc = SQLITE_NOMEM;
      }

      /* Search for a "categories" argument */
................................................................................
      if( rc==SQLITE_OK ){
        rc = unicodeSetCategories(p, zCat);
      }

      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        const char *zArg = azArg[i+1];
        if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
            rc = SQLITE_ERROR;
          }else{
            p->eRemoveDiacritic = (zArg[0] - '0');
            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
            );
          }
        }else
        if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 1);
        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          rc = fts5UnicodeAddExceptions(p, zArg, 0);
        }else
................................................................................

      if( *zCsr & 0x80 ){
        /* An non-ascii-range character. Fold it into the output buffer if
        ** it is a token character, or break out of the loop if it is not. */
        READ_UTF8(zCsr, zTerm, iCode);
        if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
 non_ascii_tokenchar:
          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
          if( iCode ) WRITE_UTF8(zOut, iCode);
        }else{
          break;
        }
      }else if( a[*zCsr]==0 ){
        /* An ascii-range separator character. End of token. */
        break; 

Changes to ext/fts5/fts5_unicode2.c.

24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

40
41
42
43


44
45


46
47
48
49
50
51
52
53
54
55
56















57
58
59
60
61
62
63
..
66
67
68
69
70
71
72

73
74
75
76
77
78
79
80
..
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
...
216
217
218
219
220
221
222

223

224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
...
752
753
754
755
756
757
758
759
760
761
762
763
764
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int fts5_remove_diacritic(int c){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 

     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 


    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
    62924, 63050, 63082, 63274, 63390, 


  };
  char aChar[] = {
    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
    'e',  'i',  'o',  'u',  'y',  















  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
  int iRes = 0;
  int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
  int iLo = 0;
  while( iHi>=iLo ){
................................................................................
      iRes = iTest;
      iLo = iTest+1;
    }else{
      iHi = iTest-1;
    }
  }
  assert( key>=aDia[iRes] );

  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}


/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
................................................................................
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
  **
  ** If the least significant bit in flags is clear, then the rule applies
  ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
  ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
    assert( iRes>=0 && c>=aEntry[iRes].iCode );
    p = &aEntry[iRes];
    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
      assert( ret>0 );
    }


    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);

  }
  
  else if( c>=66560 && c<66600 ){
    ret = c + 40;
  }

  return ret;
}


#if 0
int sqlite3Fts5UnicodeNCat(void) { 
  return 32;
}
#endif

int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
  aArray[0] = 1;
  switch( zCat[0] ){
    case 'C':
          switch( zCat[1] ){
            case 'c': aArray[1] = 1; break;
            case 'f': aArray[2] = 1; break;
................................................................................
void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
  int i = 0;
  int iTbl = 0;
  while( i<128 ){
    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
    for(; i<128 && i<n; i++){
      aAscii[i] = (u8)bToken;
    }
    iTbl++;
  }
}








|





|
|
|
>
|
<
|
|
>
>
|
<
>
>


<
<
<
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
|







 







|







 







>
|
>









<
<
<
<
<
<
<







 







|




<
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

42
43
44
45
46

47
48
49
50



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
..
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
...
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
...
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250







251
252
253
254
255
256
257
...
763
764
765
766
767
768
769
770
771
772
773
774

** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int fts5_remove_diacritic(int c, int bComplex){
  unsigned short aDia[] = {
        0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
     2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
     2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
     2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 

    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 

    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
    63182, 63242, 63274, 63310, 63368, 63390, 
  };
  char aChar[] = {



    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
  };

  unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
  int iRes = 0;
  int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
  int iLo = 0;
  while( iHi>=iLo ){
................................................................................
      iRes = iTest;
      iLo = iTest+1;
    }else{
      iHi = iTest-1;
    }
  }
  assert( key>=aDia[iRes] );
  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
}


/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
................................................................................
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
  /* Each entry in the following array defines a rule for folding a range
  ** of codepoints to lower case. The rule applies to a range of nRange
  ** codepoints starting at codepoint iCode.
  **
  ** If the least significant bit in flags is clear, then the rule applies
  ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
  ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
    assert( iRes>=0 && c>=aEntry[iRes].iCode );
    p = &aEntry[iRes];
    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
      assert( ret>0 );
    }

    if( eRemoveDiacritic ){
      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
    }
  }
  
  else if( c>=66560 && c<66600 ){
    ret = c + 40;
  }

  return ret;
}








int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
  aArray[0] = 1;
  switch( zCat[0] ){
    case 'C':
          switch( zCat[1] ){
            case 'c': aArray[1] = 1; break;
            case 'f': aArray[2] = 1; break;
................................................................................
void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
  int i = 0;
  int iTbl = 0;
  while( i<128 ){
    int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
    int n = (aFts5UnicodeData[iTbl] >> 5) + i;
    for(; i<128 && i<n; i++){
      aAscii[i] = bToken;
    }
    iTbl++;
  }
}

Changes to ext/fts5/test/fts5tokenizer.test.

185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 tokenchars');
} {1 {error in tokenizer constructor}}
do_catchsql_test 6.2 {
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
} {1 {error in tokenizer constructor}}
do_catchsql_test 6.3 {
  CREATE VIRTUAL TABLE a3 USING fts5(
    x, y, tokenize = 'unicode61 remove_diacritics 2'
  );
} {1 {error in tokenizer constructor}}
do_catchsql_test 6.4 {
  CREATE VIRTUAL TABLE a3 USING fts5(
    x, y, tokenize = 'unicode61 remove_diacritics 10'
  );
} {1 {error in tokenizer constructor}}







|







185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 tokenchars');
} {1 {error in tokenizer constructor}}
do_catchsql_test 6.2 {
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
} {1 {error in tokenizer constructor}}
do_catchsql_test 6.3 {
  CREATE VIRTUAL TABLE a3 USING fts5(
    x, y, tokenize = 'unicode61 remove_diacritics 3'
  );
} {1 {error in tokenizer constructor}}
do_catchsql_test 6.4 {
  CREATE VIRTUAL TABLE a3 USING fts5(
    x, y, tokenize = 'unicode61 remove_diacritics 10'
  );
} {1 {error in tokenizer constructor}}

Added ext/fts5/test/fts5umlaut.test.



































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# 2014 June 17
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS5 module.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5umlaut

# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  CREATE VIRTUAL TABLE t2 USING fts5(
      x, 
      tokenize="unicode61 remove_diacritics 2"
  );
}

foreach {tn q res1 res2} {
  1 "Hà Nội"                  0 1
  2 "Hà Noi"                  1 1
  3 "Ha Noi"                  1 1
  4 "Ha N\u1ed9i"             0 1
  5 "Ha N\u006fi"             1 1
  6 "Ha N\u006f\u0302i"       1 1
  7 "Ha N\u006f\u0323\u0302i" 1 1
} {
  do_execsql_test 1.$tn.1 {
    DELETE FROM t1;
    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
    SELECT count(*) FROM t1($q)
  } $res1
  do_execsql_test 1.$tn.2 {
    DELETE FROM t1;
    INSERT INTO t1(rowid, x) VALUES (1, $q);
    SELECT count(*) FROM t1('Ha Noi')
  } $res1

  do_execsql_test 1.$tn.2 {
    DELETE FROM t2;
    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
    SELECT count(*) FROM t2($q)
  } $res2
  do_execsql_test 1.$tn.2 {
    DELETE FROM t2;
    INSERT INTO t2(rowid, x) VALUES (1, $q);
    SELECT count(*) FROM t2('Ha Noi')
  } $res2
}

finish_test

Changes to ext/fts5/test/fts5unicode3.test.

32
33
34
35
36
37
38
39
40
41
42
43
44
45

46
47
48
49
50

51
52
53
54
55
56
57
58
59
60
61
62
63
..
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96










97
98
99
100
101
102
103

tl_load_casefolding_txt $CF
foreach x [an_load_unicodedata_text $UD] {
  set aNotAlnum($x) 1
}

foreach {y} [rd_load_unicodedata_text $UD] {
  foreach {code ascii} $y {}
  if {$ascii==""} {
    set int 0
  } else {
    binary scan $ascii c int
  }
  set aDiacritic($code) $int

}

proc tcl_fold {i {bRemoveDiacritic 0}} {
  global tl_lookup_table
  global aDiacritic


  if {[info exists tl_lookup_table($i)]} {
    set i $tl_lookup_table($i)
  }
  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
    set i $aDiacritic($i)
  }
  expr $i
}
db func tcl_fold tcl_fold

proc tcl_isalnum {i} {
  global aNotAlnum
................................................................................
    SELECT -1
    UNION ALL
    SELECT i+1 FROM ii WHERE i<100000
  )
  SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
} {0 {}}

do_execsql_test 1.2 {
  WITH ii(i) AS (
    SELECT -1
    UNION ALL
    SELECT i+1 FROM ii WHERE i<100000
  )
  SELECT count(*), min(i) FROM ii 
  WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
} {0 {}}











do_execsql_test 1.3 {
  WITH ii(i) AS (
    SELECT -1
    UNION ALL
    SELECT i+1 FROM ii WHERE i<100000
  )







|





|
>





>




|
|







 







|








>
>
>
>
>
>
>
>
>
>







32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
..
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

tl_load_casefolding_txt $CF
foreach x [an_load_unicodedata_text $UD] {
  set aNotAlnum($x) 1
}

foreach {y} [rd_load_unicodedata_text $UD] {
  foreach {code ascii f} $y {}
  if {$ascii==""} {
    set int 0
  } else {
    binary scan $ascii c int
  }
  set aDiacritic($code,$f) $int
  if {$f==0} { set aDiacritic($code,1) $int }
}

proc tcl_fold {i {bRemoveDiacritic 0}} {
  global tl_lookup_table
  global aDiacritic
  set f [expr $bRemoveDiacritic==2]

  if {[info exists tl_lookup_table($i)]} {
    set i $tl_lookup_table($i)
  }
  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
    set i $aDiacritic($i,$f)
  }
  expr $i
}
db func tcl_fold tcl_fold

proc tcl_isalnum {i} {
  global aNotAlnum
................................................................................
    SELECT -1
    UNION ALL
    SELECT i+1 FROM ii WHERE i<100000
  )
  SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
} {0 {}}

do_execsql_test 1.2.1 {
  WITH ii(i) AS (
    SELECT -1
    UNION ALL
    SELECT i+1 FROM ii WHERE i<100000
  )
  SELECT count(*), min(i) FROM ii 
  WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
} {0 {}}

do_execsql_test 1.2.2 {
  WITH ii(i) AS (
    SELECT -1
    UNION ALL
    SELECT i+1 FROM ii WHERE i<100000
  )
  SELECT count(*), min(i) FROM ii 
  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
} {0 {}}

do_execsql_test 1.3 {
  WITH ii(i) AS (
    SELECT -1
    UNION ALL
    SELECT i+1 FROM ii WHERE i<100000
  )

Changes to ext/rbu/rbu_common.tcl.

82
83
84
85
86
87
88
89
90

91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    sqlite3 tmpdb $rbu
    tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
    tmpdb close
  }
  set rc
}

proc do_rbu_vacuum_test {tn step} {
  forcedelete state.db

  uplevel [list do_test $tn.1 {
    if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
    while 1 {
      if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
      set state [rbu state]
      check_prestep_state test.db $state
      set rc [rbu step]
      check_poststep_state $rc test.db $state
      if {$rc!="SQLITE_OK"} break
      if {$step==1} { rbu close }
    }
    rbu close
  } {SQLITE_DONE}]

  uplevel [list do_execsql_test $tn.2 {
    PRAGMA integrity_check
  } ok]
}








|
|
>
|
|

|








|






82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
    sqlite3 tmpdb $rbu
    tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
    tmpdb close
  }
  set rc
}

proc do_rbu_vacuum_test {tn step {statedb state.db}} {
  forcedelete $statedb
  if {$statedb=="" && $step==1} breakpoint
  uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
    if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
    while 1 {
      if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
      set state [rbu state]
      check_prestep_state test.db $state
      set rc [rbu step]
      check_poststep_state $rc test.db $state
      if {$rc!="SQLITE_OK"} break
      if {$step==1} { rbu close }
    }
    rbu close
  }] {SQLITE_DONE}]

  uplevel [list do_execsql_test $tn.2 {
    PRAGMA integrity_check
  } ok]
}

Changes to ext/rbu/rbuvacuum2.test.

12
13
14
15
16
17
18

19


20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
..
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
..
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
..
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
...
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
...
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

159
160
161
162
163
164
165
...
227
228
229
230
231
232
233





234
# This file contains tests for the RBU module. More specifically, it
# contains tests to ensure that the sqlite3rbu_vacuum() API works as
# expected.
#

source [file join [file dirname [info script]] rbu_common.tcl]


foreach step {0 1} {


  set ::testprefix rbuvacuum2-$step
  
  #-------------------------------------------------------------------------
  # Test that a database that contains fts3 tables can be vacuumed.
  #
  ifcapable fts3 {
    reset_db
    do_execsql_test 1.1 {
      CREATE VIRTUAL TABLE t1 USING fts3(z, y);
      INSERT INTO t1 VALUES('fix this issue', 'at some point');
    }
  
    do_rbu_vacuum_test 1.2 $step
  
    do_execsql_test 1.3 {
      SELECT * FROM t1;
    } {{fix this issue} {at some point}}
  
    do_execsql_test 1.4 {
      SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
................................................................................
    do_execsql_test 1.5 {
      INSERT INTO t1 VALUES('a b c', 'd e f');
      INSERT INTO t1 VALUES('l h i', 'd e f');
      DELETE FROM t1 WHERE docid = 2;
      INSERT INTO t1 VALUES('a b c', 'x y z');
    }

    do_rbu_vacuum_test 1.6 $step
    do_execsql_test 1.7 {
      INSERT INTO t1(t1) VALUES('integrity-check');
      SELECT * FROM t1;
    } {
      {fix this issue} {at some point}
      {l h i} {d e f}
      {a b c} {x y z}
................................................................................
  ifcapable fts5 {
    reset_db
    do_execsql_test 2.1 {
      CREATE VIRTUAL TABLE t1 USING fts5(z, y);
      INSERT INTO t1 VALUES('fix this issue', 'at some point');
    }
  
    do_rbu_vacuum_test 2.2 $step
  
    do_execsql_test 2.3 {
      SELECT * FROM t1;
    } {{fix this issue} {at some point}}
  
    do_execsql_test 2.4 {
      SELECT rowid FROM t1 ('fix');
................................................................................
    do_execsql_test 2.5 {
      INSERT INTO t1 VALUES('a b c', 'd e f');
      INSERT INTO t1 VALUES('l h i', 'd e f');
      DELETE FROM t1 WHERE rowid = 2;
      INSERT INTO t1 VALUES('a b c', 'x y z');
    }

    do_rbu_vacuum_test 2.6 $step
    do_execsql_test 2.7 {
      INSERT INTO t1(t1) VALUES('integrity-check');
      SELECT * FROM t1;
    } {
      {fix this issue} {at some point}
      {l h i} {d e f}
      {a b c} {x y z}
................................................................................
    do_execsql_test 3.1 {
      CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
      INSERT INTO rt VALUES(1, 45, 55);
      INSERT INTO rt VALUES(2, 50, 60);
      INSERT INTO rt VALUES(3, 55, 65);
    }
  
    do_rbu_vacuum_test 3.2 $step
  
    do_execsql_test 3.3 {
      SELECT * FROM rt;
    } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
  
    do_execsql_test 3.4.1 {
      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
    } {1 2}
    do_execsql_test 3.4.2 {
      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
    } {2 3}

    do_rbu_vacuum_test 3.5 $step

    do_execsql_test 3.6.1 {
      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
    } {1 2}
    do_execsql_test 3.6.2 {
      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
    } {2 3}
................................................................................
      SELECT * FROM sqlite_master;
    } {
    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
    }

    do_rbu_vacuum_test 4.3 $step
    do_execsql_test 4.4 {
      SELECT * FROM sqlite_master;
    } {
    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
    }
  }

}
  
#-------------------------------------------------------------------------
# Test that passing a NULL value as the second argument to 
# sqlite3rbu_vacuum() causes it to:
#
#   * Use <database>-vacuum as the state db, and
................................................................................
do_test 6.3 {
  sqlite3rbu_vacuum rbu test.db test.db2
  while {[rbu step]!="SQLITE_DONE"} { rbu step }
  rbu close
  execsql { PRAGMA integrity_check }
} {ok}






finish_test







>
|
>
>
|











|







 







|







 







|







 







|







 







|












|







 







|








>







 







>
>
>
>
>

12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
..
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
..
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
..
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
...
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
...
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
...
231
232
233
234
235
236
237
238
239
240
241
242
243
# This file contains tests for the RBU module. More specifically, it
# contains tests to ensure that the sqlite3rbu_vacuum() API works as
# expected.
#

source [file join [file dirname [info script]] rbu_common.tcl]

foreach {step} {0 1} {
foreach {ttt state} {
  s state.db t test.db-vacuum n {}
} {
  set ::testprefix rbuvacuum2-$step$ttt
  
  #-------------------------------------------------------------------------
  # Test that a database that contains fts3 tables can be vacuumed.
  #
  ifcapable fts3 {
    reset_db
    do_execsql_test 1.1 {
      CREATE VIRTUAL TABLE t1 USING fts3(z, y);
      INSERT INTO t1 VALUES('fix this issue', 'at some point');
    }
  
    do_rbu_vacuum_test 1.2 $step $state
  
    do_execsql_test 1.3 {
      SELECT * FROM t1;
    } {{fix this issue} {at some point}}
  
    do_execsql_test 1.4 {
      SELECT rowid FROM t1 WHERE t1 MATCH 'fix';
................................................................................
    do_execsql_test 1.5 {
      INSERT INTO t1 VALUES('a b c', 'd e f');
      INSERT INTO t1 VALUES('l h i', 'd e f');
      DELETE FROM t1 WHERE docid = 2;
      INSERT INTO t1 VALUES('a b c', 'x y z');
    }

    do_rbu_vacuum_test 1.6 $step $state
    do_execsql_test 1.7 {
      INSERT INTO t1(t1) VALUES('integrity-check');
      SELECT * FROM t1;
    } {
      {fix this issue} {at some point}
      {l h i} {d e f}
      {a b c} {x y z}
................................................................................
  ifcapable fts5 {
    reset_db
    do_execsql_test 2.1 {
      CREATE VIRTUAL TABLE t1 USING fts5(z, y);
      INSERT INTO t1 VALUES('fix this issue', 'at some point');
    }
  
    do_rbu_vacuum_test 2.2 $step $state
  
    do_execsql_test 2.3 {
      SELECT * FROM t1;
    } {{fix this issue} {at some point}}
  
    do_execsql_test 2.4 {
      SELECT rowid FROM t1 ('fix');
................................................................................
    do_execsql_test 2.5 {
      INSERT INTO t1 VALUES('a b c', 'd e f');
      INSERT INTO t1 VALUES('l h i', 'd e f');
      DELETE FROM t1 WHERE rowid = 2;
      INSERT INTO t1 VALUES('a b c', 'x y z');
    }

    do_rbu_vacuum_test 2.6 $step $state
    do_execsql_test 2.7 {
      INSERT INTO t1(t1) VALUES('integrity-check');
      SELECT * FROM t1;
    } {
      {fix this issue} {at some point}
      {l h i} {d e f}
      {a b c} {x y z}
................................................................................
    do_execsql_test 3.1 {
      CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2);
      INSERT INTO rt VALUES(1, 45, 55);
      INSERT INTO rt VALUES(2, 50, 60);
      INSERT INTO rt VALUES(3, 55, 65);
    }
  
    do_rbu_vacuum_test 3.2 $step $state
  
    do_execsql_test 3.3 {
      SELECT * FROM rt;
    } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0}
  
    do_execsql_test 3.4.1 {
      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
    } {1 2}
    do_execsql_test 3.4.2 {
      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
    } {2 3}

    do_rbu_vacuum_test 3.5 $step $state

    do_execsql_test 3.6.1 {
      SELECT rowid FROM rt WHERE x2>51 AND x1 < 51
    } {1 2}
    do_execsql_test 3.6.2 {
      SELECT rowid FROM rt WHERE x2>59 AND x1 < 59
    } {2 3}
................................................................................
      SELECT * FROM sqlite_master;
    } {
    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
    }

    do_rbu_vacuum_test 4.3 $step $state
    do_execsql_test 4.4 {
      SELECT * FROM sqlite_master;
    } {
    table t1 t1 2 {CREATE TABLE t1(a, b, c)}
    view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1}
    trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END}
    }
  }
}
}
  
#-------------------------------------------------------------------------
# Test that passing a NULL value as the second argument to 
# sqlite3rbu_vacuum() causes it to:
#
#   * Use <database>-vacuum as the state db, and
................................................................................
do_test 6.3 {
  sqlite3rbu_vacuum rbu test.db test.db2
  while {[rbu step]!="SQLITE_DONE"} { rbu step }
  rbu close
  execsql { PRAGMA integrity_check }
} {ok}

do_test 6.4 {
  sqlite3rbu_vacuum rbu test.db test.db-vactmp
  list [catch { rbu close } msg] $msg
} {1 SQLITE_MISUSE}

finish_test

Changes to ext/rbu/sqlite3rbu.c.

2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
....
3739
3740
3741
3742
3743
3744
3745






3746
3747
3748
3749
3750
3751
3752
        zExtra = &p->zRbu[5];
        while( *zExtra ){
          if( *zExtra++=='?' ) break;
        }
        if( *zExtra=='\0' ) zExtra = 0;
      }

      zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", 
          sqlite3_db_filename(p->dbRbu, "main"),
          (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
      );

      if( zTarget==0 ){
        p->rc = SQLITE_NOMEM;
        return;
................................................................................
** Open a handle to begin or resume an RBU VACUUM operation.
*/
sqlite3rbu *sqlite3rbu_vacuum(
  const char *zTarget, 
  const char *zState
){
  if( zTarget==0 ){ return rbuMisuseError(); }






  /* TODO: Check that both arguments are non-NULL */
  return openRbuHandle(0, zTarget, zState);
}

/*
** Return the database handle used by pRbu.
*/







|







 







>
>
>
>
>
>







2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
....
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
        zExtra = &p->zRbu[5];
        while( *zExtra ){
          if( *zExtra++=='?' ) break;
        }
        if( *zExtra=='\0' ) zExtra = 0;
      }

      zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", 
          sqlite3_db_filename(p->dbRbu, "main"),
          (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra)
      );

      if( zTarget==0 ){
        p->rc = SQLITE_NOMEM;
        return;
................................................................................
** Open a handle to begin or resume an RBU VACUUM operation.
*/
sqlite3rbu *sqlite3rbu_vacuum(
  const char *zTarget, 
  const char *zState
){
  if( zTarget==0 ){ return rbuMisuseError(); }
  if( zState ){
    int n = strlen(zState);
    if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
      return rbuMisuseError();
    }
  }
  /* TODO: Check that both arguments are non-NULL */
  return openRbuHandle(0, zTarget, zState);
}

/*
** Return the database handle used by pRbu.
*/

Changes to ext/rbu/sqlite3rbu.h.

330
331
332
333
334
335
336




337
338
339
340
341
342
343
** handle specifying the same target and state databases.
**
** If the second argument passed to this function is NULL, then the
** name of the state database is "<database>-vacuum", where <database>
** is the name of the target database file. In this case, on UNIX, if the
** state database is not already present in the file-system, it is created
** with the same permissions as the target db is made.




**
** This function does not delete the state database after an RBU vacuum
** is completed, even if it created it. However, if the call to
** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
** of the state tables within the state database are zeroed. This way,
** the next call to sqlite3rbu_vacuum() opens a handle that starts a 
** new RBU vacuum operation.







>
>
>
>







330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
** handle specifying the same target and state databases.
**
** If the second argument passed to this function is NULL, then the
** name of the state database is "<database>-vacuum", where <database>
** is the name of the target database file. In this case, on UNIX, if the
** state database is not already present in the file-system, it is created
** with the same permissions as the target db is made. 
**
** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the 
** state database ends with "-vactmp". This name is reserved for internal 
** use.
**
** This function does not delete the state database after an RBU vacuum
** is completed, even if it created it. However, if the call to
** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents
** of the state tables within the state database are zeroed. This way,
** the next call to sqlite3rbu_vacuum() opens a handle that starts a 
** new RBU vacuum operation.

Changes to ext/rbu/test_rbu.c.

269
270
271
272
273
274
275

276
277
278
279
280
281
282
  if( objc!=3 && objc!=4 ){
    Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?");
    return TCL_ERROR;
  }
  zCmd = Tcl_GetString(objv[1]);
  zTarget = Tcl_GetString(objv[2]);
  if( objc==4 ) zStateDb = Tcl_GetString(objv[3]);


  pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
  Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
  Tcl_SetObjResult(interp, objv[1]);
  return TCL_OK;
}








>







269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
  if( objc!=3 && objc!=4 ){
    Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?");
    return TCL_ERROR;
  }
  zCmd = Tcl_GetString(objv[1]);
  zTarget = Tcl_GetString(objv[2]);
  if( objc==4 ) zStateDb = Tcl_GetString(objv[3]);
  if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0;

  pRbu = sqlite3rbu_vacuum(zTarget, zStateDb);
  Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0);
  Tcl_SetObjResult(interp, objv[1]);
  return TCL_OK;
}

Changes to main.mk.

995
996
997
998
999
1000
1001




1002
1003
1004
1005
1006
1007
1008

showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)

showshm$(EXE):	$(TOP)/tool/showshm.c
	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c





changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)

changesetfuzz$(EXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changesetfuzz$(EXE) \







>
>
>
>







995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012

showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)

showshm$(EXE):	$(TOP)/tool/showshm.c
	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c

index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)

changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)

changesetfuzz$(EXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changesetfuzz$(EXE) \

Changes to src/alter.c.

775
776
777
778
779
780
781


















782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
....
1354
1355
1356
1357
1358
1359
1360




1361
1362
1363
1364
1365
1366

1367
1368
1369
1370
1371
1372
1373
      pToken->pNext = pCtx->pList;
      pCtx->pList = pToken;
      pCtx->nList++;
      break;
    }
  }
}



















/*
** This is a Walker select callback. It does nothing. It is only required
** because without a dummy callback, sqlite3WalkExpr() and similar do not
** descend into sub-select statements.
*/
static int renameColumnSelectCb(Walker *pWalker, Select *p){
  UNUSED_PARAMETER(pWalker);
  UNUSED_PARAMETER(p);
  return WRC_Continue;
}

/*
** This is a Walker expression callback.
**
** For every TK_COLUMN node in the expression tree, search to see
................................................................................
/*
** Walker select callback used by "RENAME TABLE". 
*/
static int renameTableSelectCb(Walker *pWalker, Select *pSelect){
  int i;
  RenameCtx *p = pWalker->u.pRename;
  SrcList *pSrc = pSelect->pSrc;




  for(i=0; i<pSrc->nSrc; i++){
    struct SrcList_item *pItem = &pSrc->a[i];
    if( pItem->pTab==p->pTab ){
      renameTokenFind(pWalker->pParse, p, pItem->zName);
    }
  }


  return WRC_Continue;
}


/*
** This C function implements an SQL user function that is used by SQL code







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







|
<







 







>
>
>
>






>







775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807

808
809
810
811
812
813
814
....
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
      pToken->pNext = pCtx->pList;
      pCtx->pList = pToken;
      pCtx->nList++;
      break;
    }
  }
}

/*
** Iterate through the Select objects that are part of WITH clauses attached
** to select statement pSelect.
*/
static void renameWalkWith(Walker *pWalker, Select *pSelect){
  if( pSelect->pWith ){
    int i;
    for(i=0; i<pSelect->pWith->nCte; i++){
      Select *p = pSelect->pWith->a[i].pSelect;
      NameContext sNC;
      memset(&sNC, 0, sizeof(sNC));
      sNC.pParse = pWalker->pParse;
      sqlite3SelectPrep(sNC.pParse, p, &sNC);
      sqlite3WalkSelect(pWalker, p);
    }
  }
}

/*
** This is a Walker select callback. It does nothing. It is only required
** because without a dummy callback, sqlite3WalkExpr() and similar do not
** descend into sub-select statements.
*/
static int renameColumnSelectCb(Walker *pWalker, Select *p){
  renameWalkWith(pWalker, p);

  return WRC_Continue;
}

/*
** This is a Walker expression callback.
**
** For every TK_COLUMN node in the expression tree, search to see
................................................................................
/*
** Walker select callback used by "RENAME TABLE". 
*/
static int renameTableSelectCb(Walker *pWalker, Select *pSelect){
  int i;
  RenameCtx *p = pWalker->u.pRename;
  SrcList *pSrc = pSelect->pSrc;
  if( pSrc==0 ){
    assert( pWalker->pParse->db->mallocFailed );
    return WRC_Abort;
  }
  for(i=0; i<pSrc->nSrc; i++){
    struct SrcList_item *pItem = &pSrc->a[i];
    if( pItem->pTab==p->pTab ){
      renameTokenFind(pWalker->pParse, p, pItem->zName);
    }
  }
  renameWalkWith(pWalker, pSelect);

  return WRC_Continue;
}


/*
** This C function implements an SQL user function that is used by SQL code

Changes to src/btree.c.

10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
  }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){
    checkAppendMsg(&sCheck,
      "incremental_vacuum enabled with a max rootpage of zero"
    );
  }
#endif
  testcase( pBt->db->flags & SQLITE_CellSizeCk );
  pBt->db->flags &= ~SQLITE_CellSizeCk;
  for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
    i64 notUsed;
    if( aRoot[i]==0 ) continue;
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum && aRoot[i]>1 ){
      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
    }







|







10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
  }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){
    checkAppendMsg(&sCheck,
      "incremental_vacuum enabled with a max rootpage of zero"
    );
  }
#endif
  testcase( pBt->db->flags & SQLITE_CellSizeCk );
  pBt->db->flags &= ~(u64)SQLITE_CellSizeCk;
  for(i=0; (int)i<nRoot && sCheck.mxErr; i++){
    i64 notUsed;
    if( aRoot[i]==0 ) continue;
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pBt->autoVacuum && aRoot[i]>1 ){
      checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0);
    }

Changes to src/build.c.

225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
...
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653


  /* Get the VDBE program ready for execution
  */
  if( v && pParse->nErr==0 && !db->mallocFailed ){
    /* A minimum of one cursor is required if autoincrement is used
    *  See ticket [a696379c1f08866] */
    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
    sqlite3VdbeMakeReady(v, pParse);
    pParse->rc = SQLITE_DONE;
  }else{
    pParse->rc = SQLITE_ERROR;
  }
}

................................................................................
  }

  /* Delete any foreign keys attached to this table. */
  sqlite3FkDelete(db, pTable);

  /* Delete the Table structure itself.
  */
#ifdef SQLITE_ENABLE_NORMALIZE
  if( pTable->pColHash ){
    sqlite3HashClear(pTable->pColHash);
    sqlite3_free(pTable->pColHash);
  }
#endif
  sqlite3DeleteColumnNames(db, pTable);
  sqlite3DbFree(db, pTable->zName);
  sqlite3DbFree(db, pTable->zColAff);
  sqlite3SelectDelete(db, pTable->pSelect);
  sqlite3ExprListDelete(db, pTable->pCheck);
#ifndef SQLITE_OMIT_VIRTUALTABLE
  sqlite3VtabClear(db, pTable);







|







 







<
<
<
<
<
<







225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
...
634
635
636
637
638
639
640






641
642
643
644
645
646
647


  /* Get the VDBE program ready for execution
  */
  if( v && pParse->nErr==0 && !db->mallocFailed ){
    /* A minimum of one cursor is required if autoincrement is used
    *  See ticket [a696379c1f08866] */
    assert( pParse->pAinc==0 || pParse->nTab>0 );
    sqlite3VdbeMakeReady(v, pParse);
    pParse->rc = SQLITE_DONE;
  }else{
    pParse->rc = SQLITE_ERROR;
  }
}

................................................................................
  }

  /* Delete any foreign keys attached to this table. */
  sqlite3FkDelete(db, pTable);

  /* Delete the Table structure itself.
  */






  sqlite3DeleteColumnNames(db, pTable);
  sqlite3DbFree(db, pTable->zName);
  sqlite3DbFree(db, pTable->zColAff);
  sqlite3SelectDelete(db, pTable->pSelect);
  sqlite3ExprListDelete(db, pTable->pCheck);
#ifndef SQLITE_OMIT_VIRTUALTABLE
  sqlite3VtabClear(db, pTable);

Changes to src/callback.c.

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
...
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
...
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
  return match;
}

/*
** Search a FuncDefHash for a function with the given name.  Return
** a pointer to the matching FuncDef if found, or 0 if there is no match.
*/
static FuncDef *functionSearch(
  int h,               /* Hash of the name */
  const char *zFunc    /* Name of function */
){
  FuncDef *p;
  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
    if( sqlite3StrICmp(p->zName, zFunc)==0 ){
      return p;
    }
  }
  return 0;
}
#ifdef SQLITE_ENABLE_NORMALIZE
FuncDef *sqlite3FunctionSearchN(
  int h,               /* Hash of the name */
  const char *zFunc,   /* Name of function */
  int nFunc            /* Length of the name */
){
  FuncDef *p;
  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
    if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
      return p;
    }
  }
  return 0;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

/*
** Insert a new FuncDef into a FuncDefHash hash table.
*/
void sqlite3InsertBuiltinFuncs(
  FuncDef *aDef,      /* List of global functions to be inserted */
  int nDef            /* Length of the apDef[] list */
................................................................................
  int i;
  for(i=0; i<nDef; i++){
    FuncDef *pOther;
    const char *zName = aDef[i].zName;
    int nName = sqlite3Strlen30(zName);
    int h = SQLITE_FUNC_HASH(zName[0], nName);
    assert( zName[0]>='a' && zName[0]<='z' );
    pOther = functionSearch(h, zName);
    if( pOther ){
      assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] );
      aDef[i].pNext = pOther->pNext;
      pOther->pNext = &aDef[i];
    }else{
      aDef[i].pNext = 0;
      aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h];
................................................................................
  ** have fields overwritten with new information appropriate for the
  ** new function.  But the FuncDefs for built-in functions are read-only.
  ** So we must not search for built-ins when creating a new function.
  */ 
  if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
    bestScore = 0;
    h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
    p = functionSearch(h, zName);
    while( p ){
      int score = matchQuality(p, nArg, enc);
      if( score>bestScore ){
        pBest = p;
        bestScore = score;
      }
      p = p->pNext;







|











<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







|







 







|







279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297















298
299
300
301
302
303
304
...
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
...
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
  return match;
}

/*
** Search a FuncDefHash for a function with the given name.  Return
** a pointer to the matching FuncDef if found, or 0 if there is no match.
*/
FuncDef *sqlite3FunctionSearch(
  int h,               /* Hash of the name */
  const char *zFunc    /* Name of function */
){
  FuncDef *p;
  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
    if( sqlite3StrICmp(p->zName, zFunc)==0 ){
      return p;
    }
  }
  return 0;
}
















/*
** Insert a new FuncDef into a FuncDefHash hash table.
*/
void sqlite3InsertBuiltinFuncs(
  FuncDef *aDef,      /* List of global functions to be inserted */
  int nDef            /* Length of the apDef[] list */
................................................................................
  int i;
  for(i=0; i<nDef; i++){
    FuncDef *pOther;
    const char *zName = aDef[i].zName;
    int nName = sqlite3Strlen30(zName);
    int h = SQLITE_FUNC_HASH(zName[0], nName);
    assert( zName[0]>='a' && zName[0]<='z' );
    pOther = sqlite3FunctionSearch(h, zName);
    if( pOther ){
      assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] );
      aDef[i].pNext = pOther->pNext;
      pOther->pNext = &aDef[i];
    }else{
      aDef[i].pNext = 0;
      aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h];
................................................................................
  ** have fields overwritten with new information appropriate for the
  ** new function.  But the FuncDefs for built-in functions are read-only.
  ** So we must not search for built-ins when creating a new function.
  */ 
  if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
    bestScore = 0;
    h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
    p = sqlite3FunctionSearch(h, zName);
    while( p ){
      int score = matchQuality(p, nArg, enc);
      if( score>bestScore ){
        pBest = p;
        bestScore = score;
      }
      p = p->pNext;

Changes to src/expr.c.

1325
1326
1327
1328
1329
1330
1331





























1332
1333
1334
1335
1336
1337
1338
....
1493
1494
1495
1496
1497
1498
1499

1500
1501
1502
1503
1504
1505
1506
....
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
....
2417
2418
2419
2420
2421
2422
2423

2424
2425
2426
2427
2428
2429
2430
    }
  }
  return pRet;
}
#else
# define withDup(x,y) 0
#endif






























/*
** The following group of routines make deep copies of expressions,
** expression lists, ID lists, and select statements.  The copies can
** be deleted (by being passed to their respective ...Delete() routines)
** without effecting the originals.
**
................................................................................
    pNew->addrOpenEphm[0] = -1;
    pNew->addrOpenEphm[1] = -1;
    pNew->nSelectRow = p->nSelectRow;
    pNew->pWith = withDup(db, p->pWith);
#ifndef SQLITE_OMIT_WINDOWFUNC
    pNew->pWin = 0;
    pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn);

#endif
    pNew->selId = p->selId;
    *pp = pNew;
    pp = &pNew->pPrior;
    pNext = pNew;
  }

................................................................................
*/
int sqlite3IsRowid(const char *z){
  if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
  if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
  if( sqlite3StrICmp(z, "OID")==0 ) return 1;
  return 0;
}
#ifdef SQLITE_ENABLE_NORMALIZE
int sqlite3IsRowidN(const char *z, int n){
  if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1;
  if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1;
  if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1;
  return 0;
}
#endif

/*
** pX is the RHS of an IN operator.  If pX is a SELECT statement 
** that can be simplified to a direct table access, then return
** a pointer to the SELECT statement.  If pX is not a SELECT statement,
** or if the SELECT statement needs to be manifested into a transient
** table, then return NULL.
................................................................................

      if( affinity_ok ){
        /* Search for an existing index that will work for this IN operator */
        for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){
          Bitmask colUsed;      /* Columns of the index used */
          Bitmask mCol;         /* Mask for the current column */
          if( pIdx->nColumn<nExpr ) continue;

          /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute
          ** BITMASK(nExpr) without overflowing */
          testcase( pIdx->nColumn==BMS-2 );
          testcase( pIdx->nColumn==BMS-1 );
          if( pIdx->nColumn>=BMS-1 ) continue;
          if( mustBeUnique ){
            if( pIdx->nKeyCol>nExpr







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







<
<
<
<
<
<
<
<







 







>







1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
....
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
....
2175
2176
2177
2178
2179
2180
2181








2182
2183
2184
2185
2186
2187
2188
....
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
    }
  }
  return pRet;
}
#else
# define withDup(x,y) 0
#endif

#ifndef SQLITE_OMIT_WINDOWFUNC
/*
** The gatherSelectWindows() procedure and its helper routine
** gatherSelectWindowsCallback() are used to scan all the expressions
** an a newly duplicated SELECT statement and gather all of the Window
** objects found there, assembling them onto the linked list at Select->pWin.
*/
static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){
  if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){
    assert( ExprHasProperty(pExpr, EP_WinFunc) );
    pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin;
    pWalker->u.pSelect->pWin = pExpr->y.pWin;
  }
  return WRC_Continue;
}
static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){
  return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune;
}
static void gatherSelectWindows(Select *p){
  Walker w;
  w.xExprCallback = gatherSelectWindowsCallback;
  w.xSelectCallback = gatherSelectWindowsSelectCallback;
  w.xSelectCallback2 = 0;
  w.u.pSelect = p;
  sqlite3WalkSelect(&w, p);
}
#endif


/*
** The following group of routines make deep copies of expressions,
** expression lists, ID lists, and select statements.  The copies can
** be deleted (by being passed to their respective ...Delete() routines)
** without effecting the originals.
**
................................................................................
    pNew->addrOpenEphm[0] = -1;
    pNew->addrOpenEphm[1] = -1;
    pNew->nSelectRow = p->nSelectRow;
    pNew->pWith = withDup(db, p->pWith);
#ifndef SQLITE_OMIT_WINDOWFUNC
    pNew->pWin = 0;
    pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn);
    if( p->pWin ) gatherSelectWindows(pNew);
#endif
    pNew->selId = p->selId;
    *pp = pNew;
    pp = &pNew->pPrior;
    pNext = pNew;
  }

................................................................................
*/
int sqlite3IsRowid(const char *z){
  if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
  if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
  if( sqlite3StrICmp(z, "OID")==0 ) return 1;
  return 0;
}









/*
** pX is the RHS of an IN operator.  If pX is a SELECT statement 
** that can be simplified to a direct table access, then return
** a pointer to the SELECT statement.  If pX is not a SELECT statement,
** or if the SELECT statement needs to be manifested into a transient
** table, then return NULL.
................................................................................

      if( affinity_ok ){
        /* Search for an existing index that will work for this IN operator */
        for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){
          Bitmask colUsed;      /* Columns of the index used */
          Bitmask mCol;         /* Mask for the current column */
          if( pIdx->nColumn<nExpr ) continue;
          if( pIdx->pPartIdxWhere!=0 ) continue;
          /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute
          ** BITMASK(nExpr) without overflowing */
          testcase( pIdx->nColumn==BMS-2 );
          testcase( pIdx->nColumn==BMS-1 );
          if( pIdx->nColumn>=BMS-1 ) continue;
          if( mustBeUnique ){
            if( pIdx->nKeyCol>nExpr

Changes to src/hash.c.

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
...
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
...
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    h += sqlite3UpperToLower[c];
    h *= 0x9e3779b1;
  }
  return h;
}
#ifdef SQLITE_ENABLE_NORMALIZE
static unsigned int strHashN(const char *z, int n){
  unsigned int h = 0;
  int i;
  for(i=0; i<n; i++){
    /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    h += sqlite3UpperToLower[z[i]];
    h *= 0x9e3779b1;
  }
  return h;
}
#endif /* SQLITE_ENABLE_NORMALIZE */


/* Link pNew element into the hash table pH.  If pEntry!=0 then also
** insert pNew into the pEntry hash bucket.
*/
static void insertElement(
  Hash *pH,              /* The complete hash table */
................................................................................
    if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ 
      return elem;
    }
    elem = elem->next;
  }
  return &nullElement;
}
#ifdef SQLITE_ENABLE_NORMALIZE
static HashElem *findElementWithHashN(
  const Hash *pH,     /* The pH to be searched */
  const char *pKey,   /* The key we are searching for */
  int nKey,           /* Number of key bytes to use */
  unsigned int *pHash /* Write the hash value here */
){
  HashElem *elem;                /* Used to loop thru the element list */
  int count;                     /* Number of elements left to test */
  unsigned int h;                /* The computed hash */
  static HashElem nullElement = { 0, 0, 0, 0 };

  if( pH->ht ){   /*OPTIMIZATION-IF-TRUE*/
    struct _ht *pEntry;
    h = strHashN(pKey, nKey) % pH->htsize;
    pEntry = &pH->ht[h];
    elem = pEntry->chain;
    count = pEntry->count;
  }else{
    h = 0;
    elem = pH->first;
    count = pH->count;
  }
  if( pHash ) *pHash = h;
  while( count-- ){
    assert( elem!=0 );
    if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ 
      return elem;
    }
    elem = elem->next;
  }
  return &nullElement;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
  Hash *pH,         /* The pH containing "elem" */
  HashElem* elem,   /* The element to be removed from the pH */
................................................................................
** found, or NULL if there is no match.
*/
void *sqlite3HashFind(const Hash *pH, const char *pKey){
  assert( pH!=0 );
  assert( pKey!=0 );
  return findElementWithHash(pH, pKey, 0)->data;
}
#ifdef SQLITE_ENABLE_NORMALIZE
void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
  assert( pH!=0 );
  assert( pKey!=0 );
  assert( nKey>=0 );
  return findElementWithHashN(pH, pKey, nKey, 0)->data;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

/* Insert an element into the hash table pH.  The key is pKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created and NULL is returned.
**







<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







<
<
<
<
<
<
<
<







60
61
62
63
64
65
66














67
68
69
70
71
72
73
...
171
172
173
174
175
176
177


































178
179
180
181
182
183
184
...
215
216
217
218
219
220
221








222
223
224
225
226
227
228
    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    h += sqlite3UpperToLower[c];
    h *= 0x9e3779b1;
  }
  return h;
}
















/* Link pNew element into the hash table pH.  If pEntry!=0 then also
** insert pNew into the pEntry hash bucket.
*/
static void insertElement(
  Hash *pH,              /* The complete hash table */
................................................................................
    if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ 
      return elem;
    }
    elem = elem->next;
  }
  return &nullElement;
}



































/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void removeElementGivenHash(
  Hash *pH,         /* The pH containing "elem" */
  HashElem* elem,   /* The element to be removed from the pH */
................................................................................
** found, or NULL if there is no match.
*/
void *sqlite3HashFind(const Hash *pH, const char *pKey){
  assert( pH!=0 );
  assert( pKey!=0 );
  return findElementWithHash(pH, pKey, 0)->data;
}









/* Insert an element into the hash table pH.  The key is pKey
** and the data is "data".
**
** If no element exists with a matching key, then a new
** element is created and NULL is returned.
**

Changes to src/insert.c.

315
316
317
318
319
320
321

322
323
324
325
326
327
328
    aOp[3].p5 = SQLITE_JUMPIFNULL;
    aOp[4].p2 = memId+1;
    aOp[5].p3 = memId;
    aOp[6].p1 = memId;
    aOp[7].p2 = memId+2;
    aOp[7].p1 = memId;
    aOp[10].p2 = memId;

  }
}

/*
** Update the maximum rowid for an autoincrement calculation.
**
** This routine should be called when the regRowid register holds a







>







315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
    aOp[3].p5 = SQLITE_JUMPIFNULL;
    aOp[4].p2 = memId+1;
    aOp[5].p3 = memId;
    aOp[6].p1 = memId;
    aOp[7].p2 = memId+2;
    aOp[7].p1 = memId;
    aOp[10].p2 = memId;
    if( pParse->nTab==0 ) pParse->nTab = 1;
  }
}

/*
** Update the maximum rowid for an autoincrement calculation.
**
** This routine should be called when the regRowid register holds a

Changes to src/loadext.c.

646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
** default so as not to open security holes in older applications.
*/
int sqlite3_enable_load_extension(sqlite3 *db, int onoff){
  sqlite3_mutex_enter(db->mutex);
  if( onoff ){
    db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc;
  }else{
    db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
  }
  sqlite3_mutex_leave(db->mutex);
  return SQLITE_OK;
}

#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */








|







646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
** default so as not to open security holes in older applications.
*/
int sqlite3_enable_load_extension(sqlite3 *db, int onoff){
  sqlite3_mutex_enter(db->mutex);
  if( onoff ){
    db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc;
  }else{
    db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc);
  }
  sqlite3_mutex_leave(db->mutex);
  return SQLITE_OK;
}

#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */

Changes to src/main.c.

839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
....
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
....
2048
2049
2050
2051
2052
2053
2054


2055
2056
2057
2058
2059
2060
2061
....
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
      };
      unsigned int i;
      rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
      for(i=0; i<ArraySize(aFlagOp); i++){
        if( aFlagOp[i].op==op ){
          int onoff = va_arg(ap, int);
          int *pRes = va_arg(ap, int*);
          u32 oldFlags = db->flags;
          if( onoff>0 ){
            db->flags |= aFlagOp[i].mask;
          }else if( onoff==0 ){
            db->flags &= ~aFlagOp[i].mask;
          }
          if( oldFlags!=db->flags ){
            sqlite3ExpirePreparedStatements(db, 0);
          }
          if( pRes ){
            *pRes = (db->flags & aFlagOp[i].mask)!=0;
          }
................................................................................
    sqlite3ResetAllSchemasOfConnection(db);
  }
  sqlite3BtreeLeaveAll(db);

  /* Any deferred constraint violations have now been resolved. */
  db->nDeferredCons = 0;
  db->nDeferredImmCons = 0;
  db->flags &= ~SQLITE_DeferFKs;

  /* If one has been configured, invoke the rollback-hook callback */
  if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
    db->xRollbackCallback(db->pRollbackArg);
  }
}

................................................................................
    return 0;
  }
#endif
  sqlite3_mutex_enter(db->mutex);
  pOld = db->pProfileArg;
  db->xProfile = xProfile;
  db->pProfileArg = pArg;


  sqlite3_mutex_leave(db->mutex);
  return pOld;
}
#endif /* SQLITE_OMIT_DEPRECATED */
#endif /* SQLITE_OMIT_TRACE */

/*
................................................................................
    return sqlite3ErrStr(SQLITE_MISUSE_BKPT);
  }
  sqlite3_mutex_enter(db->mutex);
  if( db->mallocFailed ){
    z = sqlite3ErrStr(SQLITE_NOMEM_BKPT);
  }else{
    testcase( db->pErr==0 );
    z = (char*)sqlite3_value_text(db->pErr);
    assert( !db->mallocFailed );
    if( z==0 ){
      z = sqlite3ErrStr(db->errCode);
    }
  }
  sqlite3_mutex_leave(db->mutex);
  return z;







|



|







 







|







 







>
>







 







|







839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
....
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
....
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
....
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
      };
      unsigned int i;
      rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
      for(i=0; i<ArraySize(aFlagOp); i++){
        if( aFlagOp[i].op==op ){
          int onoff = va_arg(ap, int);
          int *pRes = va_arg(ap, int*);
          u64 oldFlags = db->flags;
          if( onoff>0 ){
            db->flags |= aFlagOp[i].mask;
          }else if( onoff==0 ){
            db->flags &= ~(u64)aFlagOp[i].mask;
          }
          if( oldFlags!=db->flags ){
            sqlite3ExpirePreparedStatements(db, 0);
          }
          if( pRes ){
            *pRes = (db->flags & aFlagOp[i].mask)!=0;
          }
................................................................................
    sqlite3ResetAllSchemasOfConnection(db);
  }
  sqlite3BtreeLeaveAll(db);

  /* Any deferred constraint violations have now been resolved. */
  db->nDeferredCons = 0;
  db->nDeferredImmCons = 0;
  db->flags &= ~(u64)SQLITE_DeferFKs;

  /* If one has been configured, invoke the rollback-hook callback */
  if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){
    db->xRollbackCallback(db->pRollbackArg);
  }
}

................................................................................
    return 0;
  }
#endif
  sqlite3_mutex_enter(db->mutex);
  pOld = db->pProfileArg;
  db->xProfile = xProfile;
  db->pProfileArg = pArg;
  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  sqlite3_mutex_leave(db->mutex);
  return pOld;
}
#endif /* SQLITE_OMIT_DEPRECATED */
#endif /* SQLITE_OMIT_TRACE */

/*
................................................................................
    return sqlite3ErrStr(SQLITE_MISUSE_BKPT);
  }
  sqlite3_mutex_enter(db->mutex);
  if( db->mallocFailed ){
    z = sqlite3ErrStr(SQLITE_NOMEM_BKPT);
  }else{
    testcase( db->pErr==0 );
    z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0;
    assert( !db->mallocFailed );
    if( z==0 ){
      z = sqlite3ErrStr(db->errCode);
    }
  }
  sqlite3_mutex_leave(db->mutex);
  return z;

Changes to src/parse.y.

675
676
677
678
679
680
681






682
683
684
685
686
687
688
....
1372
1373
1374
1375
1376
1377
1378


1379
1380


1381
1382
1383
1384
1385
1386
1387
      A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,N,U);
      if( A ){
        struct SrcList_item *pNew = &A->a[A->nSrc-1];
        struct SrcList_item *pOld = F->a;
        pNew->zName = pOld->zName;
        pNew->zDatabase = pOld->zDatabase;
        pNew->pSelect = pOld->pSelect;






        pOld->zName = pOld->zDatabase = 0;
        pOld->pSelect = 0;
      }
      sqlite3SrcListDelete(pParse->db, F);
    }else{
      Select *pSubquery;
      sqlite3SrcListShiftJoinType(F);
................................................................................
//
cmd ::= DROP INDEX ifexists(E) fullname(X).   {sqlite3DropIndex(pParse, X, E);}

///////////////////////////// The VACUUM command /////////////////////////////
//
%ifndef SQLITE_OMIT_VACUUM
%ifndef SQLITE_OMIT_ATTACH


cmd ::= VACUUM.                {sqlite3Vacuum(pParse,0);}
cmd ::= VACUUM nm(X).          {sqlite3Vacuum(pParse,&X);}


%endif  SQLITE_OMIT_ATTACH
%endif  SQLITE_OMIT_VACUUM

///////////////////////////// The PRAGMA command /////////////////////////////
//
%ifndef SQLITE_OMIT_PRAGMA
cmd ::= PRAGMA nm(X) dbnm(Z).                {sqlite3Pragma(pParse,&X,&Z,0,0);}







>
>
>
>
>
>







 







>
>
|
|
>
>







675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
....
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
      A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,N,U);
      if( A ){
        struct SrcList_item *pNew = &A->a[A->nSrc-1];
        struct SrcList_item *pOld = F->a;
        pNew->zName = pOld->zName;
        pNew->zDatabase = pOld->zDatabase;
        pNew->pSelect = pOld->pSelect;
        if( pOld->fg.isTabFunc ){
          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
          pOld->u1.pFuncArg = 0;
          pOld->fg.isTabFunc = 0;
          pNew->fg.isTabFunc = 1;
        }
        pOld->zName = pOld->zDatabase = 0;
        pOld->pSelect = 0;
      }
      sqlite3SrcListDelete(pParse->db, F);
    }else{
      Select *pSubquery;
      sqlite3SrcListShiftJoinType(F);
................................................................................
//
cmd ::= DROP INDEX ifexists(E) fullname(X).   {sqlite3DropIndex(pParse, X, E);}

///////////////////////////// The VACUUM command /////////////////////////////
//
%ifndef SQLITE_OMIT_VACUUM
%ifndef SQLITE_OMIT_ATTACH
%type vinto {Expr*}
%destructor vinto {sqlite3ExprDelete(pParse->db, $$);}
cmd ::= VACUUM vinto(Y).                {sqlite3Vacuum(pParse,0,Y);}
cmd ::= VACUUM nm(X) vinto(Y).          {sqlite3Vacuum(pParse,&X,Y);}
vinto(A) ::= INTO expr(X).              {A = X;}
vinto(A) ::= .                          {A = 0;}
%endif  SQLITE_OMIT_ATTACH
%endif  SQLITE_OMIT_VACUUM

///////////////////////////// The PRAGMA command /////////////////////////////
//
%ifndef SQLITE_OMIT_PRAGMA
cmd ::= PRAGMA nm(X) dbnm(Z).                {sqlite3Pragma(pParse,&X,&Z,0,0);}

Changes to src/pcache1.c.

98
99
100
101
102
103
104

105
106
107
108
109
110
111
...
566
567
568
569
570
571
572
573

574
575
576
577
578
579
580
...
904
905
906
907
908
909
910
911
912

913
914
915
916
917
918
919
....
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
  unsigned int iKey;             /* Key value (page number) */
  u8 isBulkLocal;                /* This page from bulk local storage */
  u8 isAnchor;                   /* This is the PGroup.lru element */
  PgHdr1 *pNext;                 /* Next in hash table chain */
  PCache1 *pCache;               /* Cache that currently owns this page */
  PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
  PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */

};

/*
** A page is pinned if it is not on the LRU list.  To be "pinned" means
** that the page is in active use and must not be deallocated.
*/
#define PAGE_IS_PINNED(p)    ((p)->pLruNext==0)
................................................................................
  assert( PAGE_IS_UNPINNED(pPage) );
  assert( pPage->pLruNext );
  assert( pPage->pLruPrev );
  assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
  pPage->pLruPrev->pLruNext = pPage->pLruNext;
  pPage->pLruNext->pLruPrev = pPage->pLruPrev;
  pPage->pLruNext = 0;
  pPage->pLruPrev = 0;

  assert( pPage->isAnchor==0 );
  assert( pPage->pCache->pGroup->lru.isAnchor==1 );
  pPage->pCache->nRecyclable--;
  return pPage;
}


................................................................................

  if( pPage ){
    unsigned int h = iKey % pCache->nHash;
    pCache->nPage++;
    pPage->iKey = iKey;
    pPage->pNext = pCache->apHash[h];
    pPage->pCache = pCache;
    pPage->pLruPrev = 0;
    pPage->pLruNext = 0;

    *(void **)pPage->page.pExtra = 0;
    pCache->apHash[h] = pPage;
    if( iKey>pCache->iMaxKey ){
      pCache->iMaxKey = iKey;
    }
  }
  return pPage;
................................................................................
 
  assert( pPage->pCache==pCache );
  pcache1EnterMutex(pGroup);

  /* It is an error to call this function if the page is already 
  ** part of the PGroup LRU list.
  */
  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
  assert( PAGE_IS_PINNED(pPage) );

  if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
    pcache1RemoveFromHash(pPage, 1);
  }else{
    /* Add the page to the PGroup LRU list. */
    PgHdr1 **ppFirst = &pGroup->lru.pLruNext;







>







 







|
>







 







|
|
>







 







|







98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
...
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
...
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
....
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
  unsigned int iKey;             /* Key value (page number) */
  u8 isBulkLocal;                /* This page from bulk local storage */
  u8 isAnchor;                   /* This is the PGroup.lru element */
  PgHdr1 *pNext;                 /* Next in hash table chain */
  PCache1 *pCache;               /* Cache that currently owns this page */
  PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
  PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
};

/*
** A page is pinned if it is not on the LRU list.  To be "pinned" means
** that the page is in active use and must not be deallocated.
*/
#define PAGE_IS_PINNED(p)    ((p)->pLruNext==0)
................................................................................
  assert( PAGE_IS_UNPINNED(pPage) );
  assert( pPage->pLruNext );
  assert( pPage->pLruPrev );
  assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
  pPage->pLruPrev->pLruNext = pPage->pLruNext;
  pPage->pLruNext->pLruPrev = pPage->pLruPrev;
  pPage->pLruNext = 0;
  /* pPage->pLruPrev = 0;
  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
  assert( pPage->isAnchor==0 );
  assert( pPage->pCache->pGroup->lru.isAnchor==1 );
  pPage->pCache->nRecyclable--;
  return pPage;
}


................................................................................

  if( pPage ){
    unsigned int h = iKey % pCache->nHash;
    pCache->nPage++;
    pPage->iKey = iKey;
    pPage->pNext = pCache->apHash[h];
    pPage->pCache = pCache;
    pPage->pLruNext = 0;
    /* pPage->pLruPrev = 0;
    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
    *(void **)pPage->page.pExtra = 0;
    pCache->apHash[h] = pPage;
    if( iKey>pCache->iMaxKey ){
      pCache->iMaxKey = iKey;
    }
  }
  return pPage;
................................................................................
 
  assert( pPage->pCache==pCache );
  pcache1EnterMutex(pGroup);

  /* It is an error to call this function if the page is already 
  ** part of the PGroup LRU list.
  */
  assert( pPage->pLruNext==0 );
  assert( PAGE_IS_PINNED(pPage) );

  if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
    pcache1RemoveFromHash(pPage, 1);
  }else{
    /* Add the page to the PGroup LRU list. */
    PgHdr1 **ppFirst = &pGroup->lru.pLruNext;

Changes to src/pragma.c.

813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
      int size = 1;
      if( sqlite3GetInt32(zRight, &size) ){
        sqlite3BtreeSetSpillSize(pDb->pBt, size);
      }
      if( sqlite3GetBoolean(zRight, size!=0) ){
        db->flags |= SQLITE_CacheSpill;
      }else{
        db->flags &= ~SQLITE_CacheSpill;
      }
      setAllPagerFlags(db);
    }
    break;
  }

  /*







|







813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
      int size = 1;
      if( sqlite3GetInt32(zRight, &size) ){
        sqlite3BtreeSetSpillSize(pDb->pBt, size);
      }
      if( sqlite3GetBoolean(zRight, size!=0) ){
        db->flags |= SQLITE_CacheSpill;
      }else{
        db->flags &= ~(u64)SQLITE_CacheSpill;
      }
      setAllPagerFlags(db);
    }
    break;
  }

  /*

Changes to src/prepare.c.

289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
...
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005

  /* Ticket #2804:  When we open a database in the newer file format,
  ** clear the legacy_file_format pragma flag so that a VACUUM will
  ** not downgrade the database and thus invalidate any descending
  ** indices that the user might have created.
  */
  if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){
    db->flags &= ~SQLITE_LegacyFileFmt;
  }

  /* Read the schema information out of the schema tables
  */
  assert( db->init.busy );
  {
    char *zSql;
................................................................................
  sqlite3BtreeLeaveAll(db);
  rc = sqlite3ApiExit(db, rc);
  assert( (rc&db->errMask)==rc );
  sqlite3_mutex_leave(db->mutex);
  return rc;
}

#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Checks if the specified token is a table, column, or function name,
** based on the databases associated with the statement being prepared.
** If the function fails, zero is returned and pRc is filled with the
** error code.
*/
static int shouldTreatAsIdentifier(
  sqlite3 *db,        /* Database handle. */
  const char *zToken, /* Pointer to start of token to be checked */
  int nToken,         /* Length of token to be checked */
  int *pRc            /* Pointer to error code upon failure */
){
  int bFound = 0;     /* Non-zero if token is an identifier name. */
  int i, j;           /* Database and column loop indexes. */
  Schema *pSchema;    /* Schema for current database. */
  Hash *pHash;        /* Hash table of tables for current database. */
  HashElem *e;        /* Hash element for hash table iteration. */
  Table *pTab;        /* Database table for columns being checked. */

  if( sqlite3IsRowidN(zToken, nToken) ){
    return 1;
  }
  if( nToken>0 ){
    int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
    if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
  }
  assert( db!=0 );
  sqlite3_mutex_enter(db->mutex);
  sqlite3BtreeEnterAll(db);
  for(i=0; i<db->nDb; i++){
    pHash = &db->aFunc;
    if( sqlite3HashFindN(pHash, zToken, nToken) ){
      bFound = 1;
      break;
    }
    pSchema = db->aDb[i].pSchema;
    if( pSchema==0 ) continue;
    pHash = &pSchema->tblHash;
    if( sqlite3HashFindN(pHash, zToken, nToken) ){
      bFound = 1;
      break;
    }
    for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
      pTab = sqliteHashData(e);
      if( pTab==0 ) continue;
      pHash = pTab->pColHash;
      if( pHash==0 ){
        pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
        if( pHash ){
          sqlite3HashInit(pHash);
          for(j=0; j<pTab->nCol; j++){
            Column *pCol = &pTab->aCol[j];
            sqlite3HashInsert(pHash, pCol->zName, pCol);
          }
        }else{
          *pRc = SQLITE_NOMEM_BKPT;
          bFound = 0;
          goto done;
        }
      }
      if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
        bFound = 1;
        goto done;
      }
    }
  }
done:
  sqlite3BtreeLeaveAll(db);
  sqlite3_mutex_leave(db->mutex);
  return bFound;
}

/*
** Attempt to estimate the final output buffer size needed for the fully
** normalized version of the specified SQL string.  This should take into
** account any potential expansion that could occur (e.g. via IN clauses
** being expanded, etc).  This size returned is the total number of bytes
** including the NUL terminator.
*/
static int estimateNormalizedSize(
  const char *zSql, /* The original SQL string */
  int nSql,         /* Length of original SQL string */
  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
){
  int nOut = nSql + 4;
  const char *z = zSql;
  while( nOut<nSql*5 ){
    while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
    if( z[0]==0 ) break;
    z++;
    if( z[0]!='N' && z[0]!='n' ) break;
    z++;
    while( sqlite3Isspace(z[0]) ){ z++; }
    if( z[0]!='(' ) break;
    z++;
    nOut += 5; /* ?,?,? */
  }
  return nOut;
}

/*
** Copy the current token into the output buffer while dealing with quoted
** identifiers.  By default, all letters will be converted into lowercase.
** If the bUpper flag is set, uppercase will be used.  The piOut argument
** will be used to update the target index into the output string.
*/
static void copyNormalizedToken(
  const char *zSql, /* The original SQL string */
  int iIn,          /* Current index into the original SQL string */
  int nToken,       /* Number of bytes in the current token */
  int tokenFlags,   /* Flags returned by the tokenizer */
  char *zOut,       /* The output string */
  int *piOut        /* Pointer to target index into the output string */
){
  int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
  int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
  int j = *piOut, k = 0;
  for(; k<nToken; k++){
    if( bQuoted ){
      if( k==0 && iIn>0 ){
        zOut[j++] = '"';
        continue;
      }else if( k==nToken-1 ){
        zOut[j++] = '"';
        continue;
      }
    }
    if( bKeyword ){
      zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
    }else{
      zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
    }
  }
  *piOut = j;
}

/*
** Perform normalization of the SQL contained in the prepared statement and
** store the result in the zNormSql field.  The schema for the associated
** databases are consulted while performing the normalization in order to
** determine if a token appears to be an identifier.  All identifiers are
** left intact in the normalized SQL and all literals are replaced with a
** single '?'.
*/
void sqlite3Normalize(
  Vdbe *pVdbe,      /* VM being reprepared */
  const char *zSql, /* The original SQL string */
  int nSql,         /* Size of the input string in bytes */
  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
){
  sqlite3 *db;           /* Database handle. */
  char *z;               /* The output string */
  int nZ;                /* Size of the output string in bytes */
  int i;                 /* Next character to read from zSql[] */
  int j;                 /* Next character to fill in on z[] */
  int tokenType = 0;     /* Type of the next token */
  int prevTokenType = 0; /* Type of the previous token, except spaces */
  int n;                 /* Size of the next token */
  int nParen = 0;        /* Nesting level of parenthesis */
  Hash inHash;           /* Table of parenthesis levels to output index. */

  db = sqlite3VdbeDb(pVdbe);
  assert( db!=0 );
  assert( pVdbe->zNormSql==0 );
  if( zSql==0 ) return;
  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
  z = sqlite3DbMallocRawNN(db, nZ);
  if( z==0 ) return;
  sqlite3HashInit(&inHash);
  for(i=j=0; i<nSql && zSql[i]; i+=n){
    int flags = 0;
    if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
    n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
    switch( tokenType ){
      case TK_SPACE: {
        break;
      }
      case TK_ILLEGAL: {
        sqlite3DbFree(db, z);
        sqlite3HashClear(&inHash);
        return;
      }
      case TK_STRING:
      case TK_INTEGER:
      case TK_FLOAT:
      case TK_VARIABLE:
      case TK_BLOB: {
        z[j++] = '?';
        break;
      }
      case TK_LP:
      case TK_RP: {
        if( tokenType==TK_LP ){
          nParen++;
          if( prevTokenType==TK_IN ){
            assert( nParen<nSql );
            sqlite3HashInsert(&inHash, zSql+nParen, SQLITE_INT_TO_PTR(j));
          }
        }else{
          int jj;
          assert( nParen<nSql );
          jj = SQLITE_PTR_TO_INT(sqlite3HashFind(&inHash, zSql+nParen));
          if( jj>0 ){
            sqlite3HashInsert(&inHash, zSql+nParen, 0);
            assert( jj+6<nZ );
            memcpy(z+jj+1, "?,?,?", 5);
            j = jj+6;
            assert( nZ-1-j>=0 );
            assert( nZ-1-j<nZ );
            memset(z+j, 0, nZ-1-j);
          }
          nParen--;
        }
        assert( nParen>=0 );
        /* Fall through */
      }
      case TK_MINUS:
      case TK_SEMI:
      case TK_PLUS:
      case TK_STAR:
      case TK_SLASH:
      case TK_REM:
      case TK_EQ:
      case TK_LE:
      case TK_NE:
      case TK_LSHIFT:
      case TK_LT:
      case TK_RSHIFT:
      case TK_GT:
      case TK_GE:
      case TK_BITOR:
      case TK_CONCAT:
      case TK_COMMA:
      case TK_BITAND:
      case TK_BITNOT:
      case TK_DOT:
      case TK_IN:
      case TK_IS:
      case TK_NOT:
      case TK_NULL:
      case TK_ID: {
        if( tokenType==TK_NULL ){
          if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
            /* NULL is a keyword in this case, not a literal value */
          }else{
            /* Here the NULL is a literal value */
            z[j++] = '?';
            break;
          }
        }
        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
          z[j++] = ' ';
        }
        if( tokenType==TK_ID ){
          int i2 = i, n2 = n, rc = SQLITE_OK;
          if( nParen>0 ){
            assert( nParen<nSql );
            sqlite3HashInsert(&inHash, zSql+nParen, 0);
          }
          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
          if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
            if( rc!=SQLITE_OK ){
              sqlite3DbFree(db, z);
              sqlite3HashClear(&inHash);
              return;
            }
            if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
              z[j++] = '?';
              break;
            }
          }
        }
        copyNormalizedToken(zSql, i, n, flags, z, &j);
        break;
      }
    }
  }
  assert( j<nZ && "one" );
  while( j>0 && z[j-1]==' ' ){ j--; }
  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
  z[j] = 0;
  assert( j<nZ && "two" );
  pVdbe->zNormSql = z;
  sqlite3HashClear(&inHash);
}
#endif /* SQLITE_ENABLE_NORMALIZE */

/*
** Rerun the compilation of a statement after a schema change.
**
** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,
** if the statement cannot be recompiled because another connection has
** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error







|







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
...
705
706
707
708
709
710
711































































































































































































































































































712
713
714
715
716
717
718

  /* Ticket #2804:  When we open a database in the newer file format,
  ** clear the legacy_file_format pragma flag so that a VACUUM will
  ** not downgrade the database and thus invalidate any descending
  ** indices that the user might have created.
  */
  if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){
    db->flags &= ~(u64)SQLITE_LegacyFileFmt;
  }

  /* Read the schema information out of the schema tables
  */
  assert( db->init.busy );
  {
    char *zSql;
................................................................................
  sqlite3BtreeLeaveAll(db);
  rc = sqlite3ApiExit(db, rc);
  assert( (rc&db->errMask)==rc );
  sqlite3_mutex_leave(db->mutex);
  return rc;
}

































































































































































































































































































/*
** Rerun the compilation of a statement after a schema change.
**
** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,
** if the statement cannot be recompiled because another connection has
** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error

Changes to src/resolve.c.

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
...
470
471
472
473
474
475
476



















477
478
479
480
481
482
483
  db = pParse->db;
  pDup = sqlite3ExprDup(db, pOrig, 0);
  if( pDup!=0 ){
    if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
    if( pExpr->op==TK_COLLATE ){
      pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
    }
    ExprSetProperty(pDup, EP_Alias);

    /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
    ** prevents ExprDelete() from deleting the Expr structure itself,
    ** allowing it to be repopulated by the memcpy() on the following line.
    ** The pExpr->u.zToken might point into memory that will be freed by the
    ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
    ** make a copy of the token before doing the sqlite3DbFree().
................................................................................
  **
  ** Because no reference was made to outer contexts, the pNC->nRef
  ** fields are not changed in any context.
  */
  if( cnt==0 && zTab==0 ){
    assert( pExpr->op==TK_ID );
    if( ExprHasProperty(pExpr,EP_DblQuoted) ){



















      pExpr->op = TK_STRING;
      pExpr->y.pTab = 0;
      return WRC_Prune;
    }
    if( sqlite3ExprIdToTrueFalse(pExpr) ){
      return WRC_Prune;
    }







|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
...
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
  db = pParse->db;
  pDup = sqlite3ExprDup(db, pOrig, 0);
  if( pDup!=0 ){
    if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
    if( pExpr->op==TK_COLLATE ){
      pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
    }
//    ExprSetProperty(pDup, EP_Alias);

    /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
    ** prevents ExprDelete() from deleting the Expr structure itself,
    ** allowing it to be repopulated by the memcpy() on the following line.
    ** The pExpr->u.zToken might point into memory that will be freed by the
    ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
    ** make a copy of the token before doing the sqlite3DbFree().
................................................................................
  **
  ** Because no reference was made to outer contexts, the pNC->nRef
  ** fields are not changed in any context.
  */
  if( cnt==0 && zTab==0 ){
    assert( pExpr->op==TK_ID );
    if( ExprHasProperty(pExpr,EP_DblQuoted) ){
      /* If a double-quoted identifier does not match any known column name,
      ** then treat it as a string.
      **
      ** This hack was added in the early days of SQLite in a misguided attempt
      ** to be compatible with MySQL 3.x, which used double-quotes for strings.
      ** I now sorely regret putting in this hack. The effect of this hack is
      ** that misspelled identifier names are silently converted into strings
      ** rather than causing an error, to the frustration of countless
      ** programmers. To all those frustrated programmers, my apologies.
      **
      ** Someday, I hope to get rid of this hack. Unfortunately there is
      ** a huge amount of legacy SQL that uses it. So for now, we just
      ** issue a warning.
      */
      sqlite3_log(SQLITE_WARNING,
        "double-quoted string literal: \"%w\"", zCol);
#ifdef SQLITE_ENABLE_NORMALIZE
      sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol);
#endif
      pExpr->op = TK_STRING;
      pExpr->y.pTab = 0;
      return WRC_Prune;
    }
    if( sqlite3ExprIdToTrueFalse(pExpr) ){
      return WRC_Prune;
    }

Changes to src/select.c.

2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
....
3457
3458
3459
3460
3461
3462
3463

3464
3465
3466
3467
3468
3469
3470
....
4021
4022
4023
4024
4025
4026
4027

4028
4029
4030
4031
4032
4033
4034
4035
/*
** Given a SELECT statement, generate a Table structure that describes
** the result set of that SELECT.
*/
Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){
  Table *pTab;
  sqlite3 *db = pParse->db;
  int savedFlags;

  savedFlags = db->flags;
  db->flags &= ~SQLITE_FullColNames;
  db->flags |= SQLITE_ShortColNames;
  sqlite3SelectPrep(pParse, pSelect, 0);
  if( pParse->nErr ) return 0;
  while( pSelect->pPrior ) pSelect = pSelect->pPrior;
  db->flags = savedFlags;
  pTab = sqlite3DbMallocZero(db, sizeof(Table) );
  if( pTab==0 ){
................................................................................
        if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
          memset(&ifNullRow, 0, sizeof(ifNullRow));
          ifNullRow.op = TK_IF_NULL_ROW;
          ifNullRow.pLeft = pCopy;
          ifNullRow.iTable = pSubst->iNewTable;
          pCopy = &ifNullRow;
        }

        pNew = sqlite3ExprDup(db, pCopy, 0);
        if( pNew && pSubst->isLeftJoin ){
          ExprSetProperty(pNew, EP_CanBeNull);
        }
        if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){
          pNew->iRightJoinTable = pExpr->iRightJoinTable;
          ExprSetProperty(pNew, EP_FromJoin);
................................................................................
      for(i=0; i<pOrderBy->nExpr; i++){
        pOrderBy->a[i].u.x.iOrderByCol = 0;
      }
      assert( pParent->pOrderBy==0 );
      pParent->pOrderBy = pOrderBy;
      pSub->pOrderBy = 0;
    }

    pWhere = sqlite3ExprDup(db, pSub->pWhere, 0);
    if( isLeftJoin>0 ){
      setJoinExpr(pWhere, iNewParent);
    }
    pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
    if( db->mallocFailed==0 ){
      SubstContext x;
      x.pParse = pParse;







|


|







 







>







 







>
|







2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
....
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
....
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
/*
** Given a SELECT statement, generate a Table structure that describes
** the result set of that SELECT.
*/
Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){
  Table *pTab;
  sqlite3 *db = pParse->db;
  u64 savedFlags;

  savedFlags = db->flags;
  db->flags &= ~(u64)SQLITE_FullColNames;
  db->flags |= SQLITE_ShortColNames;
  sqlite3SelectPrep(pParse, pSelect, 0);
  if( pParse->nErr ) return 0;
  while( pSelect->pPrior ) pSelect = pSelect->pPrior;
  db->flags = savedFlags;
  pTab = sqlite3DbMallocZero(db, sizeof(Table) );
  if( pTab==0 ){
................................................................................
        if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){
          memset(&ifNullRow, 0, sizeof(ifNullRow));
          ifNullRow.op = TK_IF_NULL_ROW;
          ifNullRow.pLeft = pCopy;
          ifNullRow.iTable = pSubst->iNewTable;
          pCopy = &ifNullRow;
        }
        testcase( ExprHasProperty(pCopy, EP_Subquery) );
        pNew = sqlite3ExprDup(db, pCopy, 0);
        if( pNew && pSubst->isLeftJoin ){
          ExprSetProperty(pNew, EP_CanBeNull);
        }
        if( pNew && ExprHasProperty(pExpr,EP_FromJoin) ){
          pNew->iRightJoinTable = pExpr->iRightJoinTable;
          ExprSetProperty(pNew, EP_FromJoin);
................................................................................
      for(i=0; i<pOrderBy->nExpr; i++){
        pOrderBy->a[i].u.x.iOrderByCol = 0;
      }
      assert( pParent->pOrderBy==0 );
      pParent->pOrderBy = pOrderBy;
      pSub->pOrderBy = 0;
    }
    pWhere = pSub->pWhere;
    pSub->pWhere = 0;
    if( isLeftJoin>0 ){
      setJoinExpr(pWhere, iNewParent);
    }
    pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere);
    if( db->mallocFailed==0 ){
      SubstContext x;
      x.pParse = pParse;

Changes to src/shell.c.in.

1003
1004
1005
1006
1007
1008
1009

1010
1011
1012
1013
1014
1015
1016
....
1062
1063
1064
1065
1066
1067
1068






1069
1070
1071
1072
1073
1074
1075
....
3366
3367
3368
3369
3370
3371
3372

3373
3374
3375
3376
3377
3378
3379
....
3488
3489
3490
3491
3492
3493
3494

3495














3496
3497
3498
3499
3500
3501
3502
....
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015






4016




















4017
4018
4019












4020
4021
4022
4023
4024
4025
4026
....
5778
5779
5780
5781
5782
5783
5784

5785
5786
5787
5788
5789
5790
5791



5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
....
5812
5813
5814
5815
5816
5817
5818




5819
5820
5821
5822
5823
5824
5825
....
7834
7835
7836
7837
7838
7839
7840

7841


7842
7843



























7844
7845
7846
7847

7848
7849
7850


7851
7852
7853

7854
7855
7856
7857

7858
7859
7860
7861
7862
7863
7864
  u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  u8 autoEQPtest;        /* autoEQP is in test mode */
  u8 statsOn;            /* True to display memory stats before each finalize */
  u8 scanstatsOn;        /* True to display scan stats before each finalize */
  u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
  u8 nEqpLevel;          /* Depth of the EQP output graph */

  unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  int outCount;          /* Revert to stdout when reaching zero */
  int cnt;               /* Number of records displayed so far */
  FILE *out;             /* Write results here */
  FILE *traceOut;        /* Output for sqlite3_trace() */
  int nErr;              /* Number of errors seen */
  int mode;              /* An output mode setting */
................................................................................
#define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
#define SHELL_OPEN_NORMAL      1      /* Normal database file */
#define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
#define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
#define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
#define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */







/*
** These are the allowed shellFlgs values
*/
#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
#define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
................................................................................
  "      http://sqlite.org/cli.html#sqlar_archive_support",
#endif
#ifndef SQLITE_OMIT_AUTHORIZATION
  ".auth ON|OFF             Show authorizer callbacks",
#endif
  ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
  "       --append            Use the appendvfs",

  ".bail on|off             Stop after hitting an error.  Default OFF",
  ".binary on|off           Turn binary output on or off.  Default OFF",
  ".cd DIRECTORY            Change the working directory to DIRECTORY",
  ".changes on|off          Show number of rows changed by SQL",
  ".check GLOB              Fail if output since .testcase does not match",
  ".clone NEWDB             Clone data into NEWDB from the existing database",
  ".databases               List names and files of attached databases",
................................................................................
#ifndef SQLITE_NOHAVE_SYSTEM
  ".system CMD ARGS...      Run CMD ARGS... in a system shell",
#endif
  ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
  ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
  ".timeout MS              Try opening locked tables for MS milliseconds",
  ".timer on|off            Turn SQL timer on or off",

  ".trace FILE|off          Output each SQL statement as it is run",














  ".vfsinfo ?AUX?           Information about the top-level VFS",
  ".vfslist                 List all available VFSes",
  ".vfsname ?AUX?           Print the name of the VFS stack",
  ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
  "     Negative values right-justify",
};

................................................................................
    if( f==0 ){
      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
    }
  }
  return f;
}

#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
/*
** A routine for handling output from sqlite3_trace().
*/
static int sql_trace_callback(
  unsigned mType,
  void *pArg,
  void *pP,
  void *pX
){
  FILE *f = (FILE*)pArg;
  UNUSED_PARAMETER(mType);
  UNUSED_PARAMETER(pP);
  if( f ){






    const char *z = (const char*)pX;




















    int i = strlen30(z);
    while( i>0 && z[i-1]==';' ){ i--; }
    utf8_printf(f, "%.*s;\n", i, z);












  }
  return 0;
}
#endif

/*
** A no-op routine that runs with the ".breakpoint" doc-command.  This is
................................................................................
   || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
  ){
    const char *zDestFile = 0;
    const char *zDb = 0;
    sqlite3 *pDest;
    sqlite3_backup *pBackup;
    int j;

    const char *zVfs = 0;
    for(j=1; j<nArg; j++){
      const char *z = azArg[j];
      if( z[0]=='-' ){
        if( z[1]=='-' ) z++;
        if( strcmp(z, "-append")==0 ){
          zVfs = "apndvfs";



        }else
        {
          utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
          return 1;
        }
      }else if( zDestFile==0 ){
        zDestFile = azArg[j];
      }else if( zDb==0 ){
        zDb = zDestFile;
        zDestFile = azArg[j];
      }else{
        raw_printf(stderr, "Usage: .backup ?DB? ?--append? FILENAME\n");
        return 1;
      }
    }
    if( zDestFile==0 ){
      raw_printf(stderr, "missing FILENAME argument on .backup\n");
      return 1;
    }
................................................................................
    rc = sqlite3_open_v2(zDestFile, &pDest, 
                  SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs);
    if( rc!=SQLITE_OK ){
      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
      close_db(pDest);
      return 1;
    }




    open_db(p, 0);
    pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
    if( pBackup==0 ){
      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
      close_db(pDest);
      return 1;
    }
................................................................................
      }
    }else{
      raw_printf(stderr, "Usage: .timer on|off\n");
      rc = 1;
    }
  }else


  if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){


    open_db(p, 0);
    if( nArg!=2 ){



























      raw_printf(stderr, "Usage: .trace FILE|off\n");
      rc = 1;
      goto meta_command_exit;
    }

    output_file_close(p->traceOut);
    p->traceOut = output_file_open(azArg[1], 0);
#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)


    if( p->traceOut==0 ){
      sqlite3_trace_v2(p->db, 0, 0, 0);
    }else{

      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
    }
#endif
  }else


#if SQLITE_USER_AUTHENTICATION
  if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
    if( nArg<2 ){
      raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
      rc = 1;
      goto meta_command_exit;







>







 







>
>
>
>
>
>







 







>







 







>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|




|
|
|
|

|
|
|
|
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
<
<
>
>
>
>
>
>
>
>
>
>
>
>







 







>







>
>
>











|







 







>
>
>
>







 







>

>
>

<
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
>
|
|
<
>
>



>
|

<

>







1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
....
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
....
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
....
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
....
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066


4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
....
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
....
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
....
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912

7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946

7947
7948
7949
7950
7951
7952
7953
7954

7955
7956
7957
7958
7959
7960
7961
7962
7963
  u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  u8 autoEQPtest;        /* autoEQP is in test mode */
  u8 statsOn;            /* True to display memory stats before each finalize */
  u8 scanstatsOn;        /* True to display scan stats before each finalize */
  u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
  u8 nEqpLevel;          /* Depth of the EQP output graph */
  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
  unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  int outCount;          /* Revert to stdout when reaching zero */
  int cnt;               /* Number of records displayed so far */
  FILE *out;             /* Write results here */
  FILE *traceOut;        /* Output for sqlite3_trace() */
  int nErr;              /* Number of errors seen */
  int mode;              /* An output mode setting */
................................................................................
#define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
#define SHELL_OPEN_NORMAL      1      /* Normal database file */
#define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
#define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
#define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
#define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */

/* Allowed values for ShellState.eTraceType
*/
#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */

/*
** These are the allowed shellFlgs values
*/
#define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
#define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
#define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
#define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
................................................................................
  "      http://sqlite.org/cli.html#sqlar_archive_support",
#endif
#ifndef SQLITE_OMIT_AUTHORIZATION
  ".auth ON|OFF             Show authorizer callbacks",
#endif
  ".backup ?DB? FILE        Backup DB (default \"main\") to FILE",
  "       --append            Use the appendvfs",
  "       --async             Write to FILE without a journal and without fsync()",
  ".bail on|off             Stop after hitting an error.  Default OFF",
  ".binary on|off           Turn binary output on or off.  Default OFF",
  ".cd DIRECTORY            Change the working directory to DIRECTORY",
  ".changes on|off          Show number of rows changed by SQL",
  ".check GLOB              Fail if output since .testcase does not match",
  ".clone NEWDB             Clone data into NEWDB from the existing database",
  ".databases               List names and files of attached databases",
................................................................................
#ifndef SQLITE_NOHAVE_SYSTEM
  ".system CMD ARGS...      Run CMD ARGS... in a system shell",
#endif
  ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
  ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
  ".timeout MS              Try opening locked tables for MS milliseconds",
  ".timer on|off            Turn SQL timer on or off",
#ifndef SQLITE_OMIT_TRACE
  ".trace ?OPTIONS?         Output each SQL statement as it is run",
  "    FILE                    Send output to FILE",
  "    stdout                  Send output to stdout",
  "    stderr                  Send output to stderr",
  "    off                     Disable tracing",
  "    --expanded              Expand query parameters",
#ifdef SQLITE_ENABLE_NORMALIZE
  "    --normalized            Normal the SQL statements",
#endif
  "    --plain                 Show SQL as it is input",
  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
#endif /* SQLITE_OMIT_TRACE */
  ".vfsinfo ?AUX?           Information about the top-level VFS",
  ".vfslist                 List all available VFSes",
  ".vfsname ?AUX?           Print the name of the VFS stack",
  ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
  "     Negative values right-justify",
};

................................................................................
    if( f==0 ){
      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
    }
  }
  return f;
}

#ifndef SQLITE_OMIT_TRACE
/*
** A routine for handling output from sqlite3_trace().
*/
static int sql_trace_callback(
  unsigned mType,         /* The trace type */
  void *pArg,             /* The ShellState pointer */
  void *pP,               /* Usually a pointer to sqlite_stmt */
  void *pX                /* Auxiliary output */
){
  ShellState *p = (ShellState*)pArg;
  sqlite3_stmt *pStmt;
  const char *zSql;
  int nSql;
  if( p->traceOut==0 ) return 0;
  if( mType==SQLITE_TRACE_CLOSE ){
    utf8_printf(p->traceOut, "-- closing database connection\n");
    return 0;
  }
  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
    zSql = (const char*)pX;
  }else{
    pStmt = (sqlite3_stmt*)pP;
    switch( p->eTraceType ){
      case SHELL_TRACE_EXPANDED: {
        zSql = sqlite3_expanded_sql(pStmt);
        break;
      }
#ifdef SQLITE_ENABLE_NORMALIZE
      case SHELL_TRACE_NORMALIZED: {
        zSql = sqlite3_normalized_sql(pStmt);
        break;
      }
#endif
      default: {
        zSql = sqlite3_sql(pStmt);
        break;
      }
    }
  }
  if( zSql==0 ) return 0;
  nSql = strlen30(zSql);


  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
  switch( mType ){
    case SQLITE_TRACE_ROW:
    case SQLITE_TRACE_STMT: {
      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
      break;
    }
    case SQLITE_TRACE_PROFILE: {
      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
      break;
    }
  }
  return 0;
}
#endif

/*
** A no-op routine that runs with the ".breakpoint" doc-command.  This is
................................................................................
   || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0)
  ){
    const char *zDestFile = 0;
    const char *zDb = 0;
    sqlite3 *pDest;
    sqlite3_backup *pBackup;
    int j;
    int bAsync = 0;
    const char *zVfs = 0;
    for(j=1; j<nArg; j++){
      const char *z = azArg[j];
      if( z[0]=='-' ){
        if( z[1]=='-' ) z++;
        if( strcmp(z, "-append")==0 ){
          zVfs = "apndvfs";
        }else
        if( strcmp(z, "-async")==0 ){
          bAsync = 1;
        }else
        {
          utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
          return 1;
        }
      }else if( zDestFile==0 ){
        zDestFile = azArg[j];
      }else if( zDb==0 ){
        zDb = zDestFile;
        zDestFile = azArg[j];
      }else{
        raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
        return 1;
      }
    }
    if( zDestFile==0 ){
      raw_printf(stderr, "missing FILENAME argument on .backup\n");
      return 1;
    }
................................................................................
    rc = sqlite3_open_v2(zDestFile, &pDest, 
                  SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs);
    if( rc!=SQLITE_OK ){
      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
      close_db(pDest);
      return 1;
    }
    if( bAsync ){
      sqlite3_exec(pDest, "PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;",
                   0, 0, 0);
    }
    open_db(p, 0);
    pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
    if( pBackup==0 ){
      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
      close_db(pDest);
      return 1;
    }
................................................................................
      }
    }else{
      raw_printf(stderr, "Usage: .timer on|off\n");
      rc = 1;
    }
  }else

#ifndef SQLITE_OMIT_TRACE
  if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
    int mType = 0;
    int jj;
    open_db(p, 0);

    for(jj=1; jj<nArg; jj++){
      const char *z = azArg[jj];
      if( z[0]=='-' ){
        if( optionMatch(z, "expanded") ){
          p->eTraceType = SHELL_TRACE_EXPANDED;
        }
#ifdef SQLITE_ENABLE_NORMALIZE
        else if( optionMatch(z, "normalized") ){
          p->eTraceType = SHELL_TRACE_NORMALIZED;
        }
#endif
        else if( optionMatch(z, "plain") ){
          p->eTraceType = SHELL_TRACE_PLAIN;
        }
        else if( optionMatch(z, "profile") ){
          mType |= SQLITE_TRACE_PROFILE;
        }
        else if( optionMatch(z, "row") ){
          mType |= SQLITE_TRACE_ROW;
        }
        else if( optionMatch(z, "stmt") ){
          mType |= SQLITE_TRACE_STMT;
        }
        else if( optionMatch(z, "close") ){
          mType |= SQLITE_TRACE_CLOSE;
        }
        else {
          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
          rc = 1;
          goto meta_command_exit;
        }
      }else{
        output_file_close(p->traceOut);
        p->traceOut = output_file_open(azArg[1], 0);

      }
    }
    if( p->traceOut==0 ){
      sqlite3_trace_v2(p->db, 0, 0, 0);
    }else{
      if( mType==0 ) mType = SQLITE_TRACE_STMT;
      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
    }

  }else
#endif /* !defined(SQLITE_OMIT_TRACE) */

#if SQLITE_USER_AUTHENTICATION
  if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
    if( nArg<2 ){
      raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
      rc = 1;
      goto meta_command_exit;

Changes to src/sqlite.h.in.

2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
....
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636

3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
** ^The callback function registered by sqlite3_profile() is invoked
** as each SQL statement finishes.  ^The profile callback contains
** the original statement text and an estimate of wall-clock time
** of how long that statement took to run.  ^The profile callback
** time is in units of nanoseconds, however the current implementation
** is only capable of millisecond resolution so the six least significant
** digits in the time are meaningless.  Future versions of SQLite
** might provide greater resolution on the profiler callback.  The
** sqlite3_profile() function is considered experimental and is
** subject to change in future versions of SQLite.
*/
SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
   void(*xTrace)(void*,const char*), void*);
SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
   void(*xProfile)(void*,const char*,sqlite3_uint64), void*);

/*
................................................................................
** and [sqlite3_prepare16_v3()] assume that the prepared statement will 
** be used just once or at most a few times and then destroyed using
** [sqlite3_finalize()] relatively soon. The current implementation acts
** on this hint by avoiding the use of [lookaside memory] so as not to
** deplete the limited store of lookaside memory. Future versions of
** SQLite may act on this hint differently.
**
** [[SQLITE_PREPARE_NORMALIZE]] ^(<dt>SQLITE_PREPARE_NORMALIZE</dt>
** <dd>The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized
** representation of the SQL statement should be calculated and then
** associated with the prepared statement, which can be obtained via
** the [sqlite3_normalized_sql()] interface.)^  The semantics used to

** normalize a SQL statement are unspecified and subject to change.
** At a minimum, literal values will be replaced with suitable
** placeholders.
** </dl>
*/
#define SQLITE_PREPARE_PERSISTENT              0x01
#define SQLITE_PREPARE_NORMALIZE               0x02

/*
** CAPI3REF: Compiling An SQL Statement







|
|
|







 







|
|
|
<
|
>
|
|
<







2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
....
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634

3635
3636
3637
3638

3639
3640
3641
3642
3643
3644
3645
** ^The callback function registered by sqlite3_profile() is invoked
** as each SQL statement finishes.  ^The profile callback contains
** the original statement text and an estimate of wall-clock time
** of how long that statement took to run.  ^The profile callback
** time is in units of nanoseconds, however the current implementation
** is only capable of millisecond resolution so the six least significant
** digits in the time are meaningless.  Future versions of SQLite
** might provide greater resolution on the profiler callback.  Invoking
** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the
** profile callback.
*/
SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
   void(*xTrace)(void*,const char*), void*);
SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
   void(*xProfile)(void*,const char*,sqlite3_uint64), void*);

/*
................................................................................
** and [sqlite3_prepare16_v3()] assume that the prepared statement will 
** be used just once or at most a few times and then destroyed using
** [sqlite3_finalize()] relatively soon. The current implementation acts
** on this hint by avoiding the use of [lookaside memory] so as not to
** deplete the limited store of lookaside memory. Future versions of
** SQLite may act on this hint differently.
**
** [[SQLITE_PREPARE_NORMALIZE]] <dt>SQLITE_PREPARE_NORMALIZE</dt>
** <dd>The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used
** to be required for any prepared statement that wanted to use the

** [sqlite3_normalized_sql()] interface.  However, the
** [sqlite3_normalized_sql()] interface is now available to all
** prepared statements, regardless of whether or not they use this
** flag.

** </dl>
*/
#define SQLITE_PREPARE_PERSISTENT              0x01
#define SQLITE_PREPARE_NORMALIZE               0x02

/*
** CAPI3REF: Compiling An SQL Statement

Changes to src/sqliteInt.h.

1352
1353
1354
1355
1356
1357
1358
1359

1360
1361

1362

1363
1364
1365
1366
1367
1368
1369
....
1415
1416
1417
1418
1419
1420
1421

1422
1423

1424
1425
1426
1427
1428
1429
1430
....
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
....
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
....
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
....
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
....
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
....
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
                               const char*);
#endif

#ifndef SQLITE_OMIT_DEPRECATED
/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
** in the style of sqlite3_trace()
*/
#define SQLITE_TRACE_LEGACY  0x80

#else
#define SQLITE_TRACE_LEGACY  0

#endif /* SQLITE_OMIT_DEPRECATED */



/*
** Each database connection is an instance of the following structure.
*/
struct sqlite3 {
  sqlite3_vfs *pVfs;            /* OS Interface */
................................................................................
  int nVdbeWrite;               /* Number of active VDBEs that read and write */
  int nVdbeExec;                /* Number of nested calls to VdbeExec() */
  int nVDestroy;                /* Number of active OP_VDestroy operations */
  int nExtension;               /* Number of loaded extensions */
  void **aExtension;            /* Array of shared library handles */
  int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
  void *pTraceArg;                          /* Argument to the trace function */

  void (*xProfile)(void*,const char*,u64);  /* Profiling function */
  void *pProfileArg;                        /* Argument to profile function */

  void *pCommitArg;                 /* Argument to xCommitCallback() */
  int (*xCommitCallback)(void*);    /* Invoked at every commit. */
  void *pRollbackArg;               /* Argument to xRollbackCallback() */
  void (*xRollbackCallback)(void*); /* Invoked at every commit. */
  void *pUpdateArg;
  void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
................................................................................
/*
** The schema for each SQL table and view is represented in memory
** by an instance of the following structure.
*/
struct Table {
  char *zName;         /* Name of the table or view */
  Column *aCol;        /* Information about each column */
#ifdef SQLITE_ENABLE_NORMALIZE
  Hash *pColHash;      /* All columns indexed by name */
#endif
  Index *pIndex;       /* List of SQL indexes on this table. */
  Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
  FKey *pFKey;         /* Linked list of all foreign keys in this table */
  char *zColAff;       /* String defining the affinity of each column */
  ExprList *pCheck;    /* All CHECK constraints */
                       /*   ... also used as column name list in a VIEW */
  int tnum;            /* Root BTree page for this table */
................................................................................
#define LOCATE_VIEW    0x01
#define LOCATE_NOERR   0x02
Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
void sqlite3Vacuum(Parse*,Token*);
int sqlite3RunVacuum(char**, sqlite3*, int);
char *sqlite3NameFromToken(sqlite3*, Token*);
int sqlite3ExprCompare(Parse*,Expr*, Expr*, int);
int sqlite3ExprCompareSkip(Expr*, Expr*, int);
int sqlite3ExprListCompare(ExprList*, ExprList*, int);
int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int);
int sqlite3ExprImpliesNonNullRow(Expr*,int);
void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
................................................................................
#ifdef SQLITE_ENABLE_CURSOR_HINTS
int sqlite3ExprContainsSubquery(Expr*);
#endif
int sqlite3ExprIsInteger(Expr*, int*);
int sqlite3ExprCanBeNull(const Expr*);
int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
int sqlite3IsRowid(const char*);
#ifdef SQLITE_ENABLE_NORMALIZE
int sqlite3IsRowidN(const char*, int);
#endif
void sqlite3GenerateRowDelete(
    Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
void sqlite3ResolvePartIdxLabel(Parse*,int);
int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int);
void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
................................................................................
void sqlite3UniqueConstraint(Parse*, int, Index*);
void sqlite3RowidConstraint(Parse*, int, Table*);
Expr *sqlite3ExprDup(sqlite3*,Expr*,int);
ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
IdList *sqlite3IdListDup(sqlite3*,IdList*);
Select *sqlite3SelectDup(sqlite3*,Select*,int);
#ifdef SQLITE_ENABLE_NORMALIZE
FuncDef *sqlite3FunctionSearchN(int,const char*,int);
#endif
void sqlite3InsertBuiltinFuncs(FuncDef*,int);
FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
void sqlite3RegisterBuiltinFunctions(void);
void sqlite3RegisterDateTimeFunctions(void);
void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
int sqlite3SafetyCheckOk(sqlite3*);
int sqlite3SafetyCheckSickOrOk(sqlite3*);
................................................................................
#endif
void sqlite3RootPageMoved(sqlite3*, int, int, int);
void sqlite3Reindex(Parse*, Token*, Token*);
void sqlite3AlterFunctions(void);
void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
int sqlite3GetToken(const unsigned char *, int *);
#ifdef SQLITE_ENABLE_NORMALIZE
int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
#endif
void sqlite3NestedParse(Parse*, const char*, ...);
void sqlite3ExpirePreparedStatements(sqlite3*, int);
int sqlite3CodeSubselect(Parse*, Expr *, int, int);
void sqlite3SelectPrep(Parse*, Select*, NameContext*);
void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
int sqlite3MatchSpanName(const char*, const char*, const char*, const char*);
int sqlite3ResolveExprNames(NameContext*, Expr*);
................................................................................
int sqlite3VtabBegin(sqlite3 *, VTable *);
FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
void sqlite3ParserReset(Parse*);
#ifdef SQLITE_ENABLE_NORMALIZE
void sqlite3Normalize(Vdbe*, const char*, int, u8);
#endif
int sqlite3Reprepare(Vdbe*);
void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
int sqlite3TempInMemory(const sqlite3*);
const char *sqlite3JournalModename(int);
#ifndef SQLITE_OMIT_WAL







|
>

|
>

>







 







>


>







 







<
<
<







 







|
|







 







<
<
<







 







<
|
<







 







<
<
<







 







|







1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
....
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
....
1954
1955
1956
1957
1958
1959
1960



1961
1962
1963
1964
1965
1966
1967
....
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
....
4021
4022
4023
4024
4025
4026
4027



4028
4029
4030
4031
4032
4033
4034
....
4047
4048
4049
4050
4051
4052
4053

4054

4055
4056
4057
4058
4059
4060
4061
....
4252
4253
4254
4255
4256
4257
4258



4259
4260
4261
4262
4263
4264
4265
....
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
                               const char*);
#endif

#ifndef SQLITE_OMIT_DEPRECATED
/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
** in the style of sqlite3_trace()
*/
#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
#else
#define SQLITE_TRACE_LEGACY          0
#define SQLITE_TRACE_XPROFILE        0
#endif /* SQLITE_OMIT_DEPRECATED */
#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */


/*
** Each database connection is an instance of the following structure.
*/
struct sqlite3 {
  sqlite3_vfs *pVfs;            /* OS Interface */
................................................................................
  int nVdbeWrite;               /* Number of active VDBEs that read and write */
  int nVdbeExec;                /* Number of nested calls to VdbeExec() */
  int nVDestroy;                /* Number of active OP_VDestroy operations */
  int nExtension;               /* Number of loaded extensions */
  void **aExtension;            /* Array of shared library handles */
  int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
  void *pTraceArg;                          /* Argument to the trace function */
#ifndef SQLITE_OMIT_DEPRECATED
  void (*xProfile)(void*,const char*,u64);  /* Profiling function */
  void *pProfileArg;                        /* Argument to profile function */
#endif
  void *pCommitArg;                 /* Argument to xCommitCallback() */
  int (*xCommitCallback)(void*);    /* Invoked at every commit. */
  void *pRollbackArg;               /* Argument to xRollbackCallback() */
  void (*xRollbackCallback)(void*); /* Invoked at every commit. */
  void *pUpdateArg;
  void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
................................................................................
/*
** The schema for each SQL table and view is represented in memory
** by an instance of the following structure.
*/
struct Table {
  char *zName;         /* Name of the table or view */
  Column *aCol;        /* Information about each column */



  Index *pIndex;       /* List of SQL indexes on this table. */
  Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
  FKey *pFKey;         /* Linked list of all foreign keys in this table */
  char *zColAff;       /* String defining the affinity of each column */
  ExprList *pCheck;    /* All CHECK constraints */
                       /*   ... also used as column name list in a VIEW */
  int tnum;            /* Root BTree page for this table */
................................................................................
#define LOCATE_VIEW    0x01
#define LOCATE_NOERR   0x02
Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
void sqlite3Vacuum(Parse*,Token*,Expr*);
int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*);
char *sqlite3NameFromToken(sqlite3*, Token*);
int sqlite3ExprCompare(Parse*,Expr*, Expr*, int);
int sqlite3ExprCompareSkip(Expr*, Expr*, int);
int sqlite3ExprListCompare(ExprList*, ExprList*, int);
int sqlite3ExprImpliesExpr(Parse*,Expr*, Expr*, int);
int sqlite3ExprImpliesNonNullRow(Expr*,int);
void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
................................................................................
#ifdef SQLITE_ENABLE_CURSOR_HINTS
int sqlite3ExprContainsSubquery(Expr*);
#endif
int sqlite3ExprIsInteger(Expr*, int*);
int sqlite3ExprCanBeNull(const Expr*);
int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
int sqlite3IsRowid(const char*);



void sqlite3GenerateRowDelete(
    Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
void sqlite3ResolvePartIdxLabel(Parse*,int);
int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int);
void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
................................................................................
void sqlite3UniqueConstraint(Parse*, int, Index*);
void sqlite3RowidConstraint(Parse*, int, Table*);
Expr *sqlite3ExprDup(sqlite3*,Expr*,int);
ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
IdList *sqlite3IdListDup(sqlite3*,IdList*);
Select *sqlite3SelectDup(sqlite3*,Select*,int);

FuncDef *sqlite3FunctionSearch(int,const char*);

void sqlite3InsertBuiltinFuncs(FuncDef*,int);
FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
void sqlite3RegisterBuiltinFunctions(void);
void sqlite3RegisterDateTimeFunctions(void);
void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
int sqlite3SafetyCheckOk(sqlite3*);
int sqlite3SafetyCheckSickOrOk(sqlite3*);
................................................................................
#endif
void sqlite3RootPageMoved(sqlite3*, int, int, int);
void sqlite3Reindex(Parse*, Token*, Token*);
void sqlite3AlterFunctions(void);
void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
int sqlite3GetToken(const unsigned char *, int *);



void sqlite3NestedParse(Parse*, const char*, ...);
void sqlite3ExpirePreparedStatements(sqlite3*, int);
int sqlite3CodeSubselect(Parse*, Expr *, int, int);
void sqlite3SelectPrep(Parse*, Select*, NameContext*);
void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
int sqlite3MatchSpanName(const char*, const char*, const char*, const char*);
int sqlite3ResolveExprNames(NameContext*, Expr*);
................................................................................
int sqlite3VtabBegin(sqlite3 *, VTable *);
FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
void sqlite3ParserReset(Parse*);
#ifdef SQLITE_ENABLE_NORMALIZE
char *sqlite3Normalize(Vdbe*, const char*);
#endif
int sqlite3Reprepare(Vdbe*);
void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
int sqlite3TempInMemory(const sqlite3*);
const char *sqlite3JournalModename(int);
#ifndef SQLITE_OMIT_WAL

Changes to src/test_hexio.c.

186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
...
203
204
205
206
207
208
209

210
211







212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227



228

229
230
231
232
233
234
235
  sqlite3_free(aOut);
  fclose(out);
  Tcl_SetObjResult(interp, Tcl_NewIntObj(written));
  return TCL_OK;
}

/*
** USAGE:   hexio_get_int   HEXDATA
**
** Interpret the HEXDATA argument as a big-endian integer.  Return
** the value of that integer.  HEXDATA can contain between 2 and 8
** hexadecimal digits.
*/
static int SQLITE_TCLAPI hexio_get_int(
  void * clientData,
................................................................................
  Tcl_Obj *CONST objv[]
){
  int val;
  int nIn, nOut;
  const unsigned char *zIn;
  unsigned char *aOut;
  unsigned char aNum[4];


  if( objc!=2 ){







    Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA");
    return TCL_ERROR;
  }
  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
  aOut = sqlite3_malloc( nIn/2 );
  if( aOut==0 ){
    return TCL_ERROR;
  }
  nOut = sqlite3TestHexToBin(zIn, nIn, aOut);
  if( nOut>=4 ){
    memcpy(aNum, aOut, 4);
  }else{
    memset(aNum, 0, sizeof(aNum));
    memcpy(&aNum[4-nOut], aOut, nOut);
  }
  sqlite3_free(aOut);



  val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];

  Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
  return TCL_OK;
}


/*
** USAGE:   hexio_render_int16   INTEGER







|







 







>

|
>
>
>
>
>
>
>
|


|












>
>
>
|
>







186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
...
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
  sqlite3_free(aOut);
  fclose(out);
  Tcl_SetObjResult(interp, Tcl_NewIntObj(written));
  return TCL_OK;
}

/*
** USAGE:   hexio_get_int [-littleendian] HEXDATA
**
** Interpret the HEXDATA argument as a big-endian integer.  Return
** the value of that integer.  HEXDATA can contain between 2 and 8
** hexadecimal digits.
*/
static int SQLITE_TCLAPI hexio_get_int(
  void * clientData,
................................................................................
  Tcl_Obj *CONST objv[]
){
  int val;
  int nIn, nOut;
  const unsigned char *zIn;
  unsigned char *aOut;
  unsigned char aNum[4];
  int bLittle = 0;

  if( objc==3 ){
    int n;
    char *z = Tcl_GetStringFromObj(objv[1], &n);
    if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){
      bLittle = 1;
    }
  }
  if( (objc-bLittle)!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA");
    return TCL_ERROR;
  }
  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn);
  aOut = sqlite3_malloc( nIn/2 );
  if( aOut==0 ){
    return TCL_ERROR;
  }
  nOut = sqlite3TestHexToBin(zIn, nIn, aOut);
  if( nOut>=4 ){
    memcpy(aNum, aOut, 4);
  }else{
    memset(aNum, 0, sizeof(aNum));
    memcpy(&aNum[4-nOut], aOut, nOut);
  }
  sqlite3_free(aOut);
  if( bLittle ){
    val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
  }else{
    val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
  }
  Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
  return TCL_OK;
}


/*
** USAGE:   hexio_render_int16   INTEGER

Changes to src/tokenize.c.

541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
...
777
778
779
780
781
782
783







































































































































    }
  }
  while( IdChar(z[i]) ){ i++; }
  *tokenType = TK_ID;
  return i;
}

#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Return the length (in bytes) of the token that begins at z[0].
** Store the token type in *tokenType before returning.  If flags has
** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
** for keywords.  Add SQLITE_TOKEN_QUOTED to flags if the token was
** actually a quoted identifier.  Add SQLITE_TOKEN_KEYWORD to flags
** if the token was recognized as a keyword; this is useful when the
** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
** to differentiate between a keyword being treated as an identifier
** (for normalization purposes) and an actual identifier.
*/
int sqlite3GetTokenNormalized(
  const unsigned char *z,
  int *tokenType,
  int *flags
){
  int n;
  unsigned char iClass = aiClass[*z];
  if( iClass==CC_KYWD ){
    int i;
    for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
    if( IdChar(z[i]) ){
      /* This token started out using characters that can appear in keywords,
      ** but z[i] is a character not allowed within keywords, so this must
      ** be an identifier instead */
      i++;
      while( IdChar(z[i]) ){ i++; }
      *tokenType = TK_ID;
      return i;
    }
    *tokenType = TK_ID;
    n = keywordCode((char*)z, i, tokenType);
    /* If the token is no longer considered to be an identifier, then it is a
    ** keyword of some kind.  Make the token back into an identifier and then
    ** set the SQLITE_TOKEN_KEYWORD flag.  Several non-identifier tokens are
    ** used verbatim, including IN, IS, NOT, and NULL. */
    switch( *tokenType ){
      case TK_ID: {
        /* do nothing, handled by caller */
        break;
      }
      case TK_IN:
      case TK_IS:
      case TK_NOT:
      case TK_NULL: {
        *flags |= SQLITE_TOKEN_KEYWORD;
        break;
      }
      default: {
        *tokenType = TK_ID;
        *flags |= SQLITE_TOKEN_KEYWORD;
        break;
      }
    }
  }else{
    n = sqlite3GetToken(z, tokenType);
    /* If the token is considered to be an identifier and the character class
    ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
    if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
      *flags |= SQLITE_TOKEN_QUOTED;
    }
  }
  return n;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

/*
** Run the parser on the given SQL string.  The parser structure is
** passed in.  An SQLITE_ status code is returned.  If an error occurs
** then an and attempt is made to write an error message into 
** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
** error message.
*/
................................................................................
    Table *p = pParse->pZombieTab;
    pParse->pZombieTab = p->pNextZombie;
    sqlite3DeleteTable(db, p);
  }
  assert( nErr==0 || pParse->rc!=SQLITE_OK );
  return nErr;
}














































































































































<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
541
542
543
544
545
546
547



































































548
549
550
551
552
553
554
...
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
    }
  }
  while( IdChar(z[i]) ){ i++; }
  *tokenType = TK_ID;
  return i;
}




































































/*
** Run the parser on the given SQL string.  The parser structure is
** passed in.  An SQLITE_ status code is returned.  If an error occurs
** then an and attempt is made to write an error message into 
** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
** error message.
*/
................................................................................
    Table *p = pParse->pZombieTab;
    pParse->pZombieTab = p->pNextZombie;
    sqlite3DeleteTable(db, p);
  }
  assert( nErr==0 || pParse->rc!=SQLITE_OK );
  return nErr;
}


#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Insert a single space character into pStr if the current string
** ends with an identifier
*/
static void addSpaceSeparator(sqlite3_str *pStr){
  if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){
    sqlite3_str_append(pStr, " ", 1);
  }
}

/*
** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
** the normalization in space obtained from sqlite3DbMalloc().  Or return
** NULL if anything goes wrong or if zSql is NULL.
*/
char *sqlite3Normalize(
  Vdbe *pVdbe,       /* VM being reprepared */
  const char *zSql   /* The original SQL string */
){
  sqlite3 *db;       /* The database connection */
  int i;             /* Next unread byte of zSql[] */
  int n;             /* length of current token */
  int tokenType;     /* type of current token */
  int prevType;      /* Previous non-whitespace token */
  int nParen;        /* Number of nested levels of parentheses */
  int iStartIN;      /* Start of RHS of IN operator in z[] */
  int nParenAtIN;    /* Value of nParent at start of RHS of IN operator */
  int j;             /* Bytes of normalized SQL generated so far */
  sqlite3_str *pStr; /* The normalized SQL string under construction */

  db = sqlite3VdbeDb(pVdbe);
  tokenType = -1;
  nParen = iStartIN = nParenAtIN = 0;
  pStr = sqlite3_str_new(db);
  assert( pStr!=0 );  /* sqlite3_str_new() never returns NULL */
  for(i=0; zSql[i] && pStr->accError==0; i+=n){
    if( tokenType!=TK_SPACE ){
      prevType = tokenType;
    }
    n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
    if( NEVER(n<=0) ) break;
    switch( tokenType ){
      case TK_SPACE: {
        break;
      }
      case TK_NULL: {
        if( prevType==TK_IS || prevType==TK_NOT ){
          sqlite3_str_append(pStr, " NULL", 5);
          break;
        }
        /* Fall through */
      }
      case TK_STRING:
      case TK_INTEGER:
      case TK_FLOAT:
      case TK_VARIABLE:
      case TK_BLOB: {
        sqlite3_str_append(pStr, "?", 1);
        break;
      }
      case TK_LP: {
        nParen++;
        if( prevType==TK_IN ){
          iStartIN = pStr->nChar;
          nParenAtIN = nParen;
        }
        sqlite3_str_append(pStr, "(", 1);
        break;
      }
      case TK_RP: {
        if( iStartIN>0 && nParen==nParenAtIN ){
          assert( pStr->nChar>=iStartIN );
          pStr->nChar = iStartIN+1;
          sqlite3_str_append(pStr, "?,?,?", 5);
          iStartIN = 0;
        }
        nParen--;
        sqlite3_str_append(pStr, ")", 1);
        break;
      }
      case TK_ID: {
        iStartIN = 0;
        j = pStr->nChar;
        if( sqlite3Isquote(zSql[i]) ){
          char *zId = sqlite3DbStrNDup(db, zSql+i, n);
          int nId;
          int eType = 0;
          if( zId==0 ) break;
          sqlite3Dequote(zId);
          if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){
            sqlite3_str_append(pStr, "?", 1);
            sqlite3DbFree(db, zId);
            break;
          }
          nId = sqlite3Strlen30(zId);
          if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){
            addSpaceSeparator(pStr);
            sqlite3_str_append(pStr, zId, nId);
          }else{
            sqlite3_str_appendf(pStr, "\"%w\"", zId);
          }
          sqlite3DbFree(db, zId);
        }else{
          addSpaceSeparator(pStr);
          sqlite3_str_append(pStr, zSql+i, n);
        }
        while( j<pStr->nChar ){
          pStr->zText[j] = sqlite3Tolower(pStr->zText[j]);
          j++;
        }
        break;
      }
      case TK_SELECT: {
        iStartIN = 0;
        /* fall through */
      }
      default: {
        if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr);
        j = pStr->nChar;
        sqlite3_str_append(pStr, zSql+i, n);
        while( j<pStr->nChar ){
          pStr->zText[j] = sqlite3Toupper(pStr->zText[j]);
          j++;
        }
        break;
      }
    }
  }
  if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1);
  return sqlite3_str_finish(pStr);
}
#endif /* SQLITE_ENABLE_NORMALIZE */

Changes to src/vacuum.c.

98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125





126
127
128


129
130
131
132
133
134
135





136
137
138
139
140
141
142
143
144
145
146
147
148

149
150
151
152
153
154
155
156
157









158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
...
185
186
187
188
189
190
191
192
193
194
195
196
197







198
199
200
201
202
203
204
205
206
207
208
209
210
211
...
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
...
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333

334

335
336
337
338

339

340
341
342
343

344

345
346
347
348
349
350
351
** the copy of step (3) were replaced by deleting the original database
** and renaming the transient database as the original.  But that will
** not work if other processes are attached to the original database.
** And a power loss in between deleting the original and renaming the
** transient would cause the database file to appear to be deleted
** following reboot.
*/
void sqlite3Vacuum(Parse *pParse, Token *pNm){
  Vdbe *v = sqlite3GetVdbe(pParse);
  int iDb = 0;
  if( v==0 ) return;
  if( pNm ){
#ifndef SQLITE_BUG_COMPATIBLE_20160819
    /* Default behavior:  Report an error if the argument to VACUUM is
    ** not recognized */
    iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm);
    if( iDb<0 ) return;
#else
    /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
    ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
    ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
    ** The buggy behavior is required for binary compatibility with some
    ** legacy applications. */
    iDb = sqlite3FindDb(pParse->db, pNm);
    if( iDb<0 ) iDb = 0;
#endif
  }
  if( iDb!=1 ){





    sqlite3VdbeAddOp1(v, OP_Vacuum, iDb);
    sqlite3VdbeUsesBtree(v, iDb);
  }


  return;
}

/*
** This routine implements the OP_Vacuum opcode of the VDBE.
*/
int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){





  int rc = SQLITE_OK;     /* Return code from service routines */
  Btree *pMain;           /* The database being vacuumed */
  Btree *pTemp;           /* The temporary database we vacuum into */
  u16 saved_mDbFlags;     /* Saved value of db->mDbFlags */
  u32 saved_flags;        /* Saved value of db->flags */
  int saved_nChange;      /* Saved value of db->nChange */
  int saved_nTotalChange; /* Saved value of db->nTotalChange */
  u8 saved_mTrace;        /* Saved trace settings */
  Db *pDb = 0;            /* Database to detach at end of vacuum */
  int isMemDb;            /* True if vacuuming a :memory: database */
  int nRes;               /* Bytes of reserved space at the end of each page */
  int nDb;                /* Number of attached databases */
  const char *zDbMain;    /* Schema name of database to vacuum */


  if( !db->autoCommit ){
    sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
    return SQLITE_ERROR;
  }
  if( db->nVdbeActive>1 ){
    sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress");
    return SQLITE_ERROR;
  }










  /* Save the current value of the database flags so that it can be 
  ** restored before returning. Then set the writable-schema flag, and
  ** disable CHECK and foreign key constraints.  */
  saved_flags = db->flags;
  saved_mDbFlags = db->mDbFlags;
  saved_nChange = db->nChange;
  saved_nTotalChange = db->nTotalChange;
  saved_mTrace = db->mTrace;
  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
  db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder
                   | SQLITE_Defensive | SQLITE_CountRows);
  db->mTrace = 0;

  zDbMain = db->aDb[iDb].zDbSName;
  pMain = db->aDb[iDb].pBt;
  isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));

................................................................................
  ** that actually made the VACUUM run slower.  Very little journalling
  ** actually occurs when doing a vacuum since the vacuum_db is initially
  ** empty.  Only the journal header is written.  Apparently it takes more
  ** time to parse and run the PRAGMA to turn journalling off than it does
  ** to write the journal header file.
  */
  nDb = db->nDb;
  rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db");
  if( rc!=SQLITE_OK ) goto end_of_vacuum;
  assert( (db->nDb-1)==nDb );
  pDb = &db->aDb[nDb];
  assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
  pTemp = pDb->pBt;








  /* The call to execSql() to attach the temp database has left the file
  ** locked (as there was more than one active statement when the transaction
  ** to read the schema was concluded. Unlock it here so that this doesn't
  ** cause problems for the call to BtreeSetPageSize() below.  */
  sqlite3BtreeCommit(pTemp);

  nRes = sqlite3BtreeGetOptimalReserve(pMain);

  /* A VACUUM cannot change the pagesize of an encrypted database. */
#ifdef SQLITE_HAS_CODEC
  if( db->nextPagesize ){
    extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*);
    int nKey;
................................................................................

  /* Begin a transaction and take an exclusive lock on the main database
  ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
  ** to ensure that we do not try to change the page-size on a WAL database.
  */
  rc = execSql(db, pzErrMsg, "BEGIN");
  if( rc!=SQLITE_OK ) goto end_of_vacuum;
  rc = sqlite3BtreeBeginTrans(pMain, 2, 0);
  if( rc!=SQLITE_OK ) goto end_of_vacuum;

  /* Do not attempt to change the page size for a WAL database */
  if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain))
                                               ==PAGER_JOURNALMODE_WAL ){
    db->nextPagesize = 0;
  }
................................................................................
       BTREE_DEFAULT_CACHE_SIZE, 0,  /* Preserve the default page cache size */
       BTREE_TEXT_ENCODING,      0,  /* Preserve the text encoding */
       BTREE_USER_VERSION,       0,  /* Preserve the user version */
       BTREE_APPLICATION_ID,     0,  /* Preserve the application id */
    };

    assert( 1==sqlite3BtreeIsInTrans(pTemp) );
    assert( 1==sqlite3BtreeIsInTrans(pMain) );

    /* Copy Btree meta values */
    for(i=0; i<ArraySize(aCopy); i+=2){
      /* GetMeta() and UpdateMeta() cannot fail in this context because
      ** we already have page 1 loaded into cache and marked dirty. */
      sqlite3BtreeGetMeta(pMain, aCopy[i], &meta);
      rc = sqlite3BtreeUpdateMeta(pTemp, aCopy[i], meta+aCopy[i+1]);
      if( NEVER(rc!=SQLITE_OK) ) goto end_of_vacuum;
    }


    rc = sqlite3BtreeCopyFile(pMain, pTemp);

    if( rc!=SQLITE_OK ) goto end_of_vacuum;
    rc = sqlite3BtreeCommit(pTemp);
    if( rc!=SQLITE_OK ) goto end_of_vacuum;
#ifndef SQLITE_OMIT_AUTOVACUUM

    sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));

#endif
  }

  assert( rc==SQLITE_OK );

  rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);


end_of_vacuum:
  /* Restore the original value of db->flags */
  db->init.iDb = 0;
  db->mDbFlags = saved_mDbFlags;
  db->flags = saved_flags;
  db->nChange = saved_nChange;







|


|





|











>
>
>
>
>
|


>
>






|
>
>
>
>
>



|
|








>









>
>
>
>
>
>
>
>
>











|







 







|





>
>
>
>
>
>
>
|
<
<
<
<
<
|







 







|







 







|










>
|
>




>
|
>




>
|
>







98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
...
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227





228
229
230
231
232
233
234
235
...
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
...
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
** the copy of step (3) were replaced by deleting the original database
** and renaming the transient database as the original.  But that will
** not work if other processes are attached to the original database.
** And a power loss in between deleting the original and renaming the
** transient would cause the database file to appear to be deleted
** following reboot.
*/
void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){
  Vdbe *v = sqlite3GetVdbe(pParse);
  int iDb = 0;
  if( v==0 ) goto build_vacuum_end;
  if( pNm ){
#ifndef SQLITE_BUG_COMPATIBLE_20160819
    /* Default behavior:  Report an error if the argument to VACUUM is
    ** not recognized */
    iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm);
    if( iDb<0 ) goto build_vacuum_end;
#else
    /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
    ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
    ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
    ** The buggy behavior is required for binary compatibility with some
    ** legacy applications. */
    iDb = sqlite3FindDb(pParse->db, pNm);
    if( iDb<0 ) iDb = 0;
#endif
  }
  if( iDb!=1 ){
    int iIntoReg = 0;
    if( pInto ){
      iIntoReg = ++pParse->nMem;
      sqlite3ExprCode(pParse, pInto, iIntoReg);
    }
    sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg);
    sqlite3VdbeUsesBtree(v, iDb);
  }
build_vacuum_end:
  sqlite3ExprDelete(pParse->db, pInto);
  return;
}

/*
** This routine implements the OP_Vacuum opcode of the VDBE.
*/
int sqlite3RunVacuum(
  char **pzErrMsg,        /* Write error message here */
  sqlite3 *db,            /* Database connection */
  int iDb,                /* Which attached DB to vacuum */
  sqlite3_value *pOut     /* Write results here, if not NULL */
){
  int rc = SQLITE_OK;     /* Return code from service routines */
  Btree *pMain;           /* The database being vacuumed */
  Btree *pTemp;           /* The temporary database we vacuum into */
  u32 saved_mDbFlags;     /* Saved value of db->mDbFlags */
  u64 saved_flags;        /* Saved value of db->flags */
  int saved_nChange;      /* Saved value of db->nChange */
  int saved_nTotalChange; /* Saved value of db->nTotalChange */
  u8 saved_mTrace;        /* Saved trace settings */
  Db *pDb = 0;            /* Database to detach at end of vacuum */
  int isMemDb;            /* True if vacuuming a :memory: database */
  int nRes;               /* Bytes of reserved space at the end of each page */
  int nDb;                /* Number of attached databases */
  const char *zDbMain;    /* Schema name of database to vacuum */
  const char *zOut;       /* Name of output file */

  if( !db->autoCommit ){
    sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
    return SQLITE_ERROR;
  }
  if( db->nVdbeActive>1 ){
    sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress");
    return SQLITE_ERROR;
  }
  if( pOut ){
    if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){
      sqlite3SetString(pzErrMsg, db, "non-text filename");
      return SQLITE_ERROR;
    }
    zOut = (const char*)sqlite3_value_text(pOut);
  }else{
    zOut = "";
  }

  /* Save the current value of the database flags so that it can be 
  ** restored before returning. Then set the writable-schema flag, and
  ** disable CHECK and foreign key constraints.  */
  saved_flags = db->flags;
  saved_mDbFlags = db->mDbFlags;
  saved_nChange = db->nChange;
  saved_nTotalChange = db->nTotalChange;
  saved_mTrace = db->mTrace;
  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
  db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
  db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder
                   | SQLITE_Defensive | SQLITE_CountRows);
  db->mTrace = 0;

  zDbMain = db->aDb[iDb].zDbSName;
  pMain = db->aDb[iDb].pBt;
  isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));

................................................................................
  ** that actually made the VACUUM run slower.  Very little journalling
  ** actually occurs when doing a vacuum since the vacuum_db is initially
  ** empty.  Only the journal header is written.  Apparently it takes more
  ** time to parse and run the PRAGMA to turn journalling off than it does
  ** to write the journal header file.
  */
  nDb = db->nDb;
  rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut);
  if( rc!=SQLITE_OK ) goto end_of_vacuum;
  assert( (db->nDb-1)==nDb );
  pDb = &db->aDb[nDb];
  assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
  pTemp = pDb->pBt;
  if( pOut ){
    sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp));
    i64 sz = 0;
    if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){
      rc = SQLITE_ERROR;
      sqlite3SetString(pzErrMsg, db, "output file already exists");
      goto end_of_vacuum;
    }





  }
  nRes = sqlite3BtreeGetOptimalReserve(pMain);

  /* A VACUUM cannot change the pagesize of an encrypted database. */
#ifdef SQLITE_HAS_CODEC
  if( db->nextPagesize ){
    extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*);
    int nKey;
................................................................................

  /* Begin a transaction and take an exclusive lock on the main database
  ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
  ** to ensure that we do not try to change the page-size on a WAL database.
  */
  rc = execSql(db, pzErrMsg, "BEGIN");
  if( rc!=SQLITE_OK ) goto end_of_vacuum;
  rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0);
  if( rc!=SQLITE_OK ) goto end_of_vacuum;

  /* Do not attempt to change the page size for a WAL database */
  if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain))
                                               ==PAGER_JOURNALMODE_WAL ){
    db->nextPagesize = 0;
  }
................................................................................
       BTREE_DEFAULT_CACHE_SIZE, 0,  /* Preserve the default page cache size */
       BTREE_TEXT_ENCODING,      0,  /* Preserve the text encoding */
       BTREE_USER_VERSION,       0,  /* Preserve the user version */
       BTREE_APPLICATION_ID,     0,  /* Preserve the application id */
    };

    assert( 1==sqlite3BtreeIsInTrans(pTemp) );
    assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) );

    /* Copy Btree meta values */
    for(i=0; i<ArraySize(aCopy); i+=2){
      /* GetMeta() and UpdateMeta() cannot fail in this context because
      ** we already have page 1 loaded into cache and marked dirty. */
      sqlite3BtreeGetMeta(pMain, aCopy[i], &meta);
      rc = sqlite3BtreeUpdateMeta(pTemp, aCopy[i], meta+aCopy[i+1]);
      if( NEVER(rc!=SQLITE_OK) ) goto end_of_vacuum;
    }

    if( pOut==0 ){
      rc = sqlite3BtreeCopyFile(pMain, pTemp);
    }
    if( rc!=SQLITE_OK ) goto end_of_vacuum;
    rc = sqlite3BtreeCommit(pTemp);
    if( rc!=SQLITE_OK ) goto end_of_vacuum;
#ifndef SQLITE_OMIT_AUTOVACUUM
    if( pOut==0 ){
      sqlite3BtreeSetAutoVacuum(pMain, sqlite3BtreeGetAutoVacuum(pTemp));
    }
#endif
  }

  assert( rc==SQLITE_OK );
  if( pOut==0 ){
    rc = sqlite3BtreeSetPageSize(pMain, sqlite3BtreeGetPageSize(pTemp), nRes,1);
  }

end_of_vacuum:
  /* Restore the original value of db->flags */
  db->init.iDb = 0;
  db->mDbFlags = saved_mDbFlags;
  db->flags = saved_flags;
  db->nChange = saved_nChange;

Changes to src/vdbe.c.

6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733




6734
6735
6736
6737

6738
6739
6740
6741
6742
6743
6744
....
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
  sqlite3VdbeChangeEncoding(pOut, encoding);
  if( rc ) goto abort_due_to_error;
  break;
};
#endif /* SQLITE_OMIT_PRAGMA */

#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
/* Opcode: Vacuum P1 * * * *
**
** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
** for an attached database.  The "temp" database may not be vacuumed.




*/
case OP_Vacuum: {
  assert( p->readOnly==0 );
  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1);

  if( rc ) goto abort_due_to_error;
  break;
}
#endif

#if !defined(SQLITE_OMIT_AUTOVACUUM)
/* Opcode: IncrVacuum P1 P2 * * *
................................................................................
  assert( pName->flags & MEM_Str );
  testcase( pName->enc==SQLITE_UTF8 );
  testcase( pName->enc==SQLITE_UTF16BE );
  testcase( pName->enc==SQLITE_UTF16LE );
  rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
  if( rc ) goto abort_due_to_error;
  rc = pVtab->pModule->xRename(pVtab, pName->z);
  if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter;
  sqlite3VtabImportErrmsg(p, pVtab);
  p->expired = 0;
  if( rc ) goto abort_due_to_error;
  break;
}
#endif








|



>
>
>
>



|
>







 







|







6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
....
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
  sqlite3VdbeChangeEncoding(pOut, encoding);
  if( rc ) goto abort_due_to_error;
  break;
};
#endif /* SQLITE_OMIT_PRAGMA */

#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
/* Opcode: Vacuum P1 P2 * * *
**
** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
** for an attached database.  The "temp" database may not be vacuumed.
**
** If P2 is not zero, then it is a register holding a string which is
** the file into which the result of vacuum should be written.  When
** P2 is zero, the vacuum overwrites the original database.
*/
case OP_Vacuum: {
  assert( p->readOnly==0 );
  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1,
                        pOp->p2 ? &aMem[pOp->p2] : 0);
  if( rc ) goto abort_due_to_error;
  break;
}
#endif

#if !defined(SQLITE_OMIT_AUTOVACUUM)
/* Opcode: IncrVacuum P1 P2 * * *
................................................................................
  assert( pName->flags & MEM_Str );
  testcase( pName->enc==SQLITE_UTF8 );
  testcase( pName->enc==SQLITE_UTF16BE );
  testcase( pName->enc==SQLITE_UTF16LE );
  rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
  if( rc ) goto abort_due_to_error;
  rc = pVtab->pModule->xRename(pVtab, pName->z);
  if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter;
  sqlite3VtabImportErrmsg(p, pVtab);
  p->expired = 0;
  if( rc ) goto abort_due_to_error;
  break;
}
#endif

Changes to src/vdbe.h.

247
248
249
250
251
252
253




254
255
256
257
258
259
260
int sqlite3VdbeReset(Vdbe*);
void sqlite3VdbeSetNumCols(Vdbe*,int);
int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*));
void sqlite3VdbeCountChanges(Vdbe*);
sqlite3 *sqlite3VdbeDb(Vdbe*);
u8 sqlite3VdbePrepareFlags(Vdbe*);
void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8);




void sqlite3VdbeSwap(Vdbe*,Vdbe*);
VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*);
sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8);
void sqlite3VdbeSetVarmask(Vdbe*, int);
#ifndef SQLITE_OMIT_TRACE
  char *sqlite3VdbeExpandSql(Vdbe*, const char*);
#endif







>
>
>
>







247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
int sqlite3VdbeReset(Vdbe*);
void sqlite3VdbeSetNumCols(Vdbe*,int);
int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*));
void sqlite3VdbeCountChanges(Vdbe*);
sqlite3 *sqlite3VdbeDb(Vdbe*);
u8 sqlite3VdbePrepareFlags(Vdbe*);
void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8);
#ifdef SQLITE_ENABLE_NORMALIZE
void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*);
int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*);
#endif
void sqlite3VdbeSwap(Vdbe*,Vdbe*);
VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*);
sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8);
void sqlite3VdbeSetVarmask(Vdbe*, int);
#ifndef SQLITE_OMIT_TRACE
  char *sqlite3VdbeExpandSql(Vdbe*, const char*);
#endif

Changes to src/vdbeInt.h.

331
332
333
334
335
336
337



338
339
340
341
342
343
344
345
346













347
348
349
350
351
352
353
...
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
...
404
405
406
407
408
409
410

411
412
413
414
415
416
417
};

/* A bitfield type for use inside of structures.  Always follow with :N where
** N is the number of bits.
*/
typedef unsigned bft;  /* Bit Field Type */




typedef struct ScanStatus ScanStatus;
struct ScanStatus {
  int addrExplain;                /* OP_Explain for loop */
  int addrLoop;                   /* Address of "loops" counter */
  int addrVisit;                  /* Address of "rows visited" counter */
  int iSelectID;                  /* The "Select-ID" for this loop */
  LogEst nEst;                    /* Estimated output rows per loop */
  char *zName;                    /* Name of table or index */
};














/*
** An instance of the virtual machine.  This structure contains the complete
** state of the virtual machine.
**
** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare()
** is really a pointer to an instance of this structure.
................................................................................
  u32 magic;              /* Magic number for sanity checking */
  int nMem;               /* Number of memory locations currently allocated */
  int nCursor;            /* Number of slots in apCsr[] */
  u32 cacheCtr;           /* VdbeCursor row cache generation counter */
  int pc;                 /* The program counter */
  int rc;                 /* Value to return */
  int nChange;            /* Number of db changes made since last reset */
  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
  i64 iCurrentTime;       /* Value of julianday('now') for this statement */
  i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
  i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */

  /* When allocating a new Vdbe object, all of the fields below should be
  ** initialized to zero or NULL */
................................................................................
  bft bIsReader:1;        /* True for statements that read */
  yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
  yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
  u32 aCounter[7];        /* Counters used by sqlite3_stmt_status() */
  char *zSql;             /* Text of the SQL statement that generated this */
#ifdef SQLITE_ENABLE_NORMALIZE
  char *zNormSql;         /* Normalization of the associated SQL statement */

#endif
  void *pFree;            /* Free this when deleting the vdbe */
  VdbeFrame *pFrame;      /* Parent frame */
  VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
  int nFrame;             /* Number of frames in pFrame list */
  u32 expmask;            /* Binding to these vars invalidates VM */
  SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */







>
>
>









>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







 







>







331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
...
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
...
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
};

/* A bitfield type for use inside of structures.  Always follow with :N where
** N is the number of bits.
*/
typedef unsigned bft;  /* Bit Field Type */

/* The ScanStatus object holds a single value for the
** sqlite3_stmt_scanstatus() interface.
*/
typedef struct ScanStatus ScanStatus;
struct ScanStatus {
  int addrExplain;                /* OP_Explain for loop */
  int addrLoop;                   /* Address of "loops" counter */
  int addrVisit;                  /* Address of "rows visited" counter */
  int iSelectID;                  /* The "Select-ID" for this loop */
  LogEst nEst;                    /* Estimated output rows per loop */
  char *zName;                    /* Name of table or index */
};

/* The DblquoteStr object holds the text of a double-quoted
** string for a prepared statement.  A linked list of these objects
** is constructed during statement parsing and is held on Vdbe.pDblStr.
** When computing a normalized SQL statement for an SQL statement, that
** list is consulted for each double-quoted identifier to see if the
** identifier should really be a string literal.
*/
typedef struct DblquoteStr DblquoteStr;
struct DblquoteStr {
  DblquoteStr *pNextStr;   /* Next string literal in the list */
  char z[8];               /* Dequoted value for the string */
};

/*
** An instance of the virtual machine.  This structure contains the complete
** state of the virtual machine.
**
** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare()
** is really a pointer to an instance of this structure.
................................................................................
  u32 magic;              /* Magic number for sanity checking */
  int nMem;               /* Number of memory locations currently allocated */
  int nCursor;            /* Number of slots in apCsr[] */
  u32 cacheCtr;           /* VdbeCursor row cache generation counter */
  int pc;                 /* The program counter */
  int rc;                 /* Value to return */
  int nChange;            /* Number of db changes made since last reset */
  int iStatement;         /* Statement number (or 0 if has no opened stmt) */
  i64 iCurrentTime;       /* Value of julianday('now') for this statement */
  i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
  i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */

  /* When allocating a new Vdbe object, all of the fields below should be
  ** initialized to zero or NULL */
................................................................................
  bft bIsReader:1;        /* True for statements that read */
  yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
  yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
  u32 aCounter[7];        /* Counters used by sqlite3_stmt_status() */
  char *zSql;             /* Text of the SQL statement that generated this */
#ifdef SQLITE_ENABLE_NORMALIZE
  char *zNormSql;         /* Normalization of the associated SQL statement */
  DblquoteStr *pDblStr;   /* List of double-quoted string literals */
#endif
  void *pFree;            /* Free this when deleting the vdbe */
  VdbeFrame *pFrame;      /* Parent frame */
  VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
  int nFrame;             /* Number of frames in pFrame list */
  u32 expmask;            /* Binding to these vars invalidates VM */
  SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */

Changes to src/vdbeapi.c.

58
59
60
61
62
63
64
65
66
67
68
69

70
71
72

73
74
75
76
77
78
79
...
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
...
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
...
625
626
627
628
629
630
631

632
633
634
635
636
637
638
639
640
641
642
643

644
645
646
647
648
649
650
...
654
655
656
657
658
659
660
661
662
663

664
665
666
667
668
669
670
....
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
....
1704
1705
1706
1707
1708
1709
1710






1711
1712
1713
1714
1715
1716
1717
1718
** Invoke the profile callback.  This routine is only called if we already
** know that the profile callback is defined and needs to be invoked.
*/
static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
  sqlite3_int64 iNow;
  sqlite3_int64 iElapse;
  assert( p->startTime>0 );
  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
  assert( db->init.busy==0 );
  assert( p->zSql!=0 );
  sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
  iElapse = (iNow - p->startTime)*1000000;

  if( db->xProfile ){
    db->xProfile(db->pProfileArg, p->zSql, iElapse);
  }

  if( db->mTrace & SQLITE_TRACE_PROFILE ){
    db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
  }
  p->startTime = 0;
}
/*
** The checkProfileCallback(DB,P) macro checks to see if a profile callback
................................................................................
  /* Check that malloc() has not failed. If it has, return early. */
  db = p->db;
  if( db->mallocFailed ){
    p->rc = SQLITE_NOMEM;
    return SQLITE_NOMEM_BKPT;
  }

  if( p->pc<=0 && p->expired ){
    p->rc = SQLITE_SCHEMA;
    rc = SQLITE_ERROR;
    goto end_of_step;
  }
  if( p->pc<0 ){
    /* If there are no other statements currently running, then
    ** reset the interrupt flag.  This prevents a call to sqlite3_interrupt
................................................................................
    }

    assert( db->nVdbeWrite>0 || db->autoCommit==0 
        || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
    );

#ifndef SQLITE_OMIT_TRACE
    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
        && !db->init.busy && p->zSql ){
      sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
    }else{
      assert( p->startTime==0 );
    }
#endif

................................................................................
#endif /* SQLITE_OMIT_EXPLAIN */
  {
    db->nVdbeExec++;
    rc = sqlite3VdbeExec(p);
    db->nVdbeExec--;
  }


#ifndef SQLITE_OMIT_TRACE
  /* If the statement completed successfully, invoke the profile callback */
  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
#endif

  if( rc==SQLITE_DONE && db->autoCommit ){
    assert( p->rc==SQLITE_OK );
    p->rc = doWalCallbacks(db);
    if( p->rc!=SQLITE_OK ){
      rc = SQLITE_ERROR;
    }
  }


  db->errCode = rc;
  if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
    p->rc = SQLITE_NOMEM_BKPT;
  }
end_of_step:
  /* At this point local variable rc holds the value that should be 
................................................................................
  ** contains the value that would be returned if sqlite3_finalize() 
  ** were called on statement p.
  */
  assert( rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR 
       || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE
  );
  assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp );
  if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 
   && rc!=SQLITE_ROW 
   && rc!=SQLITE_DONE 

  ){
    /* If this statement was prepared using saved SQL and an 
    ** error has occurred, then return the error code in p->rc to the
    ** caller. Set the error code in the database handle to the same value.
    */ 
    rc = sqlite3VdbeTransferError(p);
  }
................................................................................
    sqlite3_mutex_leave(p->db->mutex);
    return SQLITE_RANGE;
  }
  i--;
  pVar = &p->aVar[i];
  sqlite3VdbeMemRelease(pVar);
  pVar->flags = MEM_Null;
  sqlite3Error(p->db, SQLITE_OK);

  /* If the bit corresponding to this variable in Vdbe.expmask is set, then 
  ** binding a new value to this variable invalidates the current query plan.
  **
  ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host
  ** parameter in the WHERE clause might influence the choice of query plan
  ** for a statement, then the statement will be automatically recompiled,
................................................................................

#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Return the normalized SQL associated with a prepared statement.
*/
const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
  Vdbe *p = (Vdbe *)pStmt;






  return p ? p->zNormSql : 0;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
/*
** Allocate and populate an UnpackedRecord structure based on the serialized
** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure







|




>



>







 







|







 







|







 







>

|
|









>







 







<
|
|
>







 







|







 







>
>
>
>
>
>
|







58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
...
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
...
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
...
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
...
658
659
660
661
662
663
664

665
666
667
668
669
670
671
672
673
674
....
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
....
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
** Invoke the profile callback.  This routine is only called if we already
** know that the profile callback is defined and needs to be invoked.
*/
static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
  sqlite3_int64 iNow;
  sqlite3_int64 iElapse;
  assert( p->startTime>0 );
  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
  assert( db->init.busy==0 );
  assert( p->zSql!=0 );
  sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
  iElapse = (iNow - p->startTime)*1000000;
#ifndef SQLITE_OMIT_DEPRECATED  	
  if( db->xProfile ){
    db->xProfile(db->pProfileArg, p->zSql, iElapse);
  }
#endif
  if( db->mTrace & SQLITE_TRACE_PROFILE ){
    db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
  }
  p->startTime = 0;
}
/*
** The checkProfileCallback(DB,P) macro checks to see if a profile callback
................................................................................
  /* Check that malloc() has not failed. If it has, return early. */
  db = p->db;
  if( db->mallocFailed ){
    p->rc = SQLITE_NOMEM;
    return SQLITE_NOMEM_BKPT;
  }

  if( p->pc<0 && p->expired ){
    p->rc = SQLITE_SCHEMA;
    rc = SQLITE_ERROR;
    goto end_of_step;
  }
  if( p->pc<0 ){
    /* If there are no other statements currently running, then
    ** reset the interrupt flag.  This prevents a call to sqlite3_interrupt
................................................................................
    }

    assert( db->nVdbeWrite>0 || db->autoCommit==0 
        || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
    );

#ifndef SQLITE_OMIT_TRACE
    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
        && !db->init.busy && p->zSql ){
      sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
    }else{
      assert( p->startTime==0 );
    }
#endif

................................................................................
#endif /* SQLITE_OMIT_EXPLAIN */
  {
    db->nVdbeExec++;
    rc = sqlite3VdbeExec(p);
    db->nVdbeExec--;
  }

  if( rc!=SQLITE_ROW ){
#ifndef SQLITE_OMIT_TRACE
    /* If the statement completed successfully, invoke the profile callback */
    checkProfileCallback(db, p);
#endif

    if( rc==SQLITE_DONE && db->autoCommit ){
      assert( p->rc==SQLITE_OK );
      p->rc = doWalCallbacks(db);
      if( p->rc!=SQLITE_OK ){
        rc = SQLITE_ERROR;
      }
    }
  }

  db->errCode = rc;
  if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
    p->rc = SQLITE_NOMEM_BKPT;
  }
end_of_step:
  /* At this point local variable rc holds the value that should be 
................................................................................
  ** contains the value that would be returned if sqlite3_finalize() 
  ** were called on statement p.
  */
  assert( rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR 
       || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE
  );
  assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp );

  if( rc!=SQLITE_ROW 
   && rc!=SQLITE_DONE
   && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0
  ){
    /* If this statement was prepared using saved SQL and an 
    ** error has occurred, then return the error code in p->rc to the
    ** caller. Set the error code in the database handle to the same value.
    */ 
    rc = sqlite3VdbeTransferError(p);
  }
................................................................................
    sqlite3_mutex_leave(p->db->mutex);
    return SQLITE_RANGE;
  }
  i--;
  pVar = &p->aVar[i];
  sqlite3VdbeMemRelease(pVar);
  pVar->flags = MEM_Null;
  p->db->errCode = SQLITE_OK;

  /* If the bit corresponding to this variable in Vdbe.expmask is set, then 
  ** binding a new value to this variable invalidates the current query plan.
  **
  ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host
  ** parameter in the WHERE clause might influence the choice of query plan
  ** for a statement, then the statement will be automatically recompiled,
................................................................................

#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Return the normalized SQL associated with a prepared statement.
*/
const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
  Vdbe *p = (Vdbe *)pStmt;
  if( p==0 ) return 0;
  if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){
    sqlite3_mutex_enter(p->db->mutex);
    p->zNormSql = sqlite3Normalize(p, p->zSql);
    sqlite3_mutex_leave(p->db->mutex);
  }
  return p->zNormSql;
}
#endif /* SQLITE_ENABLE_NORMALIZE */

#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
/*
** Allocate and populate an UnpackedRecord structure based on the serialized
** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure

Changes to src/vdbeaux.c.

60
61
62
63
64
65
66


67
68
69
70

71










72


73
74


















75
76
77
78
79
80
81
..
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
....
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
....
3186
3187
3188
3189
3190
3191
3192







3193
3194
3195
3196
3197
3198
3199
  if( p==0 ) return;
  p->prepFlags = prepFlags;
  if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
    p->expmask = 0;
  }
  assert( p->zSql==0 );
  p->zSql = sqlite3DbStrNDup(p->db, z, n);


#ifdef SQLITE_ENABLE_NORMALIZE
  assert( p->zNormSql==0 );
  if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
    sqlite3Normalize(p, p->zSql, n, prepFlags);

    assert( p->zNormSql!=0 || p->db->mallocFailed );










  }


#endif
}



















/*
** Swap all content between two VDBE structures.
*/
void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
  Vdbe tmp, *pTmp;
  char *zTmp;
................................................................................
  pB->pNext = pTmp;
  pTmp = pA->pPrev;
  pA->pPrev = pB->pPrev;
  pB->pPrev = pTmp;
  zTmp = pA->zSql;
  pA->zSql = pB->zSql;
  pB->zSql = zTmp;
#ifdef SQLITE_ENABLE_NORMALIZE
  zTmp = pA->zNormSql;
  pA->zNormSql = pB->zNormSql;
  pB->zNormSql = zTmp;
#endif
  pB->expmask = pA->expmask;
  pB->prepFlags = pA->prepFlags;
  memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));
................................................................................
        }else if( rc!=SQLITE_OK ){
          p->rc = rc;
          sqlite3RollbackAll(db, SQLITE_OK);
          p->nChange = 0;
        }else{
          db->nDeferredCons = 0;
          db->nDeferredImmCons = 0;
          db->flags &= ~SQLITE_DeferFKs;
          sqlite3CommitInternalChanges(db);
        }
      }else{
        sqlite3RollbackAll(db, SQLITE_OK);
        p->nChange = 0;
      }
      db->nStatement = 0;
................................................................................
    sqlite3DbFree(db, p->pFree);
  }
  vdbeFreeOpArray(db, p->aOp, p->nOp);
  sqlite3DbFree(db, p->aColName);
  sqlite3DbFree(db, p->zSql);
#ifdef SQLITE_ENABLE_NORMALIZE
  sqlite3DbFree(db, p->zNormSql);







#endif
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  {
    int i;
    for(i=0; i<p->nScan; i++){
      sqlite3DbFree(db, p->aScan[i].zName);
    }







>
>

<
<
<
>
|
>
>
>
>
>
>
>
>
>
>
|
>
>

|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







 







|







 







>
>
>
>
>
>
>







60
61
62
63
64
65
66
67
68
69



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
...
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
....
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
....
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
  if( p==0 ) return;
  p->prepFlags = prepFlags;
  if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
    p->expmask = 0;
  }
  assert( p->zSql==0 );
  p->zSql = sqlite3DbStrNDup(p->db, z, n);
}

#ifdef SQLITE_ENABLE_NORMALIZE



/*
** Add a new element to the Vdbe->pDblStr list.
*/
void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){
  if( p ){
    int n = sqlite3Strlen30(z);
    DblquoteStr *pStr = sqlite3DbMallocRawNN(db,
                            sizeof(*pStr)+n+1-sizeof(pStr->z));
    if( pStr ){
      pStr->pNextStr = p->pDblStr;
      p->pDblStr = pStr;
      memcpy(pStr->z, z, n+1);
    }
  }
}
#endif

#ifdef SQLITE_ENABLE_NORMALIZE
/*
** zId of length nId is a double-quoted identifier.  Check to see if
** that identifier is really used as a string literal.
*/
int sqlite3VdbeUsesDoubleQuotedString(
  Vdbe *pVdbe,            /* The prepared statement */
  const char *zId         /* The double-quoted identifier, already dequoted */
){
  DblquoteStr *pStr;
  assert( zId!=0 );
  if( pVdbe->pDblStr==0 ) return 0;
  for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){
    if( strcmp(zId, pStr->z)==0 ) return 1;
  }
  return 0;
}
#endif

/*
** Swap all content between two VDBE structures.
*/
void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
  Vdbe tmp, *pTmp;
  char *zTmp;
................................................................................
  pB->pNext = pTmp;
  pTmp = pA->pPrev;
  pA->pPrev = pB->pPrev;
  pB->pPrev = pTmp;
  zTmp = pA->zSql;
  pA->zSql = pB->zSql;
  pB->zSql = zTmp;
#if 0
  zTmp = pA->zNormSql;
  pA->zNormSql = pB->zNormSql;
  pB->zNormSql = zTmp;
#endif
  pB->expmask = pA->expmask;
  pB->prepFlags = pA->prepFlags;
  memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));
................................................................................
        }else if( rc!=SQLITE_OK ){
          p->rc = rc;
          sqlite3RollbackAll(db, SQLITE_OK);
          p->nChange = 0;
        }else{
          db->nDeferredCons = 0;
          db->nDeferredImmCons = 0;
          db->flags &= ~(u64)SQLITE_DeferFKs;
          sqlite3CommitInternalChanges(db);
        }
      }else{
        sqlite3RollbackAll(db, SQLITE_OK);
        p->nChange = 0;
      }
      db->nStatement = 0;
................................................................................
    sqlite3DbFree(db, p->pFree);
  }
  vdbeFreeOpArray(db, p->aOp, p->nOp);
  sqlite3DbFree(db, p->aColName);
  sqlite3DbFree(db, p->zSql);
#ifdef SQLITE_ENABLE_NORMALIZE
  sqlite3DbFree(db, p->zNormSql);
  {
    DblquoteStr *pThis, *pNext;
    for(pThis=p->pDblStr; pThis; pThis=pNext){
      pNext = pThis->pNextStr;
      sqlite3DbFree(db, pThis);
    }
  }
#endif
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
  {
    int i;
    for(i=0; i<p->nScan; i++){
      sqlite3DbFree(db, p->aScan[i].zName);
    }

Changes to src/wal.c.

358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
...
469
470
471
472
473
474
475
476
477

478
479




480
481
482
483
484
485
486
487
488
489
490
491
492
....
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
....
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241

1242
1243
1244
1245
1246
1247
1248
1249
....
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
....
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
....
2247
2248
2249
2250
2251
2252
2253






























































2254
2255
2256
2257
2258
2259
2260
....
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
....
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
....
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448

2449
2450
2451
2452
2453
2454
2455
....
3076
3077
3078
3079
3080
3081
3082



3083
3084
3085

3086
3087








3088
3089

3090
3091
3092
3093
3094

3095
3096
3097
3098
3099
3100
3101
....
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
....
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
....
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130

4131
4132

4133
4134
4135
4136
4137
4138
4139
....
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
** it is held, but does not prevent a checkpointer from checkpointing 
** it.
**
** There is still a single WRITER and a single CHECKPOINTER lock. The
** recovery procedure still takes the same exclusive lock on the entire
** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks
** above use four of the six read-locking slots used by legacy wal mode.
** See the header comment for function walLockReader() for details.
**
** STARTUP/RECOVERY
**
** The read and write version fields of the database header in a wal2
** database are set to 0x03, instead of 0x02 as in legacy wal mode.
**
** The wal file format used in wal2 mode is the same as the format used
................................................................................
/*
** Values that may be stored in Wal.readLock in wal2 mode.
**
** In wal mode, the Wal.readLock member is set to -1 when no read-lock
** is held, or else is the index of the read-mark on which a lock is
** held.
**
** In wal2 mode, Wal.readLock must be set to one of the following values.
** A value of -1 still indicates that no read-lock is held, but the other

** values are symbolic. See the implementation of walLockReader() for
** details of how the symbols map to OS level locks.




*/
#define WAL_LOCK_NONE        -1
#define WAL_LOCK_PART1        1
#define WAL_LOCK_PART1_FULL2  2
#define WAL_LOCK_PART2        3
#define WAL_LOCK_PART2_FULL1  4

/* 
** This constant is used in wal2 mode only.
**
** In wal2 mode, when committing a transaction, if the current wal file 
** is sufficiently large and there are no conflicting locks held, the
** writer writes the new transaction into the start of the other wal
................................................................................
  if( pWal->exclusiveMode ) return;
  (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n,
                         SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE);
  WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal,
             walLockName(lockIdx), n));
}

/*
** This function is used to take and release read-locks in wal2 mode.
**
** Use of WAL_READ_LOCK(x) slots for (1<=x<=4).
**
** 1) Partial read of *-wal-1   (blocks checkpointer from checkpointing)
** 2) Full read of *-wal-2      (blocks writer from writing)
** 3) Partial read of *-wal-2   (blocks checkpointer from checkpointing)
** 4) Full read of *-wal-1      (blocks writer from writing)
*/
static int walLockReader(Wal *pWal, int eLock, int bLock){
  int i;                          /* Index of first readmark to lock */
  int n;                          /* Number of readmarks to lock */

  assert( pWal->hdr.iVersion==WAL_VERSION2 );
  if( pWal->exclusiveMode ) return SQLITE_OK;

  switch( eLock ){
    case WAL_LOCK_PART1      : i = 1; n = 1; break; 
    case WAL_LOCK_PART1_FULL2: i = 1; n = 2; break; 
    case WAL_LOCK_PART2      : i = 3; n = 1; break; 
    case WAL_LOCK_PART2_FULL1: i = 3; n = 2; break; 
    default: assert( !"cannot happen" );
  }

  return sqlite3OsShmLock(pWal->pDbFd, WAL_READ_LOCK(i), n,
      SQLITE_SHM_SHARED | (bLock ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) 
  );
}

/*
** Compute a hash on a page number.  The resulting hash value must land
** between 0 and (HASHTABLE_NSLOT-1).  The walHashNext() function advances
** the hash to the next value in the event of a collision.
*/
static int walHash(u32 iPage){
  assert( iPage>0 );
................................................................................

  if( 0==(iHash & 0x01) ){
    /* A frame in wal file 0 */
    *piRead = (iExternal <= HASHTABLE_NPAGE_ONE) ? iExternal :
      iExternal - (iHash/2) * HASHTABLE_NPAGE;
    return 0;
  }
  if( iHash==0 ){
    *piRead = iExternal;
    return 0;
  }else{
    *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE;
  }


  return (iHash % 2);
}

/*
** Return the number of the wal-index page that contains the hash-table
** and page-number array that contain entries corresponding to WAL frame
** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages 
** are numbered starting from 0.
................................................................................
          testcase( szPage>=65536 );
          aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
          aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
        }
      }
  
      sqlite3_free(aFrame);
    }else if( pbZero && nSize==0 ){
      *pbZero = 1;
    }
  }

  pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
  pWal->hdr.aFrameCksum[1] = aFrameCksum[1];

................................................................................
    /* If more than one frame was recovered from the log file, report an
    ** event via sqlite3_log(). This is to help with identifying performance
    ** problems caused by applications routinely shutting down without
    ** checkpointing the log file.  */
    if( pWal->hdr.nPage ){
      if( isWalMode2(pWal) ){
        sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
            "recovered (%d,%d) frames from WAL files %s[2] (%s mode)",
            walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), 
            pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal"
        );
      }else{
        sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
            "recovered %d frames from WAL file %s",
            pWal->hdr.mxFrame, pWal->zWalName
        );
      }
................................................................................
  walIndexWriteHdr(pWal);
  pInfo->nBackfill = 0;
  pInfo->nBackfillAttempted = 0;
  pInfo->aReadMark[1] = 0;
  for(i=2; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
  assert( pInfo->aReadMark[0]==0 );
}































































/*
** Copy as much content as we can from the WAL back into the database file
** in response to an sqlite3_wal_checkpoint() request or the equivalent.
**
** The amount of information copies from WAL to database might be limited
** by active readers.  This routine will never overwrite a database page
................................................................................
    sqlite3_file *pWalFd = pWal->apWalFd[iCkpt];
    mxPage = pWal->hdr.nPage;

    /* If this is a wal2 system, check for a reader holding a lock 
    ** preventing this checkpoint operation. If one is found, return
    ** early.  */
    if( bWal2 ){
      rc = walLockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1);
      if( rc!=SQLITE_OK ) return rc;
    }

    /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
    ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
    assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );

................................................................................
    /* Allocate the iterator */
    if( bWal2 || pInfo->nBackfill<mxSafeFrame ){
      assert( bWal2==0 || pInfo->nBackfill==0 );
      rc = walIteratorInit(pWal, iCkpt, pInfo->nBackfill, &pIter);
      assert( rc==SQLITE_OK || pIter==0 );
    }

    if( pIter
     && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK
    ){
      u32 nBackfill = pInfo->nBackfill;

      assert( bWal2==0 || nBackfill==0 );
      pInfo->nBackfillAttempted = mxSafeFrame;

      /* Sync the wal file being checkpointed to disk */
      rc = sqlite3OsSync(pWalFd, CKPT_SYNC_FLAGS(sync_flags));
................................................................................
        }
      }
      if( rc==SQLITE_OK ){
        pInfo->nBackfill = bWal2 ? 1 : mxSafeFrame;
      }

      /* Release the reader lock held while backfilling */
      if( bWal2 ){
        walUnlockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1);
      }
      walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
    }

    if( rc==SQLITE_BUSY ){
      /* Reset the return code so as not to report a checkpoint failure
      ** just because there are active readers.  */
      rc = SQLITE_OK;
    }

  }

  /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
  ** entire wal file has been copied into the database file, then block 
  ** until all readers have finished using the wal file. This ensures that 
  ** the next process to write to the database restarts the wal file.
  */
................................................................................
    }
  }

  assert( pWal->nWiData>0 );
  assert( pWal->apWiData[0]!=0 );
  pInfo = walCkptInfo(pWal);
  if( isWalMode2(pWal) ){



    int eLock = 1 + (walidxGetFile(&pWal->hdr)*2);
    if( pInfo->nBackfill==0 ){
      eLock += walidxGetMxFrame(&pWal->hdr, !walidxGetFile(&pWal->hdr))>0;

    }
    rc = walLockReader(pWal, eLock, 1);








    if( rc!=SQLITE_OK ){
      return rc;

    }

    walShmBarrier(pWal);
    if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
      walLockReader(pWal, eLock, 0);

      return WAL_RETRY;
    }else{
      pWal->readLock = eLock;
    }
    assert( pWal->minFrame==0 && walFramePage(pWal->minFrame)==0 );
  }else{
    u32 mxReadMark;               /* Largest aReadMark[] value */
................................................................................
/*
** Finish with a read transaction.  All this does is release the
** read-lock.
*/
void sqlite3WalEndReadTransaction(Wal *pWal){
  sqlite3WalEndWriteTransaction(pWal);
  if( pWal->readLock!=WAL_LOCK_NONE ){
    if( isWalMode2(pWal) ){
      (void)walLockReader(pWal, pWal->readLock, 0);
    }else{
      walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
    }
    pWal->readLock = WAL_LOCK_NONE;
  }
}

/* Search hash table iHash for an entry matching page number
** pgno. Each call to this function searches a single hash table
** (each hash table indexes up to HASHTABLE_NPAGE frames).
................................................................................
    pWal->hdr.aFrameCksum[1] = aWalData[2];
    walCleanupHash(pWal);
  }

  return rc;
}

/*
** This function is used in wal2 mode.
**
** This function is called when writer pWal is just about to start 
** writing out frames. The "other" wal file (wal file !pWal->hdr.iAppend)
** has been fully checkpointed. This function returns SQLITE_OK if there
** are no readers preventing the writer from switching to the other wal
** file. Or SQLITE_BUSY if there are.
*/
static int walRestartOk(Wal *pWal){
  int rc;                                        /* Return code */
  int iApp = walidxGetFile(&pWal->hdr);          /* Current WAL file */

  /* No reader can be doing a "partial" read of wal file !iApp - in that
  ** case it would not have been possible to checkpoint the file. So
  ** it is only necessary to test for "full" readers. See the comment
  ** above walLockReader() function for exactly what this means in terms
  ** of locks.  */
  int i = (iApp==0) ? 2 : 4;

  rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
  if( rc==SQLITE_OK ){
    walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
  }
  return rc;
}

/*
** This function is called just before writing a set of frames to the log
** file (see sqlite3WalFrames()). It checks to see if, instead of appending
** to the current log file, it is possible and desirable to switch to the
** other log file and write the new transaction to the start of it.
** If so, the wal-index header is updated accordingly - both in heap memory
** and in the *-shm file.
................................................................................
        / (pWal->szPage+WAL_FRAME_HDRSIZE);
      nWalSize = MAX(nWalSize, 1);
    }

    if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){
      volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
      if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){
        rc = walRestartOk(pWal);
        if( rc==SQLITE_OK ){
          iApp = !iApp;
          pWal->nCkpt++;
          walidxSetFile(&pWal->hdr, iApp);
          walidxSetMxFrame(&pWal->hdr, iApp, 0);
          sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]);
          sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]);
          walIndexWriteHdr(pWal);
          pInfo->nBackfill = 0;
          walLockReader(pWal, pWal->readLock, 0);

          pWal->readLock = iApp ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2;
          rc = walLockReader(pWal, pWal->readLock, 1);

        }else if( rc==SQLITE_BUSY ){
          rc = SQLITE_OK;
        }
      }
    }
  }else if( pWal->readLock==0 ){
    volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
................................................................................
  */
  assert( pWal->readLock!=WAL_LOCK_NONE || pWal->lockError );
  assert( pWal->readLock!=WAL_LOCK_NONE || (op<=0 && pWal->exclusiveMode==0) );

  if( op==0 ){
    if( pWal->exclusiveMode ){
      pWal->exclusiveMode = WAL_NORMAL_MODE;
      if( isWalMode2(pWal) ){
        rc = walLockReader(pWal, pWal->readLock, 1);
      }else{
        rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
      }
      if( rc!=SQLITE_OK ){
        pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
      }
      rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
    }else{
      /* Already in locking_mode=NORMAL */
      rc = 0;
    }
  }else if( op>0 ){
    assert( pWal->exclusiveMode==WAL_NORMAL_MODE );
    assert( pWal->readLock>=0 );
    if( isWalMode2(pWal) ){
      walLockReader(pWal, pWal->readLock, 0);
    }else{
      walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
    }
    pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
    rc = 1;
  }else{
    rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
  }
  return rc;
}







<







 







<
|
>
|
|
>
>
>
>




|
|







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







<
<
<
<
<
|
<
>
|







 







|







 







|

|







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







 







|
|
|







 







|
|

<







>







 







>
>
>
|
<
|
>
|
<
>
>
>
>
>
>
>
>

<
>

<


<
>







 







<
<
<
|
<







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







|

|

|
|




|
>
|
<
>







 







<
<
<
|
<











<
<
<
|
<







358
359
360
361
362
363
364

365
366
367
368
369
370
371
...
468
469
470
471
472
473
474

475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
....
1111
1112
1113
1114
1115
1116
1117






























1118
1119
1120
1121
1122
1123
1124
....
1201
1202
1203
1204
1205
1206
1207





1208

1209
1210
1211
1212
1213
1214
1215
1216
1217
....
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
....
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
....
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
....
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
....
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
....
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470

2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
....
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116

3117
3118
3119

3120
3121
3122
3123
3124
3125
3126
3127
3128

3129
3130

3131
3132

3133
3134
3135
3136
3137
3138
3139
3140
....
3459
3460
3461
3462
3463
3464
3465



3466

3467
3468
3469
3470
3471
3472
3473
....
4094
4095
4096
4097
4098
4099
4100



























4101
4102
4103
4104
4105
4106
4107
....
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140

4141
4142
4143
4144
4145
4146
4147
4148
....
4763
4764
4765
4766
4767
4768
4769



4770

4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781



4782

4783
4784
4785
4786
4787
4788
4789
** it is held, but does not prevent a checkpointer from checkpointing 
** it.
**
** There is still a single WRITER and a single CHECKPOINTER lock. The
** recovery procedure still takes the same exclusive lock on the entire
** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks
** above use four of the six read-locking slots used by legacy wal mode.

**
** STARTUP/RECOVERY
**
** The read and write version fields of the database header in a wal2
** database are set to 0x03, instead of 0x02 as in legacy wal mode.
**
** The wal file format used in wal2 mode is the same as the format used
................................................................................
/*
** Values that may be stored in Wal.readLock in wal2 mode.
**
** In wal mode, the Wal.readLock member is set to -1 when no read-lock
** is held, or else is the index of the read-mark on which a lock is
** held.
**

** In wal2 mode, a value of -1 still indicates that no read-lock is held.
** And a non-zero value still represents the index of the read-mark on
** which a lock is held. There are two differences:
**
**   1. wal2 mode never uses read-mark 0.
**
**   2. locks on each read-mark have a different interpretation, as 
**      indicated by the symbolic names below.
*/
#define WAL_LOCK_NONE        -1
#define WAL_LOCK_PART1        1
#define WAL_LOCK_PART1_FULL2  2
#define WAL_LOCK_PART2_FULL1  3
#define WAL_LOCK_PART2        4

/* 
** This constant is used in wal2 mode only.
**
** In wal2 mode, when committing a transaction, if the current wal file 
** is sufficiently large and there are no conflicting locks held, the
** writer writes the new transaction into the start of the other wal
................................................................................
  if( pWal->exclusiveMode ) return;
  (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n,
                         SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE);
  WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal,
             walLockName(lockIdx), n));
}































/*
** Compute a hash on a page number.  The resulting hash value must land
** between 0 and (HASHTABLE_NSLOT-1).  The walHashNext() function advances
** the hash to the next value in the event of a collision.
*/
static int walHash(u32 iPage){
  assert( iPage>0 );
................................................................................

  if( 0==(iHash & 0x01) ){
    /* A frame in wal file 0 */
    *piRead = (iExternal <= HASHTABLE_NPAGE_ONE) ? iExternal :
      iExternal - (iHash/2) * HASHTABLE_NPAGE;
    return 0;
  }







  *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE;
  return 1;
}

/*
** Return the number of the wal-index page that contains the hash-table
** and page-number array that contain entries corresponding to WAL frame
** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages 
** are numbered starting from 0.
................................................................................
          testcase( szPage>=65536 );
          aFrameCksum[0] = pWal->hdr.aFrameCksum[0];
          aFrameCksum[1] = pWal->hdr.aFrameCksum[1];
        }
      }
  
      sqlite3_free(aFrame);
    }else if( pbZero ){
      *pbZero = 1;
    }
  }

  pWal->hdr.aFrameCksum[0] = aFrameCksum[0];
  pWal->hdr.aFrameCksum[1] = aFrameCksum[1];

................................................................................
    /* If more than one frame was recovered from the log file, report an
    ** event via sqlite3_log(). This is to help with identifying performance
    ** problems caused by applications routinely shutting down without
    ** checkpointing the log file.  */
    if( pWal->hdr.nPage ){
      if( isWalMode2(pWal) ){
        sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
            "recovered (%d,%d) frames from WAL files %s[2] (wal2 mode)",
            walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), 
            pWal->zWalName
        );
      }else{
        sqlite3_log(SQLITE_NOTICE_RECOVER_WAL,
            "recovered %d frames from WAL file %s",
            pWal->hdr.mxFrame, pWal->zWalName
        );
      }
................................................................................
  walIndexWriteHdr(pWal);
  pInfo->nBackfill = 0;
  pInfo->nBackfillAttempted = 0;
  pInfo->aReadMark[1] = 0;
  for(i=2; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED;
  assert( pInfo->aReadMark[0]==0 );
}

/*
** This function is used in wal2 mode.
**
** This function is called when writer pWal is just about to start 
** writing out frames. Parameter iApp is the current wal file. The "other" wal
** file (wal file !iApp) has been fully checkpointed. This function returns
** SQLITE_OK if there are no readers preventing the writer from switching to
** the other wal file. Or SQLITE_BUSY if there are.
*/
static int wal2RestartOk(Wal *pWal, int iApp){
  /* The other wal file (wal file !iApp) can be overwritten if there
  ** are no readers reading from it - no "full" or "partial" locks.
  ** Technically speaking it is not possible for any reader to hold
  ** a "part" lock, as this would have prevented the file from being
  ** checkpointed. But checking anyway doesn't hurt. The following
  ** is equivalent to:
  **
  **   if( iApp==0 ) eLock = WAL_LOCK_PART1_FULL2;
  **   if( iApp==1 ) eLock = WAL_LOCK_PART1;
  */
  int eLock = 1 + (iApp==0);

  assert( WAL_LOCK_PART1==1 );
  assert( WAL_LOCK_PART1_FULL2==2 );
  assert( WAL_LOCK_PART2_FULL1==3 );
  assert( WAL_LOCK_PART2==4 );

  assert( iApp!=0 || eLock==WAL_LOCK_PART1_FULL2 );
  assert( iApp!=1 || eLock==WAL_LOCK_PART1 );

  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 3);
}
static void wal2RestartFinished(Wal *pWal, int iApp){
  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iApp==0)), 3);
}

/*
** This function is used in wal2 mode.
**
** This function is called when a checkpointer wishes to checkpoint wal
** file iCkpt. It takes the required lock and, if successful, returns
** SQLITE_OK. Otherwise, an SQLite error code (e.g. SQLITE_BUSY). If this
** function returns SQLITE_OK, it is the responsibility of the caller
** to invoke wal2CheckpointFinished() to release the lock.
*/
static int wal2CheckpointOk(Wal *pWal, int iCkpt){
  int eLock = 1 + (iCkpt*2);

  assert( WAL_LOCK_PART1==1 );
  assert( WAL_LOCK_PART1_FULL2==2 );
  assert( WAL_LOCK_PART2_FULL1==3 );
  assert( WAL_LOCK_PART2==4 );

  assert( iCkpt!=0 || eLock==WAL_LOCK_PART1 );
  assert( iCkpt!=1 || eLock==WAL_LOCK_PART2_FULL1 );

  return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 2);
}
static void wal2CheckpointFinished(Wal *pWal, int iCkpt){
  walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iCkpt*2)), 2);
}

/*
** Copy as much content as we can from the WAL back into the database file
** in response to an sqlite3_wal_checkpoint() request or the equivalent.
**
** The amount of information copies from WAL to database might be limited
** by active readers.  This routine will never overwrite a database page
................................................................................
    sqlite3_file *pWalFd = pWal->apWalFd[iCkpt];
    mxPage = pWal->hdr.nPage;

    /* If this is a wal2 system, check for a reader holding a lock 
    ** preventing this checkpoint operation. If one is found, return
    ** early.  */
    if( bWal2 ){
      rc = wal2CheckpointOk(pWal, iCkpt);
      if( rc!=SQLITE_OK ) return rc;
    }

    /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
    ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
    assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );

................................................................................
    /* Allocate the iterator */
    if( bWal2 || pInfo->nBackfill<mxSafeFrame ){
      assert( bWal2==0 || pInfo->nBackfill==0 );
      rc = walIteratorInit(pWal, iCkpt, pInfo->nBackfill, &pIter);
      assert( rc==SQLITE_OK || pIter==0 );
    }

    if( pIter && (bWal2 
     || (rc = walBusyLock(pWal, xBusy, pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
    )){
      u32 nBackfill = pInfo->nBackfill;

      assert( bWal2==0 || nBackfill==0 );
      pInfo->nBackfillAttempted = mxSafeFrame;

      /* Sync the wal file being checkpointed to disk */
      rc = sqlite3OsSync(pWalFd, CKPT_SYNC_FLAGS(sync_flags));
................................................................................
        }
      }
      if( rc==SQLITE_OK ){
        pInfo->nBackfill = bWal2 ? 1 : mxSafeFrame;
      }

      /* Release the reader lock held while backfilling */
      if( bWal2==0 ){
        walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
      }

    }

    if( rc==SQLITE_BUSY ){
      /* Reset the return code so as not to report a checkpoint failure
      ** just because there are active readers.  */
      rc = SQLITE_OK;
    }
    if( bWal2 ) wal2CheckpointFinished(pWal, iCkpt);
  }

  /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
  ** entire wal file has been copied into the database file, then block 
  ** until all readers have finished using the wal file. This ensures that 
  ** the next process to write to the database restarts the wal file.
  */
................................................................................
    }
  }

  assert( pWal->nWiData>0 );
  assert( pWal->apWiData[0]!=0 );
  pInfo = walCkptInfo(pWal);
  if( isWalMode2(pWal) ){
    /* This connection needs a "part" lock on the current wal file and, 
    ** unless pInfo->nBackfill is set to indicate that it has already been
    ** checkpointed, a "full" lock on the other wal file.  */
    int iWal = walidxGetFile(&pWal->hdr);

    int nBackfill = pInfo->nBackfill || walidxGetMxFrame(&pWal->hdr, !iWal)==0;
    int eLock = 1 + (iWal*2) + (nBackfill==iWal);


    assert( nBackfill==0 || nBackfill==1 );
    assert( iWal==0 || iWal==1 );
    assert( iWal!=0 || nBackfill!=1 || eLock==WAL_LOCK_PART1 );
    assert( iWal!=0 || nBackfill!=0 || eLock==WAL_LOCK_PART1_FULL2 );
    assert( iWal!=1 || nBackfill!=1 || eLock==WAL_LOCK_PART2 );
    assert( iWal!=1 || nBackfill!=0 || eLock==WAL_LOCK_PART2_FULL1 );

    rc = walLockShared(pWal, WAL_READ_LOCK(eLock));
    if( rc!=SQLITE_OK ){

      return (rc==SQLITE_BUSY ? WAL_RETRY : rc);
    }

    walShmBarrier(pWal);
    if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){

      walUnlockShared(pWal, WAL_READ_LOCK(eLock));
      return WAL_RETRY;
    }else{
      pWal->readLock = eLock;
    }
    assert( pWal->minFrame==0 && walFramePage(pWal->minFrame)==0 );
  }else{
    u32 mxReadMark;               /* Largest aReadMark[] value */
................................................................................
/*
** Finish with a read transaction.  All this does is release the
** read-lock.
*/
void sqlite3WalEndReadTransaction(Wal *pWal){
  sqlite3WalEndWriteTransaction(pWal);
  if( pWal->readLock!=WAL_LOCK_NONE ){



    walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));

    pWal->readLock = WAL_LOCK_NONE;
  }
}

/* Search hash table iHash for an entry matching page number
** pgno. Each call to this function searches a single hash table
** (each hash table indexes up to HASHTABLE_NPAGE frames).
................................................................................
    pWal->hdr.aFrameCksum[1] = aWalData[2];
    walCleanupHash(pWal);
  }

  return rc;
}




























/*
** This function is called just before writing a set of frames to the log
** file (see sqlite3WalFrames()). It checks to see if, instead of appending
** to the current log file, it is possible and desirable to switch to the
** other log file and write the new transaction to the start of it.
** If so, the wal-index header is updated accordingly - both in heap memory
** and in the *-shm file.
................................................................................
        / (pWal->szPage+WAL_FRAME_HDRSIZE);
      nWalSize = MAX(nWalSize, 1);
    }

    if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){
      volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
      if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){
        rc = wal2RestartOk(pWal, iApp);
        if( rc==SQLITE_OK ){
          int iNew = !iApp;
          pWal->nCkpt++;
          walidxSetFile(&pWal->hdr, iNew);
          walidxSetMxFrame(&pWal->hdr, iNew, 0);
          sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]);
          sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]);
          walIndexWriteHdr(pWal);
          pInfo->nBackfill = 0;
          wal2RestartFinished(pWal, iApp);
          walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
          pWal->readLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2;

          rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));
        }else if( rc==SQLITE_BUSY ){
          rc = SQLITE_OK;
        }
      }
    }
  }else if( pWal->readLock==0 ){
    volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
................................................................................
  */
  assert( pWal->readLock!=WAL_LOCK_NONE || pWal->lockError );
  assert( pWal->readLock!=WAL_LOCK_NONE || (op<=0 && pWal->exclusiveMode==0) );

  if( op==0 ){
    if( pWal->exclusiveMode ){
      pWal->exclusiveMode = WAL_NORMAL_MODE;



      rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock));

      if( rc!=SQLITE_OK ){
        pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
      }
      rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
    }else{
      /* Already in locking_mode=NORMAL */
      rc = 0;
    }
  }else if( op>0 ){
    assert( pWal->exclusiveMode==WAL_NORMAL_MODE );
    assert( pWal->readLock>=0 );



    walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));

    pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
    rc = 1;
  }else{
    rc = pWal->exclusiveMode==WAL_NORMAL_MODE;
  }
  return rc;
}

Changes to src/window.c.

2129
2130
2131
2132
2133
2134
2135

2136
2137
2138
2139
2140
2141
2142
Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){
  Window *pNew = 0;
  if( ALWAYS(p) ){
    pNew = sqlite3DbMallocZero(db, sizeof(Window));
    if( pNew ){
      pNew->zName = sqlite3DbStrDup(db, p->zName);
      pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0);

      pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0);
      pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0);
      pNew->eType = p->eType;
      pNew->eEnd = p->eEnd;
      pNew->eStart = p->eStart;
      pNew->pStart = sqlite3ExprDup(db, p->pStart, 0);
      pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0);







>







2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){
  Window *pNew = 0;
  if( ALWAYS(p) ){
    pNew = sqlite3DbMallocZero(db, sizeof(Window));
    if( pNew ){
      pNew->zName = sqlite3DbStrDup(db, p->zName);
      pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0);
      pNew->pFunc = p->pFunc;
      pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0);
      pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0);
      pNew->eType = p->eType;
      pNew->eEnd = p->eEnd;
      pNew->eStart = p->eStart;
      pNew->pStart = sqlite3ExprDup(db, p->pStart, 0);
      pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0);

Changes to test/altertab2.test.

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
..
81
82
83
84
85
86
87


88






















































89
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix altertab

# If SQLITE_OMIT_ALTERTABLE is defined, omit this file.
ifcapable !altertable {
  finish_test
  return
}

................................................................................
  SELECT sql FROM sqlite_master WHERE name LIKE 'c%';
} {
  {CREATE TABLE c1(x REFERENCES "p3")}
  {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")}
  {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))}
}


























































finish_test







|







 







>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
..
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix altertab2

# If SQLITE_OMIT_ALTERTABLE is defined, omit this file.
ifcapable !altertable {
  finish_test
  return
}

................................................................................
  SELECT sql FROM sqlite_master WHERE name LIKE 'c%';
} {
  {CREATE TABLE c1(x REFERENCES "p3")}
  {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")}
  {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))}
}

#-------------------------------------------------------------------------
# Table name in WITH clauses that are part of views or triggers.
#
foreach {tn schema} {
  1 {
    CREATE TABLE log_entry(col1, y);
    CREATE INDEX i1 ON log_entry(col1);
  }

  2 {
    CREATE TABLE t1(a, b, c);
    CREATE TABLE t2(x);
    CREATE TABLE log_entry(col1);
    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
      INSERT INTO t2 SELECT col1 FROM log_entry;
    END;
  }

  3 {
    CREATE TABLE t1(a, b, c);
    CREATE TABLE t2(x);
    CREATE TABLE log_entry(col1);
    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
      INSERT INTO t2
        WITH xyz(x) AS (SELECT col1 FROM log_entry)
        SELECT x FROM xyz;
    END;
  }

  4 {
    CREATE TABLE log_entry(col1);
    CREATE VIEW ttt AS
        WITH xyz(x) AS (SELECT col1 FROM log_entry)
        SELECT x FROM xyz;
  }
} {
  reset_db
  do_execsql_test 3.$tn.1 $schema
  set expect [db eval "SELECT sql FROM sqlite_master"]
  set expect [string map {log_entry {"newname"}} $expect]

  do_execsql_test 3.$tn.2 {
    ALTER TABLE log_entry RENAME TO newname;
    SELECT sql FROM sqlite_master;
  } $expect

  reset_db
  do_execsql_test 3.$tn.3 $schema
  set expect [db eval "SELECT sql FROM sqlite_master"]
  set expect [string map {col1 newname} $expect]

  do_execsql_test 3.$tn.4 {
    ALTER TABLE log_entry RENAME col1 TO newname;
    SELECT sql FROM sqlite_master;
  } $expect
}

finish_test

Changes to test/dbfuzz2.c.

74
75
76
77
78
79
80

81
82
83
84
85
86
87
  int rc;
  int i;

  if( eVerbosity>=1 ){
    printf("************** nByte=%d ***************\n", (int)nByte);
    fflush(stdout);
  }

  rc = sqlite3_open(0, &db);
  if( rc ) return 1;
  a = sqlite3_malloc64(nByte+1);
  if( a==0 ) return 1;
  memcpy(a, aData, nByte);
  sqlite3_deserialize(db, "main", a, nByte, nByte,
        SQLITE_DESERIALIZE_RESIZEABLE |







>







74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
  int rc;
  int i;

  if( eVerbosity>=1 ){
    printf("************** nByte=%d ***************\n", (int)nByte);
    fflush(stdout);
  }
  if( sqlite3_initialize() ) return 0;
  rc = sqlite3_open(0, &db);
  if( rc ) return 1;
  a = sqlite3_malloc64(nByte+1);
  if( a==0 ) return 1;
  memcpy(a, aData, nByte);
  sqlite3_deserialize(db, "main", a, nByte, nByte,
        SQLITE_DESERIALIZE_RESIZEABLE |

Added test/fts4umlaut.test.



































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# 2018 December 3
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS5 module.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts4umlaut

ifcapable !fts3 {
  finish_test
  return
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  CREATE VIRTUAL TABLE t2 USING fts4(
      x, 
      tokenize=unicode61 "remove_diacritics=2"
  );
}

foreach {tn q res1 res2} {
  1 "Hà Nội"                  0 1
  2 "Hà Noi"                  1 1
  3 "Ha Noi"                  1 1
  4 "Ha N\u1ed9i"             0 1
  5 "Ha N\u006fi"             1 1
  6 "Ha N\u006f\u0302i"       1 1
  7 "Ha N\u006f\u0323\u0302i" 1 1
} {
  do_execsql_test 1.$tn.1 {
    DELETE FROM t1;
    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
    SELECT count(*) FROM t1 WHERE t1 MATCH $q
  } $res1
  do_execsql_test 1.$tn.2 {
    DELETE FROM t1;
    INSERT INTO t1(rowid, x) VALUES (1, $q);
    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
  } $res1

  do_execsql_test 1.$tn.2 {
    DELETE FROM t2;
    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
    SELECT count(*) FROM t2 WHERE t2 MATCH $q
  } $res2
  do_execsql_test 1.$tn.2 {
    DELETE FROM t2;
    INSERT INTO t2(rowid, x) VALUES (1, $q);
    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
  } $res2
}

finish_test

Changes to test/index6.test.

386
387
388
389
390
391
392



393
















394
  CREATE INDEX t11x ON t11(a) WHERE b<>99;
  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
} {/USING INDEX t11x/}
do_execsql_test index6-11.2 {
  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
} {/USING INDEX t11x/}
  




















finish_test







>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
  CREATE INDEX t11x ON t11(a) WHERE b<>99;
  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
} {/USING INDEX t11x/}
do_execsql_test index6-11.2 {
  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
} {/USING INDEX t11x/}

# 2018-12-08
# Ticket https://www.sqlite.org/src/info/1d958d90596593a7
# NOT IN operator fails when using a partial index.
#
do_execsql_test index6-12.1 {
  DROP TABLE IF EXISTS t1;
  DROP TABLE IF EXISTS t2;
  CREATE TABLE t1(a,b);
  INSERT INTO t1 VALUES(1,1);
  INSERT INTO t1 VALUES(2,2);
  CREATE TABLE t2(x);
  INSERT INTO t2 VALUES(1);
  INSERT INTO t2 VALUES(2);
  SELECT 'one', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
  CREATE INDEX t1a ON t1(a) WHERE b=1;
  SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1);
} {}
do_execsql_test index6-12.2 {
  SELECT x FROM t2 WHERE x IN (SELECT a FROM t1) ORDER BY +x;
} {1 2}
finish_test

Changes to test/json101.test.

809
810
811
812
813
814
815



816

817













818
do_execsql_test json-14.160 {
  SELECT fullkey FROM json_tree('"hello"');
} {$}
do_execsql_test json-14.170 {
  SELECT fullkey FROM json_tree('null');
} {$}




















finish_test







>
>
>
|
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>

809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
do_execsql_test json-14.160 {
  SELECT fullkey FROM json_tree('"hello"');
} {$}
do_execsql_test json-14.170 {
  SELECT fullkey FROM json_tree('null');
} {$}

# 2018-12-03
# Make sure the table-valued functions contained within parentheses
# work correctly.
#
# Bug reported via private email. See TH3 for more information.
#
do_execsql_test json-15.100 {
  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
do_execsql_test json-15.110 {
  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
do_execsql_test json-15.120 {
  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
do_execsql_test json-15.130 {
  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}

finish_test

Changes to test/normalize.test.

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
...
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
...
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
...
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
...
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
  set STMT [sqlite3_prepare_v3 $DB \
      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 0 TAIL]

  sqlite3_bind_null $STMT 1
} {}
do_test 202 {
  sqlite3_normalized_sql $STMT
} {}
do_test 203 {
  sqlite3_finalize $STMT
} {SQLITE_OK}

do_test 210 {
  set STMT [sqlite3_prepare_v3 $DB \
      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 2 TAIL]
................................................................................
  {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
  0x2
  {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}

  430
  {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
  0x2
  {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}}

  440
  {SELECT 'a' FROM t1 WHERE 'x';}
  0x2
  {0 {SELECT?FROM t1 WHERE?;}}

  450
  {SELECT [a] FROM t1 WHERE [x];}
  0x2
  {0 {SELECT"a"FROM t1 WHERE"x";}}

  460
  {SELECT * FROM t1 WHERE x IN (x);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(x);}}

  470
................................................................................
  {SELECT * FROM t1 WHERE x IN (x,a);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}

  480
  {SELECT * FROM t1 WHERE x IN ([x],"a");}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN("x","a");}}

  500
  {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}}

  520
  {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}

  540
  {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}

  550
  {SELECT a, a+1, a||'b', a+"b" FROM t1;}
  0x2
  {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}}

  570
  {SELECT * FROM t1 WHERE x IN (1);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}

  580
................................................................................
  {SELECT "col f", [col f] FROM t1;}
  0x2
  {0 {SELECT"col f","col f"FROM t1;}}

  680
  {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];}
  0x2
  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}}

  690
  {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);}
  0x2
  {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}}

  700
................................................................................
  {SELECT x FROM t1 WHERE x = NULL;}
  0x2
  {0 {SELECT x FROM t1 WHERE x=?;}}

  760
  {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');}
  0x2
  {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}}
} {
  do_test $tnum {
    set code [catch {
      set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL]
      sqlite3_normalized_sql $STMT
    } res]
    if {[info exists STMT]} {







|







 







|









|







 







|




|









|




|







 







|







 







|







82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
...
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
...
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
...
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
...
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
  set STMT [sqlite3_prepare_v3 $DB \
      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 0 TAIL]

  sqlite3_bind_null $STMT 1
} {}
do_test 202 {
  sqlite3_normalized_sql $STMT
} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;}
do_test 203 {
  sqlite3_finalize $STMT
} {SQLITE_OK}

do_test 210 {
  set STMT [sqlite3_prepare_v3 $DB \
      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 2 TAIL]
................................................................................
  {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
  0x2
  {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}

  430
  {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
  0x2
  {0 {SELECT a FROM t1 WHERE x IN(?,?,?);}}

  440
  {SELECT 'a' FROM t1 WHERE 'x';}
  0x2
  {0 {SELECT?FROM t1 WHERE?;}}

  450
  {SELECT [a] FROM t1 WHERE [x];}
  0x2
  {0 {SELECT a FROM t1 WHERE x;}}

  460
  {SELECT * FROM t1 WHERE x IN (x);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(x);}}

  470
................................................................................
  {SELECT * FROM t1 WHERE x IN (x,a);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}

  480
  {SELECT * FROM t1 WHERE x IN ([x],"a");}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}

  500
  {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(x,a,?,sqlite_version());}}

  520
  {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}

  540
  {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN((SELECT x FROM t1));}}

  550
  {SELECT a, a+1, a||'b', a+"b" FROM t1;}
  0x2
  {0 {SELECT a,a+?,a||?,a+b FROM t1;}}

  570
  {SELECT * FROM t1 WHERE x IN (1);}
  0x2
  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}

  580
................................................................................
  {SELECT "col f", [col f] FROM t1;}
  0x2
  {0 {SELECT"col f","col f"FROM t1;}}

  680
  {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];}
  0x2
  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON t1."col f"==t2."col y";}}

  690
  {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);}
  0x2
  {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}}

  700
................................................................................
  {SELECT x FROM t1 WHERE x = NULL;}
  0x2
  {0 {SELECT x FROM t1 WHERE x=?;}}

  760
  {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');}
  0x2
  {0 {SELECT x FROM t1 WHERE x IN(x IS NOT NULL,?,?,?,b,?);}}
} {
  do_test $tnum {
    set code [catch {
      set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL]
      sqlite3_normalized_sql $STMT
    } res]
    if {[info exists STMT]} {

Changes to test/permutations.test.

426
427
428
429
430
431
432
433
434
435
436







437
438
439
440
441
442
443
# Define the coverage related test suites:
#
#   coverage-wal
#
test_suite "coverage-wal" -description {
  Coverage tests for file wal.c.
} -files {
  wal.test       wal2.test     wal3.test       walmode.test    
  walbak.test    walhook.test  walcrash2.test  walcksum.test
  walfault.test  walbig.test   walnoshm.test
  wal5.test







} 

test_suite "coverage-pager" -description {
  Coverage tests for file pager.c.
} -files {
  pager1.test    pager2.test  pagerfault.test  pagerfault2.test
  walfault.test  walbak.test  journal2.test    tkt-9d68c883.test







|
|
<
|
>
>
>
>
>
>
>







426
427
428
429
430
431
432
433
434

435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
# Define the coverage related test suites:
#
#   coverage-wal
#
test_suite "coverage-wal" -description {
  Coverage tests for file wal.c.
} -files {
wal2big.test wal2recover.test wal2rewrite.test
wal2simple.test wal2snapshot.test wal2.test

wal3.test wal4.test wal5.test
wal64k.test wal6.test wal7.test wal8.test wal9.test
walbak.test walbig.test walblock.test walcksum.test 
walfault.test walhook.test walmode.test walnoshm.test
waloverwrite.test walpersist.test walprotocol2.test
walprotocol.test walro2.test walrofault.test walro.test
walshared.test walslow.test wal.test
wal2savepoint.test wal2lock.test wal2recover2.test
} 

test_suite "coverage-pager" -description {
  Coverage tests for file pager.c.
} -files {
  pager1.test    pager2.test  pagerfault.test  pagerfault2.test
  walfault.test  walbak.test  journal2.test    tkt-9d68c883.test

Changes to test/shell1.test.

252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
} {0 {}}
do_test shell1-3.1.3 {
  catchcmd "test.db" ".backup FOO BAR"
} {1 {Error: unknown database FOO}}
do_test shell1-3.1.4 {
  # too many arguments
  catchcmd "test.db" ".backup FOO BAR BAD"
} {1 {Usage: .backup ?DB? ?--append? FILENAME}}

# .bail ON|OFF           Stop after hitting an error.  Default OFF
do_test shell1-3.2.1 {
  catchcmd "test.db" ".bail"
} {1 {Usage: .bail on|off}}
do_test shell1-3.2.2 {
  catchcmd "test.db" ".bail ON"







|







252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
} {0 {}}
do_test shell1-3.1.3 {
  catchcmd "test.db" ".backup FOO BAR"
} {1 {Error: unknown database FOO}}
do_test shell1-3.1.4 {
  # too many arguments
  catchcmd "test.db" ".backup FOO BAR BAD"
} {1 {Usage: .backup ?DB? ?OPTIONS? FILENAME}}

# .bail ON|OFF           Stop after hitting an error.  Default OFF
do_test shell1-3.2.1 {
  catchcmd "test.db" ".bail"
} {1 {Usage: .bail on|off}}
do_test shell1-3.2.2 {
  catchcmd "test.db" ".bail ON"

Changes to test/shell4.test.

103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
}]
  list [regexp {Memory Used} $res] \
       [regexp {Heap Usage} $res] \
       [regexp {Autoindex Inserts} $res]
} {1 1 1}

do_test shell4-2.1 {
  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
} {1 {Usage: .trace FILE|off}}
do_test shell4-2.2 {
  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
} {0 {}}
do_test shell4-2.3 {
  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
} {/^1 {PRAGMA.*Usage:.*}$/}
ifcapable trace {
do_test shell4-2.4 {
  catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
} {0 {CREATE TABLE t1(x);
SELECT * FROM t1;}}
do_test shell4-2.5 {
  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;"







|
|




|
|







103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
}]
  list [regexp {Memory Used} $res] \
       [regexp {Heap Usage} $res] \
       [regexp {Autoindex Inserts} $res]
} {1 1 1}

do_test shell4-2.1 {
  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
} {1 {Unknown option "--unknown" on ".trace"}}
do_test shell4-2.2 {
  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
} {0 {}}
do_test shell4-2.3 {
  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
} {/^0 {PRAGMA.*}$/}
ifcapable trace {
do_test shell4-2.4 {
  catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
} {0 {CREATE TABLE t1(x);
SELECT * FROM t1;}}
do_test shell4-2.5 {
  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;"

Changes to test/tabfunc01.test.

120
121
122
123
124
125
126





127
128
129
130
131
132
133
  SELECT * FROM temp.generate_series(1,4)
} {1 2 3 4}
do_execsql_test tabfunc01-4.3 {
  ATTACH ':memory:' AS aux1;
  CREATE TABLE aux1.t1(a,b,c);
  SELECT * FROM aux1.generate_series(1,4)
} {1 2 3 4}






# The next series of tests is verifying that virtual table are able
# to optimize the IN operator, even on terms that are not marked "omit".
# When the generate_series virtual table is compiled for the testfixture,
# the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
# causes the xBestIndex method of generate_series to leave the
# sqlite3_index_constraint_usage.omit flag set to 0, which should cause







>
>
>
>
>







120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  SELECT * FROM temp.generate_series(1,4)
} {1 2 3 4}
do_execsql_test tabfunc01-4.3 {
  ATTACH ':memory:' AS aux1;
  CREATE TABLE aux1.t1(a,b,c);
  SELECT * FROM aux1.generate_series(1,4)
} {1 2 3 4}

# 2018-12-03: Fix bug reported by by private email.
do_execsql_test tabfunc01-4.4 {
  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
} {1 3 5}

# The next series of tests is verifying that virtual table are able
# to optimize the IN operator, even on terms that are not marked "omit".
# When the generate_series virtual table is compiled for the testfixture,
# the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
# causes the xBestIndex method of generate_series to leave the
# sqlite3_index_constraint_usage.omit flag set to 0, which should cause

Added test/vacuum-into.test.











































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# 2018-12-07
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the VACUUM INTO statement.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If the VACUUM statement is disabled in the current build, skip all
# the tests in this file.
#
ifcapable {!vacuum} {
  omit_test vacuum.test {Compiled with SQLITE_OMIT_VACUUM}
  finish_test
  return
}

forcedelete out.db
do_execsql_test vacuum-into-100 {
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
  INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c;
  CREATE INDEX t1b ON t1(b);
  DELETE FROM t1 WHERE a%2;
  SELECT count(*), sum(a), sum(length(b)) FROM t1;
} {50 2550 30000}
do_execsql_test vacuum-into-110 {
  VACUUM main INTO 'out.db';
} {}
sqlite3 db2 out.db
do_test vacuum-into-120 {
  db2 eval {SELECT count(*), sum(a), sum(length(b)) FROM t1}
} {50 2550 30000}
do_catchsql_test vacuum-into-130 {
  VACUUM INTO 'out.db';
} {1 {output file already exists}}
forcedelete out2.db
do_catchsql_test vacuum-into-140 {
  VACUUM INTO 'out2.db';
} {0 {}}
do_catchsql_test vacuum-into-150 {
  VACUUM INTO 'out2.db';
} {1 {output file already exists}}

do_catchsql_test vacuum-into-200 {
  VACUUM main INTO ':memory:';
} {0 {}}

# The INTO argument can be an arbitrary expression.
#
do_execsql_test vacuum-into-300 {
  CREATE TABLE t2(name TEXT);
  INSERT INTO t2 VALUES(':memory:');
  VACUUM main INTO (SELECT name FROM t2);
} {}
do_catchsql_test vacuum-into-310 {
  VACUUM INTO null;
} {1 {non-text filename}}

finish_test

Added test/wal2big.test.















































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
# 2017 September 19
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL2" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
source $testdir/wal_common.tcl

set testprefix wal2big
ifcapable !wal {finish_test ; return }

do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c);
  CREATE INDEX t1a ON t1(a);
  CREATE INDEX t1b ON t1(b);
  CREATE INDEX t1c ON t1(c);
  PRAGMA journal_mode = wal2;
  PRAGMA journal_size_limit = 10000000;

  WITH s(i) AS (
    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
  )
  INSERT INTO t1 SELECT random(), random(), random() FROM s;
} {wal2 10000000}

do_execsql_test 1.1 {
  WITH s(i) AS (
    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000
  )
  INSERT INTO t1 SELECT random(), random(), random() FROM s;
}

do_test 1.2 {
  list [expr [file size test.db-wal]>10000000] \
       [expr [file size test.db-wal2]>10000000]
} {1 1}

do_test 1.3 {
  sqlite3 db2 test.db
  execsql {
    SELECT count(*) FROM t1;
    PRAGMA integrity_check;
  } db2
} {400000 ok}

do_test 1.4 {
  db2 close
  forcecopy test.db test.db2
  forcecopy test.db-wal test.db2-wal
  forcecopy test.db-wal2 test.db2-wal2
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    PRAGMA integrity_check;
  }
} {400000 ok}

finish_test

Added test/wal2lock.test.





















































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# 2018 December 15
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL2" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
source $testdir/wal_common.tcl

set testprefix wal2lock
ifcapable !wal {finish_test ; return }

db close
testvfs tvfs 
sqlite3 db test.db -vfs tvfs

do_execsql_test 1.0 {
  PRAGMA journal_mode = wal2;
  CREATE TABLE y1(y, yy);
  CREATE INDEX y1y ON y1(y);
  CREATE INDEX y1yy ON y1(yy);
  INSERT INTO y1 VALUES(1, 2), (3, 4), (5, 6);
} {wal2}

tvfs script vfs_callback
tvfs filter xShmLock

set ::lock [list]
proc vfs_callback {func file name lock} {
  lappend ::lock $lock
  return SQLITE_OK
}

do_execsql_test 1.1.1 {
  SELECT * FROM y1
} {1 2 3 4 5 6}
do_test 1.1.2 {
  set ::lock
} {{4 1 lock shared} {4 1 unlock shared}}

set ::bFirst 1
proc vfs_callback {func file name lock} {
  if {$::bFirst} {
    set ::bFirst 0
    return SQLITE_BUSY
  }
  return SQLITE_OK
}
do_execsql_test 1.2 {
  SELECT * FROM y1
} {1 2 3 4 5 6}

set ::bFirst 1
proc vfs_callback {func file name lock} {
  if {$::bFirst} {
    set ::bFirst 0
    return SQLITE_IOERR
  }
  return SQLITE_OK
}
do_catchsql_test 1.3 {
  SELECT * FROM y1
} {1 {disk I/O error}}

puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite"
puts "# is built on unix without HAVE_USLEEP defined, it may be much longer."
proc vfs_callback {func file name lock} { return SQLITE_BUSY }
do_catchsql_test 1.4 {
  SELECT * FROM y1
} {1 {locking protocol}}
proc vfs_callback {func file name lock} { return SQLITE_OK }

sqlite3 db2 test.db -vfs tvfs
set ::bFirst 1

proc vfs_callback {func file name lock} {
  if {$::bFirst} {
    set ::bFirst 0
    db2 eval { INSERT INTO y1 VALUES(7, 8) }
  }
}

do_execsql_test 1.5.1 {
  SELECT * FROM y1
} {1 2 3 4 5 6 7 8}
do_execsql_test 1.5.2 {
  SELECT * FROM y1
} {1 2 3 4 5 6 7 8}

db close
db2 close
tvfs delete
finish_test

Added test/wal2recover.test.

















































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
# 2018 December 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL2" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
source $testdir/wal_common.tcl

set testprefix wal2recover
ifcapable !wal {finish_test ; return }

proc db_copy {from to} {
  forcecopy $from $to
  forcecopy ${from}-wal ${to}-wal
  forcecopy ${from}-wal2 ${to}-wal2
}

do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c);
  CREATE INDEX t1a ON t1(a);
  CREATE INDEX t1b ON t1(b);
  CREATE INDEX t1c ON t1(c);
  PRAGMA journal_mode = wal2;
  PRAGMA journal_size_limit = 15000;
  PRAGMA wal_autocheckpoint = 0;
} {wal2 15000 0}

do_test 1.1 {
  for {set i 1} {$i <= 1000} {incr i} {
    execsql { INSERT INTO t1 VALUES(random(), random(), random()) }
    db_copy test.db test.db2
    sqlite3 db2 test.db
    set res [execsql {
      SELECT count(*) FROM t1;
      PRAGMA integrity_check;
    } db2]
    db2 close
    if {$res != [list $i ok]} {
      error "failure on iteration $i"
    }
  }
  set {} {}
} {}

#--------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
  CREATE TABLE t1(x UNIQUE);
  CREATE TABLE t2(x UNIQUE);
  PRAGMA journal_mode = wal2;
  PRAGMA journal_size_limit = 10000;
  PRAGMA wal_autocheckpoint = 0;
  BEGIN;
    INSERT INTO t1 VALUES(randomblob(4000));
    INSERT INTO t1 VALUES(randomblob(4000));
    INSERT INTO t1 VALUES(randomblob(4000));
  COMMIT;
  BEGIN;
    INSERT INTO t2 VALUES(randomblob(4000));
    INSERT INTO t2 VALUES(randomblob(4000));
    INSERT INTO t2 VALUES(randomblob(4000));
  COMMIT;
} {wal2 10000 0}
do_test 2.0.1 {
  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
} {5120 28328 28328}

# Test recovery with both wal files intact.
#
do_test 2.1 {
  db_copy test.db test.db2
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    SELECT count(*) FROM t2;
    PRAGMA integrity_check;
  } db2
} {3 3 ok}

do_test 2.2 {
  db2 close
  db_copy test.db test.db2
  hexio_write test.db2-wal 16 12345678
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    SELECT count(*) FROM t2;
  } db2
} {0 3}

do_test 2.3 {
  db2 close
  db_copy test.db test.db2
  hexio_write test.db2-wal2 16 12345678
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    SELECT count(*) FROM t2;
    PRAGMA integrity_check;
  } db2
} {3 0 ok}

do_test 2.4 {
  db2 close
  db_copy test.db test.db2
  forcecopy test.db-wal test.db2-wal2
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    SELECT count(*) FROM t2;
    PRAGMA integrity_check;
  } db2
} {3 0 ok}

do_test 2.5 {
  db2 close
  db_copy test.db test.db2
  forcecopy test.db-wal  test.db2-wal2
  forcecopy test.db-wal2  test.db2-wal
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    SELECT count(*) FROM t2;
    PRAGMA integrity_check;
  } db2
} {3 3 ok}

do_test 2.6 {
  db2 close
  db_copy test.db test.db2
  forcecopy test.db-wal test.db2-wal2
  close [open test.db-wal w]
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    SELECT count(*) FROM t2;
    PRAGMA integrity_check;
  } db2
} {3 0 ok}

do_test 2.7 {
  db2 close
  db_copy test.db test.db2
  forcedelete test.db2-wal
  sqlite3 db2 test.db2
  execsql {
    SELECT count(*) FROM t1;
    SELECT count(*) FROM t2;
    PRAGMA integrity_check;
  } db2
} {0 0 ok}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
  CREATE TABLE t1(a TEXT, b TEXT, c TEXT);
  CREATE INDEX t1a ON t1(a);
  CREATE INDEX t1b ON t1(b);
  CREATE INDEX t1c ON t1(c);
  PRAGMA journal_mode = wal2;
  PRAGMA journal_size_limit = 10000;
  PRAGMA wal_autocheckpoint = 0;
  PRAGMA cache_size = 5;
} {wal2 10000 0}

do_execsql_test 3.1 {
  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
  INSERT INTO t1 SELECT i, i, i FROM s;

  INSERT INTO t1 VALUES(201, 201, 201);
} {}

do_test 3.2 {
  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
} {5120 15752 4224}

do_test 3.3 {
  forcecopy test.db test.db2
  forcecopy test.db-wal test.db2-wal
  forcecopy test.db-wal2 test.db2-wal2
  sqlite3 db2 test.db2
  execsql {
    PRAGMA journal_size_limit = 10000;
    PRAGMA wal_autocheckpoint = 0;
    PRAGMA cache_size = 5;
    BEGIN;
      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
      INSERT INTO t1 SELECT i, i, i FROM s;
  } db2
  list [file size test.db2] [file size test.db2-wal] [file size test.db2-wal2]
} {5120 15752 23088}

do_test 3.4 {
  set fd [open test.db2-shm]
  fconfigure $fd -encoding binary -translation binary
  set data [read $fd]
  close $fd

  set fd [open test.db-shm w]
  fconfigure $fd -encoding binary -translation binary
  puts -nonewline $fd $data
  close $fd

  execsql {
    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
      INSERT INTO t1 SELECT i, i, i FROM s;
    SELECT count(*) FROM t1;
    PRAGMA integrity_check;
  }
} {211 ok}

do_test 3.5 {
  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
} {5120 15752 18896}


finish_test

Added test/wal2recover2.test.







































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
# 2018 December 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL2" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
source $testdir/wal_common.tcl

set testprefix wal2recover2
ifcapable !wal {finish_test ; return }

do_execsql_test 1.0 {
  CREATE TABLE t1(x);
  CREATE TABLE t2(x);
  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
    INSERT INTO t1 SELECT i FROM s;
  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 )
    INSERT INTO t2 SELECT i FROM s;

  PRAGMA journal_mode = wal2;
  PRAGMA journal_size_limit = 10000;
} {wal2 10000}

set ::L 1125750
set ::M 1126500
set ::H 1127250

do_execsql_test 1.1 {
  UPDATE t1 SET x=x+1;
  UPDATE t2 SET x=x+1 WHERE rowid<=750;

  SELECT sum(x) FROM t1;
  SELECT sum(x) FROM t2;
} [list $H $M]

do_test 1.2 {
  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
} {31744 14704 7368}

proc cksum {zIn data} {
  if {[string length $zIn]==0} {
    set s0 0
    set s1 0
  } else {
    set s0 [hexio_get_int [string range $zIn 0 7]]
    set s1 [hexio_get_int [string range $zIn 8 15]]
  }
  set n [expr [string length $data] / 8]

  for {set i 0} {$i < $n} {incr i 2} {
    set x0 [hexio_get_int -l [string range $data [expr $i*8]   [expr $i*8+7]]]
    set x1 [hexio_get_int -l [string range $data [expr $i*8+8] [expr $i*8+8+7]]]

    set s0 [expr ($s0 + $x0 + $s1) & 0xFFFFFFFF]
    set s1 [expr ($s1 + $x1 + $s0) & 0xFFFFFFFF]
  }

  return "[hexio_render_int32 $s0][hexio_render_int32 $s1]"
}

proc fix_wal_cksums {file} {
  # Fix the checksum on the wal header.
  set data [hexio_read $file 0 32]
  set cksum [cksum {} [string range $data 0 47]]
  set salt [hexio_read $file 16 8]
  hexio_write $file 24 $cksum

  # Fix the checksums for all pages in the wal file.
  set pgsz [hexio_get_int [hexio_read $file 8 4]]
  set sz [file size $file]
  for {set off 32} {$off < $sz} {incr off [expr $pgsz+24]} {
    set e [hexio_read $file $off 8]
    set cksum [cksum $cksum $e]

    set p [hexio_read $file [expr $off+24] $pgsz]
    set cksum [cksum $cksum $p]

    hexio_write $file [expr $off+8] $salt
    hexio_write $file [expr $off+16] $cksum
  }
}

proc wal_incr_hdrfield {file field} {
  switch -- $field {
    nCkpt { set offset 12 }
    salt0 { set offset 16 }
    salt1 { set offset 20 }
    default {
      error "unknown field $field - should be \"nCkpt\", \"salt0\" or \"salt1\""
    }
  }

  # Increment the value in the wal header.
  set v [hexio_get_int [hexio_read $file $offset 4]]
  incr v
  hexio_write $file $offset [hexio_render_int32 $v]
  
  # Fix various checksums
  fix_wal_cksums $file
}

proc wal_set_nckpt {file val} {
  # Increment the value in the wal header.
  hexio_write $file 12 [hexio_render_int32 $val]
  
  # Fix various checksums
  fix_wal_cksums $file
}

proc wal_set_follow {file prevfile} {
  set pgsz [hexio_get_int [hexio_read $prevfile 8 4]]
  set sz [file size $prevfile]
  set cksum [hexio_read $prevfile [expr $sz-$pgsz-8] 8]

  hexio_write $file 16 $cksum
  fix_wal_cksums $file
}

foreach {tn file field} {
  1 test.db2-wal    salt0
  2 test.db2-wal    salt1
  3 test.db2-wal    nCkpt
  4 test.db2-wal2   salt0
  5 test.db2-wal2   salt1
  6 test.db2-wal2   nCkpt
} {
  do_test 1.3.$tn {
    forcecopy test.db test.db2
    forcecopy test.db-wal test.db2-wal
    forcecopy test.db-wal2 test.db2-wal2
    wal_incr_hdrfield $file $field
    sqlite3 db2 test.db2
    execsql {
      SELECT sum(x) FROM t1;
      SELECT sum(x) FROM t2;
    } db2
  } [list $H $L]
  db2 close
}

do_test 1.4 {
  forcecopy test.db test.db2
  forcecopy test.db-wal2 test.db2-wal
  forcedelete test.db2-wal2
  sqlite3 db2 test.db2
  execsql {
    SELECT sum(x) FROM t1;
    SELECT sum(x) FROM t2;
  } db2
} [list $L $M]

do_test 1.5 {
  forcecopy test.db test.db2
  forcecopy test.db-wal2 test.db2-wal
  forcecopy test.db-wal test.db2-wal2
  sqlite3 db2 test.db2
  execsql {
    SELECT sum(x) FROM t1;
    SELECT sum(x) FROM t2;
  } db2
} [list $H $M]

foreach {tn file field} {
  1 test.db2-wal    salt0
  2 test.db2-wal    salt1
  3 test.db2-wal2   salt0
  4 test.db2-wal2   salt1
} {
  do_test 1.6.$tn {
    forcecopy test.db test.db2
    forcecopy test.db-wal2 test.db2-wal
    forcecopy test.db-wal test.db2-wal2
    wal_incr_hdrfield $file $field
    breakpoint
    sqlite3 db2 test.db2
    execsql {
      SELECT sum(x) FROM t1;
      SELECT sum(x) FROM t2;
    } db2
  } [list $H $L]
  db2 close
}

foreach {tn nCkpt1 nCkpt2 res} [list \
  1   2 1   "$H $M"                  \
  2   2 2   "$L $M"                  \
  3   3 1   "$H $L"                  \
  4   15 14 "$H $M"                  \
  5   0 15  "$H $M"                  \
  6   1 15  "$L $M"                  \
] {
  do_test 1.7.$tn {
    forcecopy test.db test.db2
    forcecopy test.db-wal2 test.db2-wal
    forcecopy test.db-wal test.db2-wal2

    wal_set_nckpt test.db2-wal2 $nCkpt2
    wal_set_nckpt test.db2-wal  $nCkpt1
    wal_set_follow test.db2-wal test.db2-wal2

    if {$tn==1} breakpoint

    sqlite3 db2 test.db2
    execsql {
      SELECT sum(x) FROM t1;
      SELECT sum(x) FROM t2;
    } db2
  } $res
  db2 close
}


finish_test

Added test/wal2savepoint.test.

























































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# 2018 December 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL2" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
source $testdir/wal_common.tcl

set testprefix wal2savepoint
ifcapable !wal {finish_test ; return }

do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c);
  CREATE INDEX t1a ON t1(a);
  CREATE INDEX t1b ON t1(b);
  CREATE INDEX t1c ON t1(c);
  PRAGMA journal_mode = wal2;
  PRAGMA journal_size_limit = 15000;
  PRAGMA wal_autocheckpoint = 0;
  PRAGMA cache_size = 5;
} {wal2 15000 0}

do_execsql_test 1.1 {
  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200)
  INSERT INTO t1 SELECT random(), random(), random() FROM s;
} {}

do_test 1.2 {
  list [file size test.db] [file size test.db-wal] [file size test.db-wal2]
} {5120 23088 0}

do_execsql_test 1.3 {
  BEGIN;
    SAVEPOINT abc;
      WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100)
      INSERT INTO t1 SELECT random(), random(), random() FROM s;
    ROLLBACK TO abc;
    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10)
    INSERT INTO t1 SELECT random(), random(), random() FROM s;
  COMMIT;
  SELECT count(*) FROM t1;
  PRAGMA integrity_check;
} {210 ok}


finish_test

Changes to test/wal2simple.test.

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
...
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
...
284
285
286
287
288
289
290
291
























































































































































































292

do_execsql_test 3.0 {
  CREATE TABLE t1(x BLOB, y INTEGER PRIMARY KEY);
  CREATE INDEX i1 ON t1(x);
  PRAGMA cache_size = 5;
  PRAGMA journal_mode = wal2;
} {wal2}

breakpoint
do_test 3.1 {
  execsql BEGIN
  for {set i 1} {$i < 1000} {incr i} {
    execsql { INSERT INTO t1 VALUES(randomblob(800), $i) }
  }
  execsql COMMIT
} {}
................................................................................

do_test 6.1 {
  for {set i 0} {$i < 10} {incr i} {
    execsql "CREATE TABLE t$i (x);"
  }
} {}

puts "[file size test.db-wal] [file size test.db-wal2]"

do_test 6.2.1 {
  foreach f [glob -nocomplain test.db2*] { forcedelete $f }
  forcecopy test.db-wal2 test.db2-wal2
  sqlite3 db2 test.db2
  db2 eval { SELECT * FROM sqlite_master }
} {}
do_test 6.2.2 {
................................................................................
do_test 6.4.2 {
  db2 eval {
    PRAGMA journal_mode = wal2;
    SELECT * FROM sqlite_master;
  }
} {wal2}
db2 close

























































































































































































finish_test








<







 







<
<







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
110
111
112
113
114
115
116

117
118
119
120
121
122
123
...
238
239
240
241
242
243
244


245
246
247
248
249
250
251
...
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
do_execsql_test 3.0 {
  CREATE TABLE t1(x BLOB, y INTEGER PRIMARY KEY);
  CREATE INDEX i1 ON t1(x);
  PRAGMA cache_size = 5;
  PRAGMA journal_mode = wal2;
} {wal2}


do_test 3.1 {
  execsql BEGIN
  for {set i 1} {$i < 1000} {incr i} {
    execsql { INSERT INTO t1 VALUES(randomblob(800), $i) }
  }
  execsql COMMIT
} {}
................................................................................

do_test 6.1 {
  for {set i 0} {$i < 10} {incr i} {
    execsql "CREATE TABLE t$i (x);"
  }
} {}



do_test 6.2.1 {
  foreach f [glob -nocomplain test.db2*] { forcedelete $f }
  forcecopy test.db-wal2 test.db2-wal2
  sqlite3 db2 test.db2
  db2 eval { SELECT * FROM sqlite_master }
} {}
do_test 6.2.2 {
................................................................................
do_test 6.4.2 {
  db2 eval {
    PRAGMA journal_mode = wal2;
    SELECT * FROM sqlite_master;
  }
} {wal2}
db2 close

#-------------------------------------------------------------------------
reset_db
sqlite3 db2 test.db
do_execsql_test 7.0 {
  PRAGMA journal_size_limit = 10000;
  PRAGMA journal_mode = wal2;
  PRAGMA wal_autocheckpoint = 0;
  BEGIN;
    CREATE TABLE t1(a);
    INSERT INTO t1 VALUES( randomblob(8000) );
  COMMIT;
} {10000 wal2 0}

do_test 7.1 {
  list [file size test.db-wal] [file size test.db-wal2]
} {9464 0}

# Connection db2 is holding a PART1 lock. 
#
#   7.2.2: Test that the PART1 does not prevent db from switching to the
#          other wal file.
#
#   7.2.3: Test that the PART1 does prevent a checkpoint of test.db-wal.
#
#   7.2.4: Test that after the PART1 is released the checkpoint is possible.
#
do_test 7.2.1 {
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
} {1}
do_test 7.2.2 {
  execsql {
    INSERT INTO t1 VALUES( randomblob(800) );
    INSERT INTO t1 VALUES( randomblob(800) );
  }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {13656 3176 1024}
do_test 7.2.3 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {13656 3176 1024}
do_test 7.2.4 {
  execsql { END } db2
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {13656 3176 11264}

# Connection db2 is holding a PART2_FULL1 lock. 
#
#   7.3.2: Test that the lock does not prevent checkpointing.
#
#   7.3.3: Test that the lock does prevent the writer from overwriting 
#          test.db-wal.
#
#   7.3.4: Test that after the PART2_FULL1 is released the writer can
#          switch wal files and overwrite test.db-wal
#
db close
db2 close
sqlite3 db test.db
sqlite3 db2 test.db
do_test 7.3.1 {
  execsql {
    PRAGMA wal_autocheckpoint = 0;
    PRAGMA journal_size_limit = 10000;
    INSERT INTO t1 VALUES(randomblob(10000));
    INSERT INTO t1 VALUES(randomblob(500));
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 3176 11264}
do_test 7.3.2 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 3176 21504}
do_test 7.3.3 {
  execsql { 
    INSERT INTO t1 VALUES(randomblob(10000));
    INSERT INTO t1 VALUES(randomblob(500));
  }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 18896 21504}
do_test 7.3.4 {
  execsql END db2
  execsql { INSERT INTO t1 VALUES(randomblob(5000)); }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 18896 21504}

# Connection db2 is holding a PART2 lock. 
#
#   7.4.2: Test that the lock does not prevent writer switching to test.db-wal.
#
#   7.3.3: Test that the lock does prevent checkpointing of test.db-wal2.
#
#   7.3.4: Test that after the PART2 is released test.db-wal2 can be
#          checkpointed.
#
db close
db2 close
sqlite3 db test.db
sqlite3 db2 test.db
do_test 7.4.1 {
  execsql {
    PRAGMA wal_autocheckpoint = 0;
    PRAGMA journal_size_limit = 10000;
    INSERT INTO t1 VALUES(randomblob(10000));
    INSERT INTO t1 VALUES(randomblob(10000));
    PRAGMA wal_checkpoint;
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 44032}
do_test 7.4.2 {
  execsql { 
    INSERT INTO t1 VALUES(randomblob(5000));
  }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 44032}
do_test 7.4.3 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 44032}
do_test 7.4.4 {
  execsql END db2
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 54272}

# Connection db2 is holding a PART1_FULL2 lock. 
#
#   7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2.
#
#   7.5.3: Test that the lock does prevent the writer from overwriting
#          test.db-wal2.
#
#   7.5.4: Test that after the PART1_FULL2 lock is released, the writer
#          can switch to test.db-wal2.
#
db close
db2 close
sqlite3 db test.db
sqlite3 db2 test.db
do_test 7.5.1 {
  execsql {
    PRAGMA wal_autocheckpoint = 0;
    PRAGMA journal_size_limit = 10000;
    INSERT INTO t1 VALUES(randomblob(10000));
    INSERT INTO t1 VALUES(randomblob(10000));
    PRAGMA wal_checkpoint;
    INSERT INTO t1 VALUES(randomblob(5000));
  }
  execsql {
    BEGIN;
      SELECT count(*) FROM t1;
  } db2
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 64512}
do_test 7.5.2 {
  execsql { PRAGMA wal_checkpoint }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {12608 12608 75776}
do_test 7.5.3.1 {
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {14704 12608 75776}
do_test 7.5.3.2 {
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {22040 12608 75776}
do_test 7.5.4 {
  execsql END db2
  execsql { INSERT INTO t1 VALUES(randomblob(5000)) }
  list [file size test.db-wal] [file size test.db-wal2] [file size test.db]
} {22040 12608 75776}


finish_test

Added test/wal2snapshot.test.









































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
# 2018 December 5
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL2" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

set testprefix wal2snapshot
ifcapable !wal {finish_test ; return }
ifcapable !snapshot {finish_test; return}

foreach {tn mode} {1 wal 2 wal2} {
  reset_db
  do_execsql_test $tn.1 "PRAGMA journal_mode = $mode" $mode

  do_execsql_test $tn.2 {
    CREATE TABLE t1(a, b);
    INSERT INTO t1 VALUES(1, 2);
    INSERT INTO t1 VALUES(3, 4);
    BEGIN;
  }

  # Check that sqlite3_snapshot_get() is an error for a wal2 db.
  #
  if {$tn==1} {
    do_test 1.3 {
      set S [sqlite3_snapshot_get db main]
      sqlite3_snapshot_free $S
    } {}
  } else {
    do_test 2.3 {
      list [catch { sqlite3_snapshot_get db main } msg] $msg
    } {1 SQLITE_ERROR}
  }
  
  # Check that sqlite3_snapshot_recover() is an error for a wal2 db.
  #
  do_execsql_test $tn.4 COMMIT
  if {$tn==1} {
    do_test 1.5 {
      sqlite3_snapshot_recover db main
    } {}
  } else {
    do_test 2.5 {
      list [catch { sqlite3_snapshot_recover db main } msg] $msg
    } {1 SQLITE_ERROR}
  }
 
  # Check that sqlite3_snapshot_open() is an error for a wal2 db.
  #
  if {$tn==1} {
    do_test 1.6 {
      execsql BEGIN
      set SNAPSHOT [sqlite3_snapshot_get_blob db main]
      sqlite3_snapshot_open_blob db main $SNAPSHOT
      execsql COMMIT
    } {}
  } else {
    do_test 2.6 {
      execsql BEGIN
      set res [
        list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg
      ]
      execsql COMMIT
      set res
    } {1 SQLITE_ERROR}
  }
}


finish_test


Changes to test/window1.test.

589
590
591
592
593
594
595
596



























597

do_execsql_test 13.5 {
  SELECT a, rank() OVER(ORDER BY b) FROM t1
    INTERSECT 
  SELECT a, rank() OVER(ORDER BY b DESC) FROM t1;
} {
}




























finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624

do_execsql_test 13.5 {
  SELECT a, rank() OVER(ORDER BY b) FROM t1
    INTERSECT 
  SELECT a, rank() OVER(ORDER BY b DESC) FROM t1;
} {
}

# 2018-12-06
# https://www.sqlite.org/src/info/f09fcd17810f65f7
# Assertion fault when window functions are used.
#
# Root cause is the query flattener invoking sqlite3ExprDup() on
# expressions that contain subqueries with window functions.  The
# sqlite3ExprDup() routine is not making correctly initializing
# Select.pWin field of the subqueries.
#
sqlite3 db :memory:
do_execsql_test 14.0 {
  SELECT * FROM(
    SELECT * FROM (SELECT 1 AS c) WHERE c IN (
        SELECT (row_number() OVER()) FROM (VALUES (0))
    )
  );
} {1}
do_execsql_test 14.1 {
  CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(12345);
  CREATE TABLE t2(c); INSERT INTO t2(c) VALUES(1);
  SELECT y, y+1, y+2 FROM (
    SELECT c IN (
      SELECT (row_number() OVER()) FROM t1
    ) AS y FROM t2
  );
} {1 2 3}

finish_test

Added tool/index_usage.c.









































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
/*
** 2018-12-04
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** 
** This file implements a utility program used to help determine which
** indexes in a database schema are used and unused, and how often specific
** indexes are used.
*/
#include "sqlite3.h"
#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#include <string.h>

static void usage(const char *argv0){
  printf("Usage: %s DATABASE LOG\n\n", argv0);
  printf(
    "DATABASE is an SQLite database against which various statements\n"
    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
    "database with this schema:\n"
    "\n"
    "    CREATE TABLE sqllog(sql TEXT);\n"
    "\n"
    "This utility program analyzes statements contained in LOG and prints\n"
    "a report showing how many times each index in DATABASE is used by the\n"
    "statements in LOG.\n"
    "\n"
    "DATABASE only needs to contain the schema used by the statements in\n"
    "LOG. The content can be removed from DATABASE.\n"
  );
  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
         "checkin number %.40s. Different versions\n"
         "of SQLite might use different indexes.\n",
         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
  exit(1);
}

int main(int argc, char **argv){
  sqlite3 *db = 0;          /* The main database */
  sqlite3_stmt *pStmt = 0;  /* a query */
  char *zSql;
  int nErr = 0;
  int rc;

  if( argc!=3 ) usage(argv[0]);
  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
  if( rc ){
    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
    goto errorOut;
  }
  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
  if( rc ){
    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
           sqlite3_errmsg(db));
    goto errorOut;
  }
  sqlite3_finalize(pStmt);
  pStmt = 0;
  rc = sqlite3_exec(db, 
     "CREATE TABLE temp.idxu(\n"
     "  tbl TEXT,\n"
     "  idx TEXT,\n"
     "  cnt INT,\n"
     "  PRIMARY KEY(idx)\n"
     ") WITHOUT ROWID;", 0, 0, 0);
  if( rc ){
    printf("Cannot create the result table - %s\n",
           sqlite3_errmsg(db));
    goto errorOut;
  }
  rc = sqlite3_exec(db,
     "INSERT INTO temp.idxu(tbl,idx,cnt)"
     " SELECT tbl_name, name, 0 FROM sqlite_master"
     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);

  /* Open the LOG database */
  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
  rc = sqlite3_exec(db, zSql, 0, 0, 0);
  sqlite3_free(zSql);
  if( rc ){
    printf("Cannot open the LOG database \"%s\" - %s\n",
           argv[2], sqlite3_errmsg(db));
    goto errorOut;
  }
  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
                          -1, &pStmt, 0);
  if( rc ){
    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
           argv[2], sqlite3_errmsg(db));
    goto errorOut;
  }

  /* Update the counts based on LOG */
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
    sqlite3_stmt *pS2;
    if( zLog==0 ) continue;
    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
    sqlite3_free(zSql);
    if( rc ){
      printf("Cannot compile LOG entry %d (%s): %s\n",
             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
      nErr++;
    }else{
      while( sqlite3_step(pS2)==SQLITE_ROW ){
        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
        const char *z1, *z2;
        int n;
        /* printf("EXPLAIN: %s\n", zExplain); */
        z1 = strstr(zExplain, " USING INDEX ");
        if( z1==0 ) continue;
        z1 += 13;
        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
        n = z2 - z1;
        zSql = sqlite3_mprintf(
          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
        );
        /* printf("sql: %s\n", zSql); */
        sqlite3_exec(db, zSql, 0, 0, 0);
        sqlite3_free(zSql);
      }
    }
    sqlite3_finalize(pS2);
  }
  sqlite3_finalize(pStmt);

  /* Generate the report */
  rc = sqlite3_prepare_v2(db,
     "SELECT tbl, idx, cnt, "
     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
     " FROM temp.idxu, main.sqlite_master"
     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
     "   AND temp.idxu.idx=main.sqlite_master.name"
     " ORDER BY cnt DESC, tbl, idx",
     -1, &pStmt, 0);
  if( rc ){
    printf("Cannot query the result table - %s\n",
           sqlite3_errmsg(db));
    goto errorOut;
  }
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    printf("%10d %s on %s(%s)\n", 
       sqlite3_column_int(pStmt, 2),
       sqlite3_column_text(pStmt, 1),
       sqlite3_column_text(pStmt, 0),
       sqlite3_column_text(pStmt, 3));
  }
  sqlite3_finalize(pStmt);
  pStmt = 0;

errorOut:
  sqlite3_finalize(pStmt);
  sqlite3_close(db);
  return nErr;
}

Changes to tool/lemon.c.

4586
4587
4588
4589
4590
4591
4592
4593

4594
4595
4596
4597
4598
4599






4600
4601
4602
4603
4604
4605
4606
  }
  tplt_xfer(lemp->name,in,out,&lineno);

  /* Generate code which executes whenever the parser stack overflows */
  tplt_print(out,lemp,lemp->overflow,&lineno);
  tplt_xfer(lemp->name,in,out,&lineno);

  /* Generate the table of rule information

  **
  ** Note: This code depends on the fact that rules are number
  ** sequentually beginning with 0.
  */
  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);






    rule_print(out, rp);
    fprintf(out," */\n"); lineno++;
  }
  tplt_xfer(lemp->name,in,out,&lineno);

  /* Generate code which execution during each REDUCE action */
  i = 0;







|
>





|
>
>
>
>
>
>







4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
  }
  tplt_xfer(lemp->name,in,out,&lineno);

  /* Generate code which executes whenever the parser stack overflows */
  tplt_print(out,lemp,lemp->overflow,&lineno);
  tplt_xfer(lemp->name,in,out,&lineno);

  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
  ** yyRuleInfoNRhs[].
  **
  ** Note: This code depends on the fact that rules are number
  ** sequentually beginning with 0.
  */
  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
     rule_print(out, rp);
    fprintf(out," */\n"); lineno++;
  }
  tplt_xfer(lemp->name,in,out,&lineno);
  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
    rule_print(out, rp);
    fprintf(out," */\n"); lineno++;
  }
  tplt_xfer(lemp->name,in,out,&lineno);

  /* Generate code which execution during each REDUCE action */
  i = 0;

Changes to tool/lempar.c.

682
683
684
685
686
687
688
689
690
691
692
693
694
695


696
697
698
699
700
701
702
...
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
...
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
...
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
  yytos = yypParser->yytos;
  yytos->stateno = yyNewState;
  yytos->major = yyMajor;
  yytos->minor.yy0 = yyMinor;
  yyTraceShift(yypParser, yyNewState, "Shift");
}

/* The following table contains information about every rule that
** is used during the reduce.
*/
static const struct {
  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
} yyRuleInfo[] = {


%%
};

static void yy_accept(yyParser*);  /* Forward Declaration */

/*
** Perform a reduce action and the shift that must immediately
................................................................................
  int yysize;                     /* Amount to pop the stack */
  ParseARG_FETCH
  (void)yyLookahead;
  (void)yyLookaheadToken;
  yymsp = yypParser->yytos;
#ifndef NDEBUG
  if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
    yysize = yyRuleInfo[yyruleno].nrhs;
    if( yysize ){
      fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
        yyTracePrompt,
        yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
    }else{
      fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
        yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
................................................................................
    }
  }
#endif /* NDEBUG */

  /* Check that the stack is large enough to grow by a single entry
  ** if the RHS of the rule is empty.  This ensures that there is room
  ** enough on the stack to push the LHS value */
  if( yyRuleInfo[yyruleno].nrhs==0 ){
#ifdef YYTRACKMAXSTACKDEPTH
    if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
      yypParser->yyhwm++;
      assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
    }
#endif
#if YYSTACKDEPTH>0 
................................................................................
  **  #line <lineno> <thisfile>
  **     break;
  */
/********** Begin reduce actions **********************************************/
%%
/********** End reduce actions ************************************************/
  };
  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
  yygoto = yyRuleInfo[yyruleno].lhs;
  yysize = yyRuleInfo[yyruleno].nrhs;
  yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);

  /* There are no SHIFTREDUCE actions on nonterminals because the table
  ** generator has simplified them to pure REDUCE actions. */
  assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );

  /* It is not possible for a REDUCE to be followed by an error */







|
|
|
|
|
|
|
>
>







 







|







 







|







 







|
|
|







682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
...
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
...
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
...
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
  yytos = yypParser->yytos;
  yytos->stateno = yyNewState;
  yytos->major = yyMajor;
  yytos->minor.yy0 = yyMinor;
  yyTraceShift(yypParser, yyNewState, "Shift");
}

/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
** of that rule */
static const YYCODETYPE yyRuleInfoLhs[] = {
%%
};

/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
** of symbols on the right-hand side of that rule. */
static const signed char yyRuleInfoNRhs[] = {
%%
};

static void yy_accept(yyParser*);  /* Forward Declaration */

/*
** Perform a reduce action and the shift that must immediately
................................................................................
  int yysize;                     /* Amount to pop the stack */
  ParseARG_FETCH
  (void)yyLookahead;
  (void)yyLookaheadToken;
  yymsp = yypParser->yytos;
#ifndef NDEBUG
  if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
    yysize = yyRuleInfoNRhs[yyruleno];
    if( yysize ){
      fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
        yyTracePrompt,
        yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
    }else{
      fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
        yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
................................................................................
    }
  }
#endif /* NDEBUG */

  /* Check that the stack is large enough to grow by a single entry
  ** if the RHS of the rule is empty.  This ensures that there is room
  ** enough on the stack to push the LHS value */
  if( yyRuleInfoNRhs[yyruleno]==0 ){
#ifdef YYTRACKMAXSTACKDEPTH
    if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
      yypParser->yyhwm++;
      assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
    }
#endif
#if YYSTACKDEPTH>0 
................................................................................
  **  #line <lineno> <thisfile>
  **     break;
  */
/********** Begin reduce actions **********************************************/
%%
/********** End reduce actions ************************************************/
  };
  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
  yygoto = yyRuleInfoLhs[yyruleno];
  yysize = yyRuleInfoNRhs[yyruleno];
  yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);

  /* There are no SHIFTREDUCE actions on nonterminals because the table
  ** generator has simplified them to pure REDUCE actions. */
  assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );

  /* It is not possible for a REDUCE to be followed by an error */