Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -2673,11 +2673,18 @@ srctree-check: $(TOP)\tool\srctree-check.tcl $(TCLSH_CMD) $(TOP)\tool\srctree-check.tcl # Testing for a release # -releasetest: +releasetest: verify-source + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release + +# xdevtest is like releasetest, except that it skips the +# dependency on verify-source so that xdevtest can be run from +# a modified source tree. +# +xdevtest: $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.50.0 +3.51.0 Index: autoconf/Makefile.in ================================================================== --- autoconf/Makefile.in +++ autoconf/Makefile.in @@ -222,11 +222,11 @@ # Flags to link the shell app either directly against sqlite3.c # (ENABLE_STATIC_SHELL==1) or libsqlite3.so (ENABLE_STATIC_SHELL==0). # ENABLE_STATIC_SHELL = @ENABLE_STATIC_SHELL@ sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS.libsqlite3) -sqlite3-shell-link-flags.0 = -L. -lsqlite3 $(LDFLAGS.zlib) +sqlite3-shell-link-flags.0 = -L. -lsqlite3 $(LDFLAGS.zlib) $(LDFLAGS.math) sqlite3-shell-deps.1 = $(TOP)/sqlite3.c sqlite3-shell-deps.0 = $(libsqlite3.DLL) # # STATIC_CLI_SHELL = 1 to statically link sqlite3$(T.exe), else # 0. Requires static versions of all requisite libraries. Primarily Index: autoconf/tea/Makefile.in ================================================================== --- autoconf/tea/Makefile.in +++ autoconf/tea/Makefile.in @@ -144,11 +144,11 @@ # tx.src is the list of source or object files to include in the # (single) compiler/linker invocation. This will initially contain any # sources passed to [teaish-src-add], but may also be appended to by # teaish.make. # -tx.src =@TEAISH_EXT_SRC@ +tx.src = @TEAISH_EXT_SRC@ # # tx.CFLAGS is typically set by teaish.make, whereas TEAISH_CFLAGS # gets set up via the configure script. # @@ -165,10 +165,15 @@ # It can also be set up from teaish.tcl using [teaish-dist-add] # and/or [teaish-src-add -dist ...]. # tx.dist.files = @TEAISH_DIST_FILES@ +# +# The base name for a distribution tar/zip file. +# +tx.dist.basename = $(tx.name.dist)-$(tx.version) + # List of deps which may trigger an auto-reconfigure. # teaish__autogen.deps = \ $(tx.makefile.in) $(teaish.makefile.in) \ $(tx.tcl) \ @@ -197,13 +202,18 @@ $(teaish.makefile): $(teaish__auto.def) $(teaish.makefile.in) \ @AUTODEPS@ @if TEAISH_TESTER_TCL_IN -@TEAISH_TESTER_TCL_IN@: -@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ -config.log: @TEAISH_TESTER_TCL@ +@TEAISH_TESTER_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TESTER_TCL_IN@ +@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ +@endif +@if TEAISH_TEST_TCL_IN +@TEAISH_TEST_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TEST_TCL_IN@ +@TEAISH_TEST_TCL@: @TEAISH_TEST_TCL_IN@ @endif # # CC variant for compiling Tcl-using sources. # @@ -215,11 +225,11 @@ # $(tx.src) must come before $(LDFLAGS...) for linking to third-party # libs to work. # CC.dll = \ $(CC.tcl) $(tx.src) $(LDFLAGS.shlib) \ - $(LDFLAGS.configure) $(LDFLAGS) $(tx.LDFLAGS) $(TCL_STUB_LIB_SPEC) + $(tx.LDFLAGS) $(LDFLAGS.configure) $(LDFLAGS) $(TCL_STUB_LIB_SPEC) @if TEAISH_ENABLE_DLL # # The rest of this makefile exists solely to support this brief # target: the extension shared lib. @@ -246,20 +256,29 @@ test-extension: # this name is reserved for use by teaish.make[.in] @if TEAISH_ENABLE_DLL test-prepre: $(tx.dll) @endif @if TEAISH_TESTER_TCL -test-core.args = @TEAISH_TESTER_TCL@ +teaish.tester.tcl = @TEAISH_TESTER_TCL@ +test-core.args = $(teaish.tester.tcl) @if TEAISH_ENABLE_DLL test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' @else test-core.args += '' '' @endif test-core.args += @TEAISH_TESTUTIL_TCL@ +# Clients may pass additional args via test.args=... +# and ::argv will be rewritten before the test script loads, to +# remove $(test-core.args) +test.args ?= test-core: test-pre - $(TCLSH) $(test-core.args) -test-prepre: @TEAISH_TESTER_TCL@ + $(TCLSH) $(test-core.args) $(test.args) +test-gdb: $(teaish.tester.tcl) + gdb --args $(TCLSH) $(test-core.args) $(test.args) +test-vg.flags ?= --leak-check=full -v --show-reachable=yes --track-origins=yes +test-vg: $(teaish.tester.tcl) + valgrind $(test-vg.flags) $(TCLSH) $(test-core.args) $(test.args) @else # !TEAISH_TESTER_TCL test-prepre: @endif # TEAISH_TESTER_TCL test-pre: test-prepre test-core: test-pre @@ -286,11 +305,11 @@ @if TEAISH_MAKEFILE rm -f @TEAISH_MAKEFILE@ @endif @endif @if TEAISH_TESTER_TCL_IN - rm -f @TEAISH_TESTER_TCL@ + rm -f $(teaish.tester.tcl) @endif @if TEAISH_PKGINDEX_TCL_IN rm -f @TEAISH_PKGINDEX_TCL@ @endif @if TEAISH_PKGINIT_TCL_IN @@ -353,14 +372,19 @@ fi $(INSTALL.noexec) "@TEAISH_TM_TCL@" "$(install-core.tmdir)/$(tx.tm.tgt)" @endif install-test: install-core @echo "Post-install test of [package require $(tx.name.pkg) $(tx.version)]..."; \ + set xtra=""; \ + if [ x != "x$(DESTDIR)" ]; then \ + xtra='set ::auto_path [linsert $$::auto_path 0 [file normalize $(DESTDIR)$(TCLLIBDIR)/..]];'; \ + fi; \ if echo \ - 'set c 0; ' \ + 'set c 0; ' $$xtra \ '@TEAISH_POSTINST_PREREQUIRE@' \ - 'if {[catch {package require $(tx.name.pkg) $(tx.version)}]} {incr c};' \ + 'if {[catch {package require $(tx.name.pkg) $(tx.version)} xc]} {incr c};' \ + 'if {$$c && "" ne $$xc} {puts $$xc; puts "auto_path=$$::auto_path"};' \ 'exit $$c' \ | $(TCLSH) ; then \ echo "passed"; \ else \ echo "FAILED"; \ @@ -404,36 +428,35 @@ # configure with similar flags to what we last configured with but we # must not pass on any extension-specific flags, as those won't be # recognized when running in --teaish-install mode, causing # the sub-configure to fail. dist.flags = --with-tclsh=$(TCLSH) -dist.reconfig = $(teaish.dir)/configure $(dist.flags) +dist.reconfig = $(teaish.dir)/configure $(tx.dist.reconfig-flags) $(dist.flags) # Temp dir for dist.zip. Must be different than dist.tgz or else # parallel builds may hose the dist. teaish__dist.tmp.zip = teaish__dist_zip # # Make a distribution zip file... # -dist.basename = $(tx.name.dist)-$(tx.version) -dist.zip = $(dist.basename).zip +dist.zip = $(tx.dist.basename).zip .PHONY: dist.zip dist.zip-core dist.zip-post #dist.zip-pre: # We apparently can't add a pre-hook here, else "make dist" rebuilds # the archive each time it's run. $(dist.zip): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.zip) - @mkdir -p $(teaish__dist.tmp.zip)/$(dist.basename) + @mkdir -p $(teaish__dist.tmp.zip)/$(tx.dist.basename) @tar cf $(teaish__dist.tmp.zip)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(dist.basename) + @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(tx.dist.basename) @if TEAISH_DIST_FULL @$(dist.reconfig) \ - --teaish-install=$(teaish__dist.tmp.zip)/$(dist.basename) \ - --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null + --teaish-install=$(teaish__dist.tmp.zip)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null @endif - @rm -f $(dist.basename)/tmp.tar $(dist.zip) - @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(dist.basename) + @rm -f $(tx.dist.basename)/tmp.tar $(dist.zip) + @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(tx.dist.basename) @rm -fr $(teaish__dist.tmp.zip) @ls -la $(dist.zip) dist.zip-core: $(dist.zip) dist.zip-post: dist.zip-core dist.zip: dist.zip-post @@ -445,27 +468,27 @@ # # Make a distribution tarball... # teaish__dist.tmp.tgz = teaish__dist_tgz -dist.tgz = $(dist.basename).tar.gz +dist.tgz = $(tx.dist.basename).tar.gz .PHONY: dist.tgz dist.tgz-core dist.tgz-post # dist.tgz-pre: # see notes in dist.zip $(dist.tgz): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.tgz) - @mkdir -p $(teaish__dist.tmp.tgz)/$(dist.basename) + @mkdir -p $(teaish__dist.tmp.tgz)/$(tx.dist.basename) @tar cf $(teaish__dist.tmp.tgz)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(dist.basename) + @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(tx.dist.basename) @if TEAISH_DIST_FULL - @rm -f $(teaish__dist.tmp.tgz)/$(dist.basename)/pkgIndex.tcl.in; # kludge + @rm -f $(teaish__dist.tmp.tgz)/$(tx.dist.basename)/pkgIndex.tcl.in; # kludge @$(dist.reconfig) \ - --teaish-install=$(teaish__dist.tmp.tgz)/$(dist.basename) \ - --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null + --teaish-install=$(teaish__dist.tmp.tgz)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null @endif - @rm -f $(dist.basename)/tmp.tar $(dist.tgz) - @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(dist.basename) + @rm -f $(tx.dist.basename)/tmp.tar $(dist.tgz) + @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(tx.dist.basename) @rm -fr $(teaish__dist.tmp.tgz) @ls -la $(dist.tgz) dist.tgz-core: $(dist.tgz) dist.tgz-post: dist.tgz-core dist.tgz: dist.tgz-post Index: autoconf/tea/README.txt ================================================================== --- autoconf/tea/README.txt +++ autoconf/tea/README.txt @@ -38,29 +38,37 @@ of the tclsh that you want the SQLite extension to work with. See step-by-step instructions at the links below for more information: https://sqlite.org/src/doc/trunk/doc/compile-for-unix.md https://sqlite.org/src/doc/trunk/doc/compile-for-windows.md + +And info about the extension's Tcl interface can be found at: + + https://sqlite.org/tclsqlite.html The whole point of the amalgamation-autoconf tarball (in which this -README.txt file is embedded) is to provide a means of compiling -SQLite that does not require first installing TCL and/or "tclsh". -The canonical Makefile in the SQLite source tree provides more -capabilities (such as the the ability to run test cases to ensure -that the build worked) and is better maintained. The only -downside of the canonical Makefile is that it requires a TCL -installation. But if you are wanting to build the TCL extension for -SQLite, then presumably you already have a TCL installation. So why -not just use the more-capable and better-maintained canoncal Makefile? - -This TEA builder is derived from code found at +README.txt file is embedded) is to provide a means of compiling SQLite +that does not require first installing TCL and/or "tclsh". The +canonical Makefile in the SQLite source tree provides more +capabilities (such as the the ability to run test cases to ensure that +the build worked) and is better maintained. The only downside of the +canonical Makefile is that it requires a TCL installation. But if you +are wanting to build the TCL extension for SQLite, then presumably you +already have a TCL installation. So why not just use the more-capable +and better-maintained canoncal Makefile? + +As of version 3.50.0, this build process uses "teaish": + + https://fossil.wanderinghorse.net/r/teaish + +which is conceptually derived from the pre-3.50 toolchain, TEA: http://core.tcl-lang.org/tclconfig http://core.tcl-lang.org/sampleextension -The SQLite developers do not understand how it works. It seems to -work for us. It might also work for you. But we cannot promise that. +It to works for us. It might also work for you. But we cannot +promise that. If you want to use this TEA builder and it works for you, that's fine. But if you have trouble, the first thing you should do is go back to using the canonical Makefile in the SQLite source tree. @@ -68,32 +76,19 @@ UNIX BUILD ========== -Building under most UNIX systems is easy, just run the configure script -and then run make. For more information about the build process, see -the tcl/unix/README file in the Tcl src dist. The following minimal -example will install the extension in the /opt/tcl directory. +Building under most UNIX systems is easy, just run the configure +script and then run make. For example: $ cd sqlite-*-tea - $ ./configure --prefix=/opt/tcl - $ make + $ ./configure --with-tcl=/path/to/tcl/install/root + $ make test $ make install WINDOWS BUILD ============= -The recommended method to build extensions under windows is to use the -Msys + Mingw build process. This provides a Unix-style build while -generating native Windows binaries. Using the Msys + Mingw build tools -means that you can use the same configure script as per the Unix build -to create a Makefile. See the tcl/win/README file for the URL of -the Msys + Mingw download. - -If you have VC++ then you may wish to use the files in the win -subdirectory and build the extension using just VC++. These files have -been designed to be as generic as possible but will require some -additional maintenance by the project developer to synchronise with -the TEA configure.in and Makefile.in files. Instructions for using the -VC++ makefile are written in the first part of the Makefile.vc -file. +On Windows this build is known to work on Cygwin and some Msys2 +environments. We do not currently support Microsoft makefiles for +native Windows builds. Index: autoconf/tea/_teaish.tester.tcl.in ================================================================== --- autoconf/tea/_teaish.tester.tcl.in +++ autoconf/tea/_teaish.tester.tcl.in @@ -19,11 +19,12 @@ if {[llength [lindex $::argv 0]] > 0} { load [file normalize [lindex $::argv 0]] [lindex $::argv 1]; # ----^^^^^^^ needed on Haiku when argv 0 is just a filename, else # load cannot find the file. } -source -encoding utf-8 [lindex $::argv 2]; # teaish/tester.tcl +set ::argv [lassign $argv - -] +source -encoding utf-8 [lindex $::argv 0]; # teaish/tester.tcl @if TEAISH_PKGINIT_TCL apply {{file} { set dir [file dirname $::argv0] source -encoding utf-8 $file }} [join {@TEAISH_PKGINIT_TCL@}] DELETED autoconf/tea/doc/sqlite3.n Index: autoconf/tea/doc/sqlite3.n ================================================================== --- autoconf/tea/doc/sqlite3.n +++ /dev/null @@ -1,15 +0,0 @@ -.TH sqlite3 n 4.1 "Tcl-Extensions" -.HS sqlite3 tcl -.BS -.SH NAME -sqlite3 \- an interface to the SQLite3 database engine -.SH SYNOPSIS -\fBsqlite3\fI command_name ?filename?\fR -.br -.SH DESCRIPTION -SQLite3 is a self-contains, zero-configuration, transactional SQL database -engine. This extension provides an easy to use interface for accessing -SQLite database files from Tcl. -.PP -For full documentation see \fIhttps://sqlite.org/\fR and -in particular \fIhttps://sqlite.org/tclsqlite.html\fR. Index: autoconf/tea/teaish.tcl ================================================================== --- autoconf/tea/teaish.tcl +++ autoconf/tea/teaish.tcl @@ -65,10 +65,11 @@ -version $version -name.dist $distname -vsatisfies 8.6- -libDir sqlite$version -pragmas $pragmas + -src generic/tclsqlite3.c } }} [teaish-get -dir] # # Must return either an empty string or a list in the form accepted by @@ -117,12 +118,10 @@ # work needed for this extension. # proc teaish-configure {} { use teaish/feature - teaish-src-add -dist -dir generic/tclsqlite3.c - if {[proj-opt-was-provided override-sqlite-version]} { teaish-pkginfo-set -version [opt-val override-sqlite-version] proj-warn "overriding sqlite version number:" [teaish-pkginfo-get -version] } elseif {[proj-opt-was-provided with-system-sqlite] && [opt-val with-system-sqlite] ne "0"} { Index: autosetup/proj.tcl ================================================================== --- autosetup/proj.tcl +++ autosetup/proj.tcl @@ -58,14 +58,15 @@ # # $proj__Config is an internal-use-only array for storing whatever generic # internal stuff we need stored. # -array set ::proj__Config { - self-tests 1 -} - +array set ::proj__Config [subst { + self-tests [get-env proj.self-tests 0] + verbose-assert [get-env proj.assert-verbose 0] + isatty [isatty? stdout] +}] # # List of dot-in files to filter in the final stages of # configuration. Some configuration steps may append to this. Each # one in this list which exists will trigger the generation of a @@ -73,41 +74,41 @@ # (which differ from the source dir in out-of-tree builds). # # See: proj-dot-ins-append and proj-dot-ins-process # set ::proj__Config(dot-in-files) [list] -set ::proj__Config(isatty) [isatty? stdout] # # @proj-warn msg # # Emits a warning message to stderr. All args are appended with a # space between each. # proc proj-warn {args} { show-notices - puts stderr [join [list "WARNING: \[[proj-scope 1]\]: " {*}$args] " "] + puts stderr [join [list "WARNING:" \[ [proj-scope 1] \]: {*}$args] " "] } +# # Internal impl of [proj-fatal] and [proj-error]. It must be called # using tailcall. -proc proj__faterr {failMode argv} { +# +proc proj__faterr {failMode args} { show-notices set lvl 1 - while {"-up" eq [lindex $argv 0]} { - set argv [lassign $argv -] + while {"-up" eq [lindex $args 0]} { + set args [lassign $args -] incr lvl } if {$failMode} { - puts stderr [join [list "FATAL: \[[proj-scope $lvl]]: " {*}$argv]] + puts stderr [join [list "FATAL:" \[ [proj-scope $lvl] \]: {*}$args]] exit 1 } else { - error [join [list "\[[proj-scope $lvl]]:" {*}$argv]] + error [join [list in \[ [proj-scope $lvl] \]: {*}$args]] } } - # # @proj-fatal ?-up...? msg... # # Emits an error message to stderr and exits with non-0. All args are @@ -116,23 +117,22 @@ # The calling scope's name is used in the error message. To instead # use the name of a call higher up in the stack, use -up once for each # additional level. # proc proj-fatal {args} { - tailcall proj__faterr 1 $args + tailcall proj__faterr 1 {*}$args } # # @proj-error ?-up...? msg... # # Works like proj-fatal but uses [error] intead of [exit]. # proc proj-error {args} { - tailcall proj__faterr 0 $args + tailcall proj__faterr 0 {*}$args } -set ::proj__Config(verbose-assert) [get-env proj-assert-verbose 0] # # @proj-assert script ?message? # # Kind of like a C assert: if uplevel of [list expr $script] is false, # a fatal error is triggered. The error message, by default, includes @@ -145,11 +145,11 @@ } if {![uplevel 1 [list expr $script]]} { if {"" eq $msg} { set msg $script } - proj-fatal "Assertion failed in \[[proj-scope 1]\]: $msg" + tailcall proj__faterr 1 "Assertion failed:" $msg } } # # @proj-bold str @@ -1663,11 +1663,11 @@ } default { proj-fatal "Too many arguments: $fileIn $args" } } - #puts "******* [proj-scope]: adding $fileIn" + #puts "******* [proj-scope]: adding [llength $fileIn]-length item: $fileIn" lappend ::proj__Config(dot-in-files) $fileIn } # # @proj-dot-ins-list @@ -1701,21 +1701,22 @@ # # -clear: after processing, empty the dot-ins list. This effectively # makes proj-dot-ins-append available for re-use. # proc proj-dot-ins-process {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -touch "" {return "-touch"} -clear 0 {expr 1} -validate 0 {expr 1} } + #puts "args=$args"; parray flags if {[llength $args] > 0} { error "Invalid argument to [proj-scope]: $args" } foreach f $::proj__Config(dot-in-files) { proj-assert {3==[llength $f]} \ - "Expecting proj-dot-ins-list to be stored in 3-entry lists" + "Expecting proj-dot-ins-list to be stored in 3-entry lists. Got: $f" lassign $f fIn fOut fScript #puts "DOING $fIn ==> $fOut" proj-make-from-dot-in {*}$flags(-touch) $fIn $fOut if {$flags(-validate)} { proj-validate-no-unresolved-ats $fOut @@ -1751,11 +1752,11 @@ foreach f $args { set lnno 1 set isMake [string match {*[Mm]ake*} $f] foreach line [proj-file-content-list $f] { if {!$isMake || ![string match "#*" [string trimleft $line]]} { - if {[regexp {(@[A-Za-z0-9_]+@)} $line match]} { + if {[regexp {(@[A-Za-z0-9_\.]+@)} $line match]} { error "Unresolved reference to $match at line $lnno of $f" } } incr lnno } @@ -1891,11 +1892,11 @@ # but if passed the -list flag it will return a list of the # individual CFLAGS. # proc proj-define-to-cflag {args} { set rv {} - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -list 0 {expr 1} -quote 0 {expr 1} -zero-undef 0 {expr 1} } foreach d $args { @@ -1999,11 +2000,11 @@ # Sets a feature-check cache entry with the given key. # # See proj-cache-key for -key's and -level's semantics, noting that # this function adds one to -level for purposes of that call. proc proj-cache-set {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -key => 0 -level => 0 } lassign $args val set key [proj-cache-key $flags(-key) [expr {1 + $flags(-level)}]] @@ -2035,11 +2036,11 @@ # "" and returns 0. # # See proj-cache-key for $key's and $addLevel's semantics, noting that # this function adds one to $addLevel for purposes of that call. proc proj-cache-check {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -key => 0 -level => 0 } lassign $args tgtVar upvar $tgtVar tgt @@ -2068,151 +2069,325 @@ } return "" } # -# @proj-parse-simple-flags ... +# @proj-parse-flags argvListName targetArrayName {prototype} # # A helper to parse flags from proc argument lists. # -# Expects a list of arguments to parse, an array name to store any -# -flag values to, and a prototype object which declares the flags. -# -# The prototype must be a list in one of the following forms: -# -# -flag defaultValue {script} -# -# -flag => defaultValue -# -----^--^ (with spaces there!) -# -# Repeated for each flag. -# -# The first form represents a basic flag with no associated -# following argument. The second form extracts its value -# from the following argument in $argvName. -# -# The first argument to this function is the name of a var holding the -# args to parse. It will be overwritten, possibly with a smaller list. -# -# The second argument the name of an array variable to create in the -# caller's scope. (Pneumonic: => points to the next argument.) -# -# For the first form of flag, $script is run in the caller's scope if -# $argv contains -flag, and the result of that script is the new value -# for $tgtArrayName(-flag). This function intercepts [return $val] -# from $script. Any empty script will result in the flag having "" -# assigned to it. -# -# The args list is only inspected until the first argument which is -# not described by $prototype. i.e. the first "non-flag" (not counting -# values consumed for flags defined like --flag=>default). +# The first argument is the name of a var holding the args to +# parse. It will be overwritten, possibly with a smaller list. +# +# The second argument is the name of an array variable to create in +# the caller's scope. +# +# The third argument, $prototype, is a description of how to handle +# the flags. Each entry in that list must be in one of the +# following forms: +# +# -flag defaultValue ?-literal|-call|-apply? +# script|number|incr|proc-name|{apply $aLambda} +# +# -flag* ...as above... +# +# -flag => defaultValue ?-call proc-name-and-args|-apply lambdaExpr? +# +# -flag* => ...as above... +# +# :PRAGMA +# +# The first two forms represents a basic flag with no associated +# following argument. The third and fourth forms, called arg-consuming +# flags, extract the value from the following argument in $argvName +# (pneumonic: => points to the next argument.). The :PRAGMA form +# offers a way to configure certain aspects of this call. +# +# If $argv contains any given flag from $prototype, its default value +# is overridden depending on several factors: +# +# - If the -literal flag is used, or the flag's script is a number, +# value is used verbatim. +# +# - Else if the -call flag is used, the argument must be a proc name +# and any leading arguments, e.g. {apply $myLambda}. The proc is passed +# the (flag, value) as arguments (non-consuming flags will get +# passed the flag's current/starting value and consuming flags will +# get the next argument). Its result becomes the result of the +# flag. +# +# - Else if -apply X is used, it's effectively shorthand for -call +# {apply X}. Its argument may either be a $lambaRef or a {{f v} +# {body}} construct. +# +# - Else if $script is one of the following values, it is treated as +# the result of... +# +# - incr: increments the current value of the flag. +# +# - Else $script is eval'd to get its result value. That result +# becomes the new flag value for $tgtArrayName(-flag). This +# function intercepts [return $val] from eval'ing $script. Any +# empty script will result in the flag having "" assigned to it. +# +# Unless the -flag has a trailing asterisk, e.g. -flag*, this function +# assumes that each flag is unique, and using a flag more than once +# causes an error to be triggered. the -flag* forms works similarly +# except that may appear in $argv any number of times: +# +# - For non-arg-consuming flags, each invocation of -flag causes the +# result of $script to overwrite the previous value. e.g. so +# {-flag* {x} {incr foo}} has a default value of x, but passing in +# -flag twice would change it to the result of incrementing foo +# twice. This form can be used to implement, e.g., increasing +# verbosity levels by passing -verbose multiple times. +# +# - For arg-consuming flags, the given flag starts with value X, but +# if the flag is provided in $argv, the default is cleared, then +# each instance of -flag causes its value to be appended to the +# result, so {-flag* => {a b c}} defaults to {a b c}, but passing +# in -flag y -flag z would change it to {y z}, not {a b c y z}.. +# +# By default, the args list is only inspected until the first argument +# which is not described by $prototype. i.e. the first "non-flag" (not +# counting values consumed for flags defined like -flag => default). +# The :all-flags pragma (see below) can modify this behavior. # # If a "--" flag is encountered, no more arguments are inspected as -# flags. If "--" is the first non-flag argument, the "--" flag is -# removed from the results but all remaining arguments are passed -# through. If "--" appears after the first non-flag, it is retained. -# -# This function assumes that each flag is unique, and using a flag -# more than once behaves in a last-one-wins fashion. -# -# Any argvName entries not described in $prototype are not treated as -# flags. -# -# Returns the number of flags it processed in $argvName. +# flags unless the :all-flags pragma (see below) is in effect. The +# first instance of "--" is removed from the target result list but +# all remaining instances of "--" are are passed through. +# +# Any argvName entries not described in $prototype are considered to +# be "non-flags" for purposes of this function, even if they +# ostensibly look like flags. +# +# Returns the number of flags it processed in $argvName, not counting +# "--". # # Example: # -# set args [list -foo -bar {blah} 8 9 10 -theEnd] -# proj-parse-simple-flags args flags { -# -foo 0 {expr 1} -# -bar => 0 -# -no-baz 2 {return 0} -# } -# -# After that $flags would contain {-foo 1 -bar {blah} -no-baz 2} +## set args [list -foo -bar {blah} -z 8 9 10 -theEnd] +## proj-parse-flags args flags { +## -foo 0 {expr 1} +## -bar => 0 +## -no-baz 1 {return 0} +## -z 0 2 +## } +# +# After that $flags would contain {-foo 1 -bar {blah} -no-baz 1 -z 2} # and $args would be {8 9 10 -theEnd}. # -# Potential TODOs: consider using lappend instead of set so that any -# given flag can be used more than once. Or add a syntax to indicate -# that multiples are allowed. Also consider searching the whole -# argv list, rather than stopping at the first non-flag +# Pragmas: +# +# Passing :PRAGMAS to this function may modify how it works. The +# following pragmas are supported (note the leading ":"): +# +# :all-flags indicates that the whole input list should be scanned, +# not stopping at the first non-flag or "--". # -proc proj-parse-simple-flags {argvName tgtArrayName prototype} { +proc proj-parse-flags {argvName tgtArrayName prototype} { upvar $argvName argv - upvar $tgtArrayName tgt - array set dflt {} - array set scripts {} - array set consuming {} + upvar $tgtArrayName outFlags + array set flags {}; # staging area + array set scripts {}; # map of -flag=>script + array set consuming {}; # map of -flag=>1 for arg-consuming flags + array set multi {}; # map of -flag=>1 for multi-time flags + array set seen {}; # map of -flag=>number of times seen + array set call {}; # map of -flag=>1 for -call entries + set incrSkip 1; # 1 if we stop at the first non-flag, else 0 + # Parse $prototype for flag definitions... set n [llength $prototype] - # Figure out what our flags are... + set checkProtoFlag { + #puts "**** checkProtoFlag #$i of $n k=$k fv=$fv" + switch -exact -- $fv { + -literal { + proj-assert {![info exists consuming($k)]} + set scripts($k) [list expr [lindex $prototype [incr i]]] + } + -apply { + set fv [lindex $prototype [incr i]] + if {2 == [llength $fv]} { + # Treat this as a lambda literal + set fv [list $fv] + } + lappend call($k) "apply $fv" + } + -call { + # arg is either a proc name or {apply $aLambda} + set fv [lindex $prototype [incr i]] + lappend call($k) $fv + } + default { + proj-assert {![info exists consuming($k)]} + set scripts($k) $fv + } + } + if {$i >= $n} { + proj-error -up "[proj-scope]: Missing argument for $k flag" + } + } for {set i 0} {$i < $n} {incr i} { set k [lindex $prototype $i] #puts "**** #$i of $n k=$k" + + # Check for :PRAGMA... + switch -exact -- $k { + :all-flags { + set incrSkip 0 + continue + } + } + proj-assert {[string match -* $k]} \ - "Invalid flag value: $k" - set v "" - set s "" + "Invalid argument: $k" + + if {[string match {*\*} $k]} { + # Re-map -foo* to -foo and flag -foo as a repeatable flag + set k [string map {* ""} $k] + incr multi($k) + } + + if {[info exists flags($k)]} { + proj-error -up "[proj-scope]: Duplicated prototype for flag $k" + } + switch -exact -- [lindex $prototype [expr {$i + 1}]] { => { + # -flag => DFLT ?-subflag arg? incr i 2 if {$i >= $n} { - proj-error "Missing argument for $k => flag" + proj-error -up "[proj-scope]: Missing argument for $k => flag" } - set consuming($k) 1 - set v [lindex $prototype $i] + incr consuming($k) + set vi [lindex $prototype $i] + if {$vi in {-apply -call}} { + proj-error -up "[proj-scope]: Missing default value for $k flag" + } else { + set fv [lindex $prototype [expr {$i + 1}]] + if {$fv in {-apply -call}} { + incr i + eval $checkProtoFlag + } + } } default { - set v [lindex $prototype [incr i]] - set s [lindex $prototype [incr i]] - set scripts($k) $s - } - } - #puts "**** #$i of $n k=$k v=$v s=$s" - set dflt($k) $v - } - # Now look for those flags in the source list - array set tgt [array get dflt] - unset dflt + # -flag VALUE ?flag? SCRIPT + set vi [lindex $prototype [incr i]] + set fv [lindex $prototype [incr i]] + eval $checkProtoFlag + } + } + #puts "**** #$i of $n k=$k vi=$vi" + set flags($k) $vi + } + #puts "-- flags"; parray flags + #puts "-- scripts"; parray scripts + #puts "-- calls"; parray call set rc 0 set rv {} set skipMode 0 set n [llength $argv] + # Now look for those flags in $argv... for {set i 0} {$i < $n} {incr i} { set arg [lindex $argv $i] + #puts "-- [proj-scope] arg=$arg" if {$skipMode} { lappend rv $arg } elseif {"--" eq $arg} { - incr skipMode - } elseif {[info exists tgt($arg)]} { - if {[info exists consuming($arg)]} { - if {$i + 1 >= $n} { - proj-assert 0 {Cannot happen - bounds already checked} - } - set tgt($arg) [lindex $argv [incr i]] - } elseif {"" eq $scripts($arg)} { - set tgt($arg) "" - } else { - #puts "**** running scripts($arg) $scripts($arg)" - set code [catch {uplevel 1 $scripts($arg)} xrc xopt] - #puts "**** tgt($arg)=$scripts($arg) code=$code rc=$rc" - if {$code in {0 2}} { - set tgt($arg) $xrc - } else { - return {*}$xopt $xrc - } + # "--" is the conventional way to end processing of args + if {[incr seen(--)] > 1} { + # Elide only the first one + lappend rv $arg + } + incr skipMode $incrSkip + } elseif {[info exists flags($arg)]} { + # A known flag... + set isMulti [info exists multi($arg)] + incr seen($arg) + if {1 < $seen($arg) && !$isMulti} { + proj-error -up [proj-scope] "$arg flag was used multiple times" + } + set vMode 0; # 0=as-is, 1=eval, 2=call + set isConsuming [info exists consuming($arg)] + if {$isConsuming} { + incr i + if {$i >= $n} { + proj-error -up [proj-scope] "is missing argument for $arg flag" + } + set vv [lindex $argv $i] + } elseif {[info exists scripts($arg)]} { + set vMode 1 + set vv $scripts($arg) + } else { + set vv $flags($arg) + } + + if {[info exists call($arg)]} { + set vMode 2 + set vv [concat {*}$call($arg) $arg $vv] + } elseif {$isConsuming} { + proj-assert {!$vMode} + # fall through + } elseif {"" eq $vv || [string is double -strict $vv]} { + set vMode 0 + } elseif {$vv in {incr}} { + set vMode 0 + switch -exact $vv { + incr { + set xx $flags($k); incr xx; set vv $xx; unset xx + } + default { + proj-error "Unhandled \$vv value $vv" + } + } + } else { + set vv [list eval $vv] + set vMode 1 + } + if {$vMode} { + set code [catch [list uplevel 1 $vv] vv xopt] + if {$code ni {0 2}} { + return {*}$xopt $vv + } + } + if {$isConsuming && $isMulti} { + if {1 == $seen($arg)} { + # On the first hit, overwrite the default with a new list. + set flags($arg) [list $vv] + } else { + # On subsequent hits, append to the list. + lappend flags($arg) $vv + } + } else { + set flags($arg) $vv } incr rc } else { - incr skipMode + # Non-flag + incr skipMode $incrSkip lappend rv $arg } } set argv $rv + array set outFlags [array get flags] + #puts "-- rv=$rv argv=$argv flags="; parray flags return $rc +}; # proj-parse-flags + +# +# Older (deprecated) name of proj-parse-flags. +# +proc proj-parse-simple-flags {args} { + tailcall proj-parse-flags {*}$args } if {$::proj__Config(self-tests)} { + set __ova $::proj__Config(verbose-assert); + set ::proj__Config(verbose-assert) 1 + puts "Running [info script] self-tests..." + # proj-cache... apply {{} { #proj-warn "Test code for proj-cache" proj-assert {![proj-cache-check -key here check]} proj-assert {"here" eq [proj-cache-key here]} proj-assert {"" eq $check} @@ -2231,6 +2406,102 @@ proj-assert {"" ne [proj-cache-remove]} proj-assert {![proj-cache-check check]} proj-assert {"" eq [proj-cache-remove]} proj-assert {"" eq $check} }} -} + + # proj-parse-flags ... + apply {{} { + set foo 3 + set argv {-a "hi - world" -b -b -b -- -a {bye bye} -- -d -D c -a "" --} + proj-parse-flags argv flags { + :all-flags + -a* => "gets overwritten" + -b* 7 {incr foo} + -d 1 0 + -D 0 1 + } + + #puts "-- argv = $argv"; parray flags; + proj-assert {"-- c --" eq $argv} + proj-assert {$flags(-a) eq "{hi - world} {bye bye} {}"} + proj-assert {$foo == 6} + proj-assert {$flags(-b) eq $foo} + proj-assert {$flags(-d) == 0} + proj-assert {$flags(-D) == 1} + set foo 0 + foreach x $flags(-a) { + proj-assert {$x in {{hi - world} {bye bye} {}}} + incr foo + } + proj-assert {3 == $foo} + + set argv {-a {hi world} -b -maybe -- -a {bye bye} -- -b c --} + set foo 0 + proj-parse-flags argv flags { + -a => "aaa" + -b 0 {incr foo} + -maybe no -literal yes + } + #parray flags; puts "--- argv = $argv" + proj-assert {"-a {bye bye} -- -b c --" eq $argv} + proj-assert {$flags(-a) eq "hi world"} + proj-assert {1 == $flags(-b)} + proj-assert {"yes" eq $flags(-maybe)} + + set argv {-f -g -a aaa -M -M -M -L -H -A AAA a b c} + set foo 0 + set myLambda {{flag val} { + proj-assert {$flag in {-f -g -M}} + #puts "myLambda flag=$flag val=$val" + incr val + }} + proc myNonLambda {flag val} { + proj-assert {$flag in {-A -a}} + #puts "myNonLambda flag=$flag val=$val" + concat $val $val + } + proj-parse-flags argv flags { + -f 0 -call {apply $myLambda} + -g 2 -apply $myLambda + -h 3 -apply $myLambda + -H 30 33 + -a => aAAAa -apply {{f v} { + set v + }} + -A => AaaaA -call myNonLambda + -B => 17 -call myNonLambda + -M* 0 -apply $myLambda + -L "" -literal $myLambda + } + rename myNonLambda "" + #puts "--- argv = $argv"; parray flags + proj-assert {$flags(-f) == 1} + proj-assert {$flags(-g) == 3} + proj-assert {$flags(-h) == 3} + proj-assert {$flags(-H) == 33} + proj-assert {$flags(-a) == {aaa}} + proj-assert {$flags(-A) eq "AAA AAA"} + proj-assert {$flags(-B) == 17} + proj-assert {$flags(-M) == 3} + proj-assert {$flags(-L) eq $myLambda} + + set argv {-touch -validate} + proj-parse-flags argv flags { + -touch "" {return "-touch"} + -validate 0 1 + } + #puts "----- argv = $argv"; parray flags + proj-assert {$flags(-touch) eq "-touch"} + proj-assert {$flags(-validate) == 1} + proj-assert {$argv eq {}} + + set argv {-i -i -i} + proj-parse-flags argv flags { + -i* 0 incr + } + proj-assert {3 == $flags(-i)} + }} + set ::proj__Config(verbose-assert) $__ova + unset __ova + puts "Done running [info script] self-tests." +}; # proj- API self-tests Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -211,11 +211,11 @@ /* ** Macros needed to provide flexible arrays in a portable way */ #ifndef offsetof -# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY #else # define FLEXARRAY 1 Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -18,10 +18,11 @@ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #include #include +#include #ifndef SQLITE_AMALGAMATION typedef unsigned char u8; typedef unsigned int u32; @@ -77,11 +78,11 @@ /* ** Macros needed to provide flexible arrays in a portable way */ #ifndef offsetof -# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY #else # define FLEXARRAY 1 Index: ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java ================================================================== --- ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java +++ ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java @@ -118,21 +118,21 @@ subclass, to fetch the current per-call UDF state. On the first call to this method for any given sqlite3_context argument, the context is set to the given initial value. On all other calls, the 2nd argument is ignored. - @see SQLFunction.PerContextState#getAggregateState + @see AggregateFunction.PerContextState#getAggregateState */ protected final ValueHolder getAggregateState(sqlite3_context cx, T initialValue){ return map.getAggregateState(cx, initialValue); } /** To be called from the implementation's xFinal() method to fetch the final state of the UDF and remove its mapping. - see SQLFunction.PerContextState#takeAggregateState + see AggregateFunction.PerContextState#takeAggregateState */ protected final T takeAggregateState(sqlite3_context cx){ return map.takeAggregateState(cx); } } Index: ext/rtree/rtree.c ================================================================== --- ext/rtree/rtree.c +++ ext/rtree/rtree.c @@ -62,10 +62,12 @@ #else #include "sqlite3.h" #endif int sqlite3GetToken(const unsigned char*,int*); /* In the SQLite core */ +#include + /* ** If building separately, we will need some setup that is normally ** found in sqliteInt.h */ #if !defined(SQLITE_AMALGAMATION) @@ -93,11 +95,11 @@ #else # define ALWAYS(X) (X) # define NEVER(X) (X) #endif #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY #else # define FLEXARRAY 1 Index: ext/wasm/GNUmakefile ================================================================== --- ext/wasm/GNUmakefile +++ ext/wasm/GNUmakefile @@ -408,11 +408,11 @@ # JS/WASM build. They are solely for use with $(bin.c-pp) itself. # # -D... flags which should be included in all invocations should be # appended to $(SQLITE.CALL.C-PP.FILTER.global). bin.c-pp := ./c-pp -$(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) +$(bin.c-pp): c-pp.c $(sqlite3.c) # $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=3 DISTCLEAN_FILES += $(bin.c-pp) @@ -594,10 +594,11 @@ emcc.flags := ifeq (1,$(emcc.verbose)) emcc.flags += -v # -v is _very_ loud but also informative about what it's doing endif + ######################################################################## # emcc flags for .c/.o. emcc.cflags := emcc.cflags += -std=c99 -fPIC @@ -613,12 +614,13 @@ emcc.jsflags += -sMODULARIZE emcc.jsflags += -sDYNAMIC_EXECUTION=0 emcc.jsflags += -sNO_POLYFILL emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.api) emcc.exportedRuntimeMethods := \ - -sEXPORTED_RUNTIME_METHODS=wasmMemory - # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY + -sEXPORTED_RUNTIME_METHODS=wasmMemory,HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAP64,HEAPU64 +# wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY +# Emscripten 4.0.7 (2025-04-15) stops exporting HEAP* by default emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY ifeq (,$(filter -O0,$(emcc_opt))) emcc.assert ?= 0 Index: ext/wasm/api/sqlite3-wasm.c ================================================================== --- ext/wasm/api/sqlite3-wasm.c +++ ext/wasm/api/sqlite3-wasm.c @@ -1155,11 +1155,11 @@ unsigned char omit; } sqlite3_index_constraint_usage; { /* Validate that the above struct sizeof()s match ** expectations. We could improve upon this by ** checking the offsetof() for each member. */ - const sqlite3_index_info siiCheck; + const sqlite3_index_info siiCheck = {0}; #define IndexSzCheck(T,M) \ (sizeof(T) == sizeof(*siiCheck.M)) if(!IndexSzCheck(sqlite3_index_constraint,aConstraint) || !IndexSzCheck(sqlite3_index_orderby,aOrderBy) || !IndexSzCheck(sqlite3_index_constraint_usage,aConstraintUsage)){ Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -1834,10 +1834,16 @@ mdevtest: srctree-check has_tclsh85 $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) sdevtest: has_tclsh85 $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) + +# Like releasetest, except it omits srctree-check and verify-source so +# that it can be used on a modified source tree. +# +xdevtest: has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) # # Validate that various generated files in the source tree # are up-to-date. # Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -1103,11 +1103,11 @@ if( pKey ){ KeyInfo *pKeyInfo = pCur->pKeyInfo; assert( nKey==(i64)(int)nKey ); pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; - sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + sqlite3VdbeRecordUnpack((int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ rc = SQLITE_CORRUPT_BKPT; }else{ rc = sqlite3BtreeIndexMoveto(pCur, pIdxKey, pRes); } @@ -3105,10 +3105,11 @@ removed = 1; } sqlite3_mutex_leave(pMainMtx); return removed; #else + UNUSED_PARAMETER( pBt ); return 1; #endif } /* @@ -3948,10 +3949,17 @@ if( rc!=SQLITE_OK ){ (void)sqlite3PagerWalWriteLock(pPager, 0); unlockBtreeIfUnused(pBt); } +#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) + if( rc==SQLITE_BUSY_TIMEOUT ){ + /* If a blocking lock timed out, break out of the loop here so that + ** the busy-handler is not invoked. */ + break; + } +#endif }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && btreeInvokeBusyHandler(pBt) ); sqlite3PagerWalDb(pPager, 0); #ifdef SQLITE_ENABLE_SETLK_TIMEOUT if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; @@ -11836,10 +11844,11 @@ ** btree as the argument handle holds an exclusive lock on the ** sqlite_schema table. Otherwise SQLITE_OK. */ int sqlite3BtreeSchemaLocked(Btree *p){ int rc; + UNUSED_PARAMETER(p); /* only used in DEBUG builds */ assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); assert( rc==SQLITE_OK || rc==SQLITE_LOCKED_SHAREDCACHE ); sqlite3BtreeLeave(p); Index: src/dbpage.c ================================================================== --- src/dbpage.c +++ src/dbpage.c @@ -225,24 +225,25 @@ ** 3 schema=?1, pgno=?2 ** ** idxStr is not used */ static int dbpageFilter( - sqlite3_vtab_cursor *pCursor, + sqlite3_vtab_cursor *pCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; int rc; sqlite3 *db = pTab->db; Btree *pBt; - (void)idxStr; - + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(argc); + /* Default setting is no rows of result */ - pCsr->pgno = 1; + pCsr->pgno = 1; pCsr->mxPgno = 0; if( idxNum & 2 ){ const char *zSchema; assert( argc>=1 ); @@ -273,12 +274,12 @@ rc = sqlite3PagerGet(pCsr->pPager, 1, &pCsr->pPage1, 0); return rc; } static int dbpageColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, int i ){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; int rc = SQLITE_OK; switch( i ){ Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -71,11 +71,13 @@ assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); return sqlite3ExprAffinity( pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr ); } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && pExpr->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(pExpr) ); return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); } if( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ assert( pExpr->op==TK_COLLATE @@ -264,11 +266,13 @@ } if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; continue; } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && p->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(p) ); p = p->x.pList->a[0].pExpr; continue; } if( op==TK_COLLATE ){ Index: src/json.c ================================================================== --- src/json.c +++ src/json.c @@ -1283,12 +1283,14 @@ nExtra = 0; }else if( szType==12 ){ nExtra = 1; }else if( szType==13 ){ nExtra = 2; - }else{ + }else if( szType==14 ){ nExtra = 4; + }else{ + nExtra = 8; } if( szPayload<=11 ){ nNeeded = 0; }else if( szPayload<=0xff ){ nNeeded = 1; Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -5187,25 +5187,24 @@ /* Check that, if this to be a blocking lock, no locks that occur later ** in the following list than the lock being obtained are already held: ** - ** 1. Checkpointer lock (ofst==1). - ** 2. Write lock (ofst==0). - ** 3. Read locks (ofst>=3 && ofst=3 && ofstexclMask|p->sharedMask); assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( - (ofst!=2) /* not RECOVER */ + (ofst!=2 || lockMask==0) && (ofst!=1 || lockMask==0 || lockMask==2) && (ofst!=0 || lockMask<3) && (ofst<3 || lockMask<(1<=3 && ofst=3 && ofstexclMask|p->sharedMask); assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( - (ofst!=2) /* not RECOVER */ + (ofst!=2 || lockMask==0) && (ofst!=1 || lockMask==0 || lockMask==2) && (ofst!=0 || lockMask<3) && (ofst<3 || lockMask<(1<pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, bWal2, &pPager->pWal ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_OK ){ + sqlite3WalDb(pPager->pWal, pPager->dbWal); + } +#endif } pagerFixMaplimit(pPager); return rc; } @@ -7890,10 +7898,11 @@ /* ** Set the database handle used by the wal layer to determine if ** blocking locks are required. */ void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + pPager->dbWal = db; if( pagerUseWal(pPager) ){ sqlite3WalDb(pPager->pWal, db); } } #endif Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -594,11 +594,11 @@ "not present in both tables", zName); return 1; } pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); - if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 && pParse->nErr==0 ){ /* This branch runs if the query contains one or more RIGHT or FULL ** JOINs. If only a single table on the left side of this join ** contains the zName column, then this branch is a no-op. ** But if there are two or more tables on the left side ** of the join, construct a coalesce() function that gathers all @@ -610,10 +610,12 @@ ** JOIN. But older versions of SQLite do not do that, so we avoid ** adding a new error so as to not break legacy applications. */ ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */ static const Token tkCoalesce = { "coalesce", 8 }; + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol, pRight->fg.isSynthUsing)!=0 ){ if( pSrc->a[iLeft].fg.isUsing==0 || sqlite3IdListIndex(pSrc->a[iLeft].u3.pUsing, zName)<0 ){ @@ -626,11 +628,17 @@ sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); } if( pFuncArgs ){ pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0); + if( pE1 ){ + pE1->affExpr = SQLITE_AFF_DEFER; + } } + }else if( (pSrc->a[i+1].fg.jointype & JT_LEFT)!=0 && pParse->nErr==0 ){ + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); } pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol); sqlite3SrcItemColumnUsed(pRight, iRightCol); pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2); assert( pE2!=0 || pEq==0 ); @@ -2103,10 +2111,14 @@ #else zType = columnType(&sNC, p, 0, 0, 0); #endif sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } +#else + UNUSED_PARAMETER(pParse); + UNUSED_PARAMETER(pTabList); + UNUSED_PARAMETER(pEList); #endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } /* @@ -4234,13 +4246,13 @@ ** other than the one FROM-clause subquery that is a candidate ** for flattening. (This is due to ticket [2f7170d73bf9abf80] ** from 2015-02-09.) ** ** (3) If the subquery is the right operand of a LEFT JOIN then -** (3a) the subquery may not be a join and -** (3b) the FROM clause of the subquery may not contain a virtual -** table and +** (3a) the subquery may not be a join +** (**) Was (3b): "the FROM clause of the subquery may not contain +** a virtual table" ** (**) Was: "The outer query may not have a GROUP BY." This case ** is now managed correctly ** (3d) the outer query may not be DISTINCT. ** See also (26) for restrictions on RIGHT JOIN. ** @@ -4452,11 +4464,11 @@ ** ** See also tickets #306, #350, and #3300. */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || IsVirtual(pSubSrc->a[0].pSTab) /* (3b) */ + /**** || IsVirtual(pSubSrc->a[0].pSTab) (3b)-omitted */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ return 0; } Index: src/shell.c.in ================================================================== --- src/shell.c.in +++ src/shell.c.in @@ -885,16 +885,25 @@ return 0x3fffffff & (int)(z2 - z); } /* ** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character. +** count as a single character for single-width characters, or as two +** characters for double-width characters. */ static int strlenChar(const char *z){ int n = 0; while( *z ){ - if( (0xc0&*(z++))!=0x80 ) n++; + if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = decodeUtf8((const u8*)z, &u); + z += len; + n += cli_wcwidth(u); + } } return n; } /* Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -763,11 +763,11 @@ /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. */ #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif /* ** Work around C99 "flex-array" syntax for pre-C99 compilers, so as ** to avoid complaints from -fsanitize=strict-bounds. @@ -2355,10 +2355,11 @@ #define SQLITE_AFF_TEXT 0x42 /* 'B' */ #define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ #define SQLITE_AFF_INTEGER 0x44 /* 'D' */ #define SQLITE_AFF_REAL 0x45 /* 'E' */ #define SQLITE_AFF_FLEXNUM 0x46 /* 'F' */ +#define SQLITE_AFF_DEFER 0x58 /* 'X' - defer computation until later */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) /* ** The SQLITE_AFF_MASK values masks off the significant bits of an @@ -2670,13 +2671,19 @@ /* ** An instance of the following structure is passed as the first ** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. ** -** Note that aSortOrder[] and aColl[] have nField+1 slots. There -** are nField slots for the columns of an index then one extra slot -** for the rowid at the end. +** The aSortOrder[] and aColl[] arrays have nAllField slots each. There +** are nKeyField slots for the columns of an index then extra slots +** for the rowid or key at the end. The aSortOrder array is located after +** the aColl[] array. +** +** If SQLITE_ENABLE_PREUPDATE_HOOK is defined, then aSortFlags might be NULL +** to indicate that this object is for use by a preupdate hook. When aSortFlags +** is NULL, then nAllField is uninitialized and no space is allocated for +** aColl[], so those fields may not be used. */ struct KeyInfo { u32 nRef; /* Number of references to this KeyInfo object */ u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ u16 nKeyField; /* Number of key columns in the index */ @@ -2684,12 +2691,21 @@ sqlite3 *db; /* The database connection */ u8 *aSortFlags; /* Sort order for each column. */ CollSeq *aColl[FLEXARRAY]; /* Collating sequence for each term of the key */ }; -/* The size (in bytes) of a KeyInfo object with up to N fields */ +/* The size (in bytes) of a KeyInfo object with up to N fields. This includes +** the main body of the KeyInfo object and the aColl[] array of N elements, +** but does not count the memory used to hold aSortFlags[]. */ #define SZ_KEYINFO(N) (offsetof(KeyInfo,aColl) + (N)*sizeof(CollSeq*)) + +/* The size of a bare KeyInfo with no aColl[] entries */ +#if FLEXARRAY+1 > 1 +# define SZ_KEYINFO_0 offsetof(KeyInfo,aColl) +#else +# define SZ_KEYINFO_0 sizeof(KeyInfo) +#endif /* ** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. */ #define KEYINFO_ORDER_DESC 0x01 /* DESC sort order */ @@ -2705,23 +2721,22 @@ ** the OP_MakeRecord opcode of the VDBE and is disassembled by the ** OP_Column opcode. ** ** An instance of this object serves as a "key" for doing a search on ** an index b+tree. The goal of the search is to find the entry that -** is closed to the key described by this object. This object might hold -** just a prefix of the key. The number of fields is given by -** pKeyInfo->nField. +** is closest to the key described by this object. This object might hold +** just a prefix of the key. The number of fields is given by nField. ** ** The r1 and r2 fields are the values to return if this key is less than ** or greater than a key in the btree, respectively. These are normally ** -1 and +1 respectively, but might be inverted to +1 and -1 if the b-tree ** is in DESC order. ** ** The key comparison functions actually return default_rc when they find ** an equals comparison. default_rc can be -1, 0, or +1. If there are ** multiple entries in the b-tree with the same key (when only looking -** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to +** at the first nField elements) then default_rc can be set to -1 to ** cause the search to find the last match, or +1 to cause the search to ** find the first match. ** ** The key comparison functions will set eqSeen to true if they ever ** get and equal results when comparing this structure to a b-tree record. @@ -2729,12 +2744,12 @@ ** before the first match or immediately after the last match. The ** eqSeen field will indicate whether or not an exact match exists in the ** b-tree. */ struct UnpackedRecord { - KeyInfo *pKeyInfo; /* Collation and sort-order information */ - Mem *aMem; /* Values */ + KeyInfo *pKeyInfo; /* Comparison info for the index that is unpacked */ + Mem *aMem; /* Values for columns of the index */ union { char *z; /* Cache of aMem[0].z for vdbeRecordCompareString() */ i64 i; /* Cache of aMem[0].u.i for vdbeRecordCompareInt() */ } u; int n; /* Cache of aMem[0].n used by vdbeRecordCompareString() */ Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -47,10 +47,14 @@ /* Compatability between Tcl8.6 and Tcl9.0 */ #if TCL_MAJOR_VERSION==9 # define CONST const #elif !defined(Tcl_Size) typedef int Tcl_Size; +# ifndef Tcl_BounceRefCount +# define Tcl_BounceRefCount(X) Tcl_IncrRefCount(X); Tcl_DecrRefCount(X) + /* https://www.tcl-lang.org/man/tcl9.0/TclLib/Object.html */ +# endif #endif /**** End copy of tclsqlite.h ****/ #include @@ -1082,11 +1086,13 @@ } rc = Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT); Tcl_DecrRefCount(pCmd); } - if( rc && rc!=TCL_RETURN ){ + if( TCL_BREAK==rc ){ + sqlite3_result_null(context); + }else if( rc && rc!=TCL_RETURN ){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); }else{ Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); Tcl_Size n; u8 *data; @@ -1100,11 +1106,11 @@ ** has no string representation. */ eType = SQLITE_BLOB; }else if( (c=='b' && pVar->bytes==0 && strcmp(zType,"boolean")==0 ) || (c=='b' && pVar->bytes==0 && strcmp(zType,"booleanString")==0 ) || (c=='w' && strcmp(zType,"wideInt")==0) - || (c=='i' && strcmp(zType,"int")==0) + || (c=='i' && strcmp(zType,"int")==0) ){ eType = SQLITE_INTEGER; }else if( c=='d' && strcmp(zType,"double")==0 ){ eType = SQLITE_FLOAT; }else{ @@ -1614,15 +1620,16 @@ Tcl_Obj *pSql; /* Object holding string zSql */ const char *zSql; /* Remaining SQL to execute */ SqlPreparedStmt *pPreStmt; /* Current statement */ int nCol; /* Number of columns returned by pStmt */ int evalFlags; /* Flags used */ - Tcl_Obj *pArray; /* Name of array variable */ + Tcl_Obj *pVarName; /* Name of target array/dict variable */ Tcl_Obj **apColName; /* Array of column names */ }; #define SQLITE_EVAL_WITHOUTNULLS 0x00001 /* Unset array(*) for NULL */ +#define SQLITE_EVAL_ASDICT 0x00002 /* Use dict instead of array */ /* ** Release any cache of column names currently held as part of ** the DbEvalContext structure passed as the first argument. */ @@ -1639,34 +1646,34 @@ } /* ** Initialize a DbEvalContext structure. ** -** If pArray is not NULL, then it contains the name of a Tcl array +** If pVarName is not NULL, then it contains the name of a Tcl array ** variable. The "*" member of this array is set to a list containing ** the names of the columns returned by the statement as part of each ** call to dbEvalStep(), in order from left to right. e.g. if the names ** of the returned columns are a, b and c, it does the equivalent of the ** tcl command: ** -** set ${pArray}(*) {a b c} +** set ${pVarName}(*) {a b c} */ static void dbEvalInit( DbEvalContext *p, /* Pointer to structure to initialize */ SqliteDb *pDb, /* Database handle */ Tcl_Obj *pSql, /* Object containing SQL script */ - Tcl_Obj *pArray, /* Name of Tcl array to set (*) element of */ + Tcl_Obj *pVarName, /* Name of Tcl array to set (*) element of */ int evalFlags /* Flags controlling evaluation */ ){ memset(p, 0, sizeof(DbEvalContext)); p->pDb = pDb; p->zSql = Tcl_GetString(pSql); p->pSql = pSql; Tcl_IncrRefCount(pSql); - if( pArray ){ - p->pArray = pArray; - Tcl_IncrRefCount(pArray); + if( pVarName ){ + p->pVarName = pVarName; + Tcl_IncrRefCount(pVarName); } p->evalFlags = evalFlags; addDatabaseRef(p->pDb); } @@ -1685,33 +1692,48 @@ int i; /* Iterator variable */ int nCol; /* Number of columns returned by pStmt */ Tcl_Obj **apColName = 0; /* Array of column names */ p->nCol = nCol = sqlite3_column_count(pStmt); - if( nCol>0 && (papColName || p->pArray) ){ + if( nCol>0 && (papColName || p->pVarName) ){ apColName = (Tcl_Obj**)Tcl_Alloc( sizeof(Tcl_Obj*)*nCol ); for(i=0; iapColName = apColName; } - /* If results are being stored in an array variable, then create - ** the array(*) entry for that array + /* If results are being stored in a variable then create the + ** array(*) or dict(*) entry for that variable. */ - if( p->pArray ){ + if( p->pVarName ){ Tcl_Interp *interp = p->pDb->interp; Tcl_Obj *pColList = Tcl_NewObj(); Tcl_Obj *pStar = Tcl_NewStringObj("*", -1); + Tcl_IncrRefCount(pColList); + Tcl_IncrRefCount(pStar); for(i=0; ipArray, pStar, pColList, 0); + if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + Tcl_ObjSetVar2(interp, p->pVarName, pStar, pColList, 0); + }else{ + Tcl_Obj * pDict = Tcl_ObjGetVar2(interp, p->pVarName, NULL, 0); + if( !pDict ){ + pDict = Tcl_NewDictObj(); + }else if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjPut(interp, pDict, pStar, pColList)==TCL_OK ){ + Tcl_ObjSetVar2(interp, p->pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); + } Tcl_DecrRefCount(pStar); + Tcl_DecrRefCount(pColList); } } if( papColName ){ *papColName = p->apColName; @@ -1749,11 +1771,11 @@ rcs = sqlite3_step(pStmt); if( rcs==SQLITE_ROW ){ return TCL_OK; } - if( p->pArray ){ + if( p->pVarName ){ dbEvalRowInfo(p, 0, 0); } rcs = sqlite3_reset(pStmt); pDb->nStep = sqlite3_stmt_status(pStmt,SQLITE_STMTSTATUS_FULLSCAN_STEP,1); @@ -1800,13 +1822,13 @@ if( p->pPreStmt ){ sqlite3_reset(p->pPreStmt->pStmt); dbReleaseStmt(p->pDb, p->pPreStmt, 0); p->pPreStmt = 0; } - if( p->pArray ){ - Tcl_DecrRefCount(p->pArray); - p->pArray = 0; + if( p->pVarName ){ + Tcl_DecrRefCount(p->pVarName); + p->pVarName = 0; } Tcl_DecrRefCount(p->pSql); dbReleaseColumnNames(p); delDatabaseRef(p->pDb); } @@ -1877,11 +1899,11 @@ #endif /* ** This function is part of the implementation of the command: ** -** $db eval SQL ?ARRAYNAME? SCRIPT +** $db eval SQL ?TGT-NAME? SCRIPT */ static int SQLITE_TCLAPI DbEvalNextCmd( ClientData data[], /* data[0] is the (DbEvalContext*) */ Tcl_Interp *interp, /* Tcl interpreter */ int result /* Result so far */ @@ -1891,28 +1913,59 @@ /* The first element of the data[] array is a pointer to a DbEvalContext ** structure allocated using Tcl_Alloc(). The second element of data[] ** is a pointer to a Tcl_Obj containing the script to run for each row ** returned by the queries encapsulated in data[0]. */ DbEvalContext *p = (DbEvalContext *)data[0]; - Tcl_Obj *pScript = (Tcl_Obj *)data[1]; - Tcl_Obj *pArray = p->pArray; + Tcl_Obj * const pScript = (Tcl_Obj *)data[1]; + Tcl_Obj * const pVarName = p->pVarName; while( (rc==TCL_OK || rc==TCL_CONTINUE) && TCL_OK==(rc = dbEvalStep(p)) ){ int i; int nCol; Tcl_Obj **apColName; dbEvalRowInfo(p, &nCol, &apColName); for(i=0; ievalFlags & SQLITE_EVAL_WITHOUTNULLS)!=0 - && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL + && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL ){ - Tcl_UnsetVar2(interp, Tcl_GetString(pArray), - Tcl_GetString(apColName[i]), 0); + /* Remove NULL-containing column from the target container... */ + if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + /* Target is an array */ + Tcl_UnsetVar2(interp, Tcl_GetString(pVarName), + Tcl_GetString(apColName[i]), 0); + }else{ + /* Target is a dict */ + Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0); + if( pDict ){ + if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjRemove(interp, pDict, apColName[i])==TCL_OK ){ + Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); + } + } + }else if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + /* Target is an array: set target(colName) = colValue */ + Tcl_ObjSetVar2(interp, pVarName, apColName[i], + dbEvalColumnValue(p,i), 0); }else{ - Tcl_ObjSetVar2(interp, pArray, apColName[i], dbEvalColumnValue(p,i), 0); + /* Target is a dict: set target(colName) = colValue */ + Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0); + if( !pDict ){ + pDict = Tcl_NewDictObj(); + }else if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjPut(interp, pDict, apColName[i], + dbEvalColumnValue(p,i))==TCL_OK ){ + Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); } } /* The required interpreter variables are now populated with the data ** from the current row. If using NRE, schedule callbacks to evaluate @@ -2017,11 +2070,11 @@ "progress", "rekey", "restore", "rollback_hook", "serialize", "status", "timeout", "total_changes", "trace", "trace_v2", "transaction", "unlock_notify", "update_hook", "version", "wal_hook", - 0 + 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BIND_FALLBACK, DB_BUSY, DB_CACHE, DB_CHANGES, DB_CLOSE, DB_COLLATE, DB_COLLATION_NEEDED, @@ -2851,35 +2904,38 @@ } break; } /* - ** $db eval ?options? $sql ?array? ?{ ...code... }? + ** $db eval ?options? $sql ?varName? ?{ ...code... }? ** - ** The SQL statement in $sql is evaluated. For each row, the values are - ** placed in elements of the array named "array" and ...code... is executed. - ** If "array" and "code" are omitted, then no callback is every invoked. - ** If "array" is an empty string, then the values are placed in variables - ** that have the same name as the fields extracted by the query. + ** The SQL statement in $sql is evaluated. For each row, the values + ** are placed in elements of the array or dict named $varName and + ** ...code... is executed. If $varName and $code are omitted, then + ** no callback is ever invoked. If $varName is an empty string, + ** then the values are placed in variables that have the same name + ** as the fields extracted by the query, and those variables are + ** accessible during the eval of $code. */ case DB_EVAL: { int evalFlags = 0; const char *zOpt; while( objc>3 && (zOpt = Tcl_GetString(objv[2]))!=0 && zOpt[0]=='-' ){ if( strcmp(zOpt, "-withoutnulls")==0 ){ evalFlags |= SQLITE_EVAL_WITHOUTNULLS; - } - else{ + }else if( strcmp(zOpt, "-asdict")==0 ){ + evalFlags |= SQLITE_EVAL_ASDICT; + }else{ Tcl_AppendResult(interp, "unknown option: \"", zOpt, "\"", (void*)0); return TCL_ERROR; } objc--; objv++; } if( objc<3 || objc>5 ){ - Tcl_WrongNumArgs(interp, 2, objv, - "?OPTIONS? SQL ?ARRAY-NAME? ?SCRIPT?"); + Tcl_WrongNumArgs(interp, 2, objv, + "?OPTIONS? SQL ?VAR-NAME? ?SCRIPT?"); return TCL_ERROR; } if( objc==3 ){ DbEvalContext sEval; @@ -2901,21 +2957,21 @@ } Tcl_DecrRefCount(pRet); }else{ ClientData cd2[2]; DbEvalContext *p; - Tcl_Obj *pArray = 0; + Tcl_Obj *pVarName = 0; Tcl_Obj *pScript; if( objc>=5 && *(char *)Tcl_GetString(objv[3]) ){ - pArray = objv[3]; + pVarName = objv[3]; } pScript = objv[objc-1]; Tcl_IncrRefCount(pScript); p = (DbEvalContext *)Tcl_Alloc(sizeof(DbEvalContext)); - dbEvalInit(p, pDb, objv[2], pArray, evalFlags); + dbEvalInit(p, pDb, objv[2], pVarName, evalFlags); cd2[0] = (void *)p; cd2[1] = (void *)pScript; rc = DbEvalNextCmd(cd2, interp, TCL_OK); } Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -8533,11 +8533,16 @@ { "WRITABLE_SCHEMA", SQLITE_DBCONFIG_WRITABLE_SCHEMA }, { "LEGACY_ALTER_TABLE", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, { "DQS_DML", SQLITE_DBCONFIG_DQS_DML }, { "DQS_DDL", SQLITE_DBCONFIG_DQS_DDL }, { "LEGACY_FILE_FORMAT", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, + { "TRUSTED_SCHEMA", SQLITE_DBCONFIG_TRUSTED_SCHEMA }, { "STMT_SCANSTATUS", SQLITE_DBCONFIG_STMT_SCANSTATUS }, + { "REVERSE_SCANORDER", SQLITE_DBCONFIG_REVERSE_SCANORDER }, + { "ATTACH_CREATE", SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE }, + { "ATTACH_WRITE", SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE }, + { "COMMENTS", SQLITE_DBCONFIG_ENABLE_COMMENTS }, }; int i; int v = 0; const char *zSetting; sqlite3 *db; Index: src/vacuum.c ================================================================== --- src/vacuum.c +++ src/vacuum.c @@ -193,11 +193,12 @@ saved_flags = db->flags; saved_mDbFlags = db->mDbFlags; saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; saved_mTrace = db->mTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments + | SQLITE_AttachCreate | SQLITE_AttachWrite; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); db->mTrace = 0; Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -2474,10 +2474,11 @@ } n = pOp->p3; pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); assert( pKeyInfo!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); p1 = pOp->p1; p2 = pOp->p2; #ifdef SQLITE_DEBUG if( aPermute ){ int k, mx = 0; @@ -5383,11 +5384,11 @@ rc = ExpandBlob(r.aMem); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); if( rc ) goto no_mem; pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo); if( pIdxKey==0 ) goto no_mem; - sqlite3VdbeRecordUnpack(pC->pKeyInfo, r.aMem->n, r.aMem->z, pIdxKey); + sqlite3VdbeRecordUnpack(r.aMem->n, r.aMem->z, pIdxKey); pIdxKey->default_rc = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &pC->seekResult); sqlite3DbFreeNN(db, pIdxKey); } if( rc!=SQLITE_OK ){ Index: src/vdbe.h ================================================================== --- src/vdbe.h +++ src/vdbe.h @@ -299,11 +299,11 @@ char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3BlobCompare(const Mem*, const Mem*); -void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); +void sqlite3VdbeRecordUnpack(int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*); typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -555,11 +555,11 @@ Mem oldipk; /* Memory cell holding "old" IPK value */ Mem *aNew; /* Array of new.* values */ Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ sqlite3_value **apDflt; /* Array of default values, if required */ - u8 keyinfoSpace[SZ_KEYINFO(0)]; /* Space to hold pKeyinfo[0] content */ + u8 keyinfoSpace[SZ_KEYINFO_0]; /* Space to hold pKeyinfo[0] content */ }; /* ** An instance of this object is used to pass an vector of values into ** OP_VFilter, the xFilter method of a virtual table. The vector is the Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -2161,11 +2161,11 @@ UnpackedRecord *pRet; /* Return value */ pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pRet ){ memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1)); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); + sqlite3VdbeRecordUnpack(nKey, pKey, pRet); } return pRet; } /* Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -4221,34 +4221,26 @@ } } return; } /* -** This routine is used to allocate sufficient space for an UnpackedRecord -** structure large enough to be used with sqlite3VdbeRecordUnpack() if -** the first argument is a pointer to KeyInfo structure pKeyInfo. -** -** The space is either allocated using sqlite3DbMallocRaw() or from within -** the unaligned buffer passed via the second and third arguments (presumably -** stack space). If the former, then *ppFree is set to a pointer that should -** be eventually freed by the caller using sqlite3DbFree(). Or, if the -** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL -** before returning. -** -** If an OOM error occurs, NULL is returned. +** Allocate sufficient space for an UnpackedRecord structure large enough +** to hold a decoded index record for pKeyInfo. +** +** The space is allocated using sqlite3DbMallocRaw(). If an OOM error +** occurs, NULL is returned. */ UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( KeyInfo *pKeyInfo /* Description of the record */ ){ UnpackedRecord *p; /* Unpacked record to return */ - int nByte; /* Number of bytes required for *p */ + u64 nByte; /* Number of bytes required for *p */ assert( sizeof(UnpackedRecord) + sizeof(Mem)*65536 < 0x7fffffff ); nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; p->aMem = (Mem*)&((char*)p)[ROUND8P(sizeof(UnpackedRecord))]; - assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nKeyField + 1; return p; } @@ -4256,11 +4248,10 @@ ** Given the nKey-byte encoding of a record in pKey[], populate the ** UnpackedRecord structure indicated by the fourth argument with the ** contents of the decoded record. */ void sqlite3VdbeRecordUnpack( - KeyInfo *pKeyInfo, /* Information about the record format */ int nKey, /* Size of the binary record */ const void *pKey, /* The binary record */ UnpackedRecord *p /* Populate this structure before returning. */ ){ const unsigned char *aKey = (const unsigned char *)pKey; @@ -4267,10 +4258,11 @@ u32 d; u32 idx; /* Offset in aKey[] to read from */ u16 u; /* Unsigned loop counter */ u32 szHdr; Mem *pMem = p->aMem; + KeyInfo *pKeyInfo = p->pKeyInfo; p->default_rc = 0; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); idx = getVarint32(aKey, szHdr); d = szHdr; @@ -4294,10 +4286,12 @@ /* In a corrupt record entry, the last pMem might have been set up using ** uninitialized memory. Overwrite its value with NULL, to prevent ** warnings from MSAN. */ sqlite3VdbeMemSetNull(pMem-1); } + testcase( u == pKeyInfo->nKeyField + 1 ); + testcase( u < pKeyInfo->nKeyField + 1 ); assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; } #ifdef SQLITE_DEBUG @@ -5153,10 +5147,11 @@ ** is an integer. ** ** The easiest way to enforce this limit is to consider only records with ** 13 fields or less. If the first field is an integer, the maximum legal ** header size is (12*5 + 1 + 1) bytes. */ + assert( p->pKeyInfo->aSortFlags!=0 ); if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; if( p->pKeyInfo->aSortFlags[0] ){ if( p->pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL ){ return sqlite3VdbeRecordCompare; @@ -5511,11 +5506,10 @@ ){ sqlite3 *db = v->db; i64 iKey2; PreUpdate preupdate; const char *zTbl = pTab->zName; - static const u8 fakeSortOrder = 0; #ifdef SQLITE_DEBUG int nRealCol; if( pTab->tabFlags & TF_WithoutRowid ){ nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn; }else if( pTab->tabFlags & TF_HasVirtual ){ @@ -5550,11 +5544,11 @@ preupdate.iNewReg = iReg; preupdate.pKeyinfo = (KeyInfo*)&preupdate.keyinfoSpace; preupdate.pKeyinfo->db = db; preupdate.pKeyinfo->enc = ENC(db); preupdate.pKeyinfo->nKeyField = pTab->nCol; - preupdate.pKeyinfo->aSortFlags = (u8*)&fakeSortOrder; + preupdate.pKeyinfo->aSortFlags = 0; /* Indicate .aColl, .nAllField uninit */ preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; preupdate.iBlobWrite = iBlobWrite; Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -764,11 +764,11 @@ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ UnpackedRecord *r2 = pTask->pUnpacked; if( *pbKey2Cached==0 ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); } @@ -791,11 +791,11 @@ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ UnpackedRecord *r2 = pTask->pUnpacked; if( !*pbKey2Cached ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); } @@ -831,10 +831,11 @@ res = vdbeSorterCompareTail( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); } }else{ + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ res = res * -1; } } @@ -894,10 +895,11 @@ }else{ if( *v2 & 0x80 ) res = +1; } } + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); if( res==0 ){ if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); @@ -967,11 +969,12 @@ assert( pCsr->pKeyInfo ); assert( !pCsr->isEphemeral ); assert( pCsr->eCurType==CURTYPE_SORTER ); assert( sizeof(KeyInfo) + UMXV(pCsr->pKeyInfo->nKeyField)*sizeof(CollSeq*) < 0x7fffffff ); - szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nKeyField+1); + assert( pCsr->pKeyInfo->nKeyField<=pCsr->pKeyInfo->nAllField ); + szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nAllField); sz = SZ_VDBESORTER(nWorker+1); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->uc.pSorter = pSorter; if( pSorter==0 ){ @@ -981,11 +984,16 @@ pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; if( nField && nWorker==0 ){ pKeyInfo->nKeyField = nField; + assert( nField<=pCsr->pKeyInfo->nAllField ); } + /* It is OK that pKeyInfo reuses the aSortFlags field from pCsr->pKeyInfo, + ** since the pCsr->pKeyInfo->aSortFlags[] array is invariant and lives + ** longer that pSorter. */ + assert( pKeyInfo->aSortFlags==pCsr->pKeyInfo->aSortFlags ); sqlite3BtreeEnter(pBt); pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt); sqlite3BtreeLeave(pBt); pSorter->nTask = nWorker + 1; pSorter->iPrev = (u8)(nWorker - 1); @@ -2761,11 +2769,11 @@ r2->nField = nKeyCol; } assert( r2->nField==nKeyCol ); pKey = vdbeSorterRowkey(pSorter, &nKey); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + sqlite3VdbeRecordUnpack(nKey, pKey, r2); for(i=0; iaMem[i].flags & MEM_Null ){ *pRes = -1; return SQLITE_OK; } Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -3735,11 +3735,10 @@ assert( rc==SQLITE_OK ); if( pWal->bShmUnreliable==0 ){ rc = walIndexReadHdr(pWal, pChanged); } #ifdef SQLITE_ENABLE_SETLK_TIMEOUT - walDisableBlocking(pWal); if( rc==SQLITE_BUSY_TIMEOUT ){ rc = SQLITE_BUSY; *pCnt |= WAL_RETRY_BLOCKED_MASK; } #endif @@ -3750,10 +3749,11 @@ ** which might cause WAL_RETRY to be returned even if BUSY_RECOVERY ** would be technically correct. But the race is benign since with ** WAL_RETRY this routine will be called again and will probably be ** right on the second iteration. */ + (void)walEnableBlocking(pWal); if( pWal->apWiData[0]==0 ){ /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. ** We assume this is a transient condition, so return WAL_RETRY. The ** xShmMap() implementation used by the default unix and win32 VFS ** modules may return SQLITE_BUSY due to a race condition in the @@ -3766,10 +3766,11 @@ rc = WAL_RETRY; }else if( rc==SQLITE_BUSY ){ rc = SQLITE_BUSY_RECOVERY; } } + walDisableBlocking(pWal); if( rc!=SQLITE_OK ){ return rc; } else if( pWal->bShmUnreliable ){ return walBeginShmUnreliable(pWal, pChanged); Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -2199,16 +2199,17 @@ } if( pLevel->iLeftJoin==0 ){ /* If a partial index is driving the loop, try to eliminate WHERE clause ** terms from the query that must be true due to the WHERE clause of - ** the partial index. + ** the partial index. This optimization does not work on an outer join, + ** as shown by: ** - ** 2019-11-02 ticket 623eff57e76d45f6: This optimization does not work - ** for a LEFT JOIN. + ** 2019-11-02 ticket 623eff57e76d45f6 (LEFT JOIN) + ** 2025-05-29 forum post 7dee41d32506c4ae (RIGHT JOIN) */ - if( pIdx->pPartIdxWhere ){ + if( pIdx->pPartIdxWhere && pLevel->pRJ==0 ){ whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC); } }else{ testcase( pIdx->pPartIdxWhere ); /* The following assert() is not a requirement, merely an observation: ADDED test/dblwidth-a.sql Index: test/dblwidth-a.sql ================================================================== --- /dev/null +++ test/dblwidth-a.sql @@ -0,0 +1,20 @@ +/* +** Run this script using "sqlite3" to confirm that the command-line +** shell properly handles the output of double-width characters. +** +** https://sqlite.org/forum/forumpost/008ac80276 +*/ +.mode box +CREATE TABLE data(word TEXT, description TEXT); +INSERT INTO data VALUES('〈οὐκέτι〉','Greek without dblwidth <...>'); +.print .mode box +SELECT * FROM data; +.mode table +.print .mode table +SELECT * FROM data; +.mode qbox +.print .mode qbox +SELECT * FROM data; +.mode column +.print .mode column +SELECT * FROM data; Index: test/fts3join.test ================================================================== --- test/fts3join.test +++ test/fts3join.test @@ -95,13 +95,10 @@ SELECT docid, * FROM ft4 WHERE ft4 MATCH ? ) AS rr ON t4.rowid=rr.docid WHERE t4.y = ?; } { QUERY PLAN - |--MATERIALIZE rr - | `--SCAN ft4 VIRTUAL TABLE INDEX 3: |--SCAN t4 - |--BLOOM FILTER ON rr (docid=?) - `--SEARCH rr USING AUTOMATIC COVERING INDEX (docid=?) LEFT-JOIN + `--SCAN ft4 VIRTUAL TABLE INDEX 3: LEFT-JOIN } finish_test Index: test/join.test ================================================================== --- test/join.test +++ test/join.test @@ -999,10 +999,25 @@ } {} do_execsql_test join-20.2 { CREATE INDEX t1x ON t1(0) WHERE NULL IN (c1); SELECT * FROM t0 LEFT JOIN t1 WHERE NULL IN (c1); } {} + +# 2025-05-29 forum post 7dee41d32506c4ae +# The complaint in the forum post appears to be the same as for the +# ticket on 2019-11-02, only for RIGHT JOIN instead of LEFT JOIN. Note +# that RIGHT JOIN did not yet exist in SQLite when the ticket was +# written and fixed. +# +do_execsql_test join-20.3 { + DROP TABLE t1; + CREATE TABLE t1(x INT); INSERT INTO t1(x) VALUES(1); + CREATE TABLE t2(y BOOLEAN); INSERT INTO t2(y) VALUES(false); + CREATE TABLE t3(z INT); INSERT INTO t3(z) VALUES(3); + CREATE INDEX t2y ON t2(y) WHERE y; + SELECT quote(z) FROM t1 RIGHT JOIN t2 ON y LEFT JOIN t3 ON y; +} {NULL} # 2019-11-30 ticket 7f39060a24b47353 # Do not allow a WHERE clause term to qualify a partial index on the # right table of a LEFT JOIN. # @@ -1286,7 +1301,45 @@ do_execsql_test join-30.3 { SELECT DISTINCT a, b FROM t0 JOIN t1 ON z=a RIGHT JOIN t2 ON a=b LEFT JOIN v5 ON false WHERE x <= y; } {} + +# 2025-05-30 https://sqlite.org/forum/forumpost/4fc70203b61c7e12 +# +# When converting a USING(x) or NATURAL into the constraint expression +# t1.x==t2.x, mark the t1.x term as EP_CanBeNull if it is the left table +# of a RIGHT JOIN. +# +reset_db +db null NULL +do_execsql_test join-31.1 { + CREATE TABLE t1(c0 INT , c1 INT); INSERT INTO t1(c0, c1) VALUES(NULL,11); + CREATE TABLE t2(c0 INT NOT NULL); + CREATE TABLE t2n(c0 INT); + CREATE TABLE t3(x INT); INSERT INTO t3(x) VALUES(3); + CREATE TABLE t4(y INT); INSERT INTO t4(y) VALUES(4); + CREATE TABLE t5(c0 INT, x INT); INSERT INTO t5 VALUES(NULL, 5); +} +do_execsql_test join-31.2 { + SELECT * FROM t2 RIGHT JOIN t3 ON true LEFT JOIN t1 USING(c0); +} {NULL 3 NULL} +do_execsql_test join-31.3 { + SELECT * FROM t2 RIGHT JOIN t3 ON true NATURAL LEFT JOIN t1; +} {NULL 3 NULL} +do_execsql_test join-31.4 { + SELECT * FROM t2n RIGHT JOIN t3 ON true LEFT JOIN t1 USING(c0); +} {NULL 3 NULL} +do_execsql_test join-31.5 { + SELECT * FROM t5 LEFT JOIN t1 USING(c0); +} {NULL 5 NULL} +do_execsql_test join-31.6 { + SELECT * FROM t3 LEFT JOIN t2 ON true LEFT JOIN t1 USING(c0); +} {3 NULL NULL} +do_execsql_test join-31.7 { + SELECT * FROM t3 LEFT JOIN t2 ON true NATURAL LEFT JOIN t1; +} {3 NULL NULL} +do_execsql_test join-31.8 { + SELECT * FROM t3 LEFT JOIN t2 ON true JOIN t4 ON true NATURAL LEFT JOIN t1; +} {3 NULL 4 NULL} finish_test Index: test/joinH.test ================================================================== --- test/joinH.test +++ test/joinH.test @@ -338,7 +338,77 @@ } {NULL NULL 1 -592 4 192 16 48} do_execsql_test 13.4 { SELECT sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t3 ON true GROUP BY a1.a ORDER BY 1; } {-1480 240 480} + +#------------------------------------------------------------------------- +# 2025-05-30 +# https://sqlite.org/forum/forumpost/5028c785b6 +# +reset_db + +do_execsql_test 14.0 { + CREATE TABLE t1(c0 INT); + CREATE TABLE t2(c0 BLOB); + CREATE TABLE t3(c0 BLOB); + CREATE TABLE t4(c4 BLOB); + INSERT INTO t1(c0) VALUES(0); + INSERT INTO t3(c0) VALUES('0'); +} + +do_execsql_test 14.1.1 { + SELECT * FROM t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3; +} {0} + +do_execsql_test 14.1.2 { + SELECT * FROM t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3 FULL JOIN t4 ON true; +} {0 {}} + +do_execsql_test 14.1.3 { + SELECT * FROM (t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3) FULL JOIN t4 ON true; +} {0 {}} + +do_execsql_test 14.1.4 { + SELECT * + FROM (t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3) AS qq FULL JOIN t4 ON true; +} {0 {}} + +do_execsql_test 14.2.1 { + SELECT * FROM t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1; +} {0} + +do_execsql_test 14.2.2 { + SELECT * FROM t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1 FULL JOIN t4 ON true; +} {0 {}} + +do_execsql_test 14.2.3 { + SELECT * FROM (t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1) FULL JOIN t4 ON true; +} {0 {}} + +do_execsql_test 14.2.4 { + SELECT * + FROM (t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1) AS qq FULL JOIN t4 ON true; +} {0 {}} + +# 2025-06-01 +# +reset_db +do_execsql_test 15.1 { + CREATE TABLE t0(c0); + CREATE TABLE t1(c0); + CREATE TABLE t2(c0); + INSERT INTO t0 VALUES ('1.0'); + INSERT INTO t2(c0) VALUES (9); + SELECT t0.c0,t2.c0 FROM (SELECT CAST(t0.c0 as REAL) AS c0 FROM t0) as subquery NATURAL LEFT JOIN t1 NATURAL JOIN t0 RIGHT JOIN t2 ON 1; +} {1.0 9} +do_execsql_test 15.2 { + CREATE TABLE x1(x COLLATE nocase); + CREATE TABLE x2(x); + CREATE TABLE x3(x); + CREATE TABLE t4(y); + INSERT INTO x1 VALUES('ABC'); + INSERT INTO x3 VALUES('abc'); + SELECT lower(x), quote(y) FROM x1 LEFT JOIN x2 USING (x) JOIN x3 USING (x) FULL JOIN t4; +} {abc NULL} finish_test Index: test/snapshot3.test ================================================================== --- test/snapshot3.test +++ test/snapshot3.test @@ -93,10 +93,13 @@ do_test 1.8 { execsql BEGIN db3 list [catch { sqlite3_snapshot_open_blob db3 main $snap } msg] $msg } {1 SQLITE_ERROR_SNAPSHOT} + +db3 close +db2 close #------------------------------------------------------------------------- reset_db do_execsql_test 2.0 { PRAGMA journal_mode = wal; Index: test/tclsqlite.test ================================================================== --- test/tclsqlite.test +++ test/tclsqlite.test @@ -7,11 +7,11 @@ # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for TCL interface to the -# SQLite library. +# SQLite library. # # Actually, all tests are based on the TCL interface, so the main # interface is pretty well tested. This file contains some addition # tests for fringe issues that the main test suite does not cover. # @@ -119,11 +119,11 @@ } {1 {wrong # args: should be "db complete SQL"}} } do_test tcl-1.14 { set v [catch {db eval} msg] lappend v $msg -} {1 {wrong # args: should be "db eval ?OPTIONS? SQL ?ARRAY-NAME? ?SCRIPT?"}} +} {1 {wrong # args: should be "db eval ?OPTIONS? SQL ?VAR-NAME? ?SCRIPT?"}} do_test tcl-1.15 { set v [catch {db function} msg] lappend v $msg } {1 {wrong # args: should be "db function NAME ?SWITCHES? SCRIPT"}} do_test tcl-1.16 { @@ -356,10 +356,23 @@ } {real} do_test tcl-9.3 { db function ret_int {return [expr {int(rand()*200)}]} execsql {SELECT typeof(ret_int())} } {integer} + +proc breakAsNullUdf args { + if {"1" eq [lindex $args 0]} {return -code break} +} +do_test tcl-9.4 { + db function banu breakAsNullUdf + execsql {SELECT typeof(banu()), typeof(banu(1))} +} {text null} +do_test tcl-9.5 { + db nullvalue banunull + db eval {SELECT banu(), banu(1)} +} {{} banunull} + # Recursive calls to the same user-defined function # ifcapable tclvar { do_test tcl-9.10 { @@ -463,21 +476,21 @@ } {2} do_test tcl-10.13 { db eval {SELECT * FROM t4} } {1 2 5 6 7} -# Now test that [db transaction] commands may be nested with +# Now test that [db transaction] commands may be nested with # the expected results. # do_test tcl-10.14 { db transaction { db eval { DELETE FROM t4; INSERT INTO t4 VALUES('one'); } - catch { + catch { db transaction { db eval { INSERT INTO t4 VALUES('two') } db transaction { db eval { INSERT INTO t4 VALUES('three') } error "throw an error!" @@ -672,15 +685,15 @@ do_test tcl-15.5 { db exists {SELECT a FROM t1 WHERE a>3} } {0} -# 2017-06-26: The --withoutnulls flag to "db eval". +# 2017-06-26: The -withoutnulls flag to "db eval". # -# In the "db eval --withoutnulls SQL ARRAY" form, NULL results cause the -# corresponding array entry to be unset. The default behavior (without -# the -withoutnulls flags) is for the corresponding array value to get +# In the "db eval -withoutnulls SQL TARGET" form, NULL results cause the +# corresponding target entry to be unset. The default behavior (without +# the -withoutnulls flags) is for the corresponding target value to get # the [db nullvalue] string. # catch {db close} forcedelete test.db sqlite3 db test.db @@ -718,68 +731,68 @@ # reset_db proc add {a b} { return [expr $a + $b] } proc ret {a} { return $a } -db function add_i -returntype integer add +db function add_i -returntype integer add db function add_r -ret real add -db function add_t -return text add -db function add_b -returntype blob add -db function add_a -returntype any add +db function add_t -return text add +db function add_b -returntype blob add +db function add_a -returntype any add -db function ret_i -returntype int ret +db function ret_i -returntype int ret db function ret_r -returntype real ret -db function ret_t -returntype text ret -db function ret_b -returntype blob ret -db function ret_a -r any ret +db function ret_t -returntype text ret +db function ret_b -returntype blob ret +db function ret_a -r any ret do_execsql_test 17.0 { SELECT quote( add_i(2, 3) ); - SELECT quote( add_r(2, 3) ); - SELECT quote( add_t(2, 3) ); - SELECT quote( add_b(2, 3) ); - SELECT quote( add_a(2, 3) ); + SELECT quote( add_r(2, 3) ); + SELECT quote( add_t(2, 3) ); + SELECT quote( add_b(2, 3) ); + SELECT quote( add_a(2, 3) ); } {5 5.0 '5' X'35' 5} do_execsql_test 17.1 { SELECT quote( add_i(2.2, 3.3) ); - SELECT quote( add_r(2.2, 3.3) ); - SELECT quote( add_t(2.2, 3.3) ); - SELECT quote( add_b(2.2, 3.3) ); - SELECT quote( add_a(2.2, 3.3) ); + SELECT quote( add_r(2.2, 3.3) ); + SELECT quote( add_t(2.2, 3.3) ); + SELECT quote( add_b(2.2, 3.3) ); + SELECT quote( add_a(2.2, 3.3) ); } {5.5 5.5 '5.5' X'352E35' 5.5} do_execsql_test 17.2 { SELECT quote( ret_i(2.5) ); - SELECT quote( ret_r(2.5) ); - SELECT quote( ret_t(2.5) ); - SELECT quote( ret_b(2.5) ); - SELECT quote( ret_a(2.5) ); + SELECT quote( ret_r(2.5) ); + SELECT quote( ret_t(2.5) ); + SELECT quote( ret_b(2.5) ); + SELECT quote( ret_a(2.5) ); } {2.5 2.5 '2.5' X'322E35' 2.5} do_execsql_test 17.3 { SELECT quote( ret_i('2.5') ); - SELECT quote( ret_r('2.5') ); - SELECT quote( ret_t('2.5') ); - SELECT quote( ret_b('2.5') ); - SELECT quote( ret_a('2.5') ); + SELECT quote( ret_r('2.5') ); + SELECT quote( ret_t('2.5') ); + SELECT quote( ret_b('2.5') ); + SELECT quote( ret_a('2.5') ); } {2.5 2.5 '2.5' X'322E35' '2.5'} do_execsql_test 17.4 { SELECT quote( ret_i('abc') ); - SELECT quote( ret_r('abc') ); - SELECT quote( ret_t('abc') ); - SELECT quote( ret_b('abc') ); - SELECT quote( ret_a('abc') ); + SELECT quote( ret_r('abc') ); + SELECT quote( ret_t('abc') ); + SELECT quote( ret_b('abc') ); + SELECT quote( ret_a('abc') ); } {'abc' 'abc' 'abc' X'616263' 'abc'} do_execsql_test 17.5 { SELECT quote( ret_i(X'616263') ); - SELECT quote( ret_r(X'616263') ); - SELECT quote( ret_t(X'616263') ); - SELECT quote( ret_b(X'616263') ); - SELECT quote( ret_a(X'616263') ); + SELECT quote( ret_r(X'616263') ); + SELECT quote( ret_t(X'616263') ); + SELECT quote( ret_b(X'616263') ); + SELECT quote( ret_a(X'616263') ); } {'abc' 'abc' 'abc' X'616263' X'616263'} do_test 17.6.1 { list [catch { db function xyz -return object ret } msg] $msg } {1 {bad type "object": must be integer, real, text, blob, or any}} @@ -846,25 +859,74 @@ do_catchsql_test 19.911 { SELECT $abc, typeof($abc), $def, typeof($def), $ghi, typeof($ghi); } {1 {invalid command name "bind_fallback_does_not_exist"}} db bind_fallback {} -#------------------------------------------------------------------------- +# 2025-05-05: the -asdict eval flag +# do_test 20.0 { + execsql {CREATE TABLE tad(a,b)} + execsql {INSERT INTO tad(a,b) VALUES('aa','bb'),('AA','BB')} + db eval -asdict { + SELECT a, b FROM tad WHERE 0 + } D {} + set D +} {* {a b}} + +do_test 20.1 { + unset D + set i 0 + set res {} + set colNames {} + db eval -asdict { + SELECT a, b FROM tad ORDER BY a + } D { + dict set D i [incr i] + lappend res $i [dict get $D a] [dict get $D b] + if {1 == $i} { + set colNames [dict get $D *] + } + } + lappend res $colNames + unset D + set res +} {1 AA BB 2 aa bb {a b}} + +do_test 20.2 { + set res {} + db eval -asdict -withoutnulls { + SELECT n, a, b FROM ( + SELECT 1 as n, 'aa' as a, NULL as b + UNION ALL + SELECT 2 as n, NULL as a, 'bb' as b + ) + ORDER BY n + } D { + dict unset D * + lappend res [dict values $D] + } + unset D + execsql {DROP TABLE tad} + set res +} {{1 aa} {2 bb}} + +#------------------------------------------------------------------------- +do_test 21.0 { db transaction { db close } } {} -do_test 20.1 { +do_test 21.1 { sqlite3 db test.db set rc [catch { db eval {SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3} { db close } } msg] list $rc $msg } {1 {invalid command name "db"}} - + + proc closedb {} { db close return 10 } @@ -872,21 +934,22 @@ sqlite3 db test.db db func closedb closedb db func func1 func1 -do_test 20.2 { +do_test 21.2 { set rc [catch { db eval { SELECT closedb(),func1() UNION ALL SELECT 20,30 UNION ALL SELECT 30,40 } } msg] list $rc $msg } {0 {10 1 20 30 30 40}} sqlite3 db :memory: -do_test 21.1 { +do_test 22.1 { catch {db eval {SELECT 1 2 3;}} msg db erroroffset } {9} + finish_test Index: test/vacuum.test ================================================================== --- test/vacuum.test +++ test/vacuum.test @@ -398,7 +398,28 @@ INSERT INTO t8 VALUES('c', 'd'); PRAGMA count_changes = 1; } } {} do_test vacuum-10.2 { execsql VACUUM } {} + +# Verify that VACUUM still works if ATTACH is disabled. +# +do_execsql_test vacuum-11.1 { + PRAGMA page_size=1024; + VACUUM; + PRAGMA page_size; +} {1024} +sqlite3_db_config db ATTACH_CREATE 0 +do_execsql_test vacuum-11.2 { + PRAGMA page_size=2048; + VACUUM; + PRAGMA page_size; +} {2048} +sqlite3_db_config db ATTACH_CREATE 1 +sqlite3_db_config db ATTACH_WRITE 0 +do_execsql_test vacuum-11.3 { + PRAGMA page_size=4096; + VACUUM; + PRAGMA page_size; +} {4096} finish_test ADDED test/walsetlk_recover.test Index: test/walsetlk_recover.test ================================================================== --- /dev/null +++ test/walsetlk_recover.test @@ -0,0 +1,104 @@ +# 2025 May 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix walsetlk_recover + +ifcapable !wal {finish_test ; return } +# ifcapable !setlk_timeout {finish_test ; return } + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); +} {wal} + +db_save_and_close +db_restore + +testfixture_nb myvar { + + testvfs tvfs -fullshm 1 + sqlite3 db test.db -vfs tvfs + tvfs script vfs_callback + tvfs filter xRead + + set ::done 0 + proc vfs_callback {method file args} { + if {$::done==0 && [string match *wal $file]} { + after 4000 + set ::done 1 + } + return "SQLITE_OK" + } + + db eval { + SELECT * FROM t1 + } + + db close +} + +# Give the [testfixture_nb] command time to start +after 1000 {set xyz 1} +vwait xyz + +testvfs tvfs -fullshm 1 +sqlite3 db test.db -vfs tvfs + +tvfs script sleep_callback +tvfs filter xSleep +set ::sleep_count 0 +proc sleep_callback {args} { + incr ::sleep_count +} + +sqlite3 db test.db -vfs tvfs +db timeout 500 +set tm [lindex [time { + catch { + db eval {SELECT * FROM t1} + } msg +}] 0] + +do_test 1.2 { set ::msg } {database is locked} +do_test 1.3.($::tm) { expr $::tm>400000 && $::tm<2000000 } 1 + +vwait myvar + +do_execsql_test 1.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6} + +db close +tvfs delete + +# All SQLite builds should pass the tests above. SQLITE_ENABLE_SETLK_TIMEOUT=1 +# builds do so without calling the VFS xSleep method. +if {$::sqlite_options(setlk_timeout)==1} { + do_test 1.5.1 { + set ::sleep_count + } 0 +} else { + do_test 1.5.2 { + expr $::sleep_count>0 + } 1 +} + +finish_test + ADDED test/walsetlk_snapshot.test Index: test/walsetlk_snapshot.test ================================================================== --- /dev/null +++ test/walsetlk_snapshot.test @@ -0,0 +1,109 @@ +# 2025 May 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix walsetlk_snapshot + +ifcapable !wal {finish_test ; return } +ifcapable !snapshot {finish_test; return} + +db close +testvfs tvfs -fullshm 1 +sqlite3 db test.db -vfs tvfs +tvfs script sleep_callback +tvfs filter xSleep +set ::sleep_count 0 +proc sleep_callback {args} { + incr ::sleep_count +} + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); +} {wal} + +do_test 1.1 { + db eval BEGIN + set ::snap [sqlite3_snapshot_get db main] + db eval { + INSERT INTO t1 VALUES(7, 8); + COMMIT; + } +} {} + +testfixture_nb myvar { + + testvfs tvfs -fullshm 1 + sqlite3 db test.db -vfs tvfs + tvfs script vfs_callback + tvfs filter {xWrite} + + set ::done 0 + proc vfs_callback {args} { + if {$::done==0} { + after 4000 + set ::done 1 + } + return "SQLITE_OK" + } + + db eval { + PRAGMA wal_checkpoint; + } + + db close +} + +# Give the [testfixture_nb] command time to start +after 1000 {set xyz 1} +vwait xyz + +db timeout 500 +set tm [lindex [time { + catch { + db eval BEGIN + sqlite3_snapshot_open db main $::snap + } msg +}] 0] + +do_test 1.2 { set ::msg } {SQLITE_BUSY} +do_test 1.3.($::tm) { expr $::tm<2000000 } 1 + +do_execsql_test 1.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8} + +sqlite3_snapshot_free $::snap + +vwait myvar + +# All SQLite builds should pass the tests above. SQLITE_ENABLE_SETLK_TIMEOUT=1 +# builds do so without calling the VFS xSleep method. +if {$::sqlite_options(setlk_timeout)==1} { + do_test 1.5.1 { + set ::sleep_count + } 0 +} else { + do_test 1.5.2 { + expr $::sleep_count>0 + } 1 +} + +finish_test + Index: tool/lemon.c ================================================================== --- tool/lemon.c +++ tool/lemon.c @@ -5923,11 +5923,11 @@ /* Hash a configuration */ PRIVATE unsigned confighash(struct config *a) { unsigned h=0; - h = h*571 + a->rp->index*37 + a->dot; + h = a->rp->index*37 + a->dot; return h; } /* There is one instance of the following structure for each ** associative array of type "x4". Index: tool/sqlite3_rsync.c ================================================================== --- tool/sqlite3_rsync.c +++ tool/sqlite3_rsync.c @@ -1536,11 +1536,11 @@ sqlite3_finalize(pCkHash); sqlite3_finalize(pCkHashN); sqlite3_finalize(pInsHash); pCkHash = 0; pInsHash = 0; - if( mxHashnPage ){ + if( mxHash<=p->nPage ){ runSql(p, "WITH RECURSIVE c(n) AS" " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" " INSERT INTO badHash SELECT n, 1 FROM c", mxHash, p->nPage); } @@ -1795,10 +1795,11 @@ reportError(p, "cannot open in-memory database: %s", sqlite3_errmsg(p->db)); closeDb(p); break; } + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); runSql(p, "ATTACH %Q AS 'replica'", p->zReplica); if( p->wrongEncoding ){ p->wrongEncoding = 0; runSql(p, "PRAGMA encoding=utf16le"); runSql(p, "ATTACH %Q AS 'replica'", p->zReplica);