Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Comment: | Merge updates from trunk. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | altShellFix |
Files: | files | file ages | folders |
SHA1: | 566b551e5a81440a5c8ff865ceb4422c |
User & Date: | mistachkin 2016-04-03 20:45:04 |
2016-04-03
| ||
20:50 | Replace the new fprintf() calls. check-in: f76c3a0c user: mistachkin tags: altShellFix | |
20:45 | Merge updates from trunk. check-in: 566b551e user: mistachkin tags: altShellFix | |
2016-04-01
| ||
17:54 | Preupdate hook documentation fixes. No changes to code. check-in: 59814f35 user: drh tags: trunk | |
2016-01-06
| ||
20:50 | Merge updates from trunk. check-in: 5cbab637 user: mistachkin tags: altShellFix | |
Changes to Makefile.in.
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 .. 77 78 79 80 81 82 83 84 85 86 87 88 89 90 ... 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 ... 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 ... 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 ... 372 373 374 375 376 377 378 379 380 381 382 383 384 385 ... 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 ... 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 ... 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 ... 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 ... 622 623 624 625 626 627 628 629 630 631 632 633 634 635 ... 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 ... 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 .... 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 .... 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 .... 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 .... 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 .... 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 |
# will run on the target platform. (BCC and TCC are usually the # same unless your are cross-compiling.) Separate CC and CFLAGS macros # are provide so that these aspects of the build process can be changed # on the "make" command-line. Ex: "make CC=clang CFLAGS=-fsanitize=undefined" # CC = @CC@ CFLAGS = @CPPFLAGS@ @CFLAGS@ TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3 # Define this for the autoconf-based build, so that the code knows it can # include the generated config.h # TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite # Define -DNDEBUG to compile without debugging (i.e., for production usage) ................................................................................ TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@ # Enable/disable loadable extensions, and other optional features # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). # The same set of OMIT and ENABLE flags should be passed to the # LEMON parser generator and the mkkeywordhash tool as well. OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ TCC += $(OPT_FEATURE_FLAGS) # Add in any optional parameters specified on the make commane line # ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". TCC += $(OPTS) ................................................................................ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ fts3_tokenize_vtab.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ utf.lo vtab.lo # Object files for the amalgamation. ................................................................................ $(TOP)/src/fkey.c \ $(TOP)/src/func.c \ $(TOP)/src/global.c \ $(TOP)/src/hash.c \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ $(TOP)/src/journal.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ $(TOP)/src/main.c \ $(TOP)/src/malloc.c \ $(TOP)/src/mem0.c \ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ ................................................................................ SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c SRC += \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h \ $(TOP)/ext/rbu/sqlite3rbu.c SRC += \ $(TOP)/ext/misc/json1.c # Generated source code files # SRC += \ keywordhash.h \ opcodes.c \ opcodes.h \ ................................................................................ $(TOP)/src/test6.c \ $(TOP)/src/test7.c \ $(TOP)/src/test8.c \ $(TOP)/src/test9.c \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ $(TOP)/src/test_func.c \ ................................................................................ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ $(TOP)/src/test_windirent.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ ................................................................................ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c # Header files used by all library source files. # HDR = \ $(TOP)/src/btree.h \ $(TOP)/src/btreeInt.h \ $(TOP)/src/hash.h \ ................................................................................ # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt # Extra compiler options for various shell tools # SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) ................................................................................ libtclsqlite3.la: tclsqlite.lo libsqlite3.la $(LTLINK) -no-undefined -o $@ tclsqlite.lo \ libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \ -rpath "$(TCLLIBDIR)" \ -version-info "8:6:8" \ -avoid-version sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h $(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \ $(TOP)/src/shell.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) ................................................................................ mv vdbe.new tsrc/vdbe.c cp fts5.c fts5.h tsrc touch .target_source sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . tclsqlite3.c: sqlite3.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c ................................................................................ hash.lo: $(TOP)/src/hash.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/hash.c insert.lo: $(TOP)/src/insert.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c journal.lo: $(TOP)/src/journal.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/journal.c legacy.lo: $(TOP)/src/legacy.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c loadext.lo: $(TOP)/src/loadext.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/loadext.c main.lo: $(TOP)/src/main.c $(HDR) ................................................................................ fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c json1.lo: $(TOP)/ext/misc/json1.c $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c # FTS5 things # FTS5_SRC = \ $(TOP)/ext/fts5/fts5.h \ ................................................................................ fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c fts5.c # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.la. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(TEXE): $(TESTFIXTURE_SRC) ................................................................................ # quicktest: ./testfixture$(TEXE) ./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # test: $(TESTPROGS) fastfuzztest ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: $(TESTPROGS) valgrindfuzz OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS) ................................................................................ showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS) showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # checksymbols: sqlite3.lo nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 echo '0 errors out of 1 tests' # Build the amalgamation-autoconf package. # amalgamation-tarball: sqlite3.c TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ $(TOP)/test/tt3_checkpoint.c \ ................................................................................ # Standard install and cleanup targets # lib_install: libsqlite3.la $(INSTALL) -d $(DESTDIR)$(libdir) $(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir) install: sqlite3$(BEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install} $(INSTALL) -d $(DESTDIR)$(bindir) $(LTINSTALL) sqlite3$(BEXE) $(DESTDIR)$(bindir) $(INSTALL) -d $(DESTDIR)$(includedir) $(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir) $(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir) $(INSTALL) -d $(DESTDIR)$(pkgconfigdir) $(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(pkgconfigdir) pkgIndex.tcl: ................................................................................ rm -f *.da *.bb *.bbg gmon.out rm -rf quota2a quota2b quota2c rm -rf tsrc .target_source rm -f tclsqlite3$(TEXE) rm -f testfixture$(TEXE) test.db rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) rm -f wordcount$(TEXE) rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix rm -f mptester mptester.exe rm -f fuzzershell fuzzershell.exe rm -f fuzzcheck fuzzcheck.exe rm -f sqldiff sqldiff.exe rm -f fts5.* fts5parse.* distclean: clean rm -f config.h config.log config.status libtool Makefile sqlite3.pc |
| > > | | > < > > > < < > > > | > | > | | > > > > > > > < < < > > > > > > > > | > > > > > > > > > | | > > | > > > | | | > > |
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 .. 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 ... 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 ... 223 224 225 226 227 228 229 230 231 232 233 234 235 236 ... 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 ... 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 ... 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 ... 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 ... 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 ... 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 ... 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 ... 741 742 743 744 745 746 747 748 749 750 751 752 753 754 .... 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 .... 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 .... 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 .... 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 .... 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 .... 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 |
# will run on the target platform. (BCC and TCC are usually the # same unless your are cross-compiling.) Separate CC and CFLAGS macros # are provide so that these aspects of the build process can be changed # on the "make" command-line. Ex: "make CC=clang CFLAGS=-fsanitize=undefined" # CC = @CC@ CFLAGS = @CPPFLAGS@ @CFLAGS@ TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session # Define this for the autoconf-based build, so that the code knows it can # include the generated config.h # TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite # Define -DNDEBUG to compile without debugging (i.e., for production usage) ................................................................................ TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@ # Enable/disable loadable extensions, and other optional features # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). # The same set of OMIT and ENABLE flags should be passed to the # LEMON parser generator and the mkkeywordhash tool as well. OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ OPT_FEATURE_FLAGS += -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK TCC += $(OPT_FEATURE_FLAGS) # Add in any optional parameters specified on the make commane line # ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". TCC += $(OPTS) ................................................................................ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ fts3_tokenize_vtab.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo json1.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ utf.lo vtab.lo # Object files for the amalgamation. ................................................................................ $(TOP)/src/fkey.c \ $(TOP)/src/func.c \ $(TOP)/src/global.c \ $(TOP)/src/hash.c \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ $(TOP)/src/main.c \ $(TOP)/src/malloc.c \ $(TOP)/src/mem0.c \ $(TOP)/src/mem1.c \ $(TOP)/src/mem2.c \ ................................................................................ SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c SRC += \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c SRC += \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/sqlite3session.h SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h \ $(TOP)/ext/rbu/sqlite3rbu.c SRC += \ $(TOP)/ext/misc/json1.c # Generated source code files # SRC += \ keywordhash.h \ opcodes.c \ opcodes.h \ ................................................................................ $(TOP)/src/test6.c \ $(TOP)/src/test7.c \ $(TOP)/src/test8.c \ $(TOP)/src/test9.c \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ $(TOP)/src/test_bestindex.c \ $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ $(TOP)/src/test_func.c \ ................................................................................ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ $(TOP)/src/test_windirent.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/ext/session/test_session.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/fts5/fts5_test_tok.c \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ ................................................................................ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c \ $(TOP)/ext/session/sqlite3session.c # Header files used by all library source files. # HDR = \ $(TOP)/src/btree.h \ $(TOP)/src/btreeInt.h \ $(TOP)/src/hash.h \ ................................................................................ # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt # Extra compiler options for various shell tools # SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) ................................................................................ libtclsqlite3.la: tclsqlite.lo libsqlite3.la $(LTLINK) -no-undefined -o $@ tclsqlite.lo \ libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \ -rpath "$(TCLLIBDIR)" \ -version-info "8:6:8" \ -avoid-version sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c $(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \ $(TOP)/src/shell.c sqlite3.c \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) srcck1$(BEXE): $(TOP)/tool/srcck1.c $(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c sourcetest: srcck1$(BEXE) sqlite3.c ./srcck1 sqlite3.c fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) ................................................................................ mv vdbe.new tsrc/vdbe.c cp fts5.c fts5.h tsrc touch .target_source sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . tclsqlite3.c: sqlite3.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c ................................................................................ hash.lo: $(TOP)/src/hash.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/hash.c insert.lo: $(TOP)/src/insert.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c legacy.lo: $(TOP)/src/legacy.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c loadext.lo: $(TOP)/src/loadext.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/loadext.c main.lo: $(TOP)/src/main.c $(HDR) ................................................................................ fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c json1.lo: $(TOP)/ext/misc/json1.c $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c # FTS5 things # FTS5_SRC = \ $(TOP)/ext/fts5/fts5.h \ ................................................................................ fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c fts5.c sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.la. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(TEXE): $(TESTFIXTURE_SRC) ................................................................................ # quicktest: ./testfixture$(TEXE) ./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # test: $(TESTPROGS) sourcetest fastfuzztest ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: $(TESTPROGS) valgrindfuzz OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS) ................................................................................ showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS) showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la $(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS) # This target will fail if the SQLite amalgamation contains any exported # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # checksymbols: sqlite3.lo nm -g --defined-only sqlite3.o | egrep -v ' sqlite3(changeset|session)?_' ; test $$? -ne 0 echo '0 errors out of 1 tests' # Build the amalgamation-autoconf package. The amalamgation-tarball target builds # a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz. # The snapshot-tarball target builds a tarball named by the SHA1 hash # amalgamation-tarball: sqlite3.c TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal snapshot-tarball: sqlite3.c TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ $(TOP)/test/tt3_checkpoint.c \ ................................................................................ # Standard install and cleanup targets # lib_install: libsqlite3.la $(INSTALL) -d $(DESTDIR)$(libdir) $(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir) install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install} $(INSTALL) -d $(DESTDIR)$(bindir) $(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir) $(INSTALL) -d $(DESTDIR)$(includedir) $(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir) $(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir) $(INSTALL) -d $(DESTDIR)$(pkgconfigdir) $(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(pkgconfigdir) pkgIndex.tcl: ................................................................................ rm -f *.da *.bb *.bbg gmon.out rm -rf quota2a quota2b quota2c rm -rf tsrc .target_source rm -f tclsqlite3$(TEXE) rm -f testfixture$(TEXE) test.db rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) rm -f wordcount$(TEXE) changeset$(TEXE) rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix rm -f mptester mptester.exe rm -f rbu rbu.exe rm -f srcck1 srcck1.exe rm -f fuzzershell fuzzershell.exe rm -f fuzzcheck fuzzcheck.exe rm -f sqldiff sqldiff.exe rm -f fts5.* fts5parse.* distclean: clean rm -f config.h config.log config.status libtool Makefile sqlite3.pc |
Changes to Makefile.msc.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 .. 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 ... 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 ... 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 ... 208 209 210 211 212 213 214 215 216 217 218 219 220 221 ... 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 ... 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 ... 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 ... 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 ... 436 437 438 439 440 441 442 443 444 445 446 447 448 449 ... 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 ... 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 ... 547 548 549 550 551 552 553 554 555 556 557 558 559 560 ... 598 599 600 601 602 603 604 605 606 607 608 609 610 611 ... 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 ... 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 ... 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 .... 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 .... 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 .... 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 .... 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 .... 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 .... 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 .... 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 .... 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 .... 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 .... 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 .... 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 .... 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 .... 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 .... 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 .... 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 |
############################################################################### # The toplevel directory of the source tree. This is the directory # that contains this "Makefile.msc". # TOP = . # Set this non-0 to create and use the SQLite amalgamation file. # !IFNDEF USE_AMALGAMATION USE_AMALGAMATION = 1 !ENDIF # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN USE_FULLWARN = 0 !ENDIF ................................................................................ # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # !IFNDEF SPLIT_AMALGAMATION SPLIT_AMALGAMATION = 0 !ENDIF # Set this non-0 to use the International Components for Unicode (ICU). # !IFNDEF USE_ICU USE_ICU = 0 !ENDIF # Set this non-0 to dynamically link to the MSVC runtime library. # !IFNDEF USE_CRT_DLL USE_CRT_DLL = 0 !ENDIF ................................................................................ # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). # !IFNDEF FOR_WINRT FOR_WINRT = 0 !ENDIF # Set this non-0 to compile binaries suitable for the UAP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). # !IFNDEF FOR_UAP FOR_UAP = 0 !ENDIF # Set this non-0 to skip attempting to look for and/or link with the Tcl # runtime library. # !IFNDEF NO_TCL NO_TCL = 0 !ENDIF # Set this to non-0 to create and use PDBs. # !IFNDEF SYMBOLS SYMBOLS = 1 !ENDIF ................................................................................ # Enable use of available compiler optimizations? Normally, this should be # non-zero. Setting this to zero, thus disabling all compiler optimizations, # can be useful for testing. # !IFNDEF OPTIMIZATIONS OPTIMIZATIONS = 2 !ENDIF # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### # This assumes that MSVC is always installed in 32-bit Program Files directory # and sets the variable for use in locating other 32-bit installs accordingly. # PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) ................................................................................ # Check for the predefined command macro CC. This should point to the compiler # binary for the target platform. If it is not defined, simply define it to # the legacy default value 'cl.exe'. # !IFNDEF CC CC = cl.exe !ENDIF # Check for the command macro LD. This should point to the linker binary for # the target platform. If it is not defined, simply define it to the legacy # default value 'link.exe'. # !IFNDEF LD LD = link.exe ................................................................................ # compiler binary for the target platform. If it is not defined, simply define # it to the legacy default value 'rc.exe'. # !IFNDEF RC RC = rc.exe !ENDIF # Check for the MSVC runtime library path macro. Othertise, this value will # default to the 'lib' directory underneath the MSVC installation directory. # !IFNDEF CRTLIBPATH CRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF CRTLIBPATH = $(CRTLIBPATH:\\=\) ................................................................................ !ELSEIF $(XCOMPILE)!=0 NCC = "$(VCINSTALLDIR)\bin\$(CC)" NCC = $(NCC:\\=\) !ELSE NCC = $(CC) !ENDIF # Check for the MSVC native runtime library path macro. Othertise, # this value will default to the 'lib' directory underneath the MSVC # installation directory. # !IFNDEF NCRTLIBPATH NCRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF NCRTLIBPATH = $(NCRTLIBPATH:\\=\) # Check for the Platform SDK library path macro. Othertise, this # value will default to the 'lib' directory underneath the Windows # SDK installation directory (the environment variable used appears # to be available when using Visual C++ 2008 or later via the # command line). # !IFNDEF NSDKLIBPATH NSDKLIBPATH = $(WINDOWSSDKDIR)\lib !ENDIF NSDKLIBPATH = $(NSDKLIBPATH:\\=\) # C compiler and options for use in building executables that # will run on the platform that is doing the build. # !IF $(USE_FULLWARN)!=0 BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS) !ELSE ................................................................................ !IF $(USE_FULLWARN)!=0 TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) !ELSE TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) # Check if we want to use the "stdcall" calling convention when compiling. # This is not supported by the compilers for non-x86 platforms. It should # also be noted here that building any target with these "stdcall" options # will most likely fail if the Tcl library is also required. This is due # to how the Tcl library functions are declared and exported (i.e. without # an explicit calling convention, which results in "cdecl"). # !IF $(USE_STDCALL)!=0 !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall ................................................................................ CORE_CCONV_OPTS = SHELL_CCONV_OPTS = !ENDIF # These are additional compiler options used for the core library. # !IFNDEF CORE_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport) !ELSE CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) !ENDIF !ENDIF # These are the additional targets that the core library should depend on # when linking. # !IFNDEF CORE_LINK_DEP !IF $(DYNAMIC_SHELL)!=0 CORE_LINK_DEP = !ELSE CORE_LINK_DEP = sqlite3.def !ENDIF !ENDIF # These are additional linker options used for the core library. # !IFNDEF CORE_LINK_OPTS !IF $(DYNAMIC_SHELL)!=0 CORE_LINK_OPTS = !ELSE CORE_LINK_OPTS = /DEF:sqlite3.def !ENDIF !ENDIF # These are additional compiler options used for the shell executable. # !IFNDEF SHELL_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) !ELSE SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) !ENDIF !ENDIF # This is the core library that the shell executable should depend on. # !IFNDEF SHELL_CORE_DEP !IF $(DYNAMIC_SHELL)!=0 SHELL_CORE_DEP = sqlite3.dll !ELSE SHELL_CORE_DEP = libsqlite3.lib !ENDIF !ENDIF # This is the core library that the shell executable should link with. # !IFNDEF SHELL_CORE_LIB !IF $(DYNAMIC_SHELL)!=0 SHELL_CORE_LIB = sqlite3.lib !ELSE SHELL_CORE_LIB = libsqlite3.lib !ENDIF !ENDIF # These are additional linker options used for the shell executable. # !IFNDEF SHELL_LINK_OPTS SHELL_LINK_OPTS = $(SHELL_CORE_LIB) ................................................................................ # !IF $(FOR_WINRT)!=0 TCC = $(TCC) -DSQLITE_OS_WINRT=1 RCC = $(RCC) -DSQLITE_OS_WINRT=1 TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP !ENDIF # Also, we need to dynamically link to the correct MSVC runtime # when compiling for WinRT (e.g. debug or release) OR if the # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 ................................................................................ BCC = $(BCC) -MTd !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT !ENDIF !ENDIF # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in # any extension header files by default. For non-amalgamation # builds, we need to make sure the compiler can find these. # !IF $(USE_AMALGAMATION)==0 TCC = $(TCC) -I$(TOP)\ext\fts3 RCC = $(RCC) -I$(TOP)\ext\fts3 TCC = $(TCC) -I$(TOP)\ext\rtree RCC = $(RCC) -I$(TOP)\ext\rtree !ENDIF # The mksqlite3c.tcl script accepts some options on the command # line. When compiling with debugging enabled, some of these # options are necessary in order to allow debugging symbols to # work correctly with Visual Studio when using the amalgamation. # ................................................................................ !IFNDEF MKSQLITE3C_ARGS !IF $(DEBUG)>1 MKSQLITE3C_ARGS = --linemacros !ELSE MKSQLITE3C_ARGS = !ENDIF !ENDIF # Define -DNDEBUG to compile without debugging (i.e., for production usage) # Omitting the define will cause extra debugging code to be inserted and # includes extra comments when "EXPLAIN stmt" is used. # !IF $(DEBUG)==0 TCC = $(TCC) -DNDEBUG BCC = $(BCC) -DNDEBUG RCC = $(RCC) -DNDEBUG !ENDIF !IF $(DEBUG)>0 || $(API_ARMOR)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 RCC = $(RCC) -DSQLITE_DEBUG=1 ................................................................................ # !IF $(DEBUG)>3 TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF !ENDIF # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment # prior to running nmake in order to match the actual installed location and # version on this machine. # !IFNDEF TCLINCDIR ................................................................................ # know the specific version we want to use. This variable (TCLSH_CMD) may be # overridden via the environment prior to running nmake in order to select a # specific Tcl shell to use. # !IFNDEF TCLSH_CMD TCLSH_CMD = tclsh85 !ENDIF # Compiler options needed for programs that use the readline() library. # !IFNDEF READLINE_FLAGS READLINE_FLAGS = -DHAVE_READLINE=0 !ENDIF ................................................................................ !IF $(USE_RPCRT4_LIB)!=0 REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1 !ENDIF # Add the required and optional SQLite compilation options into the command # lines used to invoke the MSVC code and resource compilers. # TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) # Add in any optional parameters specified on the commane line, e.g. # nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1" # TCC = $(TCC) $(OPTS) RCC = $(RCC) $(OPTS) ................................................................................ # If symbols are enabled (or compiling for debugging), enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF # If ICU support is enabled, add the compiler options for it. # !IF $(USE_ICU)!=0 TCC = $(TCC) -DSQLITE_ENABLE_ICU=1 RCC = $(RCC) -DSQLITE_ENABLE_ICU=1 TCC = $(TCC) -I$(TOP)\ext\icu RCC = $(RCC) -I$(TOP)\ext\icu TCC = $(TCC) -I$(ICUINCDIR) RCC = $(RCC) -I$(ICUINCDIR) !ENDIF # Command line prefixes for compiling code, compiling resources, # linking, etc. # LTCOMPILE = $(TCC) -Fo$@ LTRCOMPILE = $(RCC) -r LTLIB = lib.exe ................................................................................ LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" !ENDIF LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib !ENDIF # When compiling for UAP, some extra linker options are also required. # !IF $(FOR_UAP)!=0 LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib LTLINKOPTS = $(LTLINKOPTS) mincore.lib !IFDEF PSDKLIBPATH LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)" !ENDIF !ENDIF # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG $(LDOPTS) !ELSE LDFLAGS = $(LDOPTS) !ENDIF # Start with the Tcl related linker options. # !IF $(NO_TCL)==0 LTLIBPATHS = /LIBPATH:$(TCLLIBDIR) LTLIBS = $(LIBTCL) !ENDIF # If ICU support is enabled, add the linker options for it. # !IF $(USE_ICU)!=0 LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR) LTLIBS = $(LTLIBS) $(LIBICU) !ENDIF # You should not have to change anything below this line ############################################################################### # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo journal.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ utf.lo vtab.lo # Object files for the amalgamation. # LIBOBJS1 = sqlite3.lo # Determine the real value of LIBOBJ based on the 'configure' script # !IF $(USE_AMALGAMATION)==0 LIBOBJ = $(LIBOBJS0) !ELSE LIBOBJ = $(LIBOBJS1) !ENDIF # Determine if embedded resource compilation and usage are enabled. # !IF $(USE_RC)!=0 LIBRESOBJS = sqlite3res.lo !ELSE LIBRESOBJS = !ENDIF # All of the source code files. # SRC1 = \ $(TOP)\src\alter.c \ $(TOP)\src\analyze.c \ $(TOP)\src\attach.c \ $(TOP)\src\auth.c \ $(TOP)\src\backup.c \ $(TOP)\src\bitvec.c \ $(TOP)\src\btmutex.c \ $(TOP)\src\btree.c \ $(TOP)\src\btree.h \ $(TOP)\src\btreeInt.h \ $(TOP)\src\build.c \ $(TOP)\src\callback.c \ $(TOP)\src\complete.c \ $(TOP)\src\ctime.c \ $(TOP)\src\date.c \ $(TOP)\src\dbstat.c \ $(TOP)\src\delete.c \ $(TOP)\src\expr.c \ $(TOP)\src\fault.c \ $(TOP)\src\fkey.c \ $(TOP)\src\func.c \ $(TOP)\src\global.c \ $(TOP)\src\hash.c \ $(TOP)\src\hash.h \ $(TOP)\src\hwtime.h \ $(TOP)\src\insert.c \ $(TOP)\src\journal.c \ $(TOP)\src\legacy.c \ $(TOP)\src\loadext.c \ $(TOP)\src\main.c \ $(TOP)\src\malloc.c \ $(TOP)\src\mem0.c \ $(TOP)\src\mem1.c \ $(TOP)\src\mem2.c \ $(TOP)\src\mem3.c \ $(TOP)\src\mem5.c \ $(TOP)\src\memjournal.c \ $(TOP)\src\msvc.h \ $(TOP)\src\mutex.c \ $(TOP)\src\mutex.h \ $(TOP)\src\mutex_noop.c \ $(TOP)\src\mutex_unix.c \ $(TOP)\src\mutex_w32.c \ $(TOP)\src\notify.c \ $(TOP)\src\os.c \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ $(TOP)\src\os_setup.h \ $(TOP)\src\os_unix.c \ $(TOP)\src\os_win.c \ $(TOP)\src\os_win.h SRC2 = \ $(TOP)\src\pager.c \ $(TOP)\src\pager.h \ $(TOP)\src\parse.y \ $(TOP)\src\pcache.c \ $(TOP)\src\pcache.h \ $(TOP)\src\pcache1.c \ $(TOP)\src\pragma.c \ $(TOP)\src\pragma.h \ $(TOP)\src\prepare.c \ $(TOP)\src\printf.c \ $(TOP)\src\random.c \ $(TOP)\src\resolve.c \ $(TOP)\src\rowset.c \ $(TOP)\src\select.c \ $(TOP)\src\status.c \ $(TOP)\src\shell.c \ $(TOP)\src\sqlite.h.in \ $(TOP)\src\sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\table.c \ $(TOP)\src\threads.c \ $(TOP)\src\tclsqlite.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\treeview.c \ $(TOP)\src\trigger.c \ $(TOP)\src\utf.c \ $(TOP)\src\update.c \ $(TOP)\src\util.c \ $(TOP)\src\vacuum.c \ $(TOP)\src\vdbe.c \ $(TOP)\src\vdbe.h \ $(TOP)\src\vdbeapi.c \ $(TOP)\src\vdbeaux.c \ $(TOP)\src\vdbeblob.c \ $(TOP)\src\vdbemem.c \ $(TOP)\src\vdbesort.c \ $(TOP)\src\vdbetrace.c \ $(TOP)\src\vdbeInt.h \ $(TOP)\src\vtab.c \ $(TOP)\src\vxworks.h \ $(TOP)\src\wal.c \ $(TOP)\src\wal.h \ $(TOP)\src\walker.c \ $(TOP)\src\where.c \ $(TOP)\src\wherecode.c \ $(TOP)\src\whereexpr.c \ $(TOP)\src\whereInt.h # Source code for extensions # SRC3 = \ $(TOP)\ext\fts1\fts1.c \ $(TOP)\ext\fts1\fts1.h \ $(TOP)\ext\fts1\fts1_hash.c \ $(TOP)\ext\fts1\fts1_hash.h \ $(TOP)\ext\fts1\fts1_porter.c \ $(TOP)\ext\fts1\fts1_tokenizer.h \ $(TOP)\ext\fts1\fts1_tokenizer1.c \ $(TOP)\ext\fts2\fts2.c \ $(TOP)\ext\fts2\fts2.h \ $(TOP)\ext\fts2\fts2_hash.c \ $(TOP)\ext\fts2\fts2_hash.h \ $(TOP)\ext\fts2\fts2_icu.c \ $(TOP)\ext\fts2\fts2_porter.c \ $(TOP)\ext\fts2\fts2_tokenizer.h \ $(TOP)\ext\fts2\fts2_tokenizer.c \ $(TOP)\ext\fts2\fts2_tokenizer1.c SRC4 = \ $(TOP)\ext\fts3\fts3.c \ $(TOP)\ext\fts3\fts3.h \ $(TOP)\ext\fts3\fts3Int.h \ $(TOP)\ext\fts3\fts3_aux.c \ $(TOP)\ext\fts3\fts3_expr.c \ $(TOP)\ext\fts3\fts3_hash.c \ $(TOP)\ext\fts3\fts3_hash.h \ $(TOP)\ext\fts3\fts3_icu.c \ $(TOP)\ext\fts3\fts3_porter.c \ $(TOP)\ext\fts3\fts3_snippet.c \ $(TOP)\ext\fts3\fts3_tokenizer.h \ $(TOP)\ext\fts3\fts3_tokenizer.c \ $(TOP)\ext\fts3\fts3_tokenizer1.c \ $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ $(TOP)\ext\fts3\fts3_unicode.c \ $(TOP)\ext\fts3\fts3_unicode2.c \ $(TOP)\ext\fts3\fts3_write.c \ $(TOP)\ext\icu\sqliteicu.h \ $(TOP)\ext\icu\icu.c \ $(TOP)\ext\rtree\rtree.h \ $(TOP)\ext\rtree\rtree.c \ $(TOP)\ext\rbu\sqlite3rbu.h \ $(TOP)\ext\rbu\sqlite3rbu.c \ $(TOP)\ext\misc\json1.c # Generated source code files # SRC5 = \ keywordhash.h \ opcodes.c \ opcodes.h \ parse.c \ parse.h \ sqlite3.h # All source code files. # SRC = $(SRC1) $(SRC2) $(SRC3) $(SRC4) $(SRC5) # Source code to the test files. # TESTSRC = \ $(TOP)\src\test1.c \ $(TOP)\src\test2.c \ $(TOP)\src\test3.c \ ................................................................................ $(TOP)\src\test6.c \ $(TOP)\src\test7.c \ $(TOP)\src\test8.c \ $(TOP)\src\test9.c \ $(TOP)\src\test_autoext.c \ $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ $(TOP)\src\test_demovfs.c \ $(TOP)\src\test_devsym.c \ $(TOP)\src\test_fs.c \ $(TOP)\src\test_func.c \ ................................................................................ $(TOP)\src\test_tclvar.c \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vfs.c \ $(TOP)\src\test_windirent.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c \ $(TOP)\ext\rbu\test_rbu.c # Statically linked extensions # TESTEXT = \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\totype.c \ $(TOP)\ext\misc\wholenumber.c # Source code to the library files needed by the test fixture # TESTSRC2 = \ $(TOP)\src\attach.c \ $(TOP)\src\backup.c \ $(TOP)\src\bitvec.c \ $(TOP)\src\btree.c \ $(TOP)\src\build.c \ $(TOP)\src\ctime.c \ $(TOP)\src\date.c \ $(TOP)\src\dbstat.c \ $(TOP)\src\expr.c \ $(TOP)\src\func.c \ $(TOP)\src\insert.c \ $(TOP)\src\wal.c \ $(TOP)\src\main.c \ $(TOP)\src\mem5.c \ $(TOP)\src\os.c \ $(TOP)\src\os_unix.c \ $(TOP)\src\os_win.c \ $(TOP)\src\pager.c \ $(TOP)\src\pragma.c \ $(TOP)\src\prepare.c \ $(TOP)\src\printf.c \ $(TOP)\src\random.c \ $(TOP)\src\pcache.c \ $(TOP)\src\pcache1.c \ $(TOP)\src\select.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\utf.c \ $(TOP)\src\util.c \ $(TOP)\src\vdbeapi.c \ $(TOP)\src\vdbeaux.c \ $(TOP)\src\vdbe.c \ $(TOP)\src\vdbemem.c \ $(TOP)\src\vdbesort.c \ $(TOP)\src\vdbetrace.c \ $(TOP)\src\where.c \ $(TOP)\src\wherecode.c \ $(TOP)\src\whereexpr.c \ parse.c \ $(TOP)\ext\fts3\fts3.c \ $(TOP)\ext\fts3\fts3_aux.c \ $(TOP)\ext\fts3\fts3_expr.c \ $(TOP)\ext\fts3\fts3_tokenizer.c \ $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ $(TOP)\ext\fts3\fts3_unicode.c \ $(TOP)\ext\fts3\fts3_unicode2.c \ $(TOP)\ext\fts3\fts3_write.c \ $(TOP)\ext\async\sqlite3async.c # Header files used by all library source files. # HDR = \ $(TOP)\src\btree.h \ $(TOP)\src\btreeInt.h \ ................................................................................ $(TOP)\src\os_common.h \ $(TOP)\src\os_setup.h \ $(TOP)\src\os_win.h \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ parse.h \ $(TOP)\src\pragma.h \ sqlite3.h \ $(TOP)\src\sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\vdbe.h \ $(TOP)\src\vdbeInt.h \ $(TOP)\src\vxworks.h \ $(TOP)\src\whereInt.h ................................................................................ $(TOP)\ext\fts3\fts3_tokenizer.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\rtree\rtree.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\icu\sqliteicu.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\rtree\sqlite3rtree.h # executables needed for testing # TESTPROGS = \ testfixture.exe \ sqlite3.exe \ sqlite3_analyzer.exe \ sqldiff.exe # Databases containing fuzzer test cases # FUZZDATA = \ $(TOP)\test\fuzzdata1.db \ $(TOP)\test\fuzzdata2.db \ $(TOP)\test\fuzzdata3.db \ $(TOP)\test\fuzzdata4.db # Extra compiler options for various shell tools # SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) sqlite3.exe: $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h $(LTLINK) $(SHELL_COMPILE_OPTS) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \ /link /pdb:sqlite3sh.pdb $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) sqldiff.exe: $(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) fuzzershell.exe: $(TOP)\tool\fuzzershell.c sqlite3.c sqlite3.h $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) \ $(TOP)\tool\fuzzershell.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) fuzzcheck.exe: $(TOP)\test\fuzzcheck.c sqlite3.c sqlite3.h $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) mptester.exe: $(TOP)\mptest\mptest.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h $(LTLINK) $(NO_WARN) $(SHELL_COMPILE_OPTS) $(TOP)\mptest\mptest.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(SHELL_LINK_OPTS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20 MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20 mptest: mptester.exe del /Q mptest.db 2>NUL $(MPTEST1) --journalmode DELETE ................................................................................ # build on the target system. Some of the C source code and header # files are automatically generated. This target takes care of # all that automatic generation. # .target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c -rmdir /Q/S tsrc 2>NUL -mkdir tsrc for %i in ($(SRC1)) do copy /Y %i tsrc for %i in ($(SRC2)) do copy /Y %i tsrc for %i in ($(SRC3)) do copy /Y %i tsrc for %i in ($(SRC4)) do copy /Y %i tsrc for %i in ($(SRC5)) do copy /Y %i tsrc copy /Y fts5.c tsrc copy /Y fts5.h tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new move vdbe.new tsrc\vdbe.c echo > .target_source sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS) copy tsrc\shell.c . sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl # Set the source code file to be used by executables and libraries when # they need the amalgamation. # !IF $(SPLIT_AMALGAMATION)!=0 SQLITE3C = sqlite3-all.c !ELSE SQLITE3C = sqlite3.c !ENDIF # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C) # Rules to build the LEMON compiler generator # lempar.c: $(TOP)\tool\lempar.c copy $(TOP)\tool\lempar.c . lemon.exe: $(TOP)\tool\lemon.c lempar.c $(BCC) $(NO_WARN) -Daccess=_access \ ................................................................................ # opcodes.lo # parse.lo: parse.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c parse.c opcodes.lo: opcodes.c $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c opcodes.c # Rule to build the Win32 resources object file. # !IF $(USE_RC)!=0 $(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(HDR) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h for /F %%V in ('type "$(TOP)\VERSION"') do ( \ echo #define SQLITE_RESOURCE_VERSION %%V \ | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \ ) echo #endif >> sqlite3rc.h $(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc !ENDIF # Rules to build individual *.lo files from files in the src directory. # alter.lo: $(TOP)\src\alter.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\alter.c analyze.lo: $(TOP)\src\analyze.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\analyze.c ................................................................................ hash.lo: $(TOP)\src\hash.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\hash.c insert.lo: $(TOP)\src\insert.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\insert.c journal.lo: $(TOP)\src\journal.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\journal.c legacy.lo: $(TOP)\src\legacy.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c loadext.lo: $(TOP)\src\loadext.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\loadext.c main.lo: $(TOP)\src\main.c $(HDR) ................................................................................ tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)\tool\mkopcodec.tcl $(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c ................................................................................ # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl del /Q parse.y parse.h parse.h.temp 2>NUL copy $(TOP)\src\parse.y . .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h sqlite3ext.h: .target_source copy tsrc\sqlite3ext.h . mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) \ $(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h ................................................................................ fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c # FTS5 things # FTS5_SRC = \ $(TOP)\ext\fts5\fts5.h \ $(TOP)\ext\fts5\fts5Int.h \ $(TOP)\ext\fts5\fts5_aux.c \ $(TOP)\ext\fts5\fts5_buffer.c \ ................................................................................ $(TOP)\ext\fts5\fts5_unicode2.c \ $(TOP)\ext\fts5\fts5_varint.c \ $(TOP)\ext\fts5\fts5_vocab.c fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe copy $(TOP)\ext\fts5\fts5parse.y . del /Q fts5parse.h 2>NUL .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) fts5parse.y fts5parse.h: fts5parse.c fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl copy $(TOP)\ext\fts5\fts5.h . ................................................................................ fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(NO_WARN) -c fts5.c fts5.dll: fts5_ext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.lib. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) $(SHELL_CORE_DEP) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) !IF $(USE_AMALGAMATION)==0 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) !ELSE TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF testfixture.exe: $(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) extensiontest: testfixture.exe testloadext.dll @set PATH=$(LIBTCLPATH);$(PATH) ................................................................................ @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS) fulltestonly: $(TESTPROGS) fuzztest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS) fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) fastfuzztest: fuzzcheck.exe .\fuzzcheck.exe --limit-mem 100M $(FUZZDATA) # Minimal testing that runs in less than 3 minutes (on a fast machine) # quicktest: testfixture.exe @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # test: $(TESTPROGS) fastfuzztest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS) smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ echo static const char *tclsh_main_loop(void){ >> $@ echo static const char *zMainloop = >> $@ $(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@ echo ; return zMainloop; } >> $@ ................................................................................ testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) LogEst.exe: $(TOP)\tool\logest.c sqlite3.h $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL del /Q sqlite3.h opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL del /Q notasharedlib.* 2>NUL -rmdir /Q/S .deps 2>NUL -rmdir /Q/S .libs 2>NUL -rmdir /Q/S quota2a 2>NUL -rmdir /Q/S quota2b 2>NUL ................................................................................ -rmdir /Q/S quota2c 2>NUL -rmdir /Q/S tsrc 2>NUL del /Q .target_source 2>NUL del /Q tclsqlite3.exe 2>NUL del /Q testloadext.dll 2>NUL del /Q testfixture.exe test.db 2>NUL del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL del /Q mptester.exe wordcount.exe 2>NUL del /Q sqlite3.exe sqlite3.dll sqlite3.def 2>NUL del /Q sqlite3.c sqlite3-*.c 2>NUL del /Q sqlite3rc.h 2>NUL del /Q shell.c sqlite3ext.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL # Dynamic link library section. # dll: sqlite3.dll sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \ | sort >> sqlite3.def sqlite3.dll: $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) |
> > > > | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | > > > > > > > > > > | | | > > | > > | | | | > > > > > > > > > > > | | | | | | > > > > > > > > > > > | > > | | > > > | | > > > > > > > > > > > > | | > > > > > > > | | < < < < < < < < < < | < > > > | < < < < < < < < < < < < < | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < > | < < < < < < > > > | < < < < < < < > > | > > > > > > > > > > > > > > > > > > > > | < > > > > > > < < > | > | > | > < > | | | | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | > > | > > > > > > > > > | | | > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > | | | > | | > > > > > > | | < | | | | < > | | | | | > > > > > > > < < < < < < < < < > > > > | > > < < < | | | | | > > > | > > > > > | | | | | | | | | | > > > > | | | | | > > > > > > > > < > > > > | > | < | < < < < < < < < < < < < < > |
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 .. 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 ... 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 ... 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 ... 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 ... 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 ... 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 ... 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 ... 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 ... 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 ... 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 ... 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 ... 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 ... 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 ... 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 ... 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 ... 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 .... 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 .... 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 .... 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 .... 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 .... 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 .... 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 .... 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 .... 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 .... 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 .... 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 .... 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 .... 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 .... 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 .... 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 .... 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 |
############################################################################### # The toplevel directory of the source tree. This is the directory # that contains this "Makefile.msc". # TOP = . # <<mark>> # Set this non-0 to create and use the SQLite amalgamation file. # !IFNDEF USE_AMALGAMATION USE_AMALGAMATION = 1 !ENDIF # <</mark>> # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN USE_FULLWARN = 0 !ENDIF ................................................................................ # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # !IFNDEF SPLIT_AMALGAMATION SPLIT_AMALGAMATION = 0 !ENDIF # <<mark>> # Set this non-0 to use the International Components for Unicode (ICU). # !IFNDEF USE_ICU USE_ICU = 0 !ENDIF # <</mark>> # Set this non-0 to dynamically link to the MSVC runtime library. # !IFNDEF USE_CRT_DLL USE_CRT_DLL = 0 !ENDIF ................................................................................ # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). # !IFNDEF FOR_WINRT FOR_WINRT = 0 !ENDIF # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). # !IFNDEF FOR_UWP FOR_UWP = 0 !ENDIF # Set this non-0 to compile binaries suitable for the Windows 10 platform. # !IFNDEF FOR_WIN10 FOR_WIN10 = 0 !ENDIF # <<mark>> # Set this non-0 to skip attempting to look for and/or link with the Tcl # runtime library. # !IFNDEF NO_TCL NO_TCL = 0 !ENDIF # <</mark>> # Set this to non-0 to create and use PDBs. # !IFNDEF SYMBOLS SYMBOLS = 1 !ENDIF ................................................................................ # Enable use of available compiler optimizations? Normally, this should be # non-zero. Setting this to zero, thus disabling all compiler optimizations, # can be useful for testing. # !IFNDEF OPTIMIZATIONS OPTIMIZATIONS = 2 !ENDIF # Set the source code file to be used by executables and libraries when # they need the amalgamation. # !IFNDEF SQLITE3C !IF $(SPLIT_AMALGAMATION)!=0 SQLITE3C = sqlite3-all.c !ELSE SQLITE3C = sqlite3.c !ENDIF !ENDIF # Set the include code file to be used by executables and libraries when # they need SQLite. # !IFNDEF SQLITE3H SQLITE3H = sqlite3.h !ENDIF # This is the name to use for the SQLite dynamic link library (DLL). # !IFNDEF SQLITE3DLL !IF $(FOR_WIN10)!=0 SQLITE3DLL = winsqlite3.dll !ELSE SQLITE3DLL = sqlite3.dll !ENDIF !ENDIF # This is the name to use for the SQLite import library (LIB). # !IFNDEF SQLITE3LIB !IF $(FOR_WIN10)!=0 SQLITE3LIB = winsqlite3.lib !ELSE SQLITE3LIB = sqlite3.lib !ENDIF !ENDIF # This is the name to use for the SQLite shell executable (EXE). # !IFNDEF SQLITE3EXE !IF $(FOR_WIN10)!=0 SQLITE3EXE = winsqlite3shell.exe !ELSE SQLITE3EXE = sqlite3.exe !ENDIF !ENDIF # This is the argument used to set the program database (PDB) file for the # SQLite shell executable (EXE). # !IFNDEF SQLITE3EXEPDB !IF $(FOR_WIN10)!=0 SQLITE3EXEPDB = !ELSE SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF !ENDIF # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 !ENDIF # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # !IFNDEF EXT_FEATURE_FLAGS !IF $(FOR_WIN10)!=0 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1 !ELSE EXT_FEATURE_FLAGS = !ENDIF !ENDIF ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### # When compiling for the Windows 10 platform, the PLATFORM macro must be set # to an appropriate value (e.g. x86, x64, arm, arm64, etc). # !IF $(FOR_WIN10)!=0 !IFNDEF PLATFORM !ERROR Using the FOR_WIN10 option requires a value for PLATFORM. !ENDIF !ENDIF # This assumes that MSVC is always installed in 32-bit Program Files directory # and sets the variable for use in locating other 32-bit installs accordingly. # PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) ................................................................................ # Check for the predefined command macro CC. This should point to the compiler # binary for the target platform. If it is not defined, simply define it to # the legacy default value 'cl.exe'. # !IFNDEF CC CC = cl.exe !ENDIF # Check for the predefined command macro CSC. This should point to a working # C Sharp compiler binary. If it is not defined, simply define it to the # legacy default value 'csc.exe'. # !IFNDEF CSC CSC = csc.exe !ENDIF # Check for the command macro LD. This should point to the linker binary for # the target platform. If it is not defined, simply define it to the legacy # default value 'link.exe'. # !IFNDEF LD LD = link.exe ................................................................................ # compiler binary for the target platform. If it is not defined, simply define # it to the legacy default value 'rc.exe'. # !IFNDEF RC RC = rc.exe !ENDIF # Check for the MSVC runtime library path macro. Otherwise, this value will # default to the 'lib' directory underneath the MSVC installation directory. # !IFNDEF CRTLIBPATH CRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF CRTLIBPATH = $(CRTLIBPATH:\\=\) ................................................................................ !ELSEIF $(XCOMPILE)!=0 NCC = "$(VCINSTALLDIR)\bin\$(CC)" NCC = $(NCC:\\=\) !ELSE NCC = $(CC) !ENDIF # Check for the MSVC native runtime library path macro. Otherwise, # this value will default to the 'lib' directory underneath the MSVC # installation directory. # !IFNDEF NCRTLIBPATH NCRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF NCRTLIBPATH = $(NCRTLIBPATH:\\=\) # Check for the Platform SDK library path macro. Otherwise, this # value will default to the 'lib' directory underneath the Windows # SDK installation directory (the environment variable used appears # to be available when using Visual C++ 2008 or later via the # command line). # !IFNDEF NSDKLIBPATH NSDKLIBPATH = $(WINDOWSSDKDIR)\lib !ENDIF NSDKLIBPATH = $(NSDKLIBPATH:\\=\) # Check for the UCRT library path macro. Otherwise, this value will # default to the version-specific, platform-specific 'lib' directory # underneath the Windows SDK installation directory. # !IFNDEF UCRTLIBPATH UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM) !ENDIF UCRTLIBPATH = $(UCRTLIBPATH:\\=\) # C compiler and options for use in building executables that # will run on the platform that is doing the build. # !IF $(USE_FULLWARN)!=0 BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS) !ELSE ................................................................................ !IF $(USE_FULLWARN)!=0 TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) !ELSE TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) # Check if we want to use the "stdcall" calling convention when compiling. # This is not supported by the compilers for non-x86 platforms. It should # also be noted here that building any target with these "stdcall" options # will most likely fail if the Tcl library is also required. This is due # to how the Tcl library functions are declared and exported (i.e. without # an explicit calling convention, which results in "cdecl"). # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall ................................................................................ CORE_CCONV_OPTS = SHELL_CCONV_OPTS = !ENDIF # These are additional compiler options used for the core library. # !IFNDEF CORE_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport) !ELSE CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) !ENDIF !ENDIF # These are the additional targets that the core library should depend on # when linking. # !IFNDEF CORE_LINK_DEP !IF $(DYNAMIC_SHELL)!=0 CORE_LINK_DEP = !ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" CORE_LINK_DEP = sqlite3.def !ELSE CORE_LINK_DEP = !ENDIF !ENDIF # These are additional linker options used for the core library. # !IFNDEF CORE_LINK_OPTS !IF $(DYNAMIC_SHELL)!=0 CORE_LINK_OPTS = !ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" CORE_LINK_OPTS = /DEF:sqlite3.def !ELSE CORE_LINK_OPTS = !ENDIF !ENDIF # These are additional compiler options used for the shell executable. # !IFNDEF SHELL_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) !ELSE SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) !ENDIF !ENDIF # This is the source code that the shell executable should be compiled # with. # !IFNDEF SHELL_CORE_SRC !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_SRC = !ELSE SHELL_CORE_SRC = $(SQLITE3C) !ENDIF !ENDIF # This is the core library that the shell executable should depend on. # !IFNDEF SHELL_CORE_DEP !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_DEP = $(SQLITE3DLL) !ELSE SHELL_CORE_DEP = !ENDIF !ENDIF # This is the core library that the shell executable should link with. # !IFNDEF SHELL_CORE_LIB !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_LIB = $(SQLITE3LIB) !ELSE SHELL_CORE_LIB = !ENDIF !ENDIF # These are additional linker options used for the shell executable. # !IFNDEF SHELL_LINK_OPTS SHELL_LINK_OPTS = $(SHELL_CORE_LIB) ................................................................................ # !IF $(FOR_WINRT)!=0 TCC = $(TCC) -DSQLITE_OS_WINRT=1 RCC = $(RCC) -DSQLITE_OS_WINRT=1 TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP !ENDIF # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 TCC = $(TCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE !ENDIF # Also, we need to dynamically link to the correct MSVC runtime # when compiling for WinRT (e.g. debug or release) OR if the # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 ................................................................................ BCC = $(BCC) -MTd !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT !ENDIF !ENDIF # <<mark>> # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in # any extension header files by default. For non-amalgamation # builds, we need to make sure the compiler can find these. # !IF $(USE_AMALGAMATION)==0 TCC = $(TCC) -I$(TOP)\ext\fts3 RCC = $(RCC) -I$(TOP)\ext\fts3 TCC = $(TCC) -I$(TOP)\ext\rtree RCC = $(RCC) -I$(TOP)\ext\rtree TCC = $(TCC) -I$(TOP)\ext\session RCC = $(RCC) -I$(TOP)\ext\session !ENDIF # The mksqlite3c.tcl script accepts some options on the command # line. When compiling with debugging enabled, some of these # options are necessary in order to allow debugging symbols to # work correctly with Visual Studio when using the amalgamation. # ................................................................................ !IFNDEF MKSQLITE3C_ARGS !IF $(DEBUG)>1 MKSQLITE3C_ARGS = --linemacros !ELSE MKSQLITE3C_ARGS = !ENDIF !ENDIF # <</mark>> # Define -DNDEBUG to compile without debugging (i.e., for production usage) # Omitting the define will cause extra debugging code to be inserted and # includes extra comments when "EXPLAIN stmt" is used. # !IF $(DEBUG)==0 TCC = $(TCC) -DNDEBUG BCC = $(BCC) -DNDEBUG RCC = $(RCC) -DNDEBUG !ENDIF !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 RCC = $(RCC) -DSQLITE_DEBUG=1 ................................................................................ # !IF $(DEBUG)>3 TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF !ENDIF # <<mark>> # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment # prior to running nmake in order to match the actual installed location and # version on this machine. # !IFNDEF TCLINCDIR ................................................................................ # know the specific version we want to use. This variable (TCLSH_CMD) may be # overridden via the environment prior to running nmake in order to select a # specific Tcl shell to use. # !IFNDEF TCLSH_CMD TCLSH_CMD = tclsh85 !ENDIF # <</mark>> # Compiler options needed for programs that use the readline() library. # !IFNDEF READLINE_FLAGS READLINE_FLAGS = -DHAVE_READLINE=0 !ENDIF ................................................................................ !IF $(USE_RPCRT4_LIB)!=0 REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1 !ENDIF # Add the required and optional SQLite compilation options into the command # lines used to invoke the MSVC code and resource compilers. # TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) # Add in any optional parameters specified on the commane line, e.g. # nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1" # TCC = $(TCC) $(OPTS) RCC = $(RCC) $(OPTS) ................................................................................ # If symbols are enabled (or compiling for debugging), enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF # <<mark>> # If ICU support is enabled, add the compiler options for it. # !IF $(USE_ICU)!=0 TCC = $(TCC) -DSQLITE_ENABLE_ICU=1 RCC = $(RCC) -DSQLITE_ENABLE_ICU=1 TCC = $(TCC) -I$(TOP)\ext\icu RCC = $(RCC) -I$(TOP)\ext\icu TCC = $(TCC) -I$(ICUINCDIR) RCC = $(RCC) -I$(ICUINCDIR) !ENDIF # <</mark>> # Command line prefixes for compiling code, compiling resources, # linking, etc. # LTCOMPILE = $(TCC) -Fo$@ LTRCOMPILE = $(RCC) -r LTLIB = lib.exe ................................................................................ LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" !ENDIF LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib !ENDIF # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # !IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0 LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib LTLINKOPTS = $(LTLINKOPTS) mincore.lib !IFDEF PSDKLIBPATH LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)" !ENDIF !ENDIF !IF $(FOR_WIN10)!=0 LTLINKOPTS = $(LTLINKOPTS) /guard:cf "/LIBPATH:$(UCRTLIBPATH)" !IF $(DEBUG)>1 LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib !ELSE LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib !ENDIF !ENDIF # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG $(LDOPTS) !ELSE LDFLAGS = $(LDOPTS) !ENDIF # <<mark>> # Start with the Tcl related linker options. # !IF $(NO_TCL)==0 LTLIBPATHS = /LIBPATH:$(TCLLIBDIR) LTLIBS = $(LIBTCL) !ENDIF # If ICU support is enabled, add the linker options for it. # !IF $(USE_ICU)!=0 LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR) LTLIBS = $(LTLIBS) $(LIBICU) !ENDIF # <</mark>> # You should not have to change anything below this line ############################################################################### # <<mark>> # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ callback.lo complete.lo ctime.lo date.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo \ sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ utf.lo vtab.lo # <</mark>> # Object files for the amalgamation. # LIBOBJS1 = sqlite3.lo # Determine the real value of LIBOBJ based on the 'configure' script # # <<mark>> !IF $(USE_AMALGAMATION)==0 LIBOBJ = $(LIBOBJS0) !ELSE # <</mark>> LIBOBJ = $(LIBOBJS1) # <<mark>> !ENDIF # <</mark>> # Determine if embedded resource compilation and usage are enabled. # !IF $(USE_RC)!=0 LIBRESOBJS = sqlite3res.lo !ELSE LIBRESOBJS = !ENDIF # <<mark>> # Core source code files, part 1. # SRC00 = \ $(TOP)\src\alter.c \ $(TOP)\src\analyze.c \ $(TOP)\src\attach.c \ $(TOP)\src\auth.c \ $(TOP)\src\backup.c \ $(TOP)\src\bitvec.c \ $(TOP)\src\btmutex.c \ $(TOP)\src\btree.c \ $(TOP)\src\build.c \ $(TOP)\src\callback.c \ $(TOP)\src\complete.c \ $(TOP)\src\ctime.c \ $(TOP)\src\date.c \ $(TOP)\src\dbstat.c \ $(TOP)\src\delete.c \ $(TOP)\src\expr.c \ $(TOP)\src\fault.c \ $(TOP)\src\fkey.c \ $(TOP)\src\func.c \ $(TOP)\src\global.c \ $(TOP)\src\hash.c \ $(TOP)\src\insert.c \ $(TOP)\src\legacy.c \ $(TOP)\src\loadext.c \ $(TOP)\src\main.c \ $(TOP)\src\malloc.c \ $(TOP)\src\mem0.c \ $(TOP)\src\mem1.c \ $(TOP)\src\mem2.c \ $(TOP)\src\mem3.c \ $(TOP)\src\mem5.c \ $(TOP)\src\memjournal.c \ $(TOP)\src\mutex.c \ $(TOP)\src\mutex_noop.c \ $(TOP)\src\mutex_unix.c \ $(TOP)\src\mutex_w32.c \ $(TOP)\src\notify.c \ $(TOP)\src\os.c \ $(TOP)\src\os_unix.c \ $(TOP)\src\os_win.c # Core source code files, part 2. # SRC01 = \ $(TOP)\src\pager.c \ $(TOP)\src\pcache.c \ $(TOP)\src\pcache1.c \ $(TOP)\src\pragma.c \ $(TOP)\src\prepare.c \ $(TOP)\src\printf.c \ $(TOP)\src\random.c \ $(TOP)\src\resolve.c \ $(TOP)\src\rowset.c \ $(TOP)\src\select.c \ $(TOP)\src\status.c \ $(TOP)\src\table.c \ $(TOP)\src\threads.c \ $(TOP)\src\tclsqlite.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\treeview.c \ $(TOP)\src\trigger.c \ $(TOP)\src\utf.c \ $(TOP)\src\update.c \ $(TOP)\src\util.c \ $(TOP)\src\vacuum.c \ $(TOP)\src\vdbe.c \ $(TOP)\src\vdbeapi.c \ $(TOP)\src\vdbeaux.c \ $(TOP)\src\vdbeblob.c \ $(TOP)\src\vdbemem.c \ $(TOP)\src\vdbesort.c \ $(TOP)\src\vdbetrace.c \ $(TOP)\src\vtab.c \ $(TOP)\src\wal.c \ $(TOP)\src\walker.c \ $(TOP)\src\where.c \ $(TOP)\src\wherecode.c \ $(TOP)\src\whereexpr.c # Shell source code files. # SRC02 = \ $(TOP)\src\shell.c # Core miscellaneous files. # SRC03 = \ $(TOP)\src\parse.y # Core header files, part 1. # SRC04 = \ $(TOP)\src\btree.h \ $(TOP)\src\btreeInt.h \ $(TOP)\src\hash.h \ $(TOP)\src\hwtime.h \ $(TOP)\src\msvc.h \ $(TOP)\src\mutex.h \ $(TOP)\src\os.h \ $(TOP)\src\os_common.h \ $(TOP)\src\os_setup.h \ $(TOP)\src\os_win.h # Core header files, part 2. # SRC05 = \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ $(TOP)\src\pragma.h \ $(TOP)\src\sqlite.h.in \ $(TOP)\src\sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\vdbe.h \ $(TOP)\src\vdbeInt.h \ $(TOP)\src\vxworks.h \ $(TOP)\src\wal.h \ $(TOP)\src\whereInt.h # Extension source code files, part 1. # SRC06 = \ $(TOP)\ext\fts1\fts1.c \ $(TOP)\ext\fts1\fts1_hash.c \ $(TOP)\ext\fts1\fts1_porter.c \ $(TOP)\ext\fts1\fts1_tokenizer1.c \ $(TOP)\ext\fts2\fts2.c \ $(TOP)\ext\fts2\fts2_hash.c \ $(TOP)\ext\fts2\fts2_icu.c \ $(TOP)\ext\fts2\fts2_porter.c \ $(TOP)\ext\fts2\fts2_tokenizer.c \ $(TOP)\ext\fts2\fts2_tokenizer1.c # Extension source code files, part 2. # SRC07 = \ $(TOP)\ext\fts3\fts3.c \ $(TOP)\ext\fts3\fts3_aux.c \ $(TOP)\ext\fts3\fts3_expr.c \ $(TOP)\ext\fts3\fts3_hash.c \ $(TOP)\ext\fts3\fts3_icu.c \ $(TOP)\ext\fts3\fts3_porter.c \ $(TOP)\ext\fts3\fts3_snippet.c \ $(TOP)\ext\fts3\fts3_tokenizer.c \ $(TOP)\ext\fts3\fts3_tokenizer1.c \ $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ $(TOP)\ext\fts3\fts3_unicode.c \ $(TOP)\ext\fts3\fts3_unicode2.c \ $(TOP)\ext\fts3\fts3_write.c \ $(TOP)\ext\icu\icu.c \ $(TOP)\ext\rtree\rtree.c \ $(TOP)\ext\session\sqlite3session.c \ $(TOP)\ext\rbu\sqlite3rbu.c \ $(TOP)\ext\misc\json1.c # Extension header files, part 1. # SRC08 = \ $(TOP)\ext\fts1\fts1.h \ $(TOP)\ext\fts1\fts1_hash.h \ $(TOP)\ext\fts1\fts1_tokenizer.h \ $(TOP)\ext\fts2\fts2.h \ $(TOP)\ext\fts2\fts2_hash.h \ $(TOP)\ext\fts2\fts2_tokenizer.h # Extension header files, part 2. # SRC09 = \ $(TOP)\ext\fts3\fts3.h \ $(TOP)\ext\fts3\fts3Int.h \ $(TOP)\ext\fts3\fts3_hash.h \ $(TOP)\ext\fts3\fts3_tokenizer.h \ $(TOP)\ext\icu\sqliteicu.h \ $(TOP)\ext\rtree\rtree.h \ $(TOP)\ext\rbu\sqlite3rbu.h \ $(TOP)\ext\session\sqlite3session.h # Generated source code files # SRC10 = \ opcodes.c \ parse.c # Generated header files # SRC11 = \ keywordhash.h \ opcodes.h \ parse.h \ $(SQLITE3H) # All source code files. # SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11) # Source code to the test files. # TESTSRC = \ $(TOP)\src\test1.c \ $(TOP)\src\test2.c \ $(TOP)\src\test3.c \ ................................................................................ $(TOP)\src\test6.c \ $(TOP)\src\test7.c \ $(TOP)\src\test8.c \ $(TOP)\src\test9.c \ $(TOP)\src\test_autoext.c \ $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ $(TOP)\src\test_bestindex.c \ $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ $(TOP)\src\test_demovfs.c \ $(TOP)\src\test_devsym.c \ $(TOP)\src\test_fs.c \ $(TOP)\src\test_func.c \ ................................................................................ $(TOP)\src\test_tclvar.c \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vfs.c \ $(TOP)\src\test_windirent.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c \ $(TOP)\ext\rbu\test_rbu.c \ $(TOP)\ext\session\test_session.c # Statically linked extensions. # TESTEXT = \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\fts5\fts5_test_tok.c \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\totype.c \ $(TOP)\ext\misc\wholenumber.c # Source code to the library files needed by the test fixture # (non-amalgamation) # TESTSRC2 = \ $(SRC00) \ $(SRC01) \ $(SRC06) \ $(SRC07) \ $(SRC10) \ $(TOP)\ext\async\sqlite3async.c # Header files used by all library source files. # HDR = \ $(TOP)\src\btree.h \ $(TOP)\src\btreeInt.h \ ................................................................................ $(TOP)\src\os_common.h \ $(TOP)\src\os_setup.h \ $(TOP)\src\os_win.h \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ parse.h \ $(TOP)\src\pragma.h \ $(SQLITE3H) \ $(TOP)\src\sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\vdbe.h \ $(TOP)\src\vdbeInt.h \ $(TOP)\src\vxworks.h \ $(TOP)\src\whereInt.h ................................................................................ $(TOP)\ext\fts3\fts3_tokenizer.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\rtree\rtree.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\icu\sqliteicu.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\rtree\sqlite3rtree.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\session\sqlite3session.h # executables needed for testing # TESTPROGS = \ testfixture.exe \ $(SQLITE3EXE) \ sqlite3_analyzer.exe \ sqldiff.exe # Databases containing fuzzer test cases # FUZZDATA = \ $(TOP)\test\fuzzdata1.db \ $(TOP)\test\fuzzdata2.db \ $(TOP)\test\fuzzdata3.db \ $(TOP)\test\fuzzdata4.db # <</mark>> # Additional compiler options for the shell. These are only effective # when the shell is not being dynamically linked. # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS !ENDIF # <<mark>> # Extra compiler options for various test tools. # MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 # Standard options to testfixture. # TESTOPTS = --verbose=file --output=test-out.txt # Extra targets for the "all" target that require Tcl. # !IF $(NO_TCL)==0 ALL_TCL_TARGETS = libtclsqlite3.lib !ELSE ALL_TCL_TARGETS = !ENDIF # <</mark>> # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: dll libsqlite3.lib shell $(ALL_TCL_TARGETS) # Dynamic link library section. # dll: $(SQLITE3DLL) # Shell executable. # shell: $(SQLITE3EXE) # <<mark>> libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) # <</mark>> $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) # <<block2>> sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \ | sort >> sqlite3.def # <</block2>> $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <<mark>> sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) srcck1.exe: $(TOP)\tool\srcck1.c $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c sourcetest: srcck1.exe sqlite3.c srcck1.exe sqlite3.c fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) fuzzcheck.exe: $(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) mptester.exe: $(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20 MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20 mptest: mptester.exe del /Q mptest.db 2>NUL $(MPTEST1) --journalmode DELETE ................................................................................ # build on the target system. Some of the C source code and header # files are automatically generated. This target takes care of # all that automatic generation. # .target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c -rmdir /Q/S tsrc 2>NUL -mkdir tsrc for %i in ($(SRC00)) do copy /Y %i tsrc for %i in ($(SRC01)) do copy /Y %i tsrc for %i in ($(SRC02)) do copy /Y %i tsrc for %i in ($(SRC03)) do copy /Y %i tsrc for %i in ($(SRC04)) do copy /Y %i tsrc for %i in ($(SRC05)) do copy /Y %i tsrc for %i in ($(SRC06)) do copy /Y %i tsrc for %i in ($(SRC07)) do copy /Y %i tsrc for %i in ($(SRC08)) do copy /Y %i tsrc for %i in ($(SRC09)) do copy /Y %i tsrc for %i in ($(SRC10)) do copy /Y %i tsrc for %i in ($(SRC11)) do copy /Y %i tsrc copy /Y fts5.c tsrc copy /Y fts5.h tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new move vdbe.new tsrc\vdbe.c echo > .target_source sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS) copy tsrc\shell.c . copy $(TOP)\ext\session\sqlite3session.h . sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl # <</mark>> # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C) # <<mark>> # Rules to build the LEMON compiler generator # lempar.c: $(TOP)\tool\lempar.c copy $(TOP)\tool\lempar.c . lemon.exe: $(TOP)\tool\lemon.c lempar.c $(BCC) $(NO_WARN) -Daccess=_access \ ................................................................................ # opcodes.lo # parse.lo: parse.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c parse.c opcodes.lo: opcodes.c $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c opcodes.c # <</mark>> # Rule to build the Win32 resources object file. # !IF $(USE_RC)!=0 # <<block1>> $(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(SQLITE3H) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h for /F %%V in ('type "$(TOP)\VERSION"') do ( \ echo #define SQLITE_RESOURCE_VERSION %%V \ | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \ ) echo #endif >> sqlite3rc.h $(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc # <</block1>> !ENDIF # <<mark>> # Rules to build individual *.lo files from files in the src directory. # alter.lo: $(TOP)\src\alter.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\alter.c analyze.lo: $(TOP)\src\analyze.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\analyze.c ................................................................................ hash.lo: $(TOP)\src\hash.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\hash.c insert.lo: $(TOP)\src\insert.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\insert.c legacy.lo: $(TOP)\src\legacy.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c loadext.lo: $(TOP)\src\loadext.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\loadext.c main.lo: $(TOP)\src\main.c $(HDR) ................................................................................ tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)\tool\mkopcodec.tcl $(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c ................................................................................ # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl del /Q parse.y parse.h parse.h.temp 2>NUL copy $(TOP)\src\parse.y . .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h $(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) sqlite3ext.h: .target_source copy tsrc\sqlite3ext.h . mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \ $(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h ................................................................................ fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c sqlite3session.lo: $(TOP)\ext\session\sqlite3sesion.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c # FTS5 things # FTS5_SRC = \ $(TOP)\ext\fts5\fts5.h \ $(TOP)\ext\fts5\fts5Int.h \ $(TOP)\ext\fts5\fts5_aux.c \ $(TOP)\ext\fts5\fts5_buffer.c \ ................................................................................ $(TOP)\ext\fts5\fts5_unicode2.c \ $(TOP)\ext\fts5\fts5_varint.c \ $(TOP)\ext\fts5\fts5_vocab.c fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe copy $(TOP)\ext\fts5\fts5parse.y . del /Q fts5parse.h 2>NUL .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y fts5parse.h: fts5parse.c fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl copy $(TOP)\ext\fts5\fts5.h . ................................................................................ fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(NO_WARN) -c fts5.c fts5.dll: fts5_ext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo sqlite3rbu.lo: $(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rbu\sqlite3rbu.c # Rules to build the 'testfixture' application. # # If using the amalgamation, use sqlite3.c directly to build the test # fixture. Otherwise link against libsqlite3.lib. (This distinction is # necessary because the test fixture requires non-API symbols which are # hidden when the library is built via the amalgamation). # TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) !IF $(USE_AMALGAMATION)==0 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) !ELSE TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) extensiontest: testfixture.exe testloadext.dll @set PATH=$(LIBTCLPATH);$(PATH) ................................................................................ @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS) fulltestonly: $(TESTPROGS) fuzztest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe shell @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS) fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) fastfuzztest: fuzzcheck.exe .\fuzzcheck.exe --limit-mem 100M $(FUZZDATA) # Minimal testing that runs in less than 3 minutes (on a fast machine) # quicktest: testfixture.exe sourcetest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # test: $(TESTPROGS) sourcetest fastfuzztest @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS) smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ echo static const char *tclsh_main_loop(void){ >> $@ echo static const char *zMainloop = >> $@ $(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@ echo ; return zMainloop; } >> $@ ................................................................................ testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) LogEst.exe: $(TOP)\tool\logest.c $(SQLITE3H) $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) moreclean: clean del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL # <</mark>> clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL # <<mark>> del /Q sqlite3.c sqlite3.h 2>NUL del /Q opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL del /Q notasharedlib.* 2>NUL -rmdir /Q/S .deps 2>NUL -rmdir /Q/S .libs 2>NUL -rmdir /Q/S quota2a 2>NUL -rmdir /Q/S quota2b 2>NUL ................................................................................ -rmdir /Q/S quota2c 2>NUL -rmdir /Q/S tsrc 2>NUL del /Q .target_source 2>NUL del /Q tclsqlite3.exe 2>NUL del /Q testloadext.dll 2>NUL del /Q testfixture.exe test.db 2>NUL del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL del /Q sqlite3.c sqlite3-*.c 2>NUL del /Q sqlite3rc.h 2>NUL del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL # <</mark>> |
Changes to VERSION.
1 |
3.10.0
|
| |
1 |
3.13.0
|
Changes to autoconf/Makefile.am.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8 bin_PROGRAMS = sqlite3 sqlite3_SOURCES = shell.c sqlite3.h EXTRA_sqlite3_SOURCES = sqlite3.c sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ sqlite3_CFLAGS = $(AM_CFLAGS) include_HEADERS = sqlite3.h sqlite3ext.h EXTRA_DIST = sqlite3.1 tea pkgconfigdir = ${libdir}/pkgconfig pkgconfig_DATA = sqlite3.pc man_MANS = sqlite3.1 |
| | |
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8 bin_PROGRAMS = sqlite3 sqlite3_SOURCES = shell.c sqlite3.h EXTRA_sqlite3_SOURCES = sqlite3.c sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS include_HEADERS = sqlite3.h sqlite3ext.h EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs pkgconfigdir = ${libdir}/pkgconfig pkgconfig_DATA = sqlite3.pc man_MANS = sqlite3.1 |
Added autoconf/Makefile.msc.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 |
#### DO NOT EDIT #### # This makefile is automatically generated from the Makefile.msc at # the root of the canonical SQLite source tree (not the # amalgamation tarball) using the tool/mkmsvcmin.tcl # script. # # # nmake Makefile for SQLite # ############################################################################### ############################## START OF OPTIONS ############################### ############################################################################### # The toplevel directory of the source tree. This is the directory # that contains this "Makefile.msc". # TOP = . # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN USE_FULLWARN = 0 !ENDIF # Set this non-0 to use "stdcall" calling convention for the core library # and shell executable. # !IFNDEF USE_STDCALL USE_STDCALL = 0 !ENDIF # Set this non-0 to have the shell executable link against the core dynamic # link library. # !IFNDEF DYNAMIC_SHELL DYNAMIC_SHELL = 0 !ENDIF # Set this non-0 to enable extra code that attempts to detect misuse of the # SQLite API. # !IFNDEF API_ARMOR API_ARMOR = 0 !ENDIF # If necessary, create a list of harmless compiler warnings to disable when # compiling the various tools. For the SQLite source code itself, warnings, # if any, will be disabled from within it. # !IFNDEF NO_WARN !IF $(USE_FULLWARN)!=0 NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206 NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF # Set this non-0 to use the library paths and other options necessary for # Windows Phone 8.1. # !IFNDEF USE_WP81_OPTS USE_WP81_OPTS = 0 !ENDIF # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # !IFNDEF SPLIT_AMALGAMATION SPLIT_AMALGAMATION = 0 !ENDIF # Set this non-0 to dynamically link to the MSVC runtime library. # !IFNDEF USE_CRT_DLL USE_CRT_DLL = 0 !ENDIF # Set this non-0 to link to the RPCRT4 library. # !IFNDEF USE_RPCRT4_LIB USE_RPCRT4_LIB = 0 !ENDIF # Set this non-0 to generate assembly code listings for the source code # files. # !IFNDEF USE_LISTINGS USE_LISTINGS = 0 !ENDIF # Set this non-0 to attempt setting the native compiler automatically # for cross-compiling the command line tools needed during the compilation # process. # !IFNDEF XCOMPILE XCOMPILE = 0 !ENDIF # Set this non-0 to use the native libraries paths for cross-compiling # the command line tools needed during the compilation process. # !IFNDEF USE_NATIVE_LIBPATHS USE_NATIVE_LIBPATHS = 0 !ENDIF # Set this 0 to skip the compiling and embedding of version resources. # !IFNDEF USE_RC USE_RC = 1 !ENDIF # Set this non-0 to compile binaries suitable for the WinRT environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). # !IFNDEF FOR_WINRT FOR_WINRT = 0 !ENDIF # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). # !IFNDEF FOR_UWP FOR_UWP = 0 !ENDIF # Set this non-0 to compile binaries suitable for the Windows 10 platform. # !IFNDEF FOR_WIN10 FOR_WIN10 = 0 !ENDIF # Set this to non-0 to create and use PDBs. # !IFNDEF SYMBOLS SYMBOLS = 1 !ENDIF # Set this to non-0 to use the SQLite debugging heap subsystem. # !IFNDEF MEMDEBUG MEMDEBUG = 0 !ENDIF # Set this to non-0 to use the Win32 native heap subsystem. # !IFNDEF WIN32HEAP WIN32HEAP = 0 !ENDIF # Set this to non-0 to enable OSTRACE() macros, which can be useful when # debugging. # !IFNDEF OSTRACE OSTRACE = 0 !ENDIF # Set this to one of the following values to enable various debugging # features. Each level includes the debugging options from the previous # levels. Currently, the recognized values for DEBUG are: # # 0 == NDEBUG: Disables assert() and other runtime diagnostics. # 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API. # 2 == Disables NDEBUG and all optimizations and then enables PDBs. # 3 == SQLITE_DEBUG: Enables various diagnostics messages and code. # 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call. # 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros. # 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros. # !IFNDEF DEBUG DEBUG = 0 !ENDIF # Enable use of available compiler optimizations? Normally, this should be # non-zero. Setting this to zero, thus disabling all compiler optimizations, # can be useful for testing. # !IFNDEF OPTIMIZATIONS OPTIMIZATIONS = 2 !ENDIF # Set the source code file to be used by executables and libraries when # they need the amalgamation. # !IFNDEF SQLITE3C !IF $(SPLIT_AMALGAMATION)!=0 SQLITE3C = sqlite3-all.c !ELSE SQLITE3C = sqlite3.c !ENDIF !ENDIF # Set the include code file to be used by executables and libraries when # they need SQLite. # !IFNDEF SQLITE3H SQLITE3H = sqlite3.h !ENDIF # This is the name to use for the SQLite dynamic link library (DLL). # !IFNDEF SQLITE3DLL !IF $(FOR_WIN10)!=0 SQLITE3DLL = winsqlite3.dll !ELSE SQLITE3DLL = sqlite3.dll !ENDIF !ENDIF # This is the name to use for the SQLite import library (LIB). # !IFNDEF SQLITE3LIB !IF $(FOR_WIN10)!=0 SQLITE3LIB = winsqlite3.lib !ELSE SQLITE3LIB = sqlite3.lib !ENDIF !ENDIF # This is the name to use for the SQLite shell executable (EXE). # !IFNDEF SQLITE3EXE !IF $(FOR_WIN10)!=0 SQLITE3EXE = winsqlite3shell.exe !ELSE SQLITE3EXE = sqlite3.exe !ENDIF !ENDIF # This is the argument used to set the program database (PDB) file for the # SQLite shell executable (EXE). # !IFNDEF SQLITE3EXEPDB !IF $(FOR_WIN10)!=0 SQLITE3EXEPDB = !ELSE SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF !ENDIF # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 !ENDIF # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # !IFNDEF EXT_FEATURE_FLAGS !IF $(FOR_WIN10)!=0 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1 EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1 !ELSE EXT_FEATURE_FLAGS = !ENDIF !ENDIF ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### # When compiling for the Windows 10 platform, the PLATFORM macro must be set # to an appropriate value (e.g. x86, x64, arm, arm64, etc). # !IF $(FOR_WIN10)!=0 !IFNDEF PLATFORM !ERROR Using the FOR_WIN10 option requires a value for PLATFORM. !ENDIF !ENDIF # This assumes that MSVC is always installed in 32-bit Program Files directory # and sets the variable for use in locating other 32-bit installs accordingly. # PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\.. PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) # Check for the predefined command macro CC. This should point to the compiler # binary for the target platform. If it is not defined, simply define it to # the legacy default value 'cl.exe'. # !IFNDEF CC CC = cl.exe !ENDIF # Check for the predefined command macro CSC. This should point to a working # C Sharp compiler binary. If it is not defined, simply define it to the # legacy default value 'csc.exe'. # !IFNDEF CSC CSC = csc.exe !ENDIF # Check for the command macro LD. This should point to the linker binary for # the target platform. If it is not defined, simply define it to the legacy # default value 'link.exe'. # !IFNDEF LD LD = link.exe !ENDIF # Check for the predefined command macro RC. This should point to the resource # compiler binary for the target platform. If it is not defined, simply define # it to the legacy default value 'rc.exe'. # !IFNDEF RC RC = rc.exe !ENDIF # Check for the MSVC runtime library path macro. Otherwise, this value will # default to the 'lib' directory underneath the MSVC installation directory. # !IFNDEF CRTLIBPATH CRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF CRTLIBPATH = $(CRTLIBPATH:\\=\) # Check for the command macro NCC. This should point to the compiler binary # for the platform the compilation process is taking place on. If it is not # defined, simply define it to have the same value as the CC macro. When # cross-compiling, it is suggested that this macro be modified via the command # line (since nmake itself does not provide a built-in method to guess it). # For example, to use the x86 compiler when cross-compiling for x64, a command # line similar to the following could be used (all on one line): # # nmake /f Makefile.msc sqlite3.dll # XCOMPILE=1 USE_NATIVE_LIBPATHS=1 # # Alternatively, the full path and file name to the compiler binary for the # platform the compilation process is taking place may be specified (all on # one line): # # nmake /f Makefile.msc sqlite3.dll # "NCC=""%VCINSTALLDIR%\bin\cl.exe""" # USE_NATIVE_LIBPATHS=1 # !IFDEF NCC NCC = $(NCC:\\=\) !ELSEIF $(XCOMPILE)!=0 NCC = "$(VCINSTALLDIR)\bin\$(CC)" NCC = $(NCC:\\=\) !ELSE NCC = $(CC) !ENDIF # Check for the MSVC native runtime library path macro. Otherwise, # this value will default to the 'lib' directory underneath the MSVC # installation directory. # !IFNDEF NCRTLIBPATH NCRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF NCRTLIBPATH = $(NCRTLIBPATH:\\=\) # Check for the Platform SDK library path macro. Otherwise, this # value will default to the 'lib' directory underneath the Windows # SDK installation directory (the environment variable used appears # to be available when using Visual C++ 2008 or later via the # command line). # !IFNDEF NSDKLIBPATH NSDKLIBPATH = $(WINDOWSSDKDIR)\lib !ENDIF NSDKLIBPATH = $(NSDKLIBPATH:\\=\) # Check for the UCRT library path macro. Otherwise, this value will # default to the version-specific, platform-specific 'lib' directory # underneath the Windows SDK installation directory. # !IFNDEF UCRTLIBPATH UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM) !ENDIF UCRTLIBPATH = $(UCRTLIBPATH:\\=\) # C compiler and options for use in building executables that # will run on the platform that is doing the build. # !IF $(USE_FULLWARN)!=0 BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS) !ELSE BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS) !ENDIF # Check if assembly code listings should be generated for the source # code files to be compiled. # !IF $(USE_LISTINGS)!=0 BCC = $(BCC) -FAcs !ENDIF # Check if the native library paths should be used when compiling # the command line tools used during the compilation process. If # so, set the necessary macro now. # !IF $(USE_NATIVE_LIBPATHS)!=0 NLTLIBPATHS = "/LIBPATH:$(NCRTLIBPATH)" "/LIBPATH:$(NSDKLIBPATH)" !IFDEF NUCRTLIBPATH NUCRTLIBPATH = $(NUCRTLIBPATH:\\=\) NLTLIBPATHS = $(NLTLIBPATHS) "/LIBPATH:$(NUCRTLIBPATH)" !ENDIF !ENDIF # C compiler and options for use in building executables that # will run on the target platform. (BCC and TCC are usually the # same unless your are cross-compiling.) # !IF $(USE_FULLWARN)!=0 TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) !ELSE TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) # Check if we want to use the "stdcall" calling convention when compiling. # This is not supported by the compilers for non-x86 platforms. It should # also be noted here that building any target with these "stdcall" options # will most likely fail if the Tcl library is also required. This is due # to how the Tcl library functions are declared and exported (i.e. without # an explicit calling convention, which results in "cdecl"). # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = !ENDIF !ENDIF !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = !ENDIF # These are additional compiler options used for the core library. # !IFNDEF CORE_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport) !ELSE CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) !ENDIF !ENDIF # These are the additional targets that the core library should depend on # when linking. # !IFNDEF CORE_LINK_DEP !IF $(DYNAMIC_SHELL)!=0 CORE_LINK_DEP = !ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" CORE_LINK_DEP = sqlite3.def !ELSE CORE_LINK_DEP = !ENDIF !ENDIF # These are additional linker options used for the core library. # !IFNDEF CORE_LINK_OPTS !IF $(DYNAMIC_SHELL)!=0 CORE_LINK_OPTS = !ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" CORE_LINK_OPTS = /DEF:sqlite3.def !ELSE CORE_LINK_OPTS = !ENDIF !ENDIF # These are additional compiler options used for the shell executable. # !IFNDEF SHELL_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) !ELSE SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) !ENDIF !ENDIF # This is the source code that the shell executable should be compiled # with. # !IFNDEF SHELL_CORE_SRC !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_SRC = !ELSE SHELL_CORE_SRC = $(SQLITE3C) !ENDIF !ENDIF # This is the core library that the shell executable should depend on. # !IFNDEF SHELL_CORE_DEP !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_DEP = $(SQLITE3DLL) !ELSE SHELL_CORE_DEP = !ENDIF !ENDIF # This is the core library that the shell executable should link with. # !IFNDEF SHELL_CORE_LIB !IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 SHELL_CORE_LIB = $(SQLITE3LIB) !ELSE SHELL_CORE_LIB = !ENDIF !ENDIF # These are additional linker options used for the shell executable. # !IFNDEF SHELL_LINK_OPTS SHELL_LINK_OPTS = $(SHELL_CORE_LIB) !ENDIF # Check if assembly code listings should be generated for the source # code files to be compiled. # !IF $(USE_LISTINGS)!=0 TCC = $(TCC) -FAcs !ENDIF # When compiling the library for use in the WinRT environment, # the following compile-time options must be used as well to # disable use of Win32 APIs that are not available and to enable # use of Win32 APIs that are specific to Windows 8 and/or WinRT. # !IF $(FOR_WINRT)!=0 TCC = $(TCC) -DSQLITE_OS_WINRT=1 RCC = $(RCC) -DSQLITE_OS_WINRT=1 TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP !ENDIF # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 TCC = $(TCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE !ENDIF # Also, we need to dynamically link to the correct MSVC runtime # when compiling for WinRT (e.g. debug or release) OR if the # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd !ELSE TCC = $(TCC) -MD BCC = $(BCC) -MD !ENDIF !ELSE !IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT !ENDIF !ENDIF # Define -DNDEBUG to compile without debugging (i.e., for production usage) # Omitting the define will cause extra debugging code to be inserted and # includes extra comments when "EXPLAIN stmt" is used. # !IF $(DEBUG)==0 TCC = $(TCC) -DNDEBUG BCC = $(BCC) -DNDEBUG RCC = $(RCC) -DNDEBUG !ENDIF !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 RCC = $(RCC) -DSQLITE_DEBUG=1 !ENDIF !IF $(DEBUG)>4 || $(OSTRACE)!=0 TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1 !ENDIF !IF $(DEBUG)>5 TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE=1 RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE=1 !ENDIF # Prevent warnings about "insecure" MSVC runtime library functions # being used. # TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS # Prevent warnings about "deprecated" POSIX functions being used. # TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS # Use the SQLite debugging heap subsystem? # !IF $(MEMDEBUG)!=0 TCC = $(TCC) -DSQLITE_MEMDEBUG=1 RCC = $(RCC) -DSQLITE_MEMDEBUG=1 # Use native Win32 heap subsystem instead of malloc/free? # !ELSEIF $(WIN32HEAP)!=0 TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1 # Validate the heap on every call into the native Win32 heap subsystem? # !IF $(DEBUG)>3 TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF !ENDIF # Compiler options needed for programs that use the readline() library. # !IFNDEF READLINE_FLAGS READLINE_FLAGS = -DHAVE_READLINE=0 !ENDIF # The library that programs using readline() must link against. # !IFNDEF LIBREADLINE LIBREADLINE = !ENDIF # Should the database engine be compiled threadsafe # TCC = $(TCC) -DSQLITE_THREADSAFE=1 RCC = $(RCC) -DSQLITE_THREADSAFE=1 # Do threads override each others locks by default (1), or do we test (-1) # TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 # Any target libraries which libsqlite must be linked against # !IFNDEF TLIBS TLIBS = !ENDIF # Flags controlling use of the in memory btree implementation # # SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to # default to file, 2 to default to memory, and 3 to force temporary # tables to always be in memory. # TCC = $(TCC) -DSQLITE_TEMP_STORE=1 RCC = $(RCC) -DSQLITE_TEMP_STORE=1 # Enable/disable loadable extensions, and other optional features # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). # The same set of OMIT and ENABLE flags should be passed to the # LEMON parser generator and the mkkeywordhash tool as well. # These are the required SQLite compilation options used when compiling for # the Windows platform. # REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 # If we are linking to the RPCRT4 library, enable features that need it. # !IF $(USE_RPCRT4_LIB)!=0 REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1 !ENDIF # Add the required and optional SQLite compilation options into the command # lines used to invoke the MSVC code and resource compilers. # TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) # Add in any optional parameters specified on the commane line, e.g. # nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1" # TCC = $(TCC) $(OPTS) RCC = $(RCC) $(OPTS) # If compiling for debugging, add some defines. # !IF $(DEBUG)>1 TCC = $(TCC) -D_DEBUG BCC = $(BCC) -D_DEBUG RCC = $(RCC) -D_DEBUG !ENDIF # If optimizations are enabled or disabled (either implicitly or # explicitly), add the necessary flags. # !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od !ELSEIF $(OPTIMIZATIONS)>=3 TCC = $(TCC) -Ox BCC = $(BCC) -Ox !ELSEIF $(OPTIMIZATIONS)==2 TCC = $(TCC) -O2 BCC = $(BCC) -O2 !ELSEIF $(OPTIMIZATIONS)==1 TCC = $(TCC) -O1 BCC = $(BCC) -O1 !ENDIF # If symbols are enabled (or compiling for debugging), enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF # Command line prefixes for compiling code, compiling resources, # linking, etc. # LTCOMPILE = $(TCC) -Fo$@ LTRCOMPILE = $(RCC) -r LTLIB = lib.exe LTLINK = $(TCC) -Fe$@ # If requested, link to the RPCRT4 library. # !IF $(USE_RPCRT4_LIB)!=0 LTLINK = $(LTLINK) rpcrt4.lib !ENDIF # If a platform was set, force the linker to target that. # Note that the vcvars*.bat family of batch files typically # set this for you. Otherwise, the linker will attempt # to deduce the binary type based on the object files. !IFDEF PLATFORM LTLINKOPTS = /NOLOGO /MACHINE:$(PLATFORM) LTLIBOPTS = /NOLOGO /MACHINE:$(PLATFORM) !ELSE LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF # When compiling for use in the WinRT environment, the following # linker option must be used to mark the executable as runnable # only in the context of an application container. # !IF $(FOR_WINRT)!=0 LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER !IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" !IFNDEF STORELIBPATH !IF "$(PLATFORM)"=="x86" STORELIBPATH = $(CRTLIBPATH)\store !ELSEIF "$(PLATFORM)"=="x64" STORELIBPATH = $(CRTLIBPATH)\store\amd64 !ELSEIF "$(PLATFORM)"=="ARM" STORELIBPATH = $(CRTLIBPATH)\store\arm !ELSE STORELIBPATH = $(CRTLIBPATH)\store !ENDIF !ENDIF STORELIBPATH = $(STORELIBPATH:\\=\) LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" !ENDIF !ENDIF # When compiling for Windows Phone 8.1, an extra library path is # required. # !IF $(USE_WP81_OPTS)!=0 !IFNDEF WP81LIBPATH !IF "$(PLATFORM)"=="x86" WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 !ELSEIF "$(PLATFORM)"=="ARM" WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM !ELSE WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 !ENDIF !ENDIF !ENDIF # When compiling for Windows Phone 8.1, some extra linker options # are also required. # !IF $(USE_WP81_OPTS)!=0 !IFDEF WP81LIBPATH LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" !ENDIF LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib !ENDIF # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # !IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0 LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib LTLINKOPTS = $(LTLINKOPTS) mincore.lib !IFDEF PSDKLIBPATH LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)" !ENDIF !ENDIF !IF $(FOR_WIN10)!=0 LTLINKOPTS = $(LTLINKOPTS) /guard:cf "/LIBPATH:$(UCRTLIBPATH)" !IF $(DEBUG)>1 LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib !ELSE LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib !ENDIF !ENDIF # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG $(LDOPTS) !ELSE LDFLAGS = $(LDOPTS) !ENDIF # You should not have to change anything below this line ############################################################################### # Object files for the amalgamation. # LIBOBJS1 = sqlite3.lo # Determine the real value of LIBOBJ based on the 'configure' script # LIBOBJ = $(LIBOBJS1) # Determine if embedded resource compilation and usage are enabled. # !IF $(USE_RC)!=0 LIBRESOBJS = sqlite3res.lo !ELSE LIBRESOBJS = !ENDIF # Additional compiler options for the shell. These are only effective # when the shell is not being dynamically linked. # !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS !ENDIF # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: dll shell # Dynamic link library section. # dll: $(SQLITE3DLL) # Shell executable. # shell: $(SQLITE3EXE) $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) Replace.exe: $(CSC) /target:exe $(TOP)\Replace.cs sqlite3.def: Replace.exe $(LIBOBJ) echo EXPORTS > sqlite3.def dumpbin /all $(LIBOBJ) \ | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ | sort >> sqlite3.def $(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \ /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C) # Rule to build the Win32 resources object file. # !IF $(USE_RC)!=0 _HASHCHAR=^# !IF ![echo !IFNDEF VERSION > rcver.vc] && \ ![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \ ![echo !ENDIF >> rcver.vc] !INCLUDE rcver.vc !ENDIF RESOURCE_VERSION = $(VERSION:^#=) RESOURCE_VERSION = $(RESOURCE_VERSION:define=) RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=) RESOURCE_VERSION = $(RESOURCE_VERSION:"=) RESOURCE_VERSION = $(RESOURCE_VERSION:.=,) $(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h echo #endif >> sqlite3rc.h $(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc !ENDIF clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL |
Name change from autoconf/README to autoconf/README.txt.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
..
24
25
26
27
28
29
30
31
32
|
This package contains: * the SQLite library amalgamation (single file) source code distribution, * the shell.c file used to build the sqlite3 shell too, and * the sqlite3.h and sqlite3ext.h header files required to link programs and sqlite extensions against the installed libary. * autoconf/automake installation infrastucture. The generic installation instructions for autoconf/automake are found in the INSTALL file. The following SQLite specific boolean options are supported: --enable-readline use readline in shell tool [default=yes] ................................................................................ $ CFLAGS="-Os" ./configure to produce a smaller installation footprint. Other SQLite compilation parameters can also be set using CFLAGS. For example: $ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure |
<
|
>
>
|
<
<
|
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
..
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
This package contains: * the SQLite library amalgamation source code file: sqlite3.c * the sqlite3.h and sqlite3ext.h header files that define the C-language interface to the sqlite3.c library file * the shell.c file used to build the sqlite3 command-line shell program * autoconf/automake installation infrastucture for building on POSIX compliant systems * a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft Visual C++ on Windows SUMMARY OF HOW TO BUILD ======================= Unix: ./configure; make Windows: nmake /f Makefile.msc BUILDING ON POSIX ================= The generic installation instructions for autoconf/automake are found in the INSTALL file. The following SQLite specific boolean options are supported: --enable-readline use readline in shell tool [default=yes] ................................................................................ $ CFLAGS="-Os" ./configure to produce a smaller installation footprint. Other SQLite compilation parameters can also be set using CFLAGS. For example: $ CFLAGS="-Os -DSQLITE_THREADSAFE=0" ./configure BUILDING WITH MICROSOFT VISUAL C++ ================================== To compile for Windows using Microsoft Visual C++: $ nmake /f Makefile.msc Using Microsoft Visual C++ 2005 (or later) is recommended. Several Windows platform variants may be built by adding additional macros to the NMAKE command line. Building for WinRT 8.0 ---------------------- FOR_WINRT=1 Using Microsoft Visual C++ 2012 (or later) is required. When using the above, something like the following macro will need to be added to the NMAKE command line as well: "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86" Building for WinRT 8.1 ---------------------- FOR_WINRT=1 Using Microsoft Visual C++ 2013 (or later) is required. When using the above, something like the following macro will need to be added to the NMAKE command line as well: "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86" Building for UWP 10.0 --------------------- FOR_WINRT=1 FOR_UWP=1 Using Microsoft Visual C++ 2015 (or later) is required. When using the above, something like the following macros will need to be added to the NMAKE command line as well: "NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86" "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86" "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86" Building for the Windows 10 SDK ------------------------------- FOR_WIN10=1 Using Microsoft Visual C++ 2015 (or later) is required. When using the above, no other macros should be needed on the NMAKE command line. Other preprocessor defines -------------------------- Additionally, preprocessor defines may be specified by using the OPTS macro on the NMAKE command line. However, not all possible preprocessor defines may be specified in this manner as some require the amalgamation to be built with them enabled (see http://www.sqlite.org/compile.html). For example, the following will work: "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1" However, the following will not compile unless the amalgamation was built with it enabled: "OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1" |
Changes to autoconf/configure.ac.
45 46 47 48 49 50 51 52 53 54 55 56 57 58 .. 70 71 72 73 74 75 76 77 78 79 80 81 82 83 ... 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
if test x"$enable_editline" != xno ; then sLIBS=$LIBS LIBS="" AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no]) READLINE_LIBS=$LIBS if test x"$LIBS" != "x"; then AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) fi LIBS=$sLIBS fi if test x"$enable_readline" != xno ; then sLIBS=$LIBS LIBS="" AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], []) ................................................................................ AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( [--enable-threadsafe], [build a thread-safe library [default=yes]])], [], [enable_threadsafe=yes]) THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0 if test x"$enable_threadsafe" != "xno"; then THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1" AC_SEARCH_LIBS(pthread_create, pthread) fi AC_SUBST(THREADSAFE_FLAGS) #----------------------------------------------------------------------- #----------------------------------------------------------------------- # --enable-dynamic-extensions # ................................................................................ # --enable-static-shell # AC_ARG_ENABLE(static-shell, [AS_HELP_STRING( [--enable-static-shell], [statically link libsqlite3 into shell tool [default=yes]])], [], [enable_static_shell=yes]) if test x"$enable_static_shell" == "xyes"; then EXTRA_SHELL_OBJ=sqlite3.$OBJEXT else EXTRA_SHELL_OBJ=libsqlite3.la fi AC_SUBST(EXTRA_SHELL_OBJ) #----------------------------------------------------------------------- AC_CHECK_FUNCS(posix_fallocate) |
> > > | |
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 .. 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 ... 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
if test x"$enable_editline" != xno ; then sLIBS=$LIBS LIBS="" AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no]) READLINE_LIBS=$LIBS if test x"$LIBS" != "x"; then AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) else unset ac_cv_search_readline fi LIBS=$sLIBS fi if test x"$enable_readline" != xno ; then sLIBS=$LIBS LIBS="" AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], []) ................................................................................ AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( [--enable-threadsafe], [build a thread-safe library [default=yes]])], [], [enable_threadsafe=yes]) THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0 if test x"$enable_threadsafe" != "xno"; then THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1" AC_SEARCH_LIBS(pthread_create, pthread) AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi AC_SUBST(THREADSAFE_FLAGS) #----------------------------------------------------------------------- #----------------------------------------------------------------------- # --enable-dynamic-extensions # ................................................................................ # --enable-static-shell # AC_ARG_ENABLE(static-shell, [AS_HELP_STRING( [--enable-static-shell], [statically link libsqlite3 into shell tool [default=yes]])], [], [enable_static_shell=yes]) if test x"$enable_static_shell" == "xyes"; then EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT else EXTRA_SHELL_OBJ=libsqlite3.la fi AC_SUBST(EXTRA_SHELL_OBJ) #----------------------------------------------------------------------- AC_CHECK_FUNCS(posix_fallocate) |
Changes to config.h.in.
40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
#undef HAVE_MALLOC_H /* Define to 1 if you have the `malloc_usable_size' function. */ #undef HAVE_MALLOC_USABLE_SIZE /* Define to 1 if you have the <memory.h> header file. */ #undef HAVE_MEMORY_H /* Define to 1 if you have the <stdint.h> header file. */ #undef HAVE_STDINT_H /* Define to 1 if you have the <stdlib.h> header file. */ #undef HAVE_STDLIB_H |
> > > > > > > > > > > > |
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
#undef HAVE_MALLOC_H /* Define to 1 if you have the `malloc_usable_size' function. */ #undef HAVE_MALLOC_USABLE_SIZE /* Define to 1 if you have the <memory.h> header file. */ #undef HAVE_MEMORY_H /* Define to 1 if you have the pread() function. */ #undef HAVE_PREAD /* Define to 1 if you have the pread64() function. */ #undef HAVE_PREAD64 /* Define to 1 if you have the pwrite() function. */ #undef HAVE_PWRITE /* Define to 1 if you have the pwrite64() function. */ #undef HAVE_PWRITE64 /* Define to 1 if you have the <stdint.h> header file. */ #undef HAVE_STDINT_H /* Define to 1 if you have the <stdlib.h> header file. */ #undef HAVE_STDLIB_H |
Changes to configure.
1 2 3 4 5 6 7 8 9 10 ... 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 .... 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 .... 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 .... 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 .... 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 ..... 10269 10270 10271 10272 10273 10274 10275 10276 10277 10278 10279 10280 10281 10282 10283 ..... 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 ..... 12019 12020 12021 12022 12023 12024 12025 12026 12027 12028 12029 12030 12031 12032 12033 ..... 12085 12086 12087 12088 12089 12090 12091 12092 12093 12094 12095 12096 12097 12098 12099 |
#! /bin/sh # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.69 for sqlite 3.10.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # # This configure script is free software; the Free Software Foundation # gives unlimited permission to copy, distribute and modify it. ................................................................................ subdirs= MFLAGS= MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' PACKAGE_VERSION='3.10.0' PACKAGE_STRING='sqlite 3.10.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ #include <stdio.h> #ifdef HAVE_SYS_TYPES_H ................................................................................ # # Report the --help message. # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF \`configure' configures sqlite 3.10.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. Defaults for the options are specified in brackets. ................................................................................ --build=BUILD configure for building on BUILD [guessed] --host=HOST cross-compile to build programs to run on HOST [BUILD] _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in short | recursive ) echo "Configuration of sqlite 3.10.0:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] ................................................................................ cd "$ac_pwd" || { ac_status=$?; break; } done fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF sqlite configure 3.10.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF exit ................................................................................ eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by sqlite $as_me 3.10.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF exec 5>>config.log { ................................................................................ done ######### # Figure out whether or not we have these functions # for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes"; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 _ACEOF ................................................................................ fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 $as_echo "$ac_cv_search_pthread_create" >&6; } ac_res=$ac_cv_search_pthread_create if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi fi ................................................................................ test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" This file was extended by sqlite $as_me 3.10.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS CONFIG_COMMANDS = $CONFIG_COMMANDS $ $0 $@ ................................................................................ Report bugs to the package provider." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ sqlite config.status 3.10.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." |
| | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | |
1 2 3 4 5 6 7 8 9 10 ... 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 .... 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 .... 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 .... 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 .... 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 ..... 10269 10270 10271 10272 10273 10274 10275 10276 10277 10278 10279 10280 10281 10282 10283 ..... 10511 10512 10513 10514 10515 10516 10517 10518 10519 10520 10521 10522 10523 10524 10525 10526 10527 10528 10529 10530 10531 10532 10533 10534 10535 10536 10537 10538 10539 10540 10541 10542 10543 10544 10545 10546 10547 10548 10549 10550 10551 10552 10553 10554 10555 10556 10557 10558 10559 10560 10561 10562 10563 10564 10565 10566 10567 10568 10569 10570 10571 10572 10573 10574 10575 10576 10577 10578 10579 10580 ..... 12075 12076 12077 12078 12079 12080 12081 12082 12083 12084 12085 12086 12087 12088 12089 ..... 12141 12142 12143 12144 12145 12146 12147 12148 12149 12150 12151 12152 12153 12154 12155 |
#! /bin/sh # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.69 for sqlite 3.13.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # # This configure script is free software; the Free Software Foundation # gives unlimited permission to copy, distribute and modify it. ................................................................................ subdirs= MFLAGS= MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' PACKAGE_VERSION='3.13.0' PACKAGE_STRING='sqlite 3.13.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ #include <stdio.h> #ifdef HAVE_SYS_TYPES_H ................................................................................ # # Report the --help message. # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF \`configure' configures sqlite 3.13.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. Defaults for the options are specified in brackets. ................................................................................ --build=BUILD configure for building on BUILD [guessed] --host=HOST cross-compile to build programs to run on HOST [BUILD] _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in short | recursive ) echo "Configuration of sqlite 3.13.0:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] ................................................................................ cd "$ac_pwd" || { ac_status=$?; break; } done fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF sqlite configure 3.13.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF exit ................................................................................ eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by sqlite $as_me 3.13.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF exec 5>>config.log { ................................................................................ done ######### # Figure out whether or not we have these functions # for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64 do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" if eval test \"x\$"$as_ac_var"\" = x"yes"; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 _ACEOF ................................................................................ fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 $as_echo "$ac_cv_search_pthread_create" >&6; } ac_res=$ac_cv_search_pthread_create if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5 $as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; } if ${ac_cv_search_pthread_mutexattr_init+:} false; then : $as_echo_n "(cached) " >&6 else ac_func_search_save_LIBS=$LIBS cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ /* Override any GCC internal prototype to avoid an error. Use char because int might match the return type of a GCC builtin and then its argument prototype would still apply. */ #ifdef __cplusplus extern "C" #endif char pthread_mutexattr_init (); int main () { return pthread_mutexattr_init (); ; return 0; } _ACEOF for ac_lib in '' pthread; do if test -z "$ac_lib"; then ac_res="none required" else ac_res=-l$ac_lib LIBS="-l$ac_lib $ac_func_search_save_LIBS" fi if ac_fn_c_try_link "$LINENO"; then : ac_cv_search_pthread_mutexattr_init=$ac_res fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext if ${ac_cv_search_pthread_mutexattr_init+:} false; then : break fi done if ${ac_cv_search_pthread_mutexattr_init+:} false; then : else ac_cv_search_pthread_mutexattr_init=no fi rm conftest.$ac_ext LIBS=$ac_func_search_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5 $as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; } ac_res=$ac_cv_search_pthread_mutexattr_init if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi fi ................................................................................ test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" This file was extended by sqlite $as_me 3.13.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS CONFIG_COMMANDS = $CONFIG_COMMANDS $ $0 $@ ................................................................................ Report bugs to the package provider." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ sqlite config.status 3.13.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." |
Changes to configure.ac.
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
...
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
#########
# Check for needed/wanted headers
AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h])
#########
# Figure out whether or not we have these functions
#
AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime])
#########
# By default, we use the amalgamation (this may be changed below...)
#
USE_AMALGAMATION=1
#########
................................................................................
SQLITE_THREADSAFE=1
AC_MSG_RESULT([yes])
fi
AC_SUBST(SQLITE_THREADSAFE)
if test "$SQLITE_THREADSAFE" = "1"; then
AC_SEARCH_LIBS(pthread_create, pthread)
fi
##########
# Do we want to support release
#
AC_ARG_ENABLE(releasemode,
AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
|
|
>
|
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
...
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
######### # Check for needed/wanted headers AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h]) ######### # Figure out whether or not we have these functions # AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64]) ######### # By default, we use the amalgamation (this may be changed below...) # USE_AMALGAMATION=1 ######### ................................................................................ SQLITE_THREADSAFE=1 AC_MSG_RESULT([yes]) fi AC_SUBST(SQLITE_THREADSAFE) if test "$SQLITE_THREADSAFE" = "1"; then AC_SEARCH_LIBS(pthread_create, pthread) AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) fi ########## # Do we want to support release # AC_ARG_ENABLE(releasemode, AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no) |
Changes to doc/lemon.html.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 .. 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 .. 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 ... 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 ... 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 ... 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 ... 259 260 261 262 263 264 265 266 267 268 269 270 271 272 ... 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 ... 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 ... 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 ... 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 ... 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 ... 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 ... 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 ... 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 ... 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 ... 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 ... 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 ... 826 827 828 829 830 831 832 833 834 835 836 837 838 839 ... 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 ... 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 |
<html> <head> <title>The Lemon Parser Generator</title> </head> <body bgcolor=white> <h1 align=center>The Lemon Parser Generator</h1> <p>Lemon is an LALR(1) parser generator for C or C++. It does the same job as ``bison'' and ``yacc''. But lemon is not another bison or yacc clone. It uses a different grammar syntax which is designed to reduce the number of coding errors. Lemon also uses a more sophisticated parsing engine that is faster than yacc and bison and which is both reentrant and thread-safe. Furthermore, Lemon implements features that can be used to eliminate resource leaks, making is suitable for use in long-running programs such as graphical user interfaces or embedded controllers.</p> <p>This document is an introduction to the Lemon parser generator.</p> ................................................................................ <ul> <li>C code to implement the parser. <li>A header file defining an integer ID for each terminal symbol. <li>An information file that describes the states of the generated parser automaton. </ul> By default, all three of these output files are generated. The header file is suppressed if the ``-m'' command-line option is used and the report file is omitted when ``-q'' is selected.</p> <p>The grammar specification file uses a ``.y'' suffix, by convention. In the examples used in this document, we'll assume the name of the grammar file is ``gram.y''. A typical use of Lemon would be the following command: <pre> lemon gram.y </pre> This command will generate three output files named ``gram.c'', ``gram.h'' and ``gram.out''. The first is C code to implement the parser. The second is the header file that defines numerical values for all terminal symbols, and the last is the report that explains the states used by the parser automaton.</p> <h3>Command Line Options</h3> ................................................................................ You can obtain a list of the available command-line options together with a brief explanation of what each does by typing <pre> lemon -? </pre> As of this writing, the following command-line options are supported: <ul> <li><tt>-b</tt> <li><tt>-c</tt> <li><tt>-g</tt> <li><tt>-m</tt> <li><tt>-q</tt> <li><tt>-s</tt> <li><tt>-x</tt> </ul> The ``-b'' option reduces the amount of text in the report file by printing only the basis of each parser state, rather than the full configuration. The ``-c'' option suppresses action table compression. Using -c will make the parser a little larger and slower but it will detect syntax errors sooner. The ``-g'' option causes no output files to be generated at all. Instead, the input grammar file is printed on standard output but with all comments, actions and other extraneous text deleted. This is a useful way to get a quick summary of a grammar. The ``-m'' option causes the output C source file to be compatible with the ``makeheaders'' program. Makeheaders is a program that automatically generates header files from C source code. When the ``-m'' option is used, the header file is not output since the makeheaders program will take care of generated all header files automatically. The ``-q'' option suppresses the report file. Using ``-s'' causes a brief summary of parser statistics to be printed. Like this: <pre> Parser statistics: 74 terminals, 70 nonterminals, 179 rules 340 states, 2026 parser table entries, 0 conflicts </pre> Finally, the ``-x'' option causes Lemon to print its version number and then stops without attempting to read the grammar or generate a parser.</p> <h3>The Parser Interface</h3> <p>Lemon doesn't generate a complete, working program. It only generates a few subroutines that implement a parser. This section describes the interface to those subroutines. It is up to the programmer to call these subroutines in an appropriate way in order to produce a ................................................................................ must first create the parser. A new parser is created as follows: <pre> void *pParser = ParseAlloc( malloc ); </pre> The ParseAlloc() routine allocates and initializes a new parser and returns a pointer to it. The actual data structure used to represent a parser is opaque -- its internal structure is not visible or usable by the calling routine. For this reason, the ParseAlloc() routine returns a pointer to void rather than a pointer to some particular structure. The sole argument to the ParseAlloc() routine is a pointer to the subroutine used to allocate memory. Typically this means ``malloc()''.</p> <p>After a program is finished using a parser, it can reclaim all memory allocated by that parser by calling <pre> ParseFree(pParser, free); </pre> The first argument is the same pointer returned by ParseAlloc(). The ................................................................................ The first argument to the Parse() routine is the pointer returned by ParseAlloc(). The second argument is a small positive integer that tells the parse the type of the next token in the data stream. There is one token type for each terminal symbol in the grammar. The gram.h file generated by Lemon contains #define statements that map symbolic terminal symbol names into appropriate integer values. (A value of 0 for the second argument is a special flag to the parser to indicate that the end of input has been reached.) The third argument is the value of the given token. By default, the type of the third argument is integer, but the grammar will usually redefine this type to be some kind of structure. Typically the second argument will be a broad category of tokens such as ``identifier'' or ``number'' and the third argument will be the name of the identifier or the value of the number.</p> <p>The Parse() function may have either three or four arguments, depending on the grammar. If the grammar specification file request it, the Parse() function will have a fourth parameter that can be of any type chosen by the programmer. The parser doesn't do anything with this argument except to pass it through to action routines. This is a convenient mechanism for passing state information down to the action routines without having to use global variables.</p> <p>A typical use of a Lemon parser might look something like the following: ................................................................................ 15 ParseFree(pParser, free ); 16 TokenizerFree(pTokenizer); 17 return sState.treeRoot; 18 } </pre> This example shows a user-written routine that parses a file of text and returns a pointer to the parse tree. (We've omitted all error-handling from this example to keep it simple.) We assume the existence of some kind of tokenizer which is created using TokenizerCreate() on line 8 and deleted by TokenizerFree() on line 16. The GetNextToken() function on line 11 retrieves the next token from the input file and puts its type in the integer variable hTokenId. The sToken variable is assumed to be some kind of structure that contains details about each token, ................................................................................ and bison do not. </ul> These differences may cause some initial confusion for programmers with prior yacc and bison experience. But after years of experience using Lemon, I firmly believe that the Lemon way of doing things is better.</p> <h2>Input File Syntax</h2> <p>The main purpose of the grammar specification file for Lemon is to define the grammar for the parser. But the input file also specifies additional information Lemon requires to do its job. Most of the work in using Lemon is in writing an appropriate grammar file.</p> ................................................................................ declaration can occur at any point in the file. Lemon ignores whitespace (except where it is needed to separate tokens) and it honors the same commenting conventions as C and C++.</p> <h3>Terminals and Nonterminals</h3> <p>A terminal symbol (token) is any string of alphanumeric and underscore characters that begins with an upper case letter. A terminal can contain lowercase letters after the first character, but the usual convention is to make terminals all upper case. A nonterminal, on the other hand, is any string of alphanumeric and underscore characters than begins with a lower case letter. Again, the usual convention is to make nonterminals use all lower case letters.</p> ................................................................................ must have alphanumeric names.</p> <h3>Grammar Rules</h3> <p>The main component of a Lemon grammar file is a sequence of grammar rules. Each grammar rule consists of a nonterminal symbol followed by the special symbol ``::='' and then a list of terminals and/or nonterminals. The rule is terminated by a period. The list of terminals and nonterminals on the right-hand side of the rule can be empty. Rules can occur in any order, except that the left-hand side of the first rule is assumed to be the start symbol for the grammar (unless specified otherwise using the <tt>%start</tt> directive described below.) A typical sequence of grammar rules might look something like this: ................................................................................ expr ::= expr PLUS expr. expr ::= expr TIMES expr. expr ::= LPAREN expr RPAREN. expr ::= VALUE. </pre> </p> <p>There is one non-terminal in this example, ``expr'', and five terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'', ``RPAREN'' and ``VALUE''.</p> <p>Like yacc and bison, Lemon allows the grammar to specify a block of C code that will be executed whenever a grammar rule is reduced by the parser. In Lemon, this action is specified by putting the C code (contained within curly braces <tt>{...}</tt>) immediately after the period that closes the rule. ................................................................................ <pre> expr ::= expr PLUS expr. { printf("Doing an addition...\n"); } </pre> </p> <p>In order to be useful, grammar actions must normally be linked to their associated grammar rules. In yacc and bison, this is accomplished by embedding a ``$$'' in the action to stand for the value of the left-hand side of the rule and symbols ``$1'', ``$2'', and so forth to stand for the value of the terminal or nonterminal at position 1, 2 and so forth on the right-hand side of the rule. This idea is very powerful, but it is also very error-prone. The single most common source of errors in a yacc or bison grammar is to miscount the number of symbols on the right-hand side of a grammar rule and say ``$7'' when you really mean ``$8''.</p> <p>Lemon avoids the need to count grammar symbols by assigning symbolic names to each symbol in a grammar rule and then using those symbolic names in the action. In yacc or bison, one would write this: <pre> expr -> expr PLUS expr { $$ = $1 + $3; }; ................................................................................ includes a linking symbol in parentheses but that linking symbol is not actually used in the reduce action, then an error message is generated. For example, the rule <pre> expr(A) ::= expr(B) PLUS expr(C). { A = B; } </pre> will generate an error because the linking symbol ``C'' is used in the grammar rule but not in the reduce action.</p> <p>The Lemon notation for linking grammar rules to reduce actions also facilitates the use of destructors for reclaiming memory allocated by the values of terminals and nonterminals on the right-hand side of a rule.</p> <h3>Precedence Rules</h3> <p>Lemon resolves parsing ambiguities in exactly the same way as yacc and bison. A shift-reduce conflict is resolved in favor of the shift, and a reduce-reduce conflict is resolved by reducing whichever rule comes first in the grammar file.</p> <p>Just like in yacc and bison, Lemon allows a measure of control over the resolution of paring conflicts using precedence rules. A precedence value can be assigned to any terminal symbol using the %left, %right or %nonassoc directives. Terminal symbols mentioned in earlier directives have a lower precedence that terminal symbols mentioned in later directives. For example:</p> <p><pre> %left AND. %left OR. %nonassoc EQ NE GT GE LT LE. ................................................................................ <p>Lemon supports the following special directives: <ul> <li><tt>%code</tt> <li><tt>%default_destructor</tt> <li><tt>%default_type</tt> <li><tt>%destructor</tt> <li><tt>%extra_argument</tt> <li><tt>%include</tt> <li><tt>%left</tt> <li><tt>%name</tt> <li><tt>%nonassoc</tt> <li><tt>%parse_accept</tt> <li><tt>%parse_failure </tt> <li><tt>%right</tt> <li><tt>%stack_overflow</tt> <li><tt>%stack_size</tt> <li><tt>%start_symbol</tt> <li><tt>%syntax_error</tt> <li><tt>%token_destructor</tt> <li><tt>%token_prefix</tt> <li><tt>%token_type</tt> <li><tt>%type</tt> </ul> Each of these directives will be described separately in the following sections:</p> <h4>The <tt>%code</tt> directive</h4> <p>The %code directive is used to specify addition C/C++ code that is added to the end of the main output file. This is similar to the %include directive except that %include is inserted at the beginning of the main output file.</p> <p>%code is typically used to include some action routines or perhaps a tokenizer as part of the output file.</p> <h4>The <tt>%default_destructor</tt> directive</h4> <p>The %default_destructor directive specifies a destructor to use for non-terminals that do not have their own destructor specified by a separate %destructor directive. See the documentation on the %destructor directive below for additional information.</p> <p>In some grammers, many different non-terminal symbols have the same datatype and hence the same destructor. This directive is a convenience way to specify the same destructor for all those non-terminals using a single statement.</p> <h4>The <tt>%default_type</tt> directive</h4> <p>The %default_type directive specifies the datatype of non-terminal symbols that do no have their own datatype defined using a separate %type directive. See the documentation on %type below for addition information.</p> <h4>The <tt>%destructor</tt> directive</h4> <p>The %destructor directive is used to specify a destructor for a non-terminal symbol. (See also the %token_destructor directive which is used to specify a destructor for terminal symbols.)</p> <p>A non-terminal's destructor is called to dispose of the non-terminal's value whenever the non-terminal is popped from the stack. This includes all of the following circumstances: <ul> <li> When a rule reduces and the value of a non-terminal on the right-hand side is not linked to C code. ................................................................................ <pre> %type nt {void*} %destructor nt { free($$); } nt(A) ::= ID NUM. { A = malloc( 100 ); } </pre> This example is a bit contrived but it serves to illustrate how destructors work. The example shows a non-terminal named ``nt'' that holds values of type ``void*''. When the rule for an ``nt'' reduces, it sets the value of the non-terminal to space obtained from malloc(). Later, when the nt non-terminal is popped from the stack, the destructor will fire and call free() on this malloced space, thus avoiding a memory leak. (Note that the symbol ``$$'' in the destructor code is replaced by the value of the non-terminal.)</p> <p>It is important to note that the value of a non-terminal is passed to the destructor whenever the non-terminal is removed from the stack, unless the non-terminal is used in a C-code action. If the non-terminal is used by C-code, then it is assumed that the C-code will take care of destroying it if it should really be destroyed. More commonly, the value is used to build some larger structure and we don't want to destroy it, which is why the destructor is not called in this circumstance.</p> <p>By appropriate use of destructors, it is possible to build a parser using Lemon that can be used within a long-running program, such as a GUI, that will not leak memory or other resources. To do the same using yacc or bison is much more difficult.</p> <h4>The <tt>%extra_argument</tt> directive</h4> The %extra_argument directive instructs Lemon to add a 4th parameter to the parameter list of the Parse() function it generates. Lemon doesn't do anything itself with this extra argument, but it does make the argument available to C-code action routines, destructors, and so forth. For example, if the grammar file contains:</p> <p><pre> %extra_argument { MyStruct *pAbc } </pre></p> <p>Then the Parse() function generated will have an 4th parameter of type ``MyStruct*'' and all action routines will have access to a variable named ``pAbc'' that is the value of the 4th parameter in the most recent call to Parse().</p> <h4>The <tt>%include</tt> directive</h4> <p>The %include directive specifies C code that is included at the top of the generated parser. You can include any text you want -- the Lemon parser generator copies it blindly. If you have multiple %include directives in your grammar file the value of the last %include directive overwrites all the others.</p. <p>The %include directive is very handy for getting some extra #include preprocessor statements at the beginning of the generated parser. For example:</p> <p><pre> %include {#include <unistd.h>} </pre></p> <p>This might be needed, for example, if some of the C actions in the grammar call functions that are prototyed in unistd.h.</p> <h4>The <tt>%left</tt> directive</h4> The %left directive is used (along with the %right and %nonassoc directives) to declare precedences of terminal symbols. Every terminal symbol whose name appears after a %left directive but before the next period (``.'') is given the same left-associative precedence value. Subsequent %left directives have higher precedence. For example:</p> <p><pre> %left AND. %left OR. %nonassoc EQ NE GT GE LT LE. ................................................................................ directive.</p> <p>LALR(1) grammars can get into a situation where they require a large amount of stack space if you make heavy use or right-associative operators. For this reason, it is recommended that you use %left rather than %right whenever possible.</p> <h4>The <tt>%name</tt> directive</h4> <p>By default, the functions generated by Lemon all begin with the five-character string ``Parse''. You can change this string to something different using the %name directive. For instance:</p> <p><pre> %name Abcde </pre></p> <p>Putting this directive in the grammar file will cause Lemon to generate ................................................................................ <li> AbcdeTrace(), and <li> Abcde(). </ul> The %name directive allows you to generator two or more different parsers and link them all into the same executable. </p> <h4>The <tt>%nonassoc</tt> directive</h4> <p>This directive is used to assign non-associative precedence to one or more terminal symbols. See the section on precedence rules or on the %left directive for additional information.</p> <h4>The <tt>%parse_accept</tt> directive</h4> <p>The %parse_accept directive specifies a block of C code that is executed whenever the parser accepts its input string. To ``accept'' an input string means that the parser was able to process all tokens without error.</p> <p>For example:</p> <p><pre> %parse_accept { printf("parsing complete!\n"); } </pre></p> <h4>The <tt>%parse_failure</tt> directive</h4> <p>The %parse_failure directive specifies a block of C code that is executed whenever the parser fails complete. This code is not executed until the parser has tried and failed to resolve an input error using is usual error recovery strategy. The routine is only invoked when parsing is unable to continue.</p> ................................................................................ <p><pre> %parse_failure { fprintf(stderr,"Giving up. Parser is hopelessly lost...\n"); } </pre></p> <h4>The <tt>%right</tt> directive</h4> <p>This directive is used to assign right-associative precedence to one or more terminal symbols. See the section on precedence rules or on the %left directive for additional information.</p> <h4>The <tt>%stack_overflow</tt> directive</h4> <p>The %stack_overflow directive specifies a block of C code that is executed if the parser's internal stack ever overflows. Typically this just prints an error message. After a stack overflow, the parser will be unable to continue and must be reset.</p> ................................................................................ </pre> Not like this: <pre> list ::= element list. // right-recursion. Bad! list ::= . </pre> <h4>The <tt>%stack_size</tt> directive</h4> <p>If stack overflow is a problem and you can't resolve the trouble by using left-recursion, then you might want to increase the size of the parser's stack using this directive. Put an positive integer after the %stack_size directive and Lemon will generate a parse with a stack of the requested size. The default value is 100.</p> <p><pre> %stack_size 2000 </pre></p> <h4>The <tt>%start_symbol</tt> directive</h4> <p>By default, the start-symbol for the grammar that Lemon generates is the first non-terminal that appears in the grammar file. But you can choose a different start-symbol using the %start_symbol directive.</p> <p><pre> %start_symbol prog </pre></p> <h4>The <tt>%token_destructor</tt> directive</h4> <p>The %destructor directive assigns a destructor to a non-terminal symbol. (See the description of the %destructor directive above.) This directive does the same thing for all terminal symbols.</p> <p>Unlike non-terminal symbols which may each have a different data type for their values, terminals all use the same data type (defined by the %token_type directive) and so they use a common destructor. Other than that, the token destructor works just like the non-terminal destructors.</p> <h4>The <tt>%token_prefix</tt> directive</h4> <p>Lemon generates #defines that assign small integer constants to each terminal symbol in the grammar. If desired, Lemon will add a prefix specified by this directive to each of the #defines it generates. So if the default output of Lemon looked like this: ................................................................................ <pre> #define TOKEN_AND 1 #define TOKEN_MINUS 2 #define TOKEN_OR 3 #define TOKEN_PLUS 4 </pre> <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4> <p>These directives are used to specify the data types for values on the parser's stack associated with terminal and non-terminal symbols. The values of all terminal symbols must be of the same type. This turns out to be the same data type as the 3rd parameter to the Parse() function generated by Lemon. Typically, you will ................................................................................ token structure. Like this:</p> <p><pre> %token_type {Token*} </pre></p> <p>If the data type of terminals is not specified, the default value is ``int''.</p> <p>Non-terminal symbols can each have their own data types. Typically the data type of a non-terminal is a pointer to the root of a parse-tree structure that contains all information about that non-terminal. For example:</p> <p><pre> ................................................................................ on what the corresponding non-terminal or terminal symbol is. But the grammar designer should keep in mind that the size of the union will be the size of its largest element. So if you have a single non-terminal whose data type requires 1K of storage, then your 100 entry parser stack will require 100K of heap space. If you are willing and able to pay that price, fine. You just need to know.</p> <h3>Error Processing</h3> <p>After extensive experimentation over several years, it has been discovered that the error recovery strategy used by yacc is about as good as it gets. And so that is what Lemon uses.</p> <p>When a Lemon-generated parser encounters a syntax error, it first invokes the code specified by the %syntax_error directive, if any. It then enters its error recovery strategy. The error recovery strategy is to begin popping the parsers stack until it enters a state where it is permitted to shift a special non-terminal symbol named ``error''. It then shifts this non-terminal and continues parsing. But the %syntax_error routine will not be called again until at least three new tokens have been successfully shifted.</p> <p>If the parser pops its stack until the stack is empty, and it still is unable to shift the error symbol, then the %parse_failed routine is invoked and the parser resets itself to its start state, ready to begin parsing a new file. This is what will happen at the very first syntax error, of course, if there are no instances of the ``error'' non-terminal in your grammar.</p> </body> </html> |
| | | | | | > > > | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | < < < < | | | | | | > | | > > > > > > | | | | | | | | | > > > > | > > > > > > > | < > | > | > > | > | | > | | | | | | | | | < > | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | < > > > | | | | > | > | > | > | | > | > | > > > > > > | > > > > > > > > > > > | | |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 .. 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 .. 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 ... 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 ... 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 ... 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 ... 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 ... 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 ... 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 ... 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 ... 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 ... 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 ... 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 ... 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 ... 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 ... 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 ... 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 ... 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 ... 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 ... 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 ... 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 |
<html> <head> <title>The Lemon Parser Generator</title> </head> <body bgcolor=white> <h1 align=center>The Lemon Parser Generator</h1> <p>Lemon is an LALR(1) parser generator for C. It does the same job as "bison" and "yacc". But lemon is not a bison or yacc clone. Lemon uses a different grammar syntax which is designed to reduce the number of coding errors. Lemon also uses a parsing engine that is faster than yacc and bison and which is both reentrant and threadsafe. (Update: Since the previous sentence was written, bison has also been updated so that it too can generate a reentrant and threadsafe parser.) Lemon also implements features that can be used to eliminate resource leaks, making is suitable for use in long-running programs such as graphical user interfaces or embedded controllers.</p> <p>This document is an introduction to the Lemon parser generator.</p> ................................................................................ <ul> <li>C code to implement the parser. <li>A header file defining an integer ID for each terminal symbol. <li>An information file that describes the states of the generated parser automaton. </ul> By default, all three of these output files are generated. The header file is suppressed if the "-m" command-line option is used and the report file is omitted when "-q" is selected.</p> <p>The grammar specification file uses a ".y" suffix, by convention. In the examples used in this document, we'll assume the name of the grammar file is "gram.y". A typical use of Lemon would be the following command: <pre> lemon gram.y </pre> This command will generate three output files named "gram.c", "gram.h" and "gram.out". The first is C code to implement the parser. The second is the header file that defines numerical values for all terminal symbols, and the last is the report that explains the states used by the parser automaton.</p> <h3>Command Line Options</h3> ................................................................................ You can obtain a list of the available command-line options together with a brief explanation of what each does by typing <pre> lemon -? </pre> As of this writing, the following command-line options are supported: <ul> <li><b>-b</b> Show only the basis for each parser state in the report file. <li><b>-c</b> Do not compress the generated action tables. <li><b>-D<i>name</i></b> Define C preprocessor macro <i>name</i>. This macro is useable by "%ifdef" lines in the grammar file. <li><b>-g</b> Do not generate a parser. Instead write the input grammar to standard output with all comments, actions, and other extraneous text removed. <li><b>-l</b> Omit "#line" directives int the generated parser C code. <li><b>-m</b> Cause the output C source code to be compatible with the "makeheaders" program. <li><b>-p</b> Display all conflicts that are resolved by <a href='#precrules'>precedence rules</a>. <li><b>-q</b> Suppress generation of the report file. <li><b>-r</b> Do not sort or renumber the parser states as part of optimization. <li><b>-s</b> Show parser statistics before existing. <li><b>-T<i>file</i></b> Use <i>file</i> as the template for the generated C-code parser implementation. <li><b>-x</b> Print the Lemon version number. </ul> <h3>The Parser Interface</h3> <p>Lemon doesn't generate a complete, working program. It only generates a few subroutines that implement a parser. This section describes the interface to those subroutines. It is up to the programmer to call these subroutines in an appropriate way in order to produce a ................................................................................ must first create the parser. A new parser is created as follows: <pre> void *pParser = ParseAlloc( malloc ); </pre> The ParseAlloc() routine allocates and initializes a new parser and returns a pointer to it. The actual data structure used to represent a parser is opaque — its internal structure is not visible or usable by the calling routine. For this reason, the ParseAlloc() routine returns a pointer to void rather than a pointer to some particular structure. The sole argument to the ParseAlloc() routine is a pointer to the subroutine used to allocate memory. Typically this means malloc().</p> <p>After a program is finished using a parser, it can reclaim all memory allocated by that parser by calling <pre> ParseFree(pParser, free); </pre> The first argument is the same pointer returned by ParseAlloc(). The ................................................................................ The first argument to the Parse() routine is the pointer returned by ParseAlloc(). The second argument is a small positive integer that tells the parse the type of the next token in the data stream. There is one token type for each terminal symbol in the grammar. The gram.h file generated by Lemon contains #define statements that map symbolic terminal symbol names into appropriate integer values. A value of 0 for the second argument is a special flag to the parser to indicate that the end of input has been reached. The third argument is the value of the given token. By default, the type of the third argument is integer, but the grammar will usually redefine this type to be some kind of structure. Typically the second argument will be a broad category of tokens such as "identifier" or "number" and the third argument will be the name of the identifier or the value of the number.</p> <p>The Parse() function may have either three or four arguments, depending on the grammar. If the grammar specification file requests it (via the <a href='#extraarg'><tt>extra_argument</tt> directive</a>), the Parse() function will have a fourth parameter that can be of any type chosen by the programmer. The parser doesn't do anything with this argument except to pass it through to action routines. This is a convenient mechanism for passing state information down to the action routines without having to use global variables.</p> <p>A typical use of a Lemon parser might look something like the following: ................................................................................ 15 ParseFree(pParser, free ); 16 TokenizerFree(pTokenizer); 17 return sState.treeRoot; 18 } </pre> This example shows a user-written routine that parses a file of text and returns a pointer to the parse tree. (All error-handling code is omitted from this example to keep it simple.) We assume the existence of some kind of tokenizer which is created using TokenizerCreate() on line 8 and deleted by TokenizerFree() on line 16. The GetNextToken() function on line 11 retrieves the next token from the input file and puts its type in the integer variable hTokenId. The sToken variable is assumed to be some kind of structure that contains details about each token, ................................................................................ and bison do not. </ul> These differences may cause some initial confusion for programmers with prior yacc and bison experience. But after years of experience using Lemon, I firmly believe that the Lemon way of doing things is better.</p> <p><i>Updated as of 2016-02-16:</i> The text above was written in the 1990s. We are told that Bison has lately been enhanced to support the tokenizer-calls-parser paradigm used by Lemon, and to obviate the need for global variables.</p> <h2>Input File Syntax</h2> <p>The main purpose of the grammar specification file for Lemon is to define the grammar for the parser. But the input file also specifies additional information Lemon requires to do its job. Most of the work in using Lemon is in writing an appropriate grammar file.</p> ................................................................................ declaration can occur at any point in the file. Lemon ignores whitespace (except where it is needed to separate tokens) and it honors the same commenting conventions as C and C++.</p> <h3>Terminals and Nonterminals</h3> <p>A terminal symbol (token) is any string of alphanumeric and/or underscore characters that begins with an upper case letter. A terminal can contain lowercase letters after the first character, but the usual convention is to make terminals all upper case. A nonterminal, on the other hand, is any string of alphanumeric and underscore characters than begins with a lower case letter. Again, the usual convention is to make nonterminals use all lower case letters.</p> ................................................................................ must have alphanumeric names.</p> <h3>Grammar Rules</h3> <p>The main component of a Lemon grammar file is a sequence of grammar rules. Each grammar rule consists of a nonterminal symbol followed by the special symbol "::=" and then a list of terminals and/or nonterminals. The rule is terminated by a period. The list of terminals and nonterminals on the right-hand side of the rule can be empty. Rules can occur in any order, except that the left-hand side of the first rule is assumed to be the start symbol for the grammar (unless specified otherwise using the <tt>%start</tt> directive described below.) A typical sequence of grammar rules might look something like this: ................................................................................ expr ::= expr PLUS expr. expr ::= expr TIMES expr. expr ::= LPAREN expr RPAREN. expr ::= VALUE. </pre> </p> <p>There is one non-terminal in this example, "expr", and five terminal symbols or tokens: "PLUS", "TIMES", "LPAREN", "RPAREN" and "VALUE".</p> <p>Like yacc and bison, Lemon allows the grammar to specify a block of C code that will be executed whenever a grammar rule is reduced by the parser. In Lemon, this action is specified by putting the C code (contained within curly braces <tt>{...}</tt>) immediately after the period that closes the rule. ................................................................................ <pre> expr ::= expr PLUS expr. { printf("Doing an addition...\n"); } </pre> </p> <p>In order to be useful, grammar actions must normally be linked to their associated grammar rules. In yacc and bison, this is accomplished by embedding a "$$" in the action to stand for the value of the left-hand side of the rule and symbols "$1", "$2", and so forth to stand for the value of the terminal or nonterminal at position 1, 2 and so forth on the right-hand side of the rule. This idea is very powerful, but it is also very error-prone. The single most common source of errors in a yacc or bison grammar is to miscount the number of symbols on the right-hand side of a grammar rule and say "$7" when you really mean "$8".</p> <p>Lemon avoids the need to count grammar symbols by assigning symbolic names to each symbol in a grammar rule and then using those symbolic names in the action. In yacc or bison, one would write this: <pre> expr -> expr PLUS expr { $$ = $1 + $3; }; ................................................................................ includes a linking symbol in parentheses but that linking symbol is not actually used in the reduce action, then an error message is generated. For example, the rule <pre> expr(A) ::= expr(B) PLUS expr(C). { A = B; } </pre> will generate an error because the linking symbol "C" is used in the grammar rule but not in the reduce action.</p> <p>The Lemon notation for linking grammar rules to reduce actions also facilitates the use of destructors for reclaiming memory allocated by the values of terminals and nonterminals on the right-hand side of a rule.</p> <a name='precrules'></a> <h3>Precedence Rules</h3> <p>Lemon resolves parsing ambiguities in exactly the same way as yacc and bison. A shift-reduce conflict is resolved in favor of the shift, and a reduce-reduce conflict is resolved by reducing whichever rule comes first in the grammar file.</p> <p>Just like in yacc and bison, Lemon allows a measure of control over the resolution of paring conflicts using precedence rules. A precedence value can be assigned to any terminal symbol using the <a href='#pleft'>%left</a>, <a href='#pright'>%right</a> or <a href='#pnonassoc'>%nonassoc</a> directives. Terminal symbols mentioned in earlier directives have a lower precedence that terminal symbols mentioned in later directives. For example:</p> <p><pre> %left AND. %left OR. %nonassoc EQ NE GT GE LT LE. ................................................................................ <p>Lemon supports the following special directives: <ul> <li><tt>%code</tt> <li><tt>%default_destructor</tt> <li><tt>%default_type</tt> <li><tt>%destructor</tt> <li><tt>%endif</tt> <li><tt>%extra_argument</tt> <li><tt>%fallback</tt> <li><tt>%ifdef</tt> <li><tt>%ifndef</tt> <li><tt>%include</tt> <li><tt>%left</tt> <li><tt>%name</tt> <li><tt>%nonassoc</tt> <li><tt>%parse_accept</tt> <li><tt>%parse_failure </tt> <li><tt>%right</tt> <li><tt>%stack_overflow</tt> <li><tt>%stack_size</tt> <li><tt>%start_symbol</tt> <li><tt>%syntax_error</tt> <li><tt>%token_class</tt> <li><tt>%token_destructor</tt> <li><tt>%token_prefix</tt> <li><tt>%token_type</tt> <li><tt>%type</tt> <li><tt>%wildcard</tt> </ul> Each of these directives will be described separately in the following sections:</p> <a name='pcode'></a> <h4>The <tt>%code</tt> directive</h4> <p>The %code directive is used to specify addition C code that is added to the end of the main output file. This is similar to the <a href='#pinclude'>%include</a> directive except that %include is inserted at the beginning of the main output file.</p> <p>%code is typically used to include some action routines or perhaps a tokenizer or even the "main()" function as part of the output file.</p> <a name='default_destructor'></a> <h4>The <tt>%default_destructor</tt> directive</h4> <p>The %default_destructor directive specifies a destructor to use for non-terminals that do not have their own destructor specified by a separate %destructor directive. See the documentation on the <a name='#destructor'>%destructor</a> directive below for additional information.</p> <p>In some grammers, many different non-terminal symbols have the same datatype and hence the same destructor. This directive is a convenience way to specify the same destructor for all those non-terminals using a single statement.</p> <a name='default_type'></a> <h4>The <tt>%default_type</tt> directive</h4> <p>The %default_type directive specifies the datatype of non-terminal symbols that do no have their own datatype defined using a separate <a href='#ptype'>%type</a> directive. </p> <a name='destructor'></a> <h4>The <tt>%destructor</tt> directive</h4> <p>The %destructor directive is used to specify a destructor for a non-terminal symbol. (See also the <a href='#token_destructor'>%token_destructor</a> directive which is used to specify a destructor for terminal symbols.)</p> <p>A non-terminal's destructor is called to dispose of the non-terminal's value whenever the non-terminal is popped from the stack. This includes all of the following circumstances: <ul> <li> When a rule reduces and the value of a non-terminal on the right-hand side is not linked to C code. ................................................................................ <pre> %type nt {void*} %destructor nt { free($$); } nt(A) ::= ID NUM. { A = malloc( 100 ); } </pre> This example is a bit contrived but it serves to illustrate how destructors work. The example shows a non-terminal named "nt" that holds values of type "void*". When the rule for an "nt" reduces, it sets the value of the non-terminal to space obtained from malloc(). Later, when the nt non-terminal is popped from the stack, the destructor will fire and call free() on this malloced space, thus avoiding a memory leak. (Note that the symbol "$$" in the destructor code is replaced by the value of the non-terminal.)</p> <p>It is important to note that the value of a non-terminal is passed to the destructor whenever the non-terminal is removed from the stack, unless the non-terminal is used in a C-code action. If the non-terminal is used by C-code, then it is assumed that the C-code will take care of destroying it. More commonly, the value is used to build some larger structure and we don't want to destroy it, which is why the destructor is not called in this circumstance.</p> <p>Destructors help avoid memory leaks by automatically freeing allocated objects when they go out of scope. To do the same using yacc or bison is much more difficult.</p> <a name="extraarg"></a> <h4>The <tt>%extra_argument</tt> directive</h4> The %extra_argument directive instructs Lemon to add a 4th parameter to the parameter list of the Parse() function it generates. Lemon doesn't do anything itself with this extra argument, but it does make the argument available to C-code action routines, destructors, and so forth. For example, if the grammar file contains:</p> <p><pre> %extra_argument { MyStruct *pAbc } </pre></p> <p>Then the Parse() function generated will have an 4th parameter of type "MyStruct*" and all action routines will have access to a variable named "pAbc" that is the value of the 4th parameter in the most recent call to Parse().</p> <a name='pfallback'></a> <h4>The <tt>%fallback</tt> directive</h4> <p>The %fallback directive specifies an alternative meaning for one or more tokens. The alternative meaning is tried if the original token would have generated a syntax error. <p>The %fallback directive was added to support robust parsing of SQL syntax in <a href="https://www.sqlite.org/">SQLite</a>. The SQL language contains a large assortment of keywords, each of which appears as a different token to the language parser. SQL contains so many keywords, that it can be difficult for programmers to keep up with them all. Programmers will, therefore, sometimes mistakenly use an obscure language keyword for an identifier. The %fallback directive provides a mechanism to tell the parser: "If you are unable to parse this keyword, try treating it as an identifier instead." <p>The syntax of %fallback is as follows: <blockquote> <tt>%fallback</tt> <i>ID</i> <i>TOKEN...</i> <b>.</b> </blockquote> <p>In words, the %fallback directive is followed by a list of token names terminated by a period. The first token name is the fallback token - the token to which all the other tokens fall back to. The second and subsequent arguments are tokens which fall back to the token identified by the first argument. <a name='pifdef'></a> <h4>The <tt>%ifdef</tt>, <tt>%ifndef</tt>, and <tt>%endif</tt> directives.</h4> <p>The %ifdef, %ifndef, and %endif directives are similar to #ifdef, #ifndef, and #endif in the C-preprocessor, just not as general. Each of these directives must begin at the left margin. No whitespace is allowed between the "%" and the directive name. <p>Grammar text in between "%ifdef MACRO" and the next nested "%endif" is ignored unless the "-DMACRO" command-line option is used. Grammar text betwen "%ifndef MACRO" and the next nested "%endif" is included except when the "-DMACRO" command-line option is used. <p>Note that the argument to %ifdef and %ifndef must be a single preprocessor symbol name, not a general expression. There is no "%else" directive. <a name='pinclude'></a> <h4>The <tt>%include</tt> directive</h4> <p>The %include directive specifies C code that is included at the top of the generated parser. You can include any text you want -- the Lemon parser generator copies it blindly. If you have multiple %include directives in your grammar file, their values are concatenated so that all %include code ultimately appears near the top of the generated parser, in the same order as it appeared in the grammer.</p> <p>The %include directive is very handy for getting some extra #include preprocessor statements at the beginning of the generated parser. For example:</p> <p><pre> %include {#include <unistd.h>} </pre></p> <p>This might be needed, for example, if some of the C actions in the grammar call functions that are prototyed in unistd.h.</p> <a name='pleft'></a> <h4>The <tt>%left</tt> directive</h4> The %left directive is used (along with the <a href='#pright'>%right</a> and <a href='#pnonassoc'>%nonassoc</a> directives) to declare precedences of terminal symbols. Every terminal symbol whose name appears after a %left directive but before the next period (".") is given the same left-associative precedence value. Subsequent %left directives have higher precedence. For example:</p> <p><pre> %left AND. %left OR. %nonassoc EQ NE GT GE LT LE. ................................................................................ directive.</p> <p>LALR(1) grammars can get into a situation where they require a large amount of stack space if you make heavy use or right-associative operators. For this reason, it is recommended that you use %left rather than %right whenever possible.</p> <a name='pname'></a> <h4>The <tt>%name</tt> directive</h4> <p>By default, the functions generated by Lemon all begin with the five-character string "Parse". You can change this string to something different using the %name directive. For instance:</p> <p><pre> %name Abcde </pre></p> <p>Putting this directive in the grammar file will cause Lemon to generate ................................................................................ <li> AbcdeTrace(), and <li> Abcde(). </ul> The %name directive allows you to generator two or more different parsers and link them all into the same executable. </p> <a name='pnonassoc'></a> <h4>The <tt>%nonassoc</tt> directive</h4> <p>This directive is used to assign non-associative precedence to one or more terminal symbols. See the section on <a href='#precrules'>precedence rules</a> or on the <a href='#pleft'>%left</a> directive for additional information.</p> <a name='parse_accept'></a> <h4>The <tt>%parse_accept</tt> directive</h4> <p>The %parse_accept directive specifies a block of C code that is executed whenever the parser accepts its input string. To "accept" an input string means that the parser was able to process all tokens without error.</p> <p>For example:</p> <p><pre> %parse_accept { printf("parsing complete!\n"); } </pre></p> <a name='parse_failure'></a> <h4>The <tt>%parse_failure</tt> directive</h4> <p>The %parse_failure directive specifies a block of C code that is executed whenever the parser fails complete. This code is not executed until the parser has tried and failed to resolve an input error using is usual error recovery strategy. The routine is only invoked when parsing is unable to continue.</p> ................................................................................ <p><pre> %parse_failure { fprintf(stderr,"Giving up. Parser is hopelessly lost...\n"); } </pre></p> <a name='pright'></a> <h4>The <tt>%right</tt> directive</h4> <p>This directive is used to assign right-associative precedence to one or more terminal symbols. See the section on <a href='#precrules'>precedence rules</a> or on the <a href='#pleft'>%left</a> directive for additional information.</p> <a name='stack_overflow'></a> <h4>The <tt>%stack_overflow</tt> directive</h4> <p>The %stack_overflow directive specifies a block of C code that is executed if the parser's internal stack ever overflows. Typically this just prints an error message. After a stack overflow, the parser will be unable to continue and must be reset.</p> ................................................................................ </pre> Not like this: <pre> list ::= element list. // right-recursion. Bad! list ::= . </pre> <a name='stack_size'></a> <h4>The <tt>%stack_size</tt> directive</h4> <p>If stack overflow is a problem and you can't resolve the trouble by using left-recursion, then you might want to increase the size of the parser's stack using this directive. Put an positive integer after the %stack_size directive and Lemon will generate a parse with a stack of the requested size. The default value is 100.</p> <p><pre> %stack_size 2000 </pre></p> <a name='start_symbol'></a> <h4>The <tt>%start_symbol</tt> directive</h4> <p>By default, the start-symbol for the grammar that Lemon generates is the first non-terminal that appears in the grammar file. But you can choose a different start-symbol using the %start_symbol directive.</p> <p><pre> %start_symbol prog </pre></p> <a name='token_destructor'></a> <h4>The <tt>%token_destructor</tt> directive</h4> <p>The %destructor directive assigns a destructor to a non-terminal symbol. (See the description of the %destructor directive above.) This directive does the same thing for all terminal symbols.</p> <p>Unlike non-terminal symbols which may each have a different data type for their values, terminals all use the same data type (defined by the %token_type directive) and so they use a common destructor. Other than that, the token destructor works just like the non-terminal destructors.</p> <a name='token_prefix'></a> <h4>The <tt>%token_prefix</tt> directive</h4> <p>Lemon generates #defines that assign small integer constants to each terminal symbol in the grammar. If desired, Lemon will add a prefix specified by this directive to each of the #defines it generates. So if the default output of Lemon looked like this: ................................................................................ <pre> #define TOKEN_AND 1 #define TOKEN_MINUS 2 #define TOKEN_OR 3 #define TOKEN_PLUS 4 </pre> <a name='token_type'></a><a name='ptype'></a> <h4>The <tt>%token_type</tt> and <tt>%type</tt> directives</h4> <p>These directives are used to specify the data types for values on the parser's stack associated with terminal and non-terminal symbols. The values of all terminal symbols must be of the same type. This turns out to be the same data type as the 3rd parameter to the Parse() function generated by Lemon. Typically, you will ................................................................................ token structure. Like this:</p> <p><pre> %token_type {Token*} </pre></p> <p>If the data type of terminals is not specified, the default value is "int".</p> <p>Non-terminal symbols can each have their own data types. Typically the data type of a non-terminal is a pointer to the root of a parse-tree structure that contains all information about that non-terminal. For example:</p> <p><pre> ................................................................................ on what the corresponding non-terminal or terminal symbol is. But the grammar designer should keep in mind that the size of the union will be the size of its largest element. So if you have a single non-terminal whose data type requires 1K of storage, then your 100 entry parser stack will require 100K of heap space. If you are willing and able to pay that price, fine. You just need to know.</p> <a name='pwildcard'></a> <h4>The <tt>%wildcard</tt> directive</h4> <p>The %wildcard directive is followed by a single token name and a period. This directive specifies that the identified token should match any input token. <p>When the generated parser has the choice of matching an input against the wildcard token and some other token, the other token is always used. The wildcard token is only matched if there are no other alternatives. <h3>Error Processing</h3> <p>After extensive experimentation over several years, it has been discovered that the error recovery strategy used by yacc is about as good as it gets. And so that is what Lemon uses.</p> <p>When a Lemon-generated parser encounters a syntax error, it first invokes the code specified by the %syntax_error directive, if any. It then enters its error recovery strategy. The error recovery strategy is to begin popping the parsers stack until it enters a state where it is permitted to shift a special non-terminal symbol named "error". It then shifts this non-terminal and continues parsing. But the %syntax_error routine will not be called again until at least three new tokens have been successfully shifted.</p> <p>If the parser pops its stack until the stack is empty, and it still is unable to shift the error symbol, then the %parse_failed routine is invoked and the parser resets itself to its start state, ready to begin parsing a new file. This is what will happen at the very first syntax error, of course, if there are no instances of the "error" non-terminal in your grammar.</p> </body> </html> |
Changes to ext/fts3/fts3Int.h.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 |
*/ #ifndef _FTSINT_H #define _FTSINT_H #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif /* ** FTS4 is really an extension for FTS3. It is enabled using the ** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. */ #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) |
> > > > > > |
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
*/ #ifndef _FTSINT_H #define _FTSINT_H #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif /* FTS3/FTS4 require virtual tables */ #ifdef SQLITE_OMIT_VIRTUALTABLE # undef SQLITE_ENABLE_FTS3 # undef SQLITE_ENABLE_FTS4 #endif /* ** FTS4 is really an extension for FTS3. It is enabled using the ** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. */ #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) |
Changes to ext/fts3/fts3_test.c.
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 |
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
#ifdef SQLITE_ENABLE_FTS3
char aBuf[24];
int rc;
Tcl_WideInt w, w2;
int nByte, nByte2;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "INTEGER");
return TCL_ERROR;
}
|
| > |
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 |
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
#ifdef SQLITE_ENABLE_FTS3
char aBuf[24];
int rc;
Tcl_WideInt w;
sqlite3_int64 w2;
int nByte, nByte2;
if( objc!=2 ){
Tcl_WrongNumArgs(interp, 1, objv, "INTEGER");
return TCL_ERROR;
}
|
Changes to ext/fts3/fts3_tokenizer.c.
24 25 26 27 28 29 30 31 32 33 34 35 36 37 .. 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 .. 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 ... 345 346 347 348 349 350 351 352 353 354 355 356 357 358 ... 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 ... 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 ... 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 |
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ #include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include <assert.h> #include <string.h> /* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: ** ** SELECT <function-name>(<key-name>); ** SELECT <function-name>(<key-name>, <pointer>); ................................................................................ ** the string <key-name> must already exist in the has table. Otherwise, ** an error is returned. ** ** Whether or not the <pointer> argument is specified, the value returned ** is a blob containing the pointer stored as the hash data corresponding ** to string <key-name> (after the hash-table is updated, if applicable). */ static void scalarFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ Fts3Hash *pHash; void *pPtr = 0; const unsigned char *zName; ................................................................................ pHash = (Fts3Hash *)sqlite3_user_data(context); zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ void *pOld; int n = sqlite3_value_bytes(argv[1]); if( zName==0 || n!=sizeof(pPtr) ){ sqlite3_result_error(context, "argument type mismatch", -1); return; } pPtr = *(void **)sqlite3_value_blob(argv[1]); pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); if( pOld==pPtr ){ sqlite3_result_error(context, "out of memory", -1); return; } }else{ if( zName ){ pPtr = sqlite3Fts3HashFind(pHash, zName, nName); } if( !pPtr ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; } } sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } int sqlite3Fts3IsIdChar(char c){ static const char isFtsIdChar[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ ................................................................................ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); sqlite3_step(pStmt); return sqlite3_finalize(pStmt); } static int queryTokenizer( sqlite3 *db, char *zName, const sqlite3_tokenizer_module **pp ){ ................................................................................ assert( p1==p2 ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_ERROR ); assert( p2==0 ); assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ rc = registerTokenizer(db, "nosuchtokenizer", p1); assert( rc==SQLITE_OK ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_OK ); assert( p2==p1 ); sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } #endif /* ................................................................................ ** the hash table pointed to by argument pHash. The hash table must ** been initialized to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above ** scalarFunc() in this file for details) and, if ENABLE_TABLE is ** defined at compilation time, a temporary virtual table (see header ** comment above struct HashTableVtab) to the database schema. Both ** provide read/write access to the contents of *pHash. ** ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ ................................................................................ zTest2 = sqlite3_mprintf("%s_internal_test", zName); if( !zTest || !zTest2 ){ rc = SQLITE_NOMEM; } #endif if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); } if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); } #ifdef SQLITE_TEST if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); } if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); |
> > > > > > > > > > > > | > > > > < > > > | | | |
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 .. 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 .. 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 ... 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 ... 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 ... 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 ... 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 |
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ #include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include <assert.h> #include <string.h> /* ** Return true if the two-argument version of fts3_tokenizer() ** has been activated via a prior call to sqlite3_db_config(db, ** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); */ static int fts3TokenizerEnabled(sqlite3_context *context){ sqlite3 *db = sqlite3_context_db_handle(context); int isEnabled = 0; sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); return isEnabled; } /* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: ** ** SELECT <function-name>(<key-name>); ** SELECT <function-name>(<key-name>, <pointer>); ................................................................................ ** the string <key-name> must already exist in the has table. Otherwise, ** an error is returned. ** ** Whether or not the <pointer> argument is specified, the value returned ** is a blob containing the pointer stored as the hash data corresponding ** to string <key-name> (after the hash-table is updated, if applicable). */ static void fts3TokenizerFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ Fts3Hash *pHash; void *pPtr = 0; const unsigned char *zName; ................................................................................ pHash = (Fts3Hash *)sqlite3_user_data(context); zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ if( fts3TokenizerEnabled(context) ){ void *pOld; int n = sqlite3_value_bytes(argv[1]); if( zName==0 || n!=sizeof(pPtr) ){ sqlite3_result_error(context, "argument type mismatch", -1); return; } pPtr = *(void **)sqlite3_value_blob(argv[1]); pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); if( pOld==pPtr ){ sqlite3_result_error(context, "out of memory", -1); } }else{ sqlite3_result_error(context, "fts3tokenize disabled", -1); return; } }else{ if( zName ){ pPtr = sqlite3Fts3HashFind(pHash, zName, nName); } if( !pPtr ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; } } sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } int sqlite3Fts3IsIdChar(char c){ static const char isFtsIdChar[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ ................................................................................ sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); sqlite3_step(pStmt); return sqlite3_finalize(pStmt); } static int queryTokenizer( sqlite3 *db, char *zName, const sqlite3_tokenizer_module **pp ){ ................................................................................ assert( p1==p2 ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_ERROR ); assert( p2==0 ); assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ if( fts3TokenizerEnabled(context) ){ rc = registerTokenizer(db, "nosuchtokenizer", p1); assert( rc==SQLITE_OK ); rc = queryTokenizer(db, "nosuchtokenizer", &p2); assert( rc==SQLITE_OK ); assert( p2==p1 ); } sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } #endif /* ................................................................................ ** the hash table pointed to by argument pHash. The hash table must ** been initialized to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above ** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is ** defined at compilation time, a temporary virtual table (see header ** comment above struct HashTableVtab) to the database schema. Both ** provide read/write access to the contents of *pHash. ** ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ ................................................................................ zTest2 = sqlite3_mprintf("%s_internal_test", zName); if( !zTest || !zTest2 ){ rc = SQLITE_NOMEM; } #endif if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); } if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); } #ifdef SQLITE_TEST if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); } if( SQLITE_OK==rc ){ rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); |
Changes to ext/fts3/fts3_write.c.
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 .... 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 .... 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 |
/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", /* This statement is used to determine which level to read the input from ** when performing an incremental merge. It returns the absolute level number ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See ** function sqlite3Fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", ................................................................................ } if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } iNewLevel = iMaxLevel; bIgnoreEmpty = 1; }else{ ................................................................................ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. ** If one is found, set iAbsLevel to the absolute level number and ** nSeg to nMin. If no level with at least nMin segments can be found, ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); sqlite3_bind_int(pFindLevel, 1, nMin); if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); nSeg = nMin; }else{ nSeg = -1; } rc = sqlite3_reset(pFindLevel); /* If the hint read from the %_stat table is not empty, check if the ** last entry in it specifies a relative level smaller than or equal |
> | | | | > |
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 .... 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 .... 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 |
/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", /* This statement is used to determine which level to read the input from ** when performing an incremental merge. It returns the absolute level number ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ /* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " " GROUP BY level HAVING cnt>=?" " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See ** function sqlite3Fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", ................................................................................ } if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ rc = SQLITE_DONE; goto finished; } iNewLevel = iMaxLevel; bIgnoreEmpty = 1; }else{ ................................................................................ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. ** If one is found, set iAbsLevel to the absolute level number and ** nSeg to nMin. If no level with at least nMin segments can be found, ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); nSeg = sqlite3_column_int(pFindLevel, 1); assert( nSeg>=2 ); }else{ nSeg = -1; } rc = sqlite3_reset(pFindLevel); /* If the hint read from the %_stat table is not empty, check if the ** last entry in it specifies a relative level smaller than or equal |
Changes to ext/fts3/unicode/mkunicode.tcl.
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 |
puts "** The results are undefined if the value passed to this function" puts "** is less than zero." puts "*/" puts "int ${zFunc}\(int c)\{" an_print_range_array $lRange an_print_ascii_bitmap $lRange puts { if( c<128 ){ return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ int iTest = (iHi + iLo) / 2; if( key >= aEntry[iTest] ){ |
| | |
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 |
puts "** The results are undefined if the value passed to this function" puts "** is less than zero." puts "*/" puts "int ${zFunc}\(int c)\{" an_print_range_array $lRange an_print_ascii_bitmap $lRange puts { if( (unsigned int)c<128 ){ return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ int iTest = (iHi + iLo) / 2; if( key >= aEntry[iTest] ){ |
Changes to ext/fts5/fts5.h.
80 81 82 83 84 85 86 87 88 89 90 91 92 93 ... 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 ... 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 ... 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 |
** *pnToken to the number of tokens in column iCol of the current row. ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. ** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** xColumnText: ** This function attempts to retrieve the text of column iCol of the ** current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values ** of (*pz) and (*pn) are undefined. ................................................................................ ** are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within ** the query within the current row. Return SQLITE_OK if successful, or ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value ** output by xInstCount(). ** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** ** xRowid: ** Returns the rowid of the current row. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. ** ................................................................................ ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient ** to use, this API may be faster under some circumstances. To iterate ** through instances of phrase iPhrase, use the following code: ** ** Fts5PhraseIter iter; ** int iCol, iOff; ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); ** iOff>=0; ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ** ){ ** // An instance of phrase iPhrase at offset iOff of column iCol ** } ** ** The Fts5PhraseIter structure is defined above. Applications should not ** modify this structure directly - it should only be used as shown above ** with the xPhraseFirst() and xPhraseNext() API methods. ** ** xPhraseNext() ** See xPhraseFirst above. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 1 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); ................................................................................ int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); }; /* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ /************************************************************************* |
> > > > > > > > > > > > > > > > > | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | > > > |
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 ... 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 ... 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 ... 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 |
** *pnToken to the number of tokens in column iCol of the current row. ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. ** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** This function may be quite inefficient if used with an FTS5 table ** created with the "columnsize=0" option. ** ** xColumnText: ** This function attempts to retrieve the text of column iCol of the ** current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values ** of (*pz) and (*pn) are undefined. ................................................................................ ** are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within ** the query within the current row. Return SQLITE_OK if successful, or ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. If the FTS5 table is created ** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always returns 0. ** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value ** output by xInstCount(). ** ** Usually, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the ** first token of the phrase. The exception is if the table was created ** with the offsets=0 option specified. In this case *piOff is always ** set to -1. ** ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) ** if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. ** ** xRowid: ** Returns the rowid of the current row. ** ** xTokenize: ** Tokenize text using the tokenizer belonging to the FTS5 table. ** ................................................................................ ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient ** to use, this API may be faster under some circumstances. To iterate ** through instances of phrase iPhrase, use the following code: ** ** Fts5PhraseIter iter; ** int iCol, iOff; ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); ** iCol>=0; ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ** ){ ** // An instance of phrase iPhrase at offset iOff of column iCol ** } ** ** The Fts5PhraseIter structure is defined above. Applications should not ** modify this structure directly - it should only be used as shown above ** with the xPhraseFirst() and xPhraseNext() API methods (and by ** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. If the FTS5 table is created ** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** ** xPhraseNext() ** See xPhraseFirst above. ** ** xPhraseFirstColumn() ** This function and xPhraseNextColumn() are similar to the xPhraseFirst() ** and xPhraseNext() APIs described above. The difference is that instead ** of iterating through all instances of a phrase in the current row, these ** APIs are used to iterate through the set of columns in the current row ** that contain one or more instances of a specified phrase. For example: ** ** Fts5PhraseIter iter; ** int iCol; ** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); ** iCol>=0; ** pApi->xPhraseNextColumn(pFts, &iter, &iCol) ** ){ ** // Column iCol contains at least one instance of phrase iPhrase ** } ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" option. If the FTS5 table is created with either ** "detail=none" "content=" option (i.e. if it is a contentless table), ** then this API always iterates through an empty set (all calls to ** xPhraseFirstColumn() set iCol to -1). ** ** The information accessed using this API and its companion ** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext ** (or xInst/xInstCount). The chief advantage of this API is that it is ** significantly more efficient than those alternatives when used with ** "detail=column" tables. ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ void *(*xUserData)(Fts5Context*); int (*xColumnCount)(Fts5Context*); int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); ................................................................................ int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) ); int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); void *(*xGetAuxdata)(Fts5Context*, int bClear); int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); }; /* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ /************************************************************************* |
Changes to ext/fts5/fts5Int.h.
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 .. 42 43 44 45 46 47 48 49 50 51 52 53 54 55 .. 75 76 77 78 79 80 81 82 83 84 85 86 87 88 ... 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 ... 175 176 177 178 179 180 181 182 183 184 185 186 187 188 ... 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 ... 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 ... 274 275 276 277 278 279 280 281 282 283 284 285 286 287 ... 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 ... 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 ... 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 ... 429 430 431 432 433 434 435 436 437 438 439 440 441 442 ... 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 ... 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 ... 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 ... 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 ... 641 642 643 644 645 646 647 648 649 650 651 652 653 654 |
#include <assert.h> #ifndef SQLITE_AMALGAMATION typedef unsigned char u8; typedef unsigned int u32; typedef unsigned short u16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; #define ArraySize(x) (sizeof(x) / sizeof(x[0])) #define testcase(x) #define ALWAYS(x) 1 #define NEVER(x) 0 #define MIN(x,y) (((x) < (y)) ? (x) : (y)) #define MAX(x,y) (((x) > (y)) ? (x) : (y)) ................................................................................ ** Constants for the largest and smallest possible 64-bit signed integers. */ # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) #endif /* ** Maximum number of prefix indexes on single FTS5 table. This must be ** less than 32. If it is set to anything large than that, an #error ** directive in fts5_index.c will cause the build to fail. */ #define FTS5_MAX_PREFIX_INDEXES 31 ................................................................................ */ #ifdef SQLITE_DEBUG extern int sqlite3_fts5_may_be_corrupt; # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) #else # define assert_nc(x) assert(x) #endif typedef struct Fts5Global Fts5Global; typedef struct Fts5Colset Fts5Colset; /* If a NEAR() clump or phrase may only match a specific set of columns, ** then an object of the following type is used to record the set of columns. ** Each entry in the aiCol[] array is a column that may be matched. ................................................................................ u8 *abUnindexed; /* True for unindexed columns */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; ................................................................................ /* Current expected value of %_config table 'version' field */ #define FTS5_CURRENT_VERSION 4 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); void sqlite3Fts5ConfigFree(Fts5Config*); ................................................................................ typedef struct Fts5Buffer Fts5Buffer; struct Fts5Buffer { u8 *p; int n; int nSpace; }; int sqlite3Fts5BufferSize(int*, Fts5Buffer*, int); void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); void sqlite3Fts5BufferFree(Fts5Buffer*); void sqlite3Fts5BufferZero(Fts5Buffer*); void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); ................................................................................ #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) #define fts5BufferGrow(pRc,pBuf,nn) ( \ (pBuf)->n + (nn) <= (pBuf)->nSpace ? 0 : \ sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ ) /* Write and decode big-endian 32-bit integer values */ void sqlite3Fts5Put32(u8*, int); int sqlite3Fts5Get32(const u8*); ................................................................................ int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*); typedef struct Fts5PoslistWriter Fts5PoslistWriter; struct Fts5PoslistWriter { i64 iPrev; }; int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64); int sqlite3Fts5PoslistNext64( const u8 *a, int n, /* Buffer containing poslist */ int *pi, /* IN/OUT: Offset within a[] */ i64 *piOff /* IN/OUT: Current offset */ ); ................................................................................ /* Malloc utility */ void *sqlite3Fts5MallocZero(int *pRc, int nByte); char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); /* Character set tests (like isspace(), isalpha() etc.) */ int sqlite3Fts5IsBareword(char t); /* ** End of interface to code in fts5_buffer.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_index.c. fts5_index.c contains contains code ** to access the data stored in the %_data table. */ typedef struct Fts5Index Fts5Index; typedef struct Fts5IndexIter Fts5IndexIter; /* ** Values used as part of the flags argument passed to IndexQuery(). */ #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ #define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* ** Create/destroy an Fts5Index object. */ int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); int sqlite3Fts5IndexClose(Fts5Index *p); /* ** for( ** sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter); ** 0==sqlite3Fts5IterEof(pIter); ** sqlite3Fts5IterNext(pIter) ** ){ ** i64 iRowid = sqlite3Fts5IterRowid(pIter); ** } */ /* ** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. */ int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ ................................................................................ Fts5IndexIter **ppIter /* OUT: New iterator object */ ); /* ** The various operations on open token or token prefix iterators opened ** using sqlite3Fts5IndexQuery(). */ int sqlite3Fts5IterEof(Fts5IndexIter*); int sqlite3Fts5IterNext(Fts5IndexIter*); int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); i64 sqlite3Fts5IterRowid(Fts5IndexIter*); int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*); int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf); /* ** Close an iterator opened by sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter*); /* ................................................................................ */ int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* ** Functions called by the storage module as part of integrity-check. */ u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int); int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); /* ** Called during virtual module initialization to register UDF ** fts5_decode() with SQLite */ int sqlite3Fts5IndexInit(sqlite3*); ................................................................................ ** this connection since it was created. */ int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); int sqlite3Fts5IndexOptimize(Fts5Index *p); int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); /* ** End of interface to code in fts5_index.c. **************************************************************************/ ................................................................................ ** Interface to code in fts5_hash.c. */ typedef struct Fts5Hash Fts5Hash; /* ** Create a hash table, free a hash table. */ int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize); void sqlite3Fts5HashFree(Fts5Hash*); int sqlite3Fts5HashWrite( Fts5Hash*, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ ................................................................................ int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**); int sqlite3Fts5StorageClose(Fts5Storage *p); int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); int sqlite3Fts5DropAll(Fts5Config*); int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); int sqlite3Fts5StorageDelete(Fts5Storage *p, i64); int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); int sqlite3Fts5StorageIntegrity(Fts5Storage *p); int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); ................................................................................ int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); int sqlite3Fts5StorageRollback(Fts5Storage *p); int sqlite3Fts5StorageConfigValue( Fts5Storage *p, const char*, sqlite3_value*, int ); int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); int sqlite3Fts5StorageRebuild(Fts5Storage *p); int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); /* ** End of interface to code in fts5_storage.c. **************************************************************************/ /************************************************************************** ................................................................................ /* Called during startup to register a UDF with SQLite */ int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by ** the parser code in fts5parse.y. */ void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); ................................................................................ Fts5ExprNode *sqlite3Fts5ParseNode( Fts5Parse *pParse, int eType, Fts5ExprNode *pLeft, Fts5ExprNode *pRight, Fts5ExprNearset *pNear ); Fts5ExprPhrase *sqlite3Fts5ParseTerm( Fts5Parse *pParse, Fts5ExprPhrase *pPhrase, Fts5Token *pToken, int bPrefix ); |
> | > > > > > > > > > > > > > > > > > > > | | | > > > > > > > > > > > > > > > > > > > > > > > | | | | | | | > > > > > > > > > > > > > < < < < < > | | < < > > > > > > > > > | > > > > > > > > |
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 .. 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 .. 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 ... 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 ... 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 ... 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 ... 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 ... 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 ... 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 ... 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 ... 461 462 463 464 465 466 467 468 469 470 471 472 473 474 ... 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 ... 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 ... 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 ... 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 ... 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 ... 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 |
#include <assert.h> #ifndef SQLITE_AMALGAMATION typedef unsigned char u8; typedef unsigned int u32; typedef unsigned short u16; typedef short i16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; #define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) #define testcase(x) #define ALWAYS(x) 1 #define NEVER(x) 0 #define MIN(x,y) (((x) < (y)) ? (x) : (y)) #define MAX(x,y) (((x) > (y)) ? (x) : (y)) ................................................................................ ** Constants for the largest and smallest possible 64-bit signed integers. */ # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) #endif /* Truncate very long tokens to this many bytes. Hard limit is ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset ** field that occurs at the start of each leaf page (see fts5_index.c). */ #define FTS5_MAX_TOKEN_SIZE 32768 /* ** Maximum number of prefix indexes on single FTS5 table. This must be ** less than 32. If it is set to anything large than that, an #error ** directive in fts5_index.c will cause the build to fail. */ #define FTS5_MAX_PREFIX_INDEXES 31 ................................................................................ */ #ifdef SQLITE_DEBUG extern int sqlite3_fts5_may_be_corrupt; # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) #else # define assert_nc(x) assert(x) #endif /* Mark a function parameter as unused, to suppress nuisance compiler ** warnings. */ #ifndef UNUSED_PARAM # define UNUSED_PARAM(X) (void)(X) #endif #ifndef UNUSED_PARAM2 # define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y) #endif typedef struct Fts5Global Fts5Global; typedef struct Fts5Colset Fts5Colset; /* If a NEAR() clump or phrase may only match a specific set of columns, ** then an object of the following type is used to record the set of columns. ** Each entry in the aiCol[] array is a column that may be matched. ................................................................................ u8 *abUnindexed; /* True for unindexed columns */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ int nUsermerge; /* 'usermerge' setting */ int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; ................................................................................ /* Current expected value of %_config table 'version' field */ #define FTS5_CURRENT_VERSION 4 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 #define FTS5_DETAIL_FULL 0 #define FTS5_DETAIL_NONE 1 #define FTS5_DETAIL_COLUMNS 2 int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); void sqlite3Fts5ConfigFree(Fts5Config*); ................................................................................ typedef struct Fts5Buffer Fts5Buffer; struct Fts5Buffer { u8 *p; int n; int nSpace; }; int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32); void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64); void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*); void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); void sqlite3Fts5BufferFree(Fts5Buffer*); void sqlite3Fts5BufferZero(Fts5Buffer*); void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); ................................................................................ #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) #define fts5BufferGrow(pRc,pBuf,nn) ( \ (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \ sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \ ) /* Write and decode big-endian 32-bit integer values */ void sqlite3Fts5Put32(u8*, int); int sqlite3Fts5Get32(const u8*); ................................................................................ int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*); typedef struct Fts5PoslistWriter Fts5PoslistWriter; struct Fts5PoslistWriter { i64 iPrev; }; int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64); void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64); int sqlite3Fts5PoslistNext64( const u8 *a, int n, /* Buffer containing poslist */ int *pi, /* IN/OUT: Offset within a[] */ i64 *piOff /* IN/OUT: Current offset */ ); ................................................................................ /* Malloc utility */ void *sqlite3Fts5MallocZero(int *pRc, int nByte); char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); /* Character set tests (like isspace(), isalpha() etc.) */ int sqlite3Fts5IsBareword(char t); /* Bucket of terms object used by the integrity-check in offsets=0 mode. */ typedef struct Fts5Termset Fts5Termset; int sqlite3Fts5TermsetNew(Fts5Termset**); int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); void sqlite3Fts5TermsetFree(Fts5Termset*); /* ** End of interface to code in fts5_buffer.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_index.c. fts5_index.c contains contains code ** to access the data stored in the %_data table. */ typedef struct Fts5Index Fts5Index; typedef struct Fts5IndexIter Fts5IndexIter; struct Fts5IndexIter { i64 iRowid; const u8 *pData; int nData; u8 bEof; }; #define sqlite3Fts5IterEof(x) ((x)->bEof) /* ** Values used as part of the flags argument passed to IndexQuery(). */ #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ #define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* The following are used internally by the fts5_index.c module. They are ** defined here only to make it easier to avoid clashes with the flags ** above. */ #define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 #define FTS5INDEX_QUERY_NOOUTPUT 0x0020 /* ** Create/destroy an Fts5Index object. */ int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); int sqlite3Fts5IndexClose(Fts5Index *p); /* ** Return a simple checksum value based on the arguments. */ u64 sqlite3Fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, int iIdx, const char *pTerm, int nTerm ); /* ** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ int sqlite3Fts5IndexCharlenToBytelen( const char *p, int nByte, int nChar ); /* ** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. */ int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ ................................................................................ Fts5IndexIter **ppIter /* OUT: New iterator object */ ); /* ** The various operations on open token or token prefix iterators opened ** using sqlite3Fts5IndexQuery(). */ int sqlite3Fts5IterNext(Fts5IndexIter*); int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); /* ** Close an iterator opened by sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter*); /* ................................................................................ */ int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* ** Functions called by the storage module as part of integrity-check. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); /* ** Called during virtual module initialization to register UDF ** fts5_decode() with SQLite */ int sqlite3Fts5IndexInit(sqlite3*); ................................................................................ ** this connection since it was created. */ int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); int sqlite3Fts5IndexOptimize(Fts5Index *p); int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); int sqlite3Fts5IndexReset(Fts5Index *p); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); /* ** End of interface to code in fts5_index.c. **************************************************************************/ ................................................................................ ** Interface to code in fts5_hash.c. */ typedef struct Fts5Hash Fts5Hash; /* ** Create a hash table, free a hash table. */ int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); void sqlite3Fts5HashFree(Fts5Hash*); int sqlite3Fts5HashWrite( Fts5Hash*, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ ................................................................................ int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**); int sqlite3Fts5StorageClose(Fts5Storage *p); int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); int sqlite3Fts5DropAll(Fts5Config*); int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); int sqlite3Fts5StorageIntegrity(Fts5Storage *p); int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); ................................................................................ int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); int sqlite3Fts5StorageRollback(Fts5Storage *p); int sqlite3Fts5StorageConfigValue( Fts5Storage *p, const char*, sqlite3_value*, int ); int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); int sqlite3Fts5StorageRebuild(Fts5Storage *p); int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); int sqlite3Fts5StorageReset(Fts5Storage *p); /* ** End of interface to code in fts5_storage.c. **************************************************************************/ /************************************************************************** ................................................................................ /* Called during startup to register a UDF with SQLite */ int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int ); void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); void sqlite3Fts5ExprClearEof(Fts5Expr*); int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by ** the parser code in fts5parse.y. */ void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...); ................................................................................ Fts5ExprNode *sqlite3Fts5ParseNode( Fts5Parse *pParse, int eType, Fts5ExprNode *pLeft, Fts5ExprNode *pRight, Fts5ExprNearset *pNear ); Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( Fts5Parse *pParse, Fts5ExprNode *pLeft, Fts5ExprNode *pRight ); Fts5ExprPhrase *sqlite3Fts5ParseTerm( Fts5Parse *pParse, Fts5ExprPhrase *pPhrase, Fts5Token *pToken, int bPrefix ); |
Changes to ext/fts5/fts5_aux.c.
153 154 155 156 157 158 159 160 161 162 163 164 165 166 ... 387 388 389 390 391 392 393 394 395 396 397 398 399 400 ... 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 |
int nToken, /* Size of token in bytes */ int iStartOff, /* Start offset of token */ int iEndOff /* End offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; int iPos; if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; iPos = p->iPos++; if( p->iRangeEnd>0 ){ if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; ................................................................................ */ static int fts5CountCb( const Fts5ExtensionApi *pApi, Fts5Context *pFts, void *pUserData /* Pointer to sqlite3_int64 variable */ ){ sqlite3_int64 *pn = (sqlite3_int64*)pUserData; (*pn)++; return SQLITE_OK; } /* ** Set *ppData to point to the Fts5Bm25Data object for the current query. ** If the object has not already been allocated, allocate and populate it ................................................................................ { "snippet", 0, fts5SnippetFunction, 0 }, { "highlight", 0, fts5HighlightFunction, 0 }, { "bm25", 0, fts5Bm25Function, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){ rc = pApi->xCreateFunction(pApi, aBuiltin[i].zFunc, aBuiltin[i].pUserData, aBuiltin[i].xFunc, aBuiltin[i].xDestroy ); } return rc; } |
> > > | |
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 ... 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 ... 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 |
int nToken, /* Size of token in bytes */ int iStartOff, /* Start offset of token */ int iEndOff /* End offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; int iPos; UNUSED_PARAM2(pToken, nToken); if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; iPos = p->iPos++; if( p->iRangeEnd>0 ){ if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; ................................................................................ */ static int fts5CountCb( const Fts5ExtensionApi *pApi, Fts5Context *pFts, void *pUserData /* Pointer to sqlite3_int64 variable */ ){ sqlite3_int64 *pn = (sqlite3_int64*)pUserData; UNUSED_PARAM2(pApi, pFts); (*pn)++; return SQLITE_OK; } /* ** Set *ppData to point to the Fts5Bm25Data object for the current query. ** If the object has not already been allocated, allocate and populate it ................................................................................ { "snippet", 0, fts5SnippetFunction, 0 }, { "highlight", 0, fts5HighlightFunction, 0 }, { "bm25", 0, fts5Bm25Function, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){ rc = pApi->xCreateFunction(pApi, aBuiltin[i].zFunc, aBuiltin[i].pUserData, aBuiltin[i].xFunc, aBuiltin[i].xDestroy ); } return rc; } |
Changes to ext/fts5/fts5_buffer.c.
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 .. 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 ... 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 ... 288 289 290 291 292 293 294 |
****************************************************************************** */ #include "fts5Int.h" int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, int nByte){ int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; u8 *pNew; while( nNew<nByte ){ nNew = nNew * 2; } pNew = sqlite3_realloc(pBuf->p, nNew); if( pNew==0 ){ *pRc = SQLITE_NOMEM; return 1; }else{ pBuf->nSpace = nNew; pBuf->p = pNew; } return 0; } /* ** Encode value iVal as an SQLite varint and append it to the buffer object ** pBuf. If an OOM error occurs, set the error code in p. ................................................................................ ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set ** the error code in p. If an error has already occurred when this function ** is called, it is a no-op. */ void sqlite3Fts5BufferAppendBlob( int *pRc, Fts5Buffer *pBuf, int nData, const u8 *pData ){ assert( *pRc || nData>=0 ); if( fts5BufferGrow(pRc, pBuf, nData) ) return; memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } /* ** Append the nul-terminated string zStr to the buffer pBuf. This function ................................................................................ ){ memset(pIter, 0, sizeof(*pIter)); pIter->a = a; pIter->n = n; sqlite3Fts5PoslistReaderNext(pIter); return pIter->bEof; } int sqlite3Fts5PoslistWriterAppend( Fts5Buffer *pBuf, Fts5PoslistWriter *pWriter, i64 iPos ){ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; int rc = SQLITE_OK; if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){ if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ pBuf->p[pBuf->n++] = 1; pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); pWriter->iPrev = (iPos & colmask); } pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2); pWriter->iPrev = iPos; } return rc; } void *sqlite3Fts5MallocZero(int *pRc, int nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ pRet = sqlite3_malloc(nByte); if( pRet==0 && nByte>0 ){ ................................................................................ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ }; return (t & 0x80) || aBareword[(int)t]; } |
| > | > | | > > > > > > > > > > > > > > > > > > > > > | < | < < < < < < < < > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 .. 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 ... 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 ... 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 |
****************************************************************************** */ #include "fts5Int.h" int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ if( (u32)pBuf->nSpace<nByte ){ u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64; u8 *pNew; while( nNew<nByte ){ nNew = nNew * 2; } pNew = sqlite3_realloc(pBuf->p, nNew); if( pNew==0 ){ *pRc = SQLITE_NOMEM; return 1; }else{ pBuf->nSpace = nNew; pBuf->p = pNew; } } return 0; } /* ** Encode value iVal as an SQLite varint and append it to the buffer object ** pBuf. If an OOM error occurs, set the error code in p. ................................................................................ ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set ** the error code in p. If an error has already occurred when this function ** is called, it is a no-op. */ void sqlite3Fts5BufferAppendBlob( int *pRc, Fts5Buffer *pBuf, u32 nData, const u8 *pData ){ assert_nc( *pRc || nData>=0 ); if( fts5BufferGrow(pRc, pBuf, nData) ) return; memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } /* ** Append the nul-terminated string zStr to the buffer pBuf. This function ................................................................................ ){ memset(pIter, 0, sizeof(*pIter)); pIter->a = a; pIter->n = n; sqlite3Fts5PoslistReaderNext(pIter); return pIter->bEof; } /* ** Append position iPos to the position list being accumulated in buffer ** pBuf, which must be already be large enough to hold the new data. ** The previous position written to this list is *piPrev. *piPrev is set ** to iPos before returning. */ void sqlite3Fts5PoslistSafeAppend( Fts5Buffer *pBuf, i64 *piPrev, i64 iPos ){ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; if( (iPos & colmask) != (*piPrev & colmask) ){ pBuf->p[pBuf->n++] = 1; pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); *piPrev = (iPos & colmask); } pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); *piPrev = iPos; } int sqlite3Fts5PoslistWriterAppend( Fts5Buffer *pBuf, Fts5PoslistWriter *pWriter, i64 iPos ){ int rc = 0; /* Initialized only to suppress erroneous warning from Clang */ if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc; sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos); return SQLITE_OK; } void *sqlite3Fts5MallocZero(int *pRc, int nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ pRet = sqlite3_malloc(nByte); if( pRet==0 && nByte>0 ){ ................................................................................ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ }; return (t & 0x80) || aBareword[(int)t]; } /************************************************************************* */ typedef struct Fts5TermsetEntry Fts5TermsetEntry; struct Fts5TermsetEntry { char *pTerm; int nTerm; int iIdx; /* Index (main or aPrefix[] entry) */ Fts5TermsetEntry *pNext; }; struct Fts5Termset { Fts5TermsetEntry *apHash[512]; }; int sqlite3Fts5TermsetNew(Fts5Termset **pp){ int rc = SQLITE_OK; *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); return rc; } int sqlite3Fts5TermsetAdd( Fts5Termset *p, int iIdx, const char *pTerm, int nTerm, int *pbPresent ){ int rc = SQLITE_OK; *pbPresent = 0; if( p ){ int i; u32 hash = 13; Fts5TermsetEntry *pEntry; /* Calculate a hash value for this term. This is the same hash checksum ** used by the fts5_hash.c module. This is not important for correct ** operation of the module, but is necessary to ensure that some tests ** designed to produce hash table collisions really do work. */ for(i=nTerm-1; i>=0; i--){ hash = (hash << 3) ^ hash ^ pTerm[i]; } hash = (hash << 3) ^ hash ^ iIdx; hash = hash % ArraySize(p->apHash); for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ if( pEntry->iIdx==iIdx && pEntry->nTerm==nTerm && memcmp(pEntry->pTerm, pTerm, nTerm)==0 ){ *pbPresent = 1; break; } } if( pEntry==0 ){ pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); if( pEntry ){ pEntry->pTerm = (char*)&pEntry[1]; pEntry->nTerm = nTerm; pEntry->iIdx = iIdx; memcpy(pEntry->pTerm, pTerm, nTerm); pEntry->pNext = p->apHash[hash]; p->apHash[hash] = pEntry; } } } return rc; } void sqlite3Fts5TermsetFree(Fts5Termset *p){ if( p ){ u32 i; for(i=0; i<ArraySize(p->apHash); i++){ Fts5TermsetEntry *pEntry = p->apHash[i]; while( pEntry ){ Fts5TermsetEntry *pDel = pEntry; pEntry = pEntry->pNext; sqlite3_free(pDel); } } sqlite3_free(p); } } |
Changes to ext/fts5/fts5_config.c.
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 ... 191 192 193 194 195 196 197 198 199 200 201 202 203 204 ... 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 ... 340 341 342 343 344 345 346 347 348 349 350 351 352 353 ... 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 ... 496 497 498 499 500 501 502 503 504 505 506 507 508 509 ... 811 812 813 814 815 816 817 818 819 820 821 822 823 824 ... 858 859 860 861 862 863 864 865 866 867 868 869 870 871 ... 898 899 900 901 902 903 904 905 |
** ****************************************************************************** ** ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ ................................................................................ assert( 0==fts5_iswhitespace(z[0]) ); quote = z[0]; if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ fts5Dequote(z); } } /* ** Parse a "special" CREATE VIRTUAL TABLE directive and update ** configuration object pConfig as appropriate. ** ** If successful, object pConfig is updated and SQLITE_OK returned. If ** an error occurs, an SQLite error code is returned and an error message ** may be left in *pzErr. It is the responsibility of the caller to ................................................................................ } while( p[0]>='0' && p[0]<='9' && nPre<1000 ){ nPre = nPre*10 + (p[0] - '0'); p++; } if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){ *pzErr = sqlite3_mprintf("prefix length out of range (max 999)"); rc = SQLITE_ERROR; break; } pConfig->aPrefix[pConfig->nPrefix] = nPre; pConfig->nPrefix++; ................................................................................ *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); rc = SQLITE_ERROR; }else{ pConfig->bColumnsize = (zArg[0]=='1'); } return rc; } *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } /* ** Allocate an instance of the default tokenizer ("simple") at ................................................................................ memcpy(zOut, zIn, nIn+1); if( fts5_isopenquote(zOut[0]) ){ int ii = fts5Dequote(zOut); zRet = &zIn[ii]; *pbQuoted = 1; }else{ zRet = fts5ConfigSkipBareword(zIn); zOut[zRet-zIn] = '\0'; } } if( zRet==0 ){ sqlite3_free(zOut); }else{ *pzOut = zOut; } ................................................................................ nByte = nArg * (sizeof(char*) + sizeof(u8)); pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; } ................................................................................ if( nAutomerge<0 || nAutomerge>64 ){ *pbBadkey = 1; }else{ if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nAutomerge = nAutomerge; } } else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ int nCrisisMerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nCrisisMerge = sqlite3_value_int(pVal); } if( nCrisisMerge<0 ){ ................................................................................ sqlite3_stmt *p = 0; int rc = SQLITE_OK; int iVersion = 0; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); ................................................................................ } if( rc==SQLITE_OK ){ pConfig->iCookie = iCookie; } return rc; } |
< > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < |
10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 ... 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 ... 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 ... 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 ... 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 ... 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 ... 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 ... 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 ... 955 956 957 958 959 960 961 |
** ****************************************************************************** ** ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_USERMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ ................................................................................ assert( 0==fts5_iswhitespace(z[0]) ); quote = z[0]; if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ fts5Dequote(z); } } struct Fts5Enum { const char *zName; int eVal; }; typedef struct Fts5Enum Fts5Enum; static int fts5ConfigSetEnum( const Fts5Enum *aEnum, const char *zEnum, int *peVal ){ int nEnum = (int)strlen(zEnum); int i; int iVal = -1; for(i=0; aEnum[i].zName; i++){ if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ if( iVal>=0 ) return SQLITE_ERROR; iVal = aEnum[i].eVal; } } *peVal = iVal; return iVal<0 ? SQLITE_ERROR : SQLITE_OK; } /* ** Parse a "special" CREATE VIRTUAL TABLE directive and update ** configuration object pConfig as appropriate. ** ** If successful, object pConfig is updated and SQLITE_OK returned. If ** an error occurs, an SQLite error code is returned and an error message ** may be left in *pzErr. It is the responsibility of the caller to ................................................................................ } while( p[0]>='0' && p[0]<='9' && nPre<1000 ){ nPre = nPre*10 + (p[0] - '0'); p++; } if( nPre<=0 || nPre>=1000 ){ *pzErr = sqlite3_mprintf("prefix length out of range (max 999)"); rc = SQLITE_ERROR; break; } pConfig->aPrefix[pConfig->nPrefix] = nPre; pConfig->nPrefix++; ................................................................................ *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); rc = SQLITE_ERROR; }else{ pConfig->bColumnsize = (zArg[0]=='1'); } return rc; } if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ const Fts5Enum aDetail[] = { { "none", FTS5_DETAIL_NONE }, { "full", FTS5_DETAIL_FULL }, { "columns", FTS5_DETAIL_COLUMNS }, { 0, 0 } }; if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ *pzErr = sqlite3_mprintf("malformed detail=... directive"); } return rc; } *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } /* ** Allocate an instance of the default tokenizer ("simple") at ................................................................................ memcpy(zOut, zIn, nIn+1); if( fts5_isopenquote(zOut[0]) ){ int ii = fts5Dequote(zOut); zRet = &zIn[ii]; *pbQuoted = 1; }else{ zRet = fts5ConfigSkipBareword(zIn); if( zRet ){ zOut[zRet-zIn] = '\0'; } } } if( zRet==0 ){ sqlite3_free(zOut); }else{ *pzOut = zOut; } ................................................................................ nByte = nArg * (sizeof(char*) + sizeof(u8)); pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; pRet->eDetail = FTS5_DETAIL_FULL; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; } ................................................................................ if( nAutomerge<0 || nAutomerge>64 ){ *pbBadkey = 1; }else{ if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nAutomerge = nAutomerge; } } else if( 0==sqlite3_stricmp(zKey, "usermerge") ){ int nUsermerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nUsermerge = sqlite3_value_int(pVal); } if( nUsermerge<2 || nUsermerge>16 ){ *pbBadkey = 1; }else{ pConfig->nUsermerge = nUsermerge; } } else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ int nCrisisMerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nCrisisMerge = sqlite3_value_int(pVal); } if( nCrisisMerge<0 ){ ................................................................................ sqlite3_stmt *p = 0; int rc = SQLITE_OK; int iVersion = 0; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); ................................................................................ } if( rc==SQLITE_OK ){ pConfig->iCookie = iCookie; } return rc; } |
Changes to ext/fts5/fts5_expr.c.
36 37 38 39 40 41 42 43 44 45 46 47 48 49 .. 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 ... 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 ... 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 ... 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 ... 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 ... 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 ... 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 ... 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 ... 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 ... 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 ... 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 ... 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 ... 890 891 892 893 894 895 896 897 898 899 900 901 902 903 ... 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 ... 957 958 959 960 961 962 963 964 965 966 967 968 969 970 ... 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 ... 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 .... 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 .... 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 .... 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 .... 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 .... 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 .... 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 .... 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 .... 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 .... 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 .... 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 .... 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 .... 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 .... 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 .... 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 .... 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 .... 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 .... 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 .... 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 .... 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 .... 2231 2232 2233 2234 2235 2236 2237 |
#include <stdio.h> void sqlite3Fts5ParserTrace(FILE*, char*); #endif struct Fts5Expr { Fts5Index *pIndex; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ }; /* ................................................................................ ** FTS5_TERM (pNear valid) */ struct Fts5ExprNode { int eType; /* Node type */ int bEof; /* True at EOF */ int bNomatch; /* True if entry is not a match */ i64 iRowid; /* Current rowid */ Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */ /* Child nodes. For a NOT node, this array always contains 2 entries. For ** AND or OR nodes, it contains 2 or more entries. */ int nChild; /* Number of child nodes */ Fts5ExprNode *apChild[1]; /* Array of child nodes */ }; #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) /* ** An instance of the following structure represents a single search term ** or term prefix. */ struct Fts5ExprTerm { int bPrefix; /* True for a prefix term */ char *zTerm; /* nul-terminated term */ ................................................................................ assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; } } sqlite3_free(sParse.apPhrase); *pzErr = sParse.zErr; return sParse.rc; } ................................................................................ int bRetValid = 0; Fts5ExprTerm *p; assert( pTerm->pSynonym ); assert( bDesc==0 || bDesc==1 ); for(p=pTerm; p; p=p->pSynonym){ if( 0==sqlite3Fts5IterEof(p->pIter) ){ i64 iRowid = sqlite3Fts5IterRowid(p->pIter); if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){ iRet = iRowid; bRetValid = 1; } } } ................................................................................ if( pbEof && bRetValid==0 ) *pbEof = 1; return iRet; } /* ** Argument pTerm must be a synonym iterator. */ static int fts5ExprSynonymPoslist( Fts5ExprTerm *pTerm, Fts5Colset *pColset, i64 iRowid, int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; int nAlloc = 4; int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; i64 dummy; rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); if( rc!=SQLITE_OK ) goto synonym_poslist_out; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( aNew==0 ){ rc = SQLITE_NOMEM; goto synonym_poslist_out; } memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); nAlloc = nAlloc*2; if( aIter!=aStatic ) sqlite3_free(aIter); aIter = aNew; } sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]); assert( aIter[nIter].bEof==0 ); nIter++; } } assert( *pbDel==0 ); if( nIter==1 ){ *pa = (u8*)aIter[0].a; *pn = aIter[0].n; }else{ Fts5PoslistWriter writer = {0}; Fts5Buffer buf = {0,0,0}; i64 iPrev = -1; while( 1 ){ int i; i64 iMin = FTS5_LARGEST_INT64; for(i=0; i<nIter; i++){ if( aIter[i].bEof==0 ){ if( aIter[i].iPos==iPrev ){ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue; ................................................................................ } if( aIter[i].iPos<iMin ){ iMin = aIter[i].iPos; } } } if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break; rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin); iPrev = iMin; } if( rc ){ sqlite3_free(buf.p); }else{ *pa = buf.p; *pn = buf.n; *pbDel = 1; } } synonym_poslist_out: if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } ................................................................................ ** ** SQLITE_OK is returned if an error occurs, or an SQLite error code ** otherwise. It is not considered an error code if the current rowid is ** not a match. */ static int fts5ExprPhraseIsMatch( Fts5ExprNode *pNode, /* Node pPhrase belongs to */ Fts5Colset *pColset, /* Restrict matches to these columns */ Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ int *pbMatch /* OUT: Set to true if really a match */ ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int i; int rc = SQLITE_OK; fts5BufferZero(&pPhrase->poslist); /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pPhrase->nTerm>(int)ArraySize(aStatic) ){ int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( !aIter ) return SQLITE_NOMEM; } memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm); /* Initialize a term iterator for each term in the phrase */ for(i=0; i<pPhrase->nTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; int n = 0; int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ rc = fts5ExprSynonymPoslist( pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n ); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); } if( rc!=SQLITE_OK ) goto ismatch_out; sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); aIter[i].bFlag = (u8)bFlag; if( aIter[i].bEof ) goto ismatch_out; } while( 1 ){ int bMatch; ................................................................................ memset(p, 0, sizeof(Fts5LookaheadReader)); p->a = a; p->n = n; fts5LookaheadReaderNext(p); return fts5LookaheadReaderNext(p); } #if 0 static int fts5LookaheadReaderEof(Fts5LookaheadReader *p){ return (p->iPos==FTS5_LOOKAHEAD_EOF); } #endif typedef struct Fts5NearTrimmer Fts5NearTrimmer; struct Fts5NearTrimmer { Fts5LookaheadReader reader; /* Input iterator */ Fts5PoslistWriter writer; /* Writer context */ Fts5Buffer *pOut; /* Output poslist */ }; ................................................................................ int rc = *pRc; int bMatch; assert( pNear->nPhrase>1 ); /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pNear->nPhrase>(int)ArraySize(aStatic) ){ int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); }else{ memset(aStatic, 0, sizeof(aStatic)); } if( rc!=SQLITE_OK ){ *pRc = rc; ................................................................................ int bRet = a[0].pOut->n>0; *pRc = rc; if( a!=aStatic ) sqlite3_free(a); return bRet; } } /* ** Advance the first term iterator in the first phrase of pNear. Set output ** variable *pbEof to true if it reaches EOF or if an error occurs. ** ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ static int fts5ExprNearAdvanceFirst( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ int bFromValid, i64 iFrom ){ Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; int rc = SQLITE_OK; if( pTerm->pSynonym ){ int bEof = 1; Fts5ExprTerm *p; /* Find the firstest rowid any synonym points to. */ i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); /* Advance each iterator that currently points to iRowid. Or, if iFrom ** is valid - each iterator that points to a rowid before iFrom. */ for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ i64 ii = sqlite3Fts5IterRowid(p->pIter); if( ii==iRowid || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) ){ if( bFromValid ){ rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); }else{ rc = sqlite3Fts5IterNext(p->pIter); } if( rc!=SQLITE_OK ) break; if( sqlite3Fts5IterEof(p->pIter)==0 ){ bEof = 0; } }else{ bEof = 0; } } } /* Set the EOF flag if either all synonym iterators are at EOF or an ** error has occurred. */ pNode->bEof = (rc || bEof); }else{ Fts5IndexIter *pIter = pTerm->pIter; assert( Fts5NodeIsString(pNode) ); if( bFromValid ){ rc = sqlite3Fts5IterNextFrom(pIter, iFrom); }else{ rc = sqlite3Fts5IterNext(pIter); } pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); } return rc; } /* ** Advance iterator pIter until it points to a value equal to or laster ** than the initial value of *piLast. If this means the iterator points ** to a value laster than *piLast, update *piLast to the new lastest value. ** ** If the iterator reaches EOF, set *pbEof to true before returning. If ** an error occurs, set *pRc to an error code. If either *pbEof or *pRc ................................................................................ i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ int *pRc, /* OUT: Error code */ int *pbEof /* OUT: Set to true if EOF */ ){ i64 iLast = *piLast; i64 iRowid; iRowid = sqlite3Fts5IterRowid(pIter); if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ int rc = sqlite3Fts5IterNextFrom(pIter, iLast); if( rc || sqlite3Fts5IterEof(pIter) ){ *pRc = rc; *pbEof = 1; return 1; } iRowid = sqlite3Fts5IterRowid(pIter); assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); } *piLast = iRowid; return 0; } ................................................................................ int rc = SQLITE_OK; i64 iLast = *piLast; Fts5ExprTerm *p; int bEof = 0; for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ i64 iRowid = sqlite3Fts5IterRowid(p->pIter); if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ rc = sqlite3Fts5IterNextFrom(p->pIter, iLast); } } } if( rc!=SQLITE_OK ){ ................................................................................ static int fts5ExprNearTest( int *pRc, Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ ){ Fts5ExprNearset *pNear = pNode->pNear; int rc = *pRc; int i; /* Check that each phrase in the nearset matches the current row. ** Populate the pPhrase->poslist buffers at the same time. If any ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; }else{ rc = sqlite3Fts5IterPoslistBuffer( pPhrase->aTerm[0].pIter, &pPhrase->poslist ); } } *pRc = rc; if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ return 1; } return 0; } static int fts5ExprTokenTest( Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ ){ /* As this "NEAR" object is actually a single phrase that consists ** of a single term only, grab pointers into the poslist managed by the ** fts5_index.c iterator object. This is much faster than synthesizing ** a new poslist the way we have to for more complicated phrase or NEAR ** expressions. */ Fts5ExprNearset *pNear = pNode->pNear; Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; Fts5Colset *pColset = pNear->pColset; int rc; assert( pNode->eType==FTS5_TERM ); assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); assert( pPhrase->aTerm[0].pSynonym==0 ); rc = sqlite3Fts5IterPoslist(pIter, pColset, (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid ); pNode->bNomatch = (pPhrase->poslist.n==0); return rc; } /* ** All individual term iterators in pNear are guaranteed to be valid when ** this function is called. This function checks if all term iterators ** point to the same rowid, and if not, advances them until they do. ** If an EOF is reached before this happens, *pbEof is set to true before ** returning. ** ** SQLITE_OK is returned if an error occurs, or an SQLite error code ** otherwise. It is not considered an error code if an iterator reaches ** EOF. */ static int fts5ExprNearNextMatch( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; int rc = SQLITE_OK; i64 iLast; /* Lastest rowid any iterator points to */ int i, j; /* Phrase and token index, respectively */ int bMatch; /* True if all terms are at the same rowid */ const int bDesc = pExpr->bDesc; /* Check that this node should not be FTS5_TERM */ assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 || pNear->apPhrase[0]->aTerm[0].pSynonym ); /* Initialize iLast, the "lastest" rowid any iterator points to. If the ** iterator skips through rowids in the default ascending order, this means ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ if( pLeft->aTerm[0].pSynonym ){ iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); }else{ iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); } do { bMatch = 1; for(i=0; i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; j<pPhrase->nTerm; j++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; if( pTerm->pSynonym ){ i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; bMatch = 0; if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ pNode->bEof = 1; return rc; } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; i64 iRowid = sqlite3Fts5IterRowid(pIter); if( iRowid==iLast ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; } } } } }while( bMatch==0 ); pNode->iRowid = iLast; pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode)); return rc; } /* ** Initialize all term iterators in the pNear object. If any term is found ** to match no documents at all, return immediately without initializing any ** further iterators. */ static int fts5ExprNearInitAll( ................................................................................ Fts5Expr *pExpr, Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; int i, j; int rc = SQLITE_OK; for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; j<pPhrase->nTerm; j++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; Fts5ExprTerm *p; int bEof = 1; ................................................................................ return rc; } } } return rc; } /* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */ static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*); /* ** If pExpr is an ASC iterator, this function returns a value with the ** same sign as: ** ** (iLhs - iRhs) ** ................................................................................ return (iLhs < iRhs); } } static void fts5ExprSetEof(Fts5ExprNode *pNode){ int i; pNode->bEof = 1; for(i=0; i<pNode->nChild; i++){ fts5ExprSetEof(pNode->apChild[i]); } } static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ ................................................................................ for(i=0; i<pNode->nChild; i++){ fts5ExprNodeZeroPoslist(pNode->apChild[i]); } } } static int fts5ExprNodeNext(Fts5Expr*, Fts5ExprNode*, int, i64); /* ** Argument pNode is an FTS5_AND node. */ static int fts5ExprAndNextRowid( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pAnd /* FTS5_AND node to advance */ ){ int iChild; i64 iLast = pAnd->iRowid; int rc = SQLITE_OK; int bMatch; ................................................................................ assert( pAnd->bEof==0 ); do { pAnd->bNomatch = 0; bMatch = 1; for(iChild=0; iChild<pAnd->nChild; iChild++){ Fts5ExprNode *pChild = pAnd->apChild[iChild]; if( 0 && pChild->eType==FTS5_STRING ){ /* TODO */ }else{ int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); if( cmp>0 ){ /* Advance pChild until it points to iLast or laster */ rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); if( rc!=SQLITE_OK ) return rc; } } /* If the child node is now at EOF, so is the parent AND node. Otherwise, ** the child node is guaranteed to have advanced at least as far as ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the ** new lastest rowid seen so far. */ assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 ); if( pChild->bEof ){ ................................................................................ if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){ fts5ExprNodeZeroPoslist(pAnd); } pAnd->iRowid = iLast; return SQLITE_OK; } /* ** Compare the values currently indicated by the two nodes as follows: ** ** res = (*p1) - (*p2) ** ** Nodes that point to values that come later in the iteration order are ** considered to be larger. Nodes at EOF are the largest of all. ** ** This means that if the iteration order is ASC, then numerically larger ** rowids are considered larger. Or if it is the default DESC, numerically ** smaller rowids are larger. */ static int fts5NodeCompare( Fts5Expr *pExpr, Fts5ExprNode *p1, Fts5ExprNode *p2 ){ if( p2->bEof ) return -1; if( p1->bEof ) return +1; return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); } /* ** Advance node iterator pNode, part of expression pExpr. If argument ** bFromValid is zero, then pNode is advanced exactly once. Or, if argument ** bFromValid is non-zero, then pNode is advanced until it is at or past ** rowid value iFrom. Whether "past" means "less than" or "greater than" ** depends on whether this is an ASC or DESC iterator. */ static int fts5ExprNodeNext( Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom ){ int rc = SQLITE_OK; if( pNode->bEof==0 ){ switch( pNode->eType ){ case FTS5_STRING: { rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom); break; }; case FTS5_TERM: { Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; if( bFromValid ){ rc = sqlite3Fts5IterNextFrom(pIter, iFrom); }else{ rc = sqlite3Fts5IterNext(pIter); } if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ assert( rc==SQLITE_OK ); rc = fts5ExprTokenTest(pExpr, pNode); }else{ pNode->bEof = 1; } return rc; }; case FTS5_AND: { Fts5ExprNode *pLeft = pNode->apChild[0]; rc = fts5ExprNodeNext(pExpr, pLeft, bFromValid, iFrom); break; } case FTS5_OR: { int i; i64 iLast = pNode->iRowid; for(i=0; rc==SQLITE_OK && i<pNode->nChild; i++){ Fts5ExprNode *p1 = pNode->apChild[i]; assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); if( p1->bEof==0 ){ if( (p1->iRowid==iLast) || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) ){ rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); } } } break; } default: assert( pNode->eType==FTS5_NOT ); { assert( pNode->nChild==2 ); rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); break; } } if( rc==SQLITE_OK ){ rc = fts5ExprNodeNextMatch(pExpr, pNode); } } /* Assert that if bFromValid was true, either: ** ** a) an error occurred, or ** b) the node is now at EOF, or ** c) the node is now at or past rowid iFrom. */ assert( bFromValid==0 || rc!=SQLITE_OK /* a */ || pNode->bEof /* b */ || pNode->iRowid==iFrom || pExpr->bDesc==(pNode->iRowid<iFrom) /* c */ ); return rc; } /* ** If pNode currently points to a match, this function returns SQLITE_OK ** without modifying it. Otherwise, pNode is advanced until it does point ** to a match or EOF is reached. */ static int fts5ExprNodeNextMatch( Fts5Expr *pExpr, /* Expression of which pNode is a part */ Fts5ExprNode *pNode /* Expression node to test */ ){ int rc = SQLITE_OK; if( pNode->bEof==0 ){ switch( pNode->eType ){ case FTS5_STRING: { /* Advance the iterators until they all point to the same rowid */ rc = fts5ExprNearNextMatch(pExpr, pNode); break; } case FTS5_TERM: { rc = fts5ExprTokenTest(pExpr, pNode); break; } case FTS5_AND: { rc = fts5ExprAndNextRowid(pExpr, pNode); break; } case FTS5_OR: { Fts5ExprNode *pNext = pNode->apChild[0]; int i; for(i=1; i<pNode->nChild; i++){ Fts5ExprNode *pChild = pNode->apChild[i]; int cmp = fts5NodeCompare(pExpr, pNext, pChild); if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ pNext = pChild; } } pNode->iRowid = pNext->iRowid; pNode->bEof = pNext->bEof; pNode->bNomatch = pNext->bNomatch; break; } default: assert( pNode->eType==FTS5_NOT ); { Fts5ExprNode *p1 = pNode->apChild[0]; Fts5ExprNode *p2 = pNode->apChild[1]; assert( pNode->nChild==2 ); while( rc==SQLITE_OK && p1->bEof==0 ){ int cmp = fts5NodeCompare(pExpr, p1, p2); if( cmp>0 ){ ................................................................................ cmp = fts5NodeCompare(pExpr, p1, p2); } assert( rc!=SQLITE_OK || cmp<=0 ); if( cmp || p2->bNomatch ) break; rc = fts5ExprNodeNext(pExpr, p1, 0, 0); } pNode->bEof = p1->bEof; pNode->iRowid = p1->iRowid; break; } } } return rc; } ................................................................................ ** ** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. ** It is not an error if there are no matches. */ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ int rc = SQLITE_OK; pNode->bEof = 0; if( Fts5NodeIsString(pNode) ){ /* Initialize all term iterators in the NEAR object. */ rc = fts5ExprNearInitAll(pExpr, pNode); }else{ int i; for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){ rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]); } pNode->iRowid = pNode->apChild[0]->iRowid; } if( rc==SQLITE_OK ){ rc = fts5ExprNodeNextMatch(pExpr, pNode); } return rc; } /* ** Begin iterating through the set of documents in index pIdx matched by ................................................................................ ** equal to iFirst. ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ Fts5ExprNode *pRoot = p->pRoot; int rc = SQLITE_OK; if( pRoot ){ p->pIndex = pIdx; p->bDesc = bDesc; rc = fts5ExprNodeFirst(p, pRoot); /* If not at EOF but the current rowid occurs earlier than iFirst in ** the iteration order, move to document iFirst or later. */ if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); } /* If the iterator is not at a real match, skip forward until it is. */ while( pRoot->bNomatch && rc==SQLITE_OK && pRoot->bEof==0 ){ rc = fts5ExprNodeNext(p, pRoot, 0, 0); } } return rc; } /* ** Move to the next document ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){ int rc; Fts5ExprNode *pRoot = p->pRoot; do { rc = fts5ExprNodeNext(p, pRoot, 0, 0); }while( pRoot->bNomatch && pRoot->bEof==0 && rc==SQLITE_OK ); if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ pRoot->bEof = 1; } return rc; } int sqlite3Fts5ExprEof(Fts5Expr *p){ return (p->pRoot==0 || p->pRoot->bEof); } i64 sqlite3Fts5ExprRowid(Fts5Expr *p){ return p->pRoot->iRowid; } static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){ ................................................................................ int i; for(i=0; i<pPhrase->nTerm; i++){ Fts5ExprTerm *pSyn; Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; sqlite3_free(pTerm->zTerm); sqlite3Fts5IterClose(pTerm->pIter); for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ pNext = pSyn->pSynonym; sqlite3Fts5IterClose(pSyn->pIter); sqlite3_free(pSyn); } } if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); sqlite3_free(pPhrase); } } ................................................................................ } if( pRet==0 ){ assert( pParse->rc!=SQLITE_OK ); sqlite3Fts5ParseNearsetFree(pNear); sqlite3Fts5ParsePhraseFree(pPhrase); }else{ pRet->apPhrase[pRet->nPhrase++] = pPhrase; } return pRet; } typedef struct TokenCtx TokenCtx; struct TokenCtx { ................................................................................ int iUnused1, /* Start offset of token */ int iUnused2 /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; /* If an error has already occurred, this is a no-op */ if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; assert( pPhrase==0 || pPhrase->nTerm>0 ); if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; int nByte = sizeof(Fts5ExprTerm) + nToken+1; pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); if( pSyn==0 ){ rc = SQLITE_NOMEM; }else{ memset(pSyn, 0, nByte); pSyn->zTerm = (char*)&pSyn[1]; memcpy(pSyn->zTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; } }else{ Fts5ExprTerm *pTerm; if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ ................................................................................ rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); if( rc || (rc = sCtx.rc) ){ pParse->rc = rc; fts5ExprPhraseFree(sCtx.pPhrase); sCtx.pPhrase = 0; }else if( sCtx.pPhrase ){ if( pAppend==0 ){ if( (pParse->nPhrase % 8)==0 ){ int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); Fts5ExprPhrase **apNew; apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte); if( apNew==0 ){ ................................................................................ return 0; } pParse->apPhrase = apNew; } pParse->nPhrase++; } pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; assert( sCtx.pPhrase->nTerm>0 ); sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix; } return sCtx.pPhrase; } /* ** Create a new FTS5 expression by cloning phrase iPhrase of the ** expression passed as the second argument. */ int sqlite3Fts5ExprClonePhrase( Fts5Config *pConfig, Fts5Expr *pExpr, int iPhrase, Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ int i; /* Used to iterate through phrase terms */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ pOrig = pExpr->apExprPhrase[iPhrase]; pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); if( rc==SQLITE_OK ){ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase*)); } if( rc==SQLITE_OK ){ pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, ................................................................................ sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; } } if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; pNew->nPhrase = 1; pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->nPhrase = 1; sCtx.pPhrase->pNode = pNew->pRoot; if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ pNew->pRoot->eType = FTS5_TERM; }else{ pNew->pRoot->eType = FTS5_STRING; } }else{ sqlite3Fts5ExprFree(pNew); fts5ExprPhraseFree(sCtx.pPhrase); pNew = 0; } ................................................................................ } void sqlite3Fts5ParseSetDistance( Fts5Parse *pParse, Fts5ExprNearset *pNear, Fts5Token *p ){ int nNear = 0; int i; if( p->n ){ for(i=0; i<p->n; i++){ char c = (char)p->p[i]; if( c<'0' || c>'9' ){ sqlite3Fts5ParseError( ................................................................................ nNear = nNear * 10 + (p->p[i] - '0'); } }else{ nNear = FTS5_DEFAULT_NEARDIST; } pNear->nNear = nNear; } /* ** The second argument passed to this function may be NULL, or it may be ** an existing Fts5Colset object. This function returns a pointer to ** a new colset object containing the contents of (p) with new value column ** number iCol appended. ** ................................................................................ } void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, Fts5ExprNearset *pNear, Fts5Colset *pColset ){ if( pNear ){ pNear->pColset = pColset; }else{ sqlite3_free(pColset); } } static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); p->nChild += pSub->nChild; sqlite3_free(pSub); ................................................................................ nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ pRet->eType = eType; pRet->pNear = pNear; if( eType==FTS5_STRING ){ int iPhrase; for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 && pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ pRet->eType = FTS5_TERM; } }else{ fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); } } } if( pRet==0 ){ assert( pParse->rc!=SQLITE_OK ); sqlite3Fts5ParseNodeFree(pLeft); sqlite3Fts5ParseNodeFree(pRight); sqlite3Fts5ParseNearsetFree(pNear); } return pRet; } static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ int nByte = 0; Fts5ExprTerm *p; char *zQuoted; ................................................................................ for(i=0; i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; zRet = fts5PrintfAppend(zRet, " {"); for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ char *zTerm = pPhrase->aTerm[iTerm].zTerm; zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); } if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); if( zRet==0 ) return 0; } }else{ ................................................................................ } return zRet; } static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ char *zRet = 0; if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ Fts5ExprNearset *pNear = pExpr->pNear; int i; int iTerm; if( pNear->pColset ){ int iCol = pNear->pColset->aiCol[0]; ................................................................................ for(i=0; i<pExpr->nChild; i++){ char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]); if( z==0 ){ sqlite3_free(zRet); zRet = 0; }else{ int e = pExpr->apChild[i]->eType; int b = (e!=FTS5_STRING && e!=FTS5_TERM); zRet = fts5PrintfAppend(zRet, "%s%s%z%s", (i==0 ? "" : zOp), (b?"(":""), z, (b?")":"") ); } if( zRet==0 ) break; } ................................................................................ rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr); } if( rc==SQLITE_OK ){ char *zText; if( pExpr->pRoot==0 ){ zText = sqlite3_mprintf(""); }else if( bTcl ){ zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot); }else{ zText = fts5ExprPrint(pConfig, pExpr->pRoot); } if( zText==0 ){ ................................................................................ { "fts5_isalnum", fts5ExprIsAlnum }, { "fts5_fold", fts5ExprFold }, }; int i; int rc = SQLITE_OK; void *pCtx = (void*)pGlobal; for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aFunc); i++){ struct Fts5ExprFunc *p = &aFunc[i]; rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); } /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */ #ifndef NDEBUG (void)sqlite3Fts5ParserTrace; ................................................................................ nRet = pPhrase->poslist.n; }else{ *pa = 0; nRet = 0; } return nRet; } |
> > > > > > > > > > > > > > > > > | > > > > | | < < > | | < < < < | < < > | | < < | | < < | < | > | | | > > > < > > < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | | > > > > > > > > > > > > > > > | | | < | < > < | | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < > < < < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < | | | < < < < < < < < < < < < < < < < < < < < | | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | > | | < > > | | < > > > > > > > > > > > > > > > > > > > < | | | | | > > > | | | > > < < < < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | < > > > | < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 .. 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 ... 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 ... 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 ... 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 ... 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 ... 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 ... 510 511 512 513 514 515 516 517 518 519 520 521 522 523 ... 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 ... 624 625 626 627 628 629 630 631 632 633 634 635 636 637 ... 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 ... 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 ... 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 ... 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 ... 785 786 787 788 789 790 791 792 793 794 795 796 797 798 ... 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 ... 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 .... 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 .... 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 .... 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 .... 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 .... 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 .... 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 .... 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 .... 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 .... 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 .... 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 .... 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 .... 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 .... 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 .... 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 .... 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 .... 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 .... 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 .... 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 .... 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 .... 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 .... 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 |
#include <stdio.h> void sqlite3Fts5ParserTrace(FILE*, char*); #endif struct Fts5Expr { Fts5Index *pIndex; Fts5Config *pConfig; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ int nPhrase; /* Number of phrases in expression */ Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ }; /* ................................................................................ ** FTS5_TERM (pNear valid) */ struct Fts5ExprNode { int eType; /* Node type */ int bEof; /* True at EOF */ int bNomatch; /* True if entry is not a match */ /* Next method for this node. */ int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); i64 iRowid; /* Current rowid */ Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */ /* Child nodes. For a NOT node, this array always contains 2 entries. For ** AND or OR nodes, it contains 2 or more entries. */ int nChild; /* Number of child nodes */ Fts5ExprNode *apChild[1]; /* Array of child nodes */ }; #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) /* ** Invoke the xNext method of an Fts5ExprNode object. This macro should be ** used as if it has the same signature as the xNext() methods themselves. */ #define fts5ExprNodeNext(a,b,c,d) (b)->xNext((a), (b), (c), (d)) /* ** An instance of the following structure represents a single search term ** or term prefix. */ struct Fts5ExprTerm { int bPrefix; /* True for a prefix term */ char *zTerm; /* nul-terminated term */ ................................................................................ assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ if( !sParse.pExpr ){ const int nByte = sizeof(Fts5ExprNode); pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); if( pNew->pRoot ){ pNew->pRoot->bEof = 1; } }else{ pNew->pRoot = sParse.pExpr; } pNew->pIndex = 0; pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; } }else{ sqlite3Fts5ParseNodeFree(sParse.pExpr); } sqlite3_free(sParse.apPhrase); *pzErr = sParse.zErr; return sParse.rc; } ................................................................................ int bRetValid = 0; Fts5ExprTerm *p; assert( pTerm->pSynonym ); assert( bDesc==0 || bDesc==1 ); for(p=pTerm; p; p=p->pSynonym){ if( 0==sqlite3Fts5IterEof(p->pIter) ){ i64 iRowid = p->pIter->iRowid; if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){ iRet = iRowid; bRetValid = 1; } } } ................................................................................ if( pbEof && bRetValid==0 ) *pbEof = 1; return iRet; } /* ** Argument pTerm must be a synonym iterator. */ static int fts5ExprSynonymList( Fts5ExprTerm *pTerm, i64 iRowid, Fts5Buffer *pBuf, /* Use this buffer for space if required */ u8 **pa, int *pn ){ Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; int nAlloc = 4; int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){ if( pIter->nData==0 ) continue; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( aNew==0 ){ rc = SQLITE_NOMEM; goto synonym_poslist_out; } memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); nAlloc = nAlloc*2; if( aIter!=aStatic ) sqlite3_free(aIter); aIter = aNew; } sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]); assert( aIter[nIter].bEof==0 ); nIter++; } } if( nIter==1 ){ *pa = (u8*)aIter[0].a; *pn = aIter[0].n; }else{ Fts5PoslistWriter writer = {0}; i64 iPrev = -1; fts5BufferZero(pBuf); while( 1 ){ int i; i64 iMin = FTS5_LARGEST_INT64; for(i=0; i<nIter; i++){ if( aIter[i].bEof==0 ){ if( aIter[i].iPos==iPrev ){ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue; ................................................................................ } if( aIter[i].iPos<iMin ){ iMin = aIter[i].iPos; } } } if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break; rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin); iPrev = iMin; } if( rc==SQLITE_OK ){ *pa = pBuf->p; *pn = pBuf->n; } } synonym_poslist_out: if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } ................................................................................ ** ** SQLITE_OK is returned if an error occurs, or an SQLite error code ** otherwise. It is not considered an error code if the current rowid is ** not a match. */ static int fts5ExprPhraseIsMatch( Fts5ExprNode *pNode, /* Node pPhrase belongs to */ Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ int *pbMatch /* OUT: Set to true if really a match */ ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int i; int rc = SQLITE_OK; fts5BufferZero(&pPhrase->poslist); /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pPhrase->nTerm>ArraySize(aStatic) ){ int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( !aIter ) return SQLITE_NOMEM; } memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm); /* Initialize a term iterator for each term in the phrase */ for(i=0; i<pPhrase->nTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; int n = 0; int bFlag = 0; u8 *a = 0; if( pTerm->pSynonym ){ Fts5Buffer buf = {0, 0, 0}; rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n); if( rc ){ sqlite3_free(a); goto ismatch_out; } if( a==buf.p ) bFlag = 1; }else{ a = (u8*)pTerm->pIter->pData; n = pTerm->pIter->nData; } sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); aIter[i].bFlag = (u8)bFlag; if( aIter[i].bEof ) goto ismatch_out; } while( 1 ){ int bMatch; ................................................................................ memset(p, 0, sizeof(Fts5LookaheadReader)); p->a = a; p->n = n; fts5LookaheadReaderNext(p); return fts5LookaheadReaderNext(p); } typedef struct Fts5NearTrimmer Fts5NearTrimmer; struct Fts5NearTrimmer { Fts5LookaheadReader reader; /* Input iterator */ Fts5PoslistWriter writer; /* Writer context */ Fts5Buffer *pOut; /* Output poslist */ }; ................................................................................ int rc = *pRc; int bMatch; assert( pNear->nPhrase>1 ); /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pNear->nPhrase>ArraySize(aStatic) ){ int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); }else{ memset(aStatic, 0, sizeof(aStatic)); } if( rc!=SQLITE_OK ){ *pRc = rc; ................................................................................ int bRet = a[0].pOut->n>0; *pRc = rc; if( a!=aStatic ) sqlite3_free(a); return bRet; } } /* ** Advance iterator pIter until it points to a value equal to or laster ** than the initial value of *piLast. If this means the iterator points ** to a value laster than *piLast, update *piLast to the new lastest value. ** ** If the iterator reaches EOF, set *pbEof to true before returning. If ** an error occurs, set *pRc to an error code. If either *pbEof or *pRc ................................................................................ i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ int *pRc, /* OUT: Error code */ int *pbEof /* OUT: Set to true if EOF */ ){ i64 iLast = *piLast; i64 iRowid; iRowid = pIter->iRowid; if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ int rc = sqlite3Fts5IterNextFrom(pIter, iLast); if( rc || sqlite3Fts5IterEof(pIter) ){ *pRc = rc; *pbEof = 1; return 1; } iRowid = pIter->iRowid; assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); } *piLast = iRowid; return 0; } ................................................................................ int rc = SQLITE_OK; i64 iLast = *piLast; Fts5ExprTerm *p; int bEof = 0; for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ i64 iRowid = p->pIter->iRowid; if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){ rc = sqlite3Fts5IterNextFrom(p->pIter, iLast); } } } if( rc!=SQLITE_OK ){ ................................................................................ static int fts5ExprNearTest( int *pRc, Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ ){ Fts5ExprNearset *pNear = pNode->pNear; int rc = *pRc; if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5ExprTerm *pTerm; Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; pPhrase->poslist.n = 0; for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ Fts5IndexIter *pIter = pTerm->pIter; if( sqlite3Fts5IterEof(pIter)==0 ){ if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){ pPhrase->poslist.n = 1; } } } return pPhrase->poslist.n; }else{ int i; /* Check that each phrase in the nearset matches the current row. ** Populate the pPhrase->poslist buffers at the same time. If any ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch); if( bMatch==0 ) break; }else{ Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData); } } *pRc = rc; if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ return 1; } return 0; } } /* ** Initialize all term iterators in the pNear object. If any term is found ** to match no documents at all, return immediately without initializing any ** further iterators. */ static int fts5ExprNearInitAll( ................................................................................ Fts5Expr *pExpr, Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; int i, j; int rc = SQLITE_OK; assert( pNode->bNomatch==0 ); for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; j<pPhrase->nTerm; j++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; Fts5ExprTerm *p; int bEof = 1; ................................................................................ return rc; } } } return rc; } /* ** If pExpr is an ASC iterator, this function returns a value with the ** same sign as: ** ** (iLhs - iRhs) ** ................................................................................ return (iLhs < iRhs); } } static void fts5ExprSetEof(Fts5ExprNode *pNode){ int i; pNode->bEof = 1; pNode->bNomatch = 0; for(i=0; i<pNode->nChild; i++){ fts5ExprSetEof(pNode->apChild[i]); } } static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ ................................................................................ for(i=0; i<pNode->nChild; i++){ fts5ExprNodeZeroPoslist(pNode->apChild[i]); } } } /* ** Compare the values currently indicated by the two nodes as follows: ** ** res = (*p1) - (*p2) ** ** Nodes that point to values that come later in the iteration order are ** considered to be larger. Nodes at EOF are the largest of all. ** ** This means that if the iteration order is ASC, then numerically larger ** rowids are considered larger. Or if it is the default DESC, numerically ** smaller rowids are larger. */ static int fts5NodeCompare( Fts5Expr *pExpr, Fts5ExprNode *p1, Fts5ExprNode *p2 ){ if( p2->bEof ) return -1; if( p1->bEof ) return +1; return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); } /* ** All individual term iterators in pNear are guaranteed to be valid when ** this function is called. This function checks if all term iterators ** point to the same rowid, and if not, advances them until they do. ** If an EOF is reached before this happens, *pbEof is set to true before ** returning. ** ** SQLITE_OK is returned if an error occurs, or an SQLite error code ** otherwise. It is not considered an error code if an iterator reaches ** EOF. */ static int fts5ExprNodeTest_STRING( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; int rc = SQLITE_OK; i64 iLast; /* Lastest rowid any iterator points to */ int i, j; /* Phrase and token index, respectively */ int bMatch; /* True if all terms are at the same rowid */ const int bDesc = pExpr->bDesc; /* Check that this node should not be FTS5_TERM */ assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 || pNear->apPhrase[0]->aTerm[0].pSynonym ); /* Initialize iLast, the "lastest" rowid any iterator points to. If the ** iterator skips through rowids in the default ascending order, this means ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ if( pLeft->aTerm[0].pSynonym ){ iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); }else{ iLast = pLeft->aTerm[0].pIter->iRowid; } do { bMatch = 1; for(i=0; i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; j<pPhrase->nTerm; j++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; if( pTerm->pSynonym ){ i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; bMatch = 0; if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ pNode->bNomatch = 0; pNode->bEof = 1; return rc; } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; if( pIter->iRowid==iLast ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; } } } } }while( bMatch==0 ); pNode->iRowid = iLast; pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK); assert( pNode->bEof==0 || pNode->bNomatch==0 ); return rc; } /* ** Advance the first term iterator in the first phrase of pNear. Set output ** variable *pbEof to true if it reaches EOF or if an error occurs. ** ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ static int fts5ExprNodeNext_STRING( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ int bFromValid, i64 iFrom ){ Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; int rc = SQLITE_OK; pNode->bNomatch = 0; if( pTerm->pSynonym ){ int bEof = 1; Fts5ExprTerm *p; /* Find the firstest rowid any synonym points to. */ i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); /* Advance each iterator that currently points to iRowid. Or, if iFrom ** is valid - each iterator that points to a rowid before iFrom. */ for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ i64 ii = p->pIter->iRowid; if( ii==iRowid || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) ){ if( bFromValid ){ rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); }else{ rc = sqlite3Fts5IterNext(p->pIter); } if( rc!=SQLITE_OK ) break; if( sqlite3Fts5IterEof(p->pIter)==0 ){ bEof = 0; } }else{ bEof = 0; } } } /* Set the EOF flag if either all synonym iterators are at EOF or an ** error has occurred. */ pNode->bEof = (rc || bEof); }else{ Fts5IndexIter *pIter = pTerm->pIter; assert( Fts5NodeIsString(pNode) ); if( bFromValid ){ rc = sqlite3Fts5IterNextFrom(pIter, iFrom); }else{ rc = sqlite3Fts5IterNext(pIter); } pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); } if( pNode->bEof==0 ){ assert( rc==SQLITE_OK ); rc = fts5ExprNodeTest_STRING(pExpr, pNode); } return rc; } static int fts5ExprNodeTest_TERM( Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ ){ /* As this "NEAR" object is actually a single phrase that consists ** of a single term only, grab pointers into the poslist managed by the ** fts5_index.c iterator object. This is much faster than synthesizing ** a new poslist the way we have to for more complicated phrase or NEAR ** expressions. */ Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; assert( pNode->eType==FTS5_TERM ); assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); assert( pPhrase->aTerm[0].pSynonym==0 ); pPhrase->poslist.n = pIter->nData; if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ pPhrase->poslist.p = (u8*)pIter->pData; } pNode->iRowid = pIter->iRowid; pNode->bNomatch = (pPhrase->poslist.n==0); return SQLITE_OK; } /* ** xNext() method for a node of type FTS5_TERM. */ static int fts5ExprNodeNext_TERM( Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom ){ int rc; Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; assert( pNode->bEof==0 ); if( bFromValid ){ rc = sqlite3Fts5IterNextFrom(pIter, iFrom); }else{ rc = sqlite3Fts5IterNext(pIter); } if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ rc = fts5ExprNodeTest_TERM(pExpr, pNode); }else{ pNode->bEof = 1; pNode->bNomatch = 0; } return rc; } static void fts5ExprNodeTest_OR( Fts5Expr *pExpr, /* Expression of which pNode is a part */ Fts5ExprNode *pNode /* Expression node to test */ ){ Fts5ExprNode *pNext = pNode->apChild[0]; int i; for(i=1; i<pNode->nChild; i++){ Fts5ExprNode *pChild = pNode->apChild[i]; int cmp = fts5NodeCompare(pExpr, pNext, pChild); if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ pNext = pChild; } } pNode->iRowid = pNext->iRowid; pNode->bEof = pNext->bEof; pNode->bNomatch = pNext->bNomatch; } static int fts5ExprNodeNext_OR( Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom ){ int i; i64 iLast = pNode->iRowid; for(i=0; i<pNode->nChild; i++){ Fts5ExprNode *p1 = pNode->apChild[i]; assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); if( p1->bEof==0 ){ if( (p1->iRowid==iLast) || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) ){ int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); if( rc!=SQLITE_OK ) return rc; } } } fts5ExprNodeTest_OR(pExpr, pNode); return SQLITE_OK; } /* ** Argument pNode is an FTS5_AND node. */ static int fts5ExprNodeTest_AND( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pAnd /* FTS5_AND node to advance */ ){ int iChild; i64 iLast = pAnd->iRowid; int rc = SQLITE_OK; int bMatch; ................................................................................ assert( pAnd->bEof==0 ); do { pAnd->bNomatch = 0; bMatch = 1; for(iChild=0; iChild<pAnd->nChild; iChild++){ Fts5ExprNode *pChild = pAnd->apChild[iChild]; int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); if( cmp>0 ){ /* Advance pChild until it points to iLast or laster */ rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); if( rc!=SQLITE_OK ) return rc; } /* If the child node is now at EOF, so is the parent AND node. Otherwise, ** the child node is guaranteed to have advanced at least as far as ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the ** new lastest rowid seen so far. */ assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 ); if( pChild->bEof ){ ................................................................................ if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){ fts5ExprNodeZeroPoslist(pAnd); } pAnd->iRowid = iLast; return SQLITE_OK; } static int fts5ExprNodeNext_AND( Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom ){ int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_AND(pExpr, pNode); } return rc; } static int fts5ExprNodeTest_NOT( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pNode /* FTS5_NOT node to advance */ ){ int rc = SQLITE_OK; Fts5ExprNode *p1 = pNode->apChild[0]; Fts5ExprNode *p2 = pNode->apChild[1]; assert( pNode->nChild==2 ); while( rc==SQLITE_OK && p1->bEof==0 ){ int cmp = fts5NodeCompare(pExpr, p1, p2); if( cmp>0 ){ ................................................................................ cmp = fts5NodeCompare(pExpr, p1, p2); } assert( rc!=SQLITE_OK || cmp<=0 ); if( cmp || p2->bNomatch ) break; rc = fts5ExprNodeNext(pExpr, p1, 0, 0); } pNode->bEof = p1->bEof; pNode->bNomatch = p1->bNomatch; pNode->iRowid = p1->iRowid; if( p1->bEof ){ fts5ExprNodeZeroPoslist(p2); } return rc; } static int fts5ExprNodeNext_NOT( Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom ){ int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_NOT(pExpr, pNode); } return rc; } /* ** If pNode currently points to a match, this function returns SQLITE_OK ** without modifying it. Otherwise, pNode is advanced until it does point ** to a match or EOF is reached. */ static int fts5ExprNodeTest( Fts5Expr *pExpr, /* Expression of which pNode is a part */ Fts5ExprNode *pNode /* Expression node to test */ ){ int rc = SQLITE_OK; if( pNode->bEof==0 ){ switch( pNode->eType ){ case FTS5_STRING: { rc = fts5ExprNodeTest_STRING(pExpr, pNode); break; } case FTS5_TERM: { rc = fts5ExprNodeTest_TERM(pExpr, pNode); break; } case FTS5_AND: { rc = fts5ExprNodeTest_AND(pExpr, pNode); break; } case FTS5_OR: { fts5ExprNodeTest_OR(pExpr, pNode); break; } default: assert( pNode->eType==FTS5_NOT ); { rc = fts5ExprNodeTest_NOT(pExpr, pNode); break; } } } return rc; } ................................................................................ ** ** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. ** It is not an error if there are no matches. */ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ int rc = SQLITE_OK; pNode->bEof = 0; pNode->bNomatch = 0; if( Fts5NodeIsString(pNode) ){ /* Initialize all term iterators in the NEAR object. */ rc = fts5ExprNearInitAll(pExpr, pNode); }else if( pNode->xNext==0 ){ pNode->bEof = 1; }else{ int i; int nEof = 0; for(i=0; i<pNode->nChild && rc==SQLITE_OK; i++){ Fts5ExprNode *pChild = pNode->apChild[i]; rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]); assert( pChild->bEof==0 || pChild->bEof==1 ); nEof += pChild->bEof; } pNode->iRowid = pNode->apChild[0]->iRowid; switch( pNode->eType ){ case FTS5_AND: if( nEof>0 ) fts5ExprSetEof(pNode); break; case FTS5_OR: if( pNode->nChild==nEof ) fts5ExprSetEof(pNode); break; default: assert( pNode->eType==FTS5_NOT ); pNode->bEof = pNode->apChild[0]->bEof; break; } } if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest(pExpr, pNode); } return rc; } /* ** Begin iterating through the set of documents in index pIdx matched by ................................................................................ ** equal to iFirst. ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ Fts5ExprNode *pRoot = p->pRoot; int rc; /* Return code */ p->pIndex = pIdx; p->bDesc = bDesc; rc = fts5ExprNodeFirst(p, pRoot); /* If not at EOF but the current rowid occurs earlier than iFirst in ** the iteration order, move to document iFirst or later. */ if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); } /* If the iterator is not at a real match, skip forward until it is. */ while( pRoot->bNomatch ){ assert( pRoot->bEof==0 && rc==SQLITE_OK ); rc = fts5ExprNodeNext(p, pRoot, 0, 0); } return rc; } /* ** Move to the next document ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){ int rc; Fts5ExprNode *pRoot = p->pRoot; assert( pRoot->bEof==0 && pRoot->bNomatch==0 ); do { rc = fts5ExprNodeNext(p, pRoot, 0, 0); assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) ); }while( pRoot->bNomatch ); if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ pRoot->bEof = 1; } return rc; } int sqlite3Fts5ExprEof(Fts5Expr *p){ return p->pRoot->bEof; } i64 sqlite3Fts5ExprRowid(Fts5Expr *p){ return p->pRoot->iRowid; } static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){ ................................................................................ int i; for(i=0; i<pPhrase->nTerm; i++){ Fts5ExprTerm *pSyn; Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; sqlite3_free(pTerm->zTerm); sqlite3Fts5IterClose(pTerm->pIter); for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ pNext = pSyn->pSynonym; sqlite3Fts5IterClose(pSyn->pIter); fts5BufferFree((Fts5Buffer*)&pSyn[1]); sqlite3_free(pSyn); } } if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); sqlite3_free(pPhrase); } } ................................................................................ } if( pRet==0 ){ assert( pParse->rc!=SQLITE_OK ); sqlite3Fts5ParseNearsetFree(pNear); sqlite3Fts5ParsePhraseFree(pPhrase); }else{ if( pRet->nPhrase>0 ){ Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); if( pPhrase->nTerm==0 ){ fts5ExprPhraseFree(pPhrase); pRet->nPhrase--; pParse->nPhrase--; pPhrase = pLast; }else if( pLast->nTerm==0 ){ fts5ExprPhraseFree(pLast); pParse->apPhrase[pParse->nPhrase-2] = pPhrase; pParse->nPhrase--; pRet->nPhrase--; } } pRet->apPhrase[pRet->nPhrase++] = pPhrase; } return pRet; } typedef struct TokenCtx TokenCtx; struct TokenCtx { ................................................................................ int iUnused1, /* Start offset of token */ int iUnused2 /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; UNUSED_PARAM2(iUnused1, iUnused2); /* If an error has already occurred, this is a no-op */ if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); if( pSyn==0 ){ rc = SQLITE_NOMEM; }else{ memset(pSyn, 0, nByte); pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); memcpy(pSyn->zTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; } }else{ Fts5ExprTerm *pTerm; if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ ................................................................................ rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); if( rc || (rc = sCtx.rc) ){ pParse->rc = rc; fts5ExprPhraseFree(sCtx.pPhrase); sCtx.pPhrase = 0; }else{ if( pAppend==0 ){ if( (pParse->nPhrase % 8)==0 ){ int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); Fts5ExprPhrase **apNew; apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte); if( apNew==0 ){ ................................................................................ return 0; } pParse->apPhrase = apNew; } pParse->nPhrase++; } if( sCtx.pPhrase==0 ){ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); }else if( sCtx.pPhrase->nTerm ){ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix; } pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; } return sCtx.pPhrase; } /* ** Create a new FTS5 expression by cloning phrase iPhrase of the ** expression passed as the second argument. */ int sqlite3Fts5ExprClonePhrase( Fts5Expr *pExpr, int iPhrase, Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ int i; /* Used to iterate through phrase terms */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ pOrig = pExpr->apExprPhrase[iPhrase]; pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); if( rc==SQLITE_OK ){ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase*)); } if( rc==SQLITE_OK ){ pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, ................................................................................ sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; } } if( rc==SQLITE_OK ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; pNew->pConfig = pExpr->pConfig; pNew->nPhrase = 1; pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->nPhrase = 1; sCtx.pPhrase->pNode = pNew->pRoot; if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ pNew->pRoot->eType = FTS5_TERM; pNew->pRoot->xNext = fts5ExprNodeNext_TERM; }else{ pNew->pRoot->eType = FTS5_STRING; pNew->pRoot->xNext = fts5ExprNodeNext_STRING; } }else{ sqlite3Fts5ExprFree(pNew); fts5ExprPhraseFree(sCtx.pPhrase); pNew = 0; } ................................................................................ } void sqlite3Fts5ParseSetDistance( Fts5Parse *pParse, Fts5ExprNearset *pNear, Fts5Token *p ){ if( pNear ){ int nNear = 0; int i; if( p->n ){ for(i=0; i<p->n; i++){ char c = (char)p->p[i]; if( c<'0' || c>'9' ){ sqlite3Fts5ParseError( ................................................................................ nNear = nNear * 10 + (p->p[i] - '0'); } }else{ nNear = FTS5_DEFAULT_NEARDIST; } pNear->nNear = nNear; } } /* ** The second argument passed to this function may be NULL, or it may be ** an existing Fts5Colset object. This function returns a pointer to ** a new colset object containing the contents of (p) with new value column ** number iCol appended. ** ................................................................................ } void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, Fts5ExprNearset *pNear, Fts5Colset *pColset ){ if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ pParse->rc = SQLITE_ERROR; pParse->zErr = sqlite3_mprintf( "fts5: column queries are not supported (detail=none)" ); sqlite3_free(pColset); return; } if( pNear ){ pNear->pColset = pColset; }else{ sqlite3_free(pColset); } } static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ switch( pNode->eType ){ case FTS5_STRING: { Fts5ExprNearset *pNear = pNode->pNear; if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 && pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ pNode->eType = FTS5_TERM; pNode->xNext = fts5ExprNodeNext_TERM; }else{ pNode->xNext = fts5ExprNodeNext_STRING; } break; }; case FTS5_OR: { pNode->xNext = fts5ExprNodeNext_OR; break; }; case FTS5_AND: { pNode->xNext = fts5ExprNodeNext_AND; break; }; default: assert( pNode->eType==FTS5_NOT ); { pNode->xNext = fts5ExprNodeNext_NOT; break; }; } } static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); p->nChild += pSub->nChild; sqlite3_free(pSub); ................................................................................ nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ pRet->eType = eType; pRet->pNear = pNear; fts5ExprAssignXNext(pRet); if( eType==FTS5_STRING ){ int iPhrase; for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; if( pNear->apPhrase[iPhrase]->nTerm==0 ){ pRet->xNext = 0; pRet->eType = FTS5_EOF; } } if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1) ){ assert( pParse->rc==SQLITE_OK ); pParse->rc = SQLITE_ERROR; assert( pParse->zErr==0 ); pParse->zErr = sqlite3_mprintf( "fts5: %s queries are not supported (detail!=full)", pNear->nPhrase==1 ? "phrase": "NEAR" ); sqlite3_free(pRet); pRet = 0; } }else{ fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); } } } if( pRet==0 ){ assert( pParse->rc!=SQLITE_OK ); sqlite3Fts5ParseNodeFree(pLeft); sqlite3Fts5ParseNodeFree(pRight); sqlite3Fts5ParseNearsetFree(pNear); } return pRet; } Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( Fts5Parse *pParse, /* Parse context */ Fts5ExprNode *pLeft, /* Left hand child expression */ Fts5ExprNode *pRight /* Right hand child expression */ ){ Fts5ExprNode *pRet = 0; Fts5ExprNode *pPrev; if( pParse->rc ){ sqlite3Fts5ParseNodeFree(pLeft); sqlite3Fts5ParseNodeFree(pRight); }else{ assert( pLeft->eType==FTS5_STRING || pLeft->eType==FTS5_TERM || pLeft->eType==FTS5_EOF || pLeft->eType==FTS5_AND ); assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF ); if( pLeft->eType==FTS5_AND ){ pPrev = pLeft->apChild[pLeft->nChild-1]; }else{ pPrev = pLeft; } assert( pPrev->eType==FTS5_STRING || pPrev->eType==FTS5_TERM || pPrev->eType==FTS5_EOF ); if( pRight->eType==FTS5_EOF ){ assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); sqlite3Fts5ParseNodeFree(pRight); pRet = pLeft; pParse->nPhrase--; } else if( pPrev->eType==FTS5_EOF ){ Fts5ExprPhrase **ap; if( pPrev==pLeft ){ pRet = pRight; }else{ pLeft->apChild[pLeft->nChild-1] = pRight; pRet = pLeft; } ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase]; assert( ap[0]==pPrev->pNear->apPhrase[0] ); memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase); pParse->nPhrase--; sqlite3Fts5ParseNodeFree(pPrev); } else{ pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0); } } return pRet; } static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ int nByte = 0; Fts5ExprTerm *p; char *zQuoted; ................................................................................ for(i=0; i<pNear->nPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; zRet = fts5PrintfAppend(zRet, " {"); for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ char *zTerm = pPhrase->aTerm[iTerm].zTerm; zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); if( pPhrase->aTerm[iTerm].bPrefix ){ zRet = fts5PrintfAppend(zRet, "*"); } } if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); if( zRet==0 ) return 0; } }else{ ................................................................................ } return zRet; } static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ char *zRet = 0; if( pExpr->eType==0 ){ return sqlite3_mprintf("\"\""); }else if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ Fts5ExprNearset *pNear = pExpr->pNear; int i; int iTerm; if( pNear->pColset ){ int iCol = pNear->pColset->aiCol[0]; ................................................................................ for(i=0; i<pExpr->nChild; i++){ char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]); if( z==0 ){ sqlite3_free(zRet); zRet = 0; }else{ int e = pExpr->apChild[i]->eType; int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF); zRet = fts5PrintfAppend(zRet, "%s%s%z%s", (i==0 ? "" : zOp), (b?"(":""), z, (b?")":"") ); } if( zRet==0 ) break; } ................................................................................ rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr); } if( rc==SQLITE_OK ){ char *zText; if( pExpr->pRoot->xNext==0 ){ zText = sqlite3_mprintf(""); }else if( bTcl ){ zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot); }else{ zText = fts5ExprPrint(pConfig, pExpr->pRoot); } if( zText==0 ){ ................................................................................ { "fts5_isalnum", fts5ExprIsAlnum }, { "fts5_fold", fts5ExprFold }, }; int i; int rc = SQLITE_OK; void *pCtx = (void*)pGlobal; for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){ struct Fts5ExprFunc *p = &aFunc[i]; rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); } /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */ #ifndef NDEBUG (void)sqlite3Fts5ParserTrace; ................................................................................ nRet = pPhrase->poslist.n; }else{ *pa = 0; nRet = 0; } return nRet; } struct Fts5PoslistPopulator { Fts5PoslistWriter writer; int bOk; /* True if ok to populate */ int bMiss; }; Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ Fts5PoslistPopulator *pRet; pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); if( pRet ){ int i; memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); for(i=0; i<pExpr->nPhrase; i++){ Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; assert( pExpr->apExprPhrase[i]->nTerm==1 ); if( bLive && (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) ){ pRet[i].bMiss = 1; }else{ pBuf->n = 0; } } } return pRet; } struct Fts5ExprCtx { Fts5Expr *pExpr; Fts5PoslistPopulator *aPopulator; i64 iOff; }; typedef struct Fts5ExprCtx Fts5ExprCtx; /* ** TODO: Make this more efficient! */ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ int i; for(i=0; i<pColset->nCol; i++){ if( pColset->aiCol[i]==iCol ) return 1; } return 0; } static int fts5ExprPopulatePoslistsCb( void *pCtx, /* Copy of 2nd argument to xTokenize() */ int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Pointer to buffer containing token */ int nToken, /* Size of token in bytes */ int iUnused1, /* Byte offset of token within input text */ int iUnused2 /* Byte offset of end of token within input text */ ){ Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; Fts5Expr *pExpr = p->pExpr; int i; UNUSED_PARAM2(iUnused1, iUnused2); if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; i<pExpr->nPhrase; i++){ Fts5ExprTerm *pTerm; if( p->aPopulator[i].bOk==0 ) continue; for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ int nTerm = (int)strlen(pTerm->zTerm); if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) && memcmp(pTerm->zTerm, pToken, nTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); if( rc ) return rc; break; } } } return SQLITE_OK; } int sqlite3Fts5ExprPopulatePoslists( Fts5Config *pConfig, Fts5Expr *pExpr, Fts5PoslistPopulator *aPopulator, int iCol, const char *z, int n ){ int i; Fts5ExprCtx sCtx; sCtx.pExpr = pExpr; sCtx.aPopulator = aPopulator; sCtx.iOff = (((i64)iCol) << 32) - 1; for(i=0; i<pExpr->nPhrase; i++){ Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; Fts5Colset *pColset = pNode->pNear->pColset; if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) || aPopulator[i].bMiss ){ aPopulator[i].bOk = 0; }else{ aPopulator[i].bOk = 1; } } return sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); } static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ pNode->pNear->apPhrase[0]->poslist.n = 0; }else{ int i; for(i=0; i<pNode->nChild; i++){ fts5ExprClearPoslists(pNode->apChild[i]); } } } static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ pNode->iRowid = iRowid; pNode->bEof = 0; switch( pNode->eType ){ case FTS5_TERM: case FTS5_STRING: return (pNode->pNear->apPhrase[0]->poslist.n>0); case FTS5_AND: { int i; for(i=0; i<pNode->nChild; i++){ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ fts5ExprClearPoslists(pNode); return 0; } } break; } case FTS5_OR: { int i; int bRet = 0; for(i=0; i<pNode->nChild; i++){ if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ bRet = 1; } } return bRet; } default: { assert( pNode->eType==FTS5_NOT ); if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) ){ fts5ExprClearPoslists(pNode); return 0; } break; } } return 1; } void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ fts5ExprCheckPoslists(pExpr->pRoot, iRowid); } static void fts5ExprClearEof(Fts5ExprNode *pNode){ int i; for(i=0; i<pNode->nChild; i++){ fts5ExprClearEof(pNode->apChild[i]); } pNode->bEof = 0; } void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ fts5ExprClearEof(pExpr->pRoot); } /* ** This function is only called for detail=columns tables. */ int sqlite3Fts5ExprPhraseCollist( Fts5Expr *pExpr, int iPhrase, const u8 **ppCollist, int *pnCollist ){ Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; Fts5ExprNode *pNode = pPhrase->pNode; int rc = SQLITE_OK; assert( iPhrase>=0 && iPhrase<pExpr->nPhrase ); assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid && pPhrase->poslist.n>0 ){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; if( pTerm->pSynonym ){ Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1]; rc = fts5ExprSynonymList( pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist ); }else{ *ppCollist = pPhrase->aTerm[0].pIter->pData; *pnCollist = pPhrase->aTerm[0].pIter->nData; } }else{ *ppCollist = 0; *pnCollist = 0; } return rc; } |
Changes to ext/fts5/fts5_hash.c.
22 23 24 25 26 27 28 29 30 31 32 33 34 35 .. 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 .. 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 ... 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 ... 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 ... 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 ... 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 |
** This file contains the implementation of an in-memory hash table used ** to accumuluate "term -> doclist" content before it is flused to a level-0 ** segment. */ struct Fts5Hash { int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ Fts5HashEntry *pScan; /* Current ordered scan item */ Fts5HashEntry **aSlot; /* Array of hash slots */ }; ................................................................................ struct Fts5HashEntry { Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ Fts5HashEntry *pScanNext; /* Next entry in sorted order */ int nAlloc; /* Total size of allocation */ int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ u8 bDel; /* Set delete-flag @ iSzPoslist */ int iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ char zKey[8]; /* Nul-terminated entry key */ }; /* ** Size of Fts5HashEntry without the zKey[] array. ................................................................................ #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) /* ** Allocate a new hash table. */ int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ int nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); if( pNew->aSlot==0 ){ sqlite3_free(pNew); *ppNew = 0; ................................................................................ sqlite3_free(apOld); pHash->nSlot = nNew; pHash->aSlot = apNew; return SQLITE_OK; } static void fts5HashAddPoslistSize(Fts5HashEntry *p){ if( p->iSzPoslist ){ u8 *pPtr = (u8*)p; int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ int nPos = nSz*2 + p->bDel; /* Value of nPos field */ assert( p->bDel==0 || p->bDel==1 ); if( nPos<=127 ){ pPtr[p->iSzPoslist] = (u8)nPos; }else{ int nByte = sqlite3Fts5GetVarintLen((u32)nPos); memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); p->nData += (nByte-1); } p->bDel = 0; p->iSzPoslist = 0; } } int sqlite3Fts5HashWrite( Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ unsigned int iHash; Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( p->zKey[0]==bByte && memcmp(&p->zKey[1], pToken, nToken)==0 && p->zKey[nToken+1]==0 ){ break; } } /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); p->nAlloc = nByte; p->zKey[0] = bByte; memcpy(&p->zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); p->iSzPoslist = p->nData; p->nData += 1; p->iRowid = iRowid; p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; nIncr += p->nData; } /* Check there is enough space to append a new entry. Worst case scenario ** is: ** ** + 9 bytes for a new rowid, ** + 4 byte reserved for the "poslist size" varint. ** + 1 byte for a "new column" byte, ** + 3 bytes for a new column number (16-bit max) as a varint, ** + 5 bytes for the new position offset (32-bit max). ................................................................................ pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); if( pNew==0 ) return SQLITE_NOMEM; pNew->nAlloc = nNew; for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); *pp = pNew; p = pNew; } pPtr = (u8*)p; nIncr -= p->nData; /* If this is a new rowid, append the 4-byte size field for the previous ** entry, and the new rowid for this entry. */ if( iRowid!=p->iRowid ){ fts5HashAddPoslistSize(p); p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); p->iSzPoslist = p->nData; p->nData += 1; p->iCol = 0; p->iPos = 0; p->iRowid = iRowid; } if( iCol>=0 ){ /* Append a new column value, if necessary */ assert( iCol>=p->iCol ); if( iCol!=p->iCol ){ pPtr[p->nData++] = 0x01; p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); p->iCol = iCol; p->iPos = 0; } /* Append the new position offset */ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); p->iPos = iPos; }else{ /* This is a delete. Set the delete flag. */ p->bDel = 1; } nIncr += p->nData; *pHash->pnByte += nIncr; return SQLITE_OK; } /* ** Arguments pLeft and pRight point to linked-lists of hash-entry objects, ................................................................................ Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } if( p ){ fts5HashAddPoslistSize(p); *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *ppDoclist = 0; *pnDoclist = 0; } ................................................................................ const char **pzTerm, /* OUT: term (nul-terminated) */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = (int)strlen(p->zKey); fts5HashAddPoslistSize(p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *pzTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } } |
> > < > | | > | > > > > > > > > > < > > > > > > > > > > > > > > > > < > > > > < < < < > > > > > | > > > | > > > > | < | > > > > | > > > | < > | < > > > > > > > > | | > | > | | > > < > | | |
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 .. 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 .. 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 ... 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 ... 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 ... 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 ... 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 |
** This file contains the implementation of an in-memory hash table used ** to accumuluate "term -> doclist" content before it is flused to a level-0 ** segment. */ struct Fts5Hash { int eDetail; /* Copy of Fts5Config.eDetail */ int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ Fts5HashEntry *pScan; /* Current ordered scan item */ Fts5HashEntry **aSlot; /* Array of hash slots */ }; ................................................................................ struct Fts5HashEntry { Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ Fts5HashEntry *pScanNext; /* Next entry in sorted order */ int nAlloc; /* Total size of allocation */ int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ int nKey; /* Length of zKey[] in bytes */ u8 bDel; /* Set delete-flag @ iSzPoslist */ u8 bContent; /* Set content-flag (detail=none mode) */ i16 iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ char zKey[8]; /* Nul-terminated entry key */ }; /* ** Size of Fts5HashEntry without the zKey[] array. ................................................................................ #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) /* ** Allocate a new hash table. */ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ int nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; pNew->eDetail = pConfig->eDetail; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); if( pNew->aSlot==0 ){ sqlite3_free(pNew); *ppNew = 0; ................................................................................ sqlite3_free(apOld); pHash->nSlot = nNew; pHash->aSlot = apNew; return SQLITE_OK; } static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ if( p->iSzPoslist ){ u8 *pPtr = (u8*)p; if( pHash->eDetail==FTS5_DETAIL_NONE ){ assert( p->nData==p->iSzPoslist ); if( p->bDel ){ pPtr[p->nData++] = 0x00; if( p->bContent ){ pPtr[p->nData++] = 0x00; } } }else{ int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ int nPos = nSz*2 + p->bDel; /* Value of nPos field */ assert( p->bDel==0 || p->bDel==1 ); if( nPos<=127 ){ pPtr[p->iSzPoslist] = (u8)nPos; }else{ int nByte = sqlite3Fts5GetVarintLen((u32)nPos); memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); p->nData += (nByte-1); } } p->iSzPoslist = 0; p->bDel = 0; p->bContent = 0; } } /* ** Add an entry to the in-memory hash table. The key is the concatenation ** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). ** ** (bByte || pToken) -> (iRowid,iCol,iPos) ** ** Or, if iCol is negative, then the value is a delete marker. */ int sqlite3Fts5HashWrite( Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ unsigned int iHash; Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ int bNew; /* If non-delete entry should be written */ bNew = (pHash->eDetail==FTS5_DETAIL_FULL); /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( p->zKey[0]==bByte && p->nKey==nToken && memcmp(&p->zKey[1], pToken, nToken)==0 ){ break; } } /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ /* Figure out how much space to allocate */ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; /* Grow the Fts5Hash.aSlot[] array if necessary. */ if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } /* Allocate new Fts5HashEntry and add it to the hash table. */ p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); p->nAlloc = nByte; p->zKey[0] = bByte; memcpy(&p->zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->nKey = nToken; p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; /* Add the first rowid field to the hash-entry */ p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); p->iRowid = iRowid; p->iSzPoslist = p->nData; if( pHash->eDetail!=FTS5_DETAIL_NONE ){ p->nData += 1; p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); } nIncr += p->nData; }else{ /* Appending to an existing hash-entry. Check that there is enough ** space to append the largest possible new entry. Worst case scenario ** is: ** ** + 9 bytes for a new rowid, ** + 4 byte reserved for the "poslist size" varint. ** + 1 byte for a "new column" byte, ** + 3 bytes for a new column number (16-bit max) as a varint, ** + 5 bytes for the new position offset (32-bit max). ................................................................................ pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); if( pNew==0 ) return SQLITE_NOMEM; pNew->nAlloc = nNew; for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); *pp = pNew; p = pNew; } nIncr -= p->nData; } assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); pPtr = (u8*)p; /* If this is a new rowid, append the 4-byte size field for the previous ** entry, and the new rowid for this entry. */ if( iRowid!=p->iRowid ){ fts5HashAddPoslistSize(pHash, p); p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); p->iRowid = iRowid; bNew = 1; p->iSzPoslist = p->nData; if( pHash->eDetail!=FTS5_DETAIL_NONE ){ p->nData += 1; p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); p->iPos = 0; } } if( iCol>=0 ){ if( pHash->eDetail==FTS5_DETAIL_NONE ){ p->bContent = 1; }else{ /* Append a new column value, if necessary */ assert( iCol>=p->iCol ); if( iCol!=p->iCol ){ if( pHash->eDetail==FTS5_DETAIL_FULL ){ pPtr[p->nData++] = 0x01; p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); p->iCol = iCol; p->iPos = 0; }else{ bNew = 1; p->iCol = iPos = iCol; } } /* Append the new position offset, if necessary */ if( bNew ){ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); p->iPos = iPos; } } }else{ /* This is a delete. Set the delete flag. */ p->bDel = 1; } nIncr += p->nData; *pHash->pnByte += nIncr; return SQLITE_OK; } /* ** Arguments pLeft and pRight point to linked-lists of hash-entry objects, ................................................................................ Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } if( p ){ fts5HashAddPoslistSize(pHash, p); *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *ppDoclist = 0; *pnDoclist = 0; } ................................................................................ const char **pzTerm, /* OUT: term (nul-terminated) */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ int nTerm = (int)strlen(p->zKey); fts5HashAddPoslistSize(pHash, p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ *pzTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } } |
Changes to ext/fts5/fts5_index.c.
257 258 259 260 261 262 263 264 265 266 267 268 269 270 ... 299 300 301 302 303 304 305 306 307 308 309 310 311 312 ... 429 430 431 432 433 434 435 436 437 438 439 440 441 442 ... 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 ... 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 ... 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 ... 697 698 699 700 701 702 703 704 705 706 707 708 709 710 ... 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 ... 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 .... 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 .... 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 .... 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 .... 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 .... 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 .... 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 .... 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 .... 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 .... 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 .... 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 .... 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 .... 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 .... 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 .... 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 .... 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 .... 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 .... 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 .... 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 .... 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 .... 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 .... 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 .... 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 .... 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 .... 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 .... 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 .... 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 .... 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 .... 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 .... 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 .... 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 .... 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 .... 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 .... 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 .... 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 .... 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 .... 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 .... 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 .... 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 .... 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 .... 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 .... 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 .... 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 .... 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 .... 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 .... 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 .... 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 .... 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 .... 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 .... 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 .... 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 .... 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 .... 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 .... 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 .... 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 .... 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 |
#define FTS5_DATA_ZERO_PADDING 8 #define FTS5_DATA_PADDING 20 typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; typedef struct Fts5PageWriter Fts5PageWriter; typedef struct Fts5SegIter Fts5SegIter; typedef struct Fts5DoclistIter Fts5DoclistIter; typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; ................................................................................ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ }; struct Fts5DoclistIter { u8 *aEof; /* Pointer to 1 byte past end of doclist */ /* Output variables. aPoslist==0 at EOF */ i64 iRowid; ................................................................................ Fts5StructureSegment *pSeg; /* Segment to iterate through */ int flags; /* Mask of configuration flags */ int iLeafPgno; /* Current leaf page number */ Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ int iLeafOffset; /* Byte offset within current leaf */ /* The page and offset from which the current term was read. The offset ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; int iTermLeafOffset; int iPgidxOff; /* Next offset in pgidx */ int iEndofDoclist; ................................................................................ Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ /* Variables populated based on current entry. */ Fts5Buffer term; /* Current term */ i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ int bDel; /* True if the delete flag is set */ }; /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. */ #define ASSERT_SZLEAF_OK(x) assert( \ (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ ) #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 /* ** Argument is a pointer to an Fts5Data structure that contains a leaf ** page. This macro evaluates to true if the leaf contains no terms, or ** false if it contains at least one term. */ #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) ................................................................................ ** aFirst[1] contains the index in aSeg[] of the iterator that points to ** the smallest key overall. aFirst[0] is unused. ** ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. */ struct Fts5IndexIter { Fts5Index *pIndex; /* Index that owns this iterator */ Fts5Structure *pStruct; /* Database structure for this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ u8 bSkipEmpty; /* True to skip deleted entries */ u8 bEof; /* True at EOF */ u8 bFiltered; /* True if column-filter already applied */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; ................................................................................ */ static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ int nCmp = MIN(pLeft->n, pRight->n); int res = memcmp(pLeft->p, pRight->p, nCmp); return (res==0 ? (pLeft->n - pRight->n) : res); } #ifdef SQLITE_DEBUG static int fts5BlobCompare( const u8 *pLeft, int nLeft, const u8 *pRight, int nRight ){ int nCmp = MIN(nLeft, nRight); int res = memcmp(pLeft, pRight, nCmp); return (res==0 ? (nLeft - nRight) : res); } #endif static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ int ret; fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); return ret; } /* ................................................................................ p->rc = rc; p->nRead++; } assert( (pRet==0)==(p->rc!=SQLITE_OK) ); return pRet; } /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). */ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); ................................................................................ pRet->nRef = 1; pRet->nLevel = nLevel; pRet->nSegment = nSegment; i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){ Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl]; int nTotal; int iSeg; i += fts5GetVarint32(&pData[i], pLvl->nMerge); i += fts5GetVarint32(&pData[i], nTotal); assert( nTotal>=pLvl->nMerge ); pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, nTotal * sizeof(Fts5StructureSegment) ); if( rc==SQLITE_OK ){ pLvl->nSeg = nTotal; for(iSeg=0; iSeg<nTotal; iSeg++){ i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); } }else{ fts5StructureRelease(pRet); pRet = 0; } } } *ppOut = pRet; return rc; } /* ** ................................................................................ } pLvl->aSeg = aNew; }else{ *pRc = SQLITE_NOMEM; } } } /* ** Read, deserialize and return the structure record. ** ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array ** are over-allocated as described for function fts5StructureDecode() ** above. ** ** If an error occurs, NULL is returned and an error code left in the ** Fts5Index handle. If an error has already occurred when this function ** is called, it is a no-op. */ static Fts5Structure *fts5StructureRead(Fts5Index *p){ Fts5Config *pConfig = p->pConfig; Fts5Structure *pRet = 0; /* Object to return */ int iCookie; /* Configuration cookie */ Fts5Data *pData; pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); if( p->rc ) return 0; /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; } return pRet; } /* ** Return the total number of segments in index structure pStruct. This ** function is only ever used as part of assert() conditions. */ #ifdef SQLITE_DEBUG ................................................................................ ** ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ int nSz; ASSERT_SZLEAF_OK(pIter->pLeaf); fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; pIter->iLeafOffset = iOff; assert_nc( pIter->nPos>=0 ); } } static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); ................................................................................ */ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; /* Offset to read at */ int nNew; /* Bytes of new data */ iOff += fts5GetVarint32(&a[iOff], nNew); pIter->term.n = nKeep; fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); iOff += nNew; pIter->iTermLeafOffset = iOff; pIter->iTermLeafPgno = pIter->iLeafPgno; pIter->iLeafOffset = iOff; ................................................................................ int nExtra; pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); pIter->iEndofDoclist += nExtra; } fts5SegIterLoadRowid(p, pIter); } /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when ** this function returns. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ................................................................................ ** at EOF already. */ assert( pIter->pLeaf==0 ); return; } if( p->rc==SQLITE_OK ){ memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; fts5SegIterNextPage(p, pIter); } if( p->rc==SQLITE_OK ){ pIter->iLeafOffset = 4; ................................................................................ ** This function advances the iterator so that it points to the last ** relevant rowid on the page and, if necessary, initializes the ** aRowidOffset[] and iRowidOffset variables. At this point the iterator ** is in its regular state - Fts5SegIter.iLeafOffset points to the first ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; int iRowidOffset = 0; if( n>pIter->iEndofDoclist ){ n = pIter->iEndofDoclist; } ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; int nPos; int bDummy; i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); i += nPos; if( i>=n ) break; i += fts5GetVarint(&a[i], (u64*)&iDelta); pIter->iRowid += iDelta; if( iRowidOffset>=pIter->nRowidOffset ){ int nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; break; } ................................................................................ } /* ** Return true if the iterator passed as the second argument currently ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } /* ** Advance iterator pIter to the next entry. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It ** is not considered an error if the iterator reaches EOF. If an error has ** already occurred when this function is called, it is a no-op. */ static void fts5SegIterNext( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ assert( pbNewTerm==0 || *pbNewTerm==0 ); if( p->rc==SQLITE_OK ){ if( pIter->flags & FTS5_SEGITER_REVERSE ){ assert( pIter->pNextLeaf==0 ); if( pIter->iRowidOffset>0 ){ u8 *a = pIter->pLeaf->p; int iOff; int nPos; int bDummy; i64 iDelta; pIter->iRowidOffset--; pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset]; iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy); iOff += nPos; fts5GetVarint(&a[iOff], (u64*)&iDelta); pIter->iRowid -= iDelta; fts5SegIterLoadNPos(p, pIter); }else{ fts5SegIterReverseNewPage(p, pIter); } }else{ Fts5Data *pLeaf = pIter->pLeaf; int iOff; int bNewTerm = 0; int nKeep = 0; /* Search for the end of the position list within the current page. */ u8 *a = pLeaf->p; int n = pLeaf->szLeaf; ASSERT_SZLEAF_OK(pLeaf); iOff = pIter->iLeafOffset + pIter->nPos; if( iOff<n ){ /* The next entry is on the current page. */ assert_nc( iOff<=pIter->iEndofDoclist ); ................................................................................ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], iOff ); pIter->iLeafOffset = iOff; pIter->iEndofDoclist = iOff; bNewTerm = 1; } if( iOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; return; } } } /* Check if the iterator is now at EOF. If so, return early. */ ................................................................................ fts5SegIterLoadTerm(p, pIter, nKeep); fts5SegIterLoadNPos(p, pIter); if( pbNewTerm ) *pbNewTerm = 1; } }else{ /* The following could be done by calling fts5SegIterLoadNPos(). But ** this block is particularly performance critical, so equivalent ** code is inlined. */ int nSz; assert( p->rc==SQLITE_OK ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; assert_nc( pIter->nPos>=0 ); } } } } } #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } /* ** Iterator pIter currently points to the first rowid in a doclist. This ** function sets the iterator up so that iterates in reverse order through ** the doclist. */ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ ................................................................................ pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ /* Currently, Fts5SegIter.iLeafOffset points to the first byte of ** position-list content for the current rowid. Back it up so that it ** points to the start of the position-list size field. */ pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to ** see where said rowid really is. */ if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ int pgno; Fts5StructureSegment *pSeg = pIter->pSeg; ................................................................................ ){ return; } pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } #define fts5IndexSkipVarint(a, iOff) { \ int iEnd = iOff+9; \ while( (a[iOff++] & 0x80) && iOff<iEnd ); \ } /* ** The iterator object passed as the second argument currently contains ** no valid values except for the Fts5SegIter.pLeaf member variable. This ** function searches the leaf page for a term matching (pTerm/nTerm). ** ** If the specified term is found on the page, then the iterator is left ** pointing to it. If argument bGe is zero and the term is not found, ................................................................................ int bEndOfPage = 0; assert( p->rc==SQLITE_OK ); iPgidx = szLeaf; iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; while( 1 ){ /* Figure out how many new bytes are in this term */ fts5FastGetVarint32(a, iOff, nNew); if( nKeep<nMatch ){ goto search_failed; ................................................................................ pIter->iEndofDoclist = iTermOff + nExtra; } pIter->iPgidxOff = iPgidx; fts5SegIterLoadRowid(p, pIter); fts5SegIterLoadNPos(p, pIter); } /* ** Initialize the object pIter to point to term pTerm/nTerm within segment ** pSeg. If there is no such term in the index, the iterator is set to EOF. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterSeekInit( Fts5Index *p, /* FTS5 backend */ Fts5Buffer *pBuf, /* Buffer to use for loading pages */ const u8 *pTerm, int nTerm, /* Term to seek to */ int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5StructureSegment *pSeg, /* Description of segment */ Fts5SegIter *pIter /* Object to populate */ ){ int iPg = 1; int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ static int nCall = 0; nCall++; assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ if( p->pIdxSelect==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( "SELECT pgno FROM '%q'.'%q_idx' WHERE " "segid=? AND term<=? ORDER BY term DESC LIMIT 1", pConfig->zDb, pConfig->zName )); } if( p->rc ) return; sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid); sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){ i64 val = sqlite3_column_int(p->pIdxSelect, 0); iPg = (int)(val>>1); bDlidx = (val & 0x0001); } p->rc = sqlite3_reset(p->pIdxSelect); if( iPg<pSeg->pgnoFirst ){ iPg = pSeg->pgnoFirst; bDlidx = 0; } pIter->iLeafPgno = iPg - 1; ................................................................................ fts5SegIterLoadDlidx(p, pIter); } if( flags & FTS5INDEX_QUERY_DESC ){ fts5SegIterReverse(p, pIter); } } } /* Either: ** ** 1) an error has occurred, or ** 2) the iterator points to EOF, or ** 3) the iterator points to an entry with term (pTerm/nTerm), or ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points ................................................................................ sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf==0 ) return; pLeaf->p = (u8*)pList; pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); pIter->iEndofDoclist = pLeaf->nn+1; if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; fts5SegIterReverseInitPage(p, pIter); }else{ fts5SegIterLoadNPos(p, pIter); } } } /* ** Zero the iterator passed as the only argument. */ static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); ................................................................................ /* ** This function is used as part of the big assert() procedure implemented by ** fts5AssertMultiIterSetup(). It ensures that the result currently stored ** in *pRes is the correct result of comparing the current positions of the ** two iterators. */ static void fts5AssertComparisonResult( Fts5IndexIter *pIter, Fts5SegIter *p1, Fts5SegIter *p2, Fts5CResult *pRes ){ int i1 = p1 - pIter->aSeg; int i2 = p2 - pIter->aSeg; ................................................................................ /* ** This function is a no-op unless SQLITE_DEBUG is defined when this module ** is compiled. In that case, this function is essentially an assert() ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int i; assert( (pFirst->pLeaf==0)==pIter->bEof ); /* Check that pIter->iSwitchRowid is set correctly. */ for(i=0; i<pIter->nSeg; i++){ Fts5SegIter *p1 = &pIter->aSeg[i]; assert( p1==pFirst || p1->pLeaf==0 || fts5BufferCompare(&pFirst->term, &p1->term) ................................................................................ ** Do the comparison necessary to populate pIter->aFirst[iOut]. ** ** If the returned value is non-zero, then it is the index of an entry ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing ** to a key that is a duplicate of another, higher priority, ** segment-iterator in the pSeg->aSeg[] array. */ static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){ int i1; /* Index of left-hand Fts5SegIter */ int i2; /* Index of right-hand Fts5SegIter */ int iRes; Fts5SegIter *p1; /* Left-hand Fts5SegIter */ Fts5SegIter *p2; /* Right-hand Fts5SegIter */ Fts5CResult *pRes = &pIter->aFirst[iOut]; ................................................................................ pIter->iLeafPgno = iLeafPgno+1; fts5SegIterReverseNewPage(p, pIter); bMove = 0; } } do{ if( bMove ) fts5SegIterNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; bMove = 1; }while( p->rc==SQLITE_OK ); } /* ** Free the iterator object passed as the second argument. */ static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){ if( pIter ){ int i; for(i=0; i<pIter->nSeg; i++){ fts5SegIterClear(&pIter->aSeg[i]); } fts5StructureRelease(pIter->pStruct); fts5BufferFree(&pIter->poslist); sqlite3_free(pIter); } } static void fts5MultiIterAdvanced( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */ int iChanged, /* Index of sub-iterator just advanced */ int iMinset /* Minimum entry in aFirst[] to set */ ){ int i; for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ int iEq; if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ fts5SegIterNext(p, &pIter->aSeg[iEq], 0); i = pIter->nSeg + iEq; } } } /* ** Sub-iterator iChanged of iterator pIter has just been advanced. It still ................................................................................ ** If it does so successfully, 0 is returned. Otherwise 1. ** ** If non-zero is returned, the caller should call fts5MultiIterAdvanced() ** on the iterator instead. That function does the same as this one, except ** that it deals with more complicated cases as well. */ static int fts5MultiIterAdvanceRowid( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5IndexIter *pIter, /* Iterator to update aFirst[] array for */ int iChanged /* Index of sub-iterator just advanced */ ){ Fts5SegIter *pNew = &pIter->aSeg[iChanged]; if( pNew->iRowid==pIter->iSwitchRowid || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev ){ int i; ................................................................................ pRes->iFirst = (u16)(pNew - pIter->aSeg); if( i==1 ) break; pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ]; } } return 0; } /* ** Set the pIter->bEof variable based on the state of the sub-iterators. */ static void fts5MultiIterSetEof(Fts5IndexIter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; pIter->bEof = pSeg->pLeaf==0; pIter->iSwitchRowid = pSeg->iRowid; } /* ** Move the iterator to the next entry. ** ** If an error occurs, an error code is left in Fts5Index.rc. It is not ** considered an error if the iterator reaches EOF, or if it is already at ** EOF when this function is called. */ static void fts5MultiIterNext( Fts5Index *p, Fts5IndexIter *pIter, int bFrom, /* True if argument iFrom is valid */ i64 iFrom /* Advance at least as far as this */ ){ if( p->rc==SQLITE_OK ){ int bUseFrom = bFrom; do { int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; assert( p->rc==SQLITE_OK ); if( bUseFrom && pSeg->pDlidx ){ fts5SegIterNextFrom(p, pSeg, iFrom); }else{ fts5SegIterNext(p, pSeg, &bNewTerm); } if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(p, pIter, iFirst) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); } fts5AssertMultiIterSetup(p, pIter); bUseFrom = 0; }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) ); } } static void fts5MultiIterNext2( Fts5Index *p, Fts5IndexIter *pIter, int *pbNewTerm /* OUT: True if *might* be new term */ ){ assert( pIter->bSkipEmpty ); if( p->rc==SQLITE_OK ){ do { int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; fts5SegIterNext(p, pSeg, &bNewTerm); if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(p, pIter, iFirst) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); *pbNewTerm = 1; }else{ *pbNewTerm = 0; } fts5AssertMultiIterSetup(p, pIter); }while( fts5MultiIterIsEmpty(p, pIter) ); } } static Fts5IndexIter *fts5MultiIterAlloc( Fts5Index *p, /* FTS5 backend to iterate within */ int nSeg ){ Fts5IndexIter *pNew; int nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2); pNew = fts5IdxMalloc(p, sizeof(Fts5IndexIter) + /* pNew */ sizeof(Fts5SegIter) * (nSlot-1) + /* pNew->aSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); if( pNew ){ pNew->nSeg = nSlot; pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; pNew->pIndex = p; } return pNew; } /* ** Allocate a new Fts5IndexIter object. ** ** The new object will be used to iterate through data in structure pStruct. ** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel ** is zero or greater, data from the first nSegment segments on level iLevel ** is merged. ** ** The iterator initially points to the first term/rowid entry in the ** iterated data. */ static void fts5MultiIterNew( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Structure *pStruct, /* Structure of specific index */ int bSkipEmpty, /* True to ignore delete-keys */ int flags, /* FTS5INDEX_QUERY_XXX flags */ const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ int iLevel, /* Level to iterate (-1 for all) */ int nSegment, /* Number of segments to merge (iLevel>=0) */ Fts5IndexIter **ppOut /* New object */ ){ int nSeg = 0; /* Number of segment-iters in use */ int iIter = 0; /* */ int iSeg; /* Used to iterate through segments */ Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */ Fts5StructureLevel *pLvl; Fts5IndexIter *pNew; assert( (pTerm==0 && nTerm==0) || iLevel<0 ); /* Allocate space for the new multi-seg-iterator. */ if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; nSeg += (p->pHash ? 1 : 0); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } } *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); if( pNew==0 ) return; pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); pNew->bSkipEmpty = (u8)bSkipEmpty; pNew->pStruct = pStruct; fts5StructureRef(pStruct); /* Initialize each of the component segment iterators. */ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; if( p->pHash ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); } for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){ for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; Fts5SegIter *pIter = &pNew->aSeg[iIter++]; if( pTerm==0 ){ fts5SegIterInit(p, pSeg, pIter); }else{ fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter); } } } }else{ pLvl = &pStruct->aLevel[iLevel]; for(iSeg=nSeg-1; iSeg>=0; iSeg--){ fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); } } assert( iIter==nSeg ); /* If the above was successful, each component iterators now points ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ for(iIter=pNew->nSeg-1; iIter>0; iIter--){ int iEq; if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ fts5SegIterNext(p, &pNew->aSeg[iEq], 0); fts5MultiIterAdvanced(p, pNew, iEq, iIter); } } fts5MultiIterSetEof(pNew); fts5AssertMultiIterSetup(p, pNew); if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ fts5MultiIterNext(p, pNew, 0, 0); } }else{ fts5MultiIterFree(p, pNew); *ppOut = 0; } fts5BufferFree(&buf); } /* ** Create an Fts5IndexIter that iterates through the doclist provided ** as the second argument. */ static void fts5MultiIterNew2( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Data *pData, /* Doclist to iterate through */ int bDesc, /* True for descending rowid order */ Fts5IndexIter **ppOut /* New object */ ){ Fts5IndexIter *pNew; pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; pNew->bFiltered = 1; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); pIter->iEndofDoclist = pData->nn; pNew->aFirst[1].iFirst = 1; if( bDesc ){ pNew->bRev = 1; pIter->flags |= FTS5_SEGITER_REVERSE; fts5SegIterReverseInitPage(p, pIter); }else{ fts5SegIterLoadNPos(p, pIter); } pData = 0; }else{ pNew->bEof = 1; } *ppOut = pNew; } fts5DataRelease(pData); } /* ** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){ assert( p->rc || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof ); return (p->rc || pIter->bEof); } /* ** Return the rowid of the entry that the iterator currently points ** to. If the iterator points to EOF when this function is called the ** results are undefined. */ static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){ assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; } /* ** Move the iterator to the next entry at or following iMatch. */ static void fts5MultiIterNextFrom( Fts5Index *p, Fts5IndexIter *pIter, i64 iMatch ){ while( 1 ){ i64 iRowid; fts5MultiIterNext(p, pIter, 1, iMatch); if( fts5MultiIterEof(p, pIter) ) break; iRowid = fts5MultiIterRowid(pIter); if( pIter->bRev==0 && iRowid>=iMatch ) break; if( pIter->bRev!=0 && iRowid<=iMatch ) break; } } /* ** Return a pointer to a buffer containing the term associated with the ** entry that the iterator currently points to. */ static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){ Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; *pn = p->term.n; return p->term.p; } static void fts5ChunkIterate( Fts5Index *p, /* Index object */ Fts5SegIter *pSeg, /* Poslist of this iterator */ void *pCtx, /* Context pointer for xChunk callback */ void (*xChunk)(Fts5Index*, void*, const u8*, int) ................................................................................ ){ int nRem = pSeg->nPos; /* Number of bytes still to come */ Fts5Data *pData = 0; u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); int pgno = pSeg->iLeafPgno; int pgnoSave = 0; if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ pgnoSave = pgno+1; } while( 1 ){ xChunk(p, pCtx, pChunk, nChunk); ................................................................................ pSeg->pNextLeaf = pData; pData = 0; } } } } /* ** Allocate a new segment-id for the structure pStruct. The new segment ** id must be between 1 and 65335 inclusive, and must not be used by ** any currently existing segment. If a free segment id cannot be found, ** SQLITE_FULL is returned. ** ................................................................................ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ int iSegid = 0; if( p->rc==SQLITE_OK ){ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ p->rc = SQLITE_FULL; }else{ while( iSegid==0 ){ int iLvl, iSeg; sqlite3_randomness(sizeof(u32), (void*)&iSegid); iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ iSegid = 0; } } } } } } return iSegid; } /* ................................................................................ if( p->pHash ){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; } } /* ** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares ** with buffer (nOld/pOld). */ static int fts5PrefixCompress( int nOld, const u8 *pOld, int nNew, const u8 *pNew ){ int i; assert( fts5BlobCompare(pOld, nOld, pNew, nNew)<0 ); for(i=0; i<nOld; i++){ if( pOld[i]!=pNew[i] ) break; } return i; } static void fts5WriteDlidxClear( ................................................................................ } } static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); ................................................................................ ** Usually, the previous term is available in pPage->term. The exception ** is if this is the first term written in an incremental-merge step. ** In this case the previous term is not available, so just write a ** copy of (pTerm/nTerm) into the parent node. This is slightly ** inefficient, but still correct. */ int n = nTerm; if( pPage->term.n ){ n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); } fts5WriteBtreeTerm(p, pWriter, n, pTerm); pPage = &pWriter->writer; } }else{ nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); } /* Append the number of bytes of new data, then the term data itself ** to the page. */ fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix); fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]); ................................................................................ /* ** Append a rowid and position-list size field to the writers output. */ static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, i64 iRowid, int nPos ){ if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ fts5WriteFlushLeaf(p, pWriter); } ................................................................................ }else{ assert( p->rc || iRowid>pWriter->iPrevRowid ); fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; pWriter->bFirstRowidInPage = 0; fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos); } } static void fts5WriteAppendPoslistData( Fts5Index *p, Fts5SegWriter *pWriter, const u8 *aData, ................................................................................ Fts5PageWriter *pLeaf = &pWriter->writer; if( p->rc==SQLITE_OK ){ assert( pLeaf->pgno>=1 ); if( pLeaf->buf.n>4 ){ fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; fts5WriteFlushBtree(p, pWriter); } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); fts5BufferFree(&pLeaf->pgidx); fts5BufferFree(&pWriter->btterm); for(i=0; i<pWriter->nDlidx; i++){ sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); ................................................................................ } /* ** Iterator pIter was used to iterate through the input segments of on an ** incremental merge operation. This function is called if the incremental ** merge step has finished but the input has not been completely exhausted. */ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ int i; Fts5Buffer buf; memset(&buf, 0, sizeof(Fts5Buffer)); for(i=0; i<pIter->nSeg; i++){ Fts5SegIter *pSeg = &pIter->aSeg[i]; if( pSeg->pSeg==0 ){ /* no-op */ ................................................................................ Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ int iLvl, /* Level to read input from */ int *pnRem /* Write up to this many output leaves */ ){ Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; Fts5StructureLevel *pLvlOut; Fts5IndexIter *pIter = 0; /* Iterator to read input data */ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ assert( iLvl<pStruct->nLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); memset(&writer, 0, sizeof(Fts5SegWriter)); memset(&term, 0, sizeof(Fts5Buffer)); if( pLvl->nMerge ){ ................................................................................ /* Read input from all segments in the input level */ nInput = pLvl->nSeg; } bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); assert( iLvl>=0 ); for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int nPos; /* position-list size field value */ int nTerm; const u8 *pTerm; ................................................................................ /* This is a new term. Append a term to the output segment. */ fts5WriteAppendTerm(p, &writer, nTerm, pTerm); fts5BufferSet(&p->rc, &term, nTerm, pTerm); } /* Append the rowid to the output */ /* WRITEPOSLISTSIZE */ nPos = pSegIter->nPos*2 + pSegIter->bDel; fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos); /* Append the position-list data to the output */ fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); } /* Flush the last leaf page to disk. Set the output segment b-tree height ** and last leaf page number at the same time. */ fts5WriteFinish(p, &writer, &pSeg->pgnoLast); if( fts5MultiIterEof(p, pIter) ){ int i; ................................................................................ } }else{ assert( pSeg->pgnoLast>0 ); fts5TrimSegments(p, pIter); pLvl->nMerge = nInput; } fts5MultiIterFree(p, pIter); fts5BufferFree(&term); if( pnRem ) *pnRem -= writer.nLeafWritten; } /* ** Do up to nPg pages of automerge work on the index. */ static void fts5IndexMerge( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ int nPg /* Pages of work to do */ ){ int nRem = nPg; Fts5Structure *pStruct = *ppStruct; while( nRem>0 && p->rc==SQLITE_OK ){ int iLvl; /* To iterate through levels */ int iBestLvl = 0; /* Level offering the most input segments */ int nBest = 0; /* Number of input segments on best level */ /* Set iBestLvl to the level to read input segments from. */ ................................................................................ /* If nBest is still 0, then the index must be empty. */ #ifdef SQLITE_DEBUG for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){ assert( pStruct->aLevel[iLvl].nSeg==0 ); } #endif if( nBest<p->pConfig->nAutomerge && pStruct->aLevel[iBestLvl].nMerge==0 ){ break; } fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } } *ppStruct = pStruct; } /* ** A total of nLeaf leaf pages of data has just been flushed to a level-0 ** segment. This function updates the write-counter accordingly and, if ** necessary, performs incremental merge work. ** ................................................................................ /* Update the write-counter. While doing so, set nWork. */ nWrite = pStruct->nWriteCounter; nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit)); pStruct->nWriteCounter += nLeaf; nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); fts5IndexMerge(p, ppStruct, nRem); } } static void fts5IndexCrisismerge( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ ){ ................................................................................ int iSegid; int pgnoLast = 0; /* Last leaf page number in segment */ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); if( iSegid ){ const int pgsz = p->pConfig->pgsz; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); ................................................................................ i64 iDelta = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOff<nDoclist ){ int nPos; int nCopy; int bDummy; iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta); nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); nCopy += nPos; iRowid += iDelta; if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); }else{ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); } assert( pBuf->n<=pBuf->nSpace ); if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy ** it in one go. */ fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ /* The entire poslist will not fit on this leaf. So it needs ** to be broken into sections. The only qualification being ................................................................................ } if( iPos>=nCopy ) break; } } iOff += nCopy; } } /* TODO2: Doclist terminator written here. */ /* pBuf->p[pBuf->n++] = '\0'; */ assert( pBuf->n<=pBuf->nSpace ); sqlite3Fts5HashScanNext(pHash); } sqlite3Fts5HashClear(pHash); ................................................................................ if( p->nPendingData ){ assert( p->pHash ); p->nPendingData = 0; fts5FlushOneHash(p); } } int sqlite3Fts5IndexOptimize(Fts5Index *p){ Fts5Structure *pStruct; Fts5Structure *pNew = 0; int nSeg = 0; assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); pStruct = fts5StructureRead(p); if( pStruct ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; if( nSeg>1 ){ int nByte = sizeof(Fts5Structure); nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); } } if( pNew ){ Fts5StructureLevel *pLvl; int nByte = nSeg * sizeof(Fts5StructureSegment); pNew->nLevel = pStruct->nLevel+1; pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; pLvl = &pNew->aLevel[pStruct->nLevel]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ int iLvl, iSeg; int iSegOut = 0; for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; iSegOut++; } } pNew->nSegment = pLvl->nSeg = nSeg; }else{ sqlite3_free(pNew); pNew = 0; } } if( pNew ){ int iLvl = pNew->nLevel-1; while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ int nRem = FTS5_OPT_WORK_UNIT; fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); } fts5StructureWrite(p, pNew); fts5StructureRelease(pNew); } fts5StructureRelease(pStruct); return fts5IndexReturn(p); } int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pStruct; pStruct = fts5StructureRead(p); if( pStruct && pStruct->nLevel ){ fts5IndexMerge(p, &pStruct, nMerge); fts5StructureWrite(p, pStruct); } fts5StructureRelease(pStruct); return fts5IndexReturn(p); } static void fts5PoslistCallback( Fts5Index *p, void *pContext, const u8 *pChunk, int nChunk ){ assert_nc( nChunk>=0 ); if( nChunk>0 ){ fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); } } typedef struct PoslistCallbackCtx PoslistCallbackCtx; struct PoslistCallbackCtx { Fts5Buffer *pBuf; /* Append to this buffer */ Fts5Colset *pColset; /* Restrict matches to this column */ int eState; /* See above */ }; /* ** TODO: Make this more efficient! */ static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ int i; for(i=0; i<pColset->nCol; i++){ if( pColset->aiCol[i]==iCol ) return 1; } return 0; } static void fts5PoslistFilterCallback( Fts5Index *p, void *pContext, const u8 *pChunk, int nChunk ){ PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; assert_nc( nChunk>=0 ); if( nChunk>0 ){ /* Search through to find the first varint with value 1. This is the ** start of the next columns hits. */ int i = 0; int iStart = 0; if( pCtx->eState==2 ){ int iCol; fts5FastGetVarint32(pChunk, i, iCol); if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ pCtx->eState = 1; fts5BufferSafeAppendVarint(pCtx->pBuf, 1); }else{ pCtx->eState = 0; } } do { while( i<nChunk && pChunk[i]!=0x01 ){ while( pChunk[i] & 0x80 ) i++; i++; } if( pCtx->eState ){ fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); } if( i<nChunk ){ int iCol; iStart = i; i++; if( i>=nChunk ){ pCtx->eState = 2; }else{ fts5FastGetVarint32(pChunk, i, iCol); pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); if( pCtx->eState ){ fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); iStart = i; } } } }while( i<nChunk ); } } /* ** Iterator pIter currently points to a valid entry (not EOF). This ** function appends the position list data for the current entry to ** buffer pBuf. It does not make a copy of the position-list size ** field. */ static void fts5SegiterPoslist( Fts5Index *p, Fts5SegIter *pSeg, Fts5Colset *pColset, Fts5Buffer *pBuf ){ if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ PoslistCallbackCtx sCtx; sCtx.pBuf = pBuf; sCtx.pColset = pColset; sCtx.eState = fts5IndexColsetTest(pColset, 0); assert( sCtx.eState==0 || sCtx.eState==1 ); fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); } } } /* ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ** the position list contains entries for column iCol, then (*pa) is set ** to point to the sub-position-list for that column and the number of ** bytes in it returned. Or, if the argument position list does not ** contain any entries for column iCol, return 0. */ static int fts5IndexExtractCol( const u8 **pa, /* IN/OUT: Pointer to poslist */ int n, /* IN: Size of poslist in bytes */ int iCol /* Column to extract from poslist */ ){ int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ const u8 *p = *pa; const u8 *pEnd = &p[n]; /* One byte past end of position list */ u8 prev = 0; while( iCol>iCurrent ){ /* Advance pointer p until it points to pEnd or an 0x01 byte that is ** not part of a varint */ while( (prev & 0x80) || *p!=0x01 ){ prev = *p++; if( p==pEnd ) return 0; } *pa = p++; p += fts5GetVarint32(p, iCurrent); } if( iCol!=iCurrent ) return 0; /* Advance pointer p until it points to pEnd or an 0x01 byte that is ** not part of a varint */ assert( (prev & 0x80)==0 ); while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){ prev = *p++; } return p - (*pa); } /* ** Iterator pMulti currently points to a valid entry (not EOF). This ** function appends the following to buffer pBuf: ** ** * The varint iDelta, and ** * the position list that currently points to, including the size field. ** ** If argument pColset is NULL, then the position list is filtered according ** to pColset before being appended to the buffer. If this means there are ** no entries in the position list, nothing is appended to the buffer (not ** even iDelta). ** ** If an error occurs, an error code is left in p->rc. */ static int fts5AppendPoslist( Fts5Index *p, i64 iDelta, Fts5IndexIter *pMulti, Fts5Colset *pColset, Fts5Buffer *pBuf ){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && (pColset==0 || pColset->nCol==1) ){ const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; int nPos; if( pColset ){ nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); if( nPos==0 ) return 1; }else{ nPos = pSeg->nPos; } assert( nPos>0 ); fts5BufferSafeAppendVarint(pBuf, iDelta); fts5BufferSafeAppendVarint(pBuf, nPos*2); fts5BufferSafeAppendBlob(pBuf, pPos, nPos); }else{ int iSv1; int iSv2; int iData; /* Append iDelta */ iSv1 = pBuf->n; fts5BufferSafeAppendVarint(pBuf, iDelta); /* WRITEPOSLISTSIZE */ iSv2 = pBuf->n; fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); iData = pBuf->n; fts5SegiterPoslist(p, pSeg, pColset, pBuf); if( pColset ){ int nActual = pBuf->n - iData; if( nActual!=pSeg->nPos ){ if( nActual==0 ){ pBuf->n = iSv1; return 1; }else{ int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); } } } } } } return 0; } static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; assert( pIter->aPoslist ); if( p>=pIter->aEof ){ pIter->aPoslist = 0; ................................................................................ #endif #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ (iLastRowid) = (iRowid); \ } /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. ** ** If an error occurs, an error code is left in p->rc. If an error has ................................................................................ Fts5Buffer *p1, /* First list to merge */ Fts5Buffer *p2 /* Second list to merge */ ){ if( p2->n ){ i64 iLastRowid = 0; Fts5DoclistIter i1; Fts5DoclistIter i2; Fts5Buffer out; Fts5Buffer tmp; memset(&out, 0, sizeof(out)); memset(&tmp, 0, sizeof(tmp)); sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){ if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){ /* Copy entry from i1 */ fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize); fts5DoclistIterNext(&i1); } else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){ /* Copy entry from i2 */ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize); fts5DoclistIterNext(&i2); } else{ i64 iPos1 = 0; i64 iPos2 = 0; int iOff1 = 0; int iOff2 = 0; u8 *a1 = &i1.aPoslist[i1.nSize]; u8 *a2 = &i2.aPoslist[i2.nSize]; Fts5PoslistWriter writer; memset(&writer, 0, sizeof(writer)); /* Merge the two position lists. */ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferZero(&tmp); sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){ i64 iNew; if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){ iNew = iPos1; sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); }else{ iNew = iPos2; sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); if( iPos1==iPos2 ){ sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1); } } p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); } /* WRITEPOSLISTSIZE */ fts5BufferSafeAppendVarint(&out, tmp.n * 2); fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); fts5DoclistIterNext(&i1); fts5DoclistIterNext(&i2); } } fts5BufferSet(&p->rc, p1, out.n, out.p); fts5BufferFree(&tmp); fts5BufferFree(&out); } } static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ Fts5Buffer tmp = *p1; *p1 = *p2; *p2 = tmp; } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ Fts5IndexIter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ const int flags = FTS5INDEX_QUERY_SCAN; int i; i64 iLastRowid = 0; Fts5IndexIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; Fts5Buffer doclist; int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) ){ i64 iRowid = fts5MultiIterRowid(p1); int nTerm; const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break; } if( doclist.n>0 && iRowid<=iLastRowid ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ assert( i<nBuf ); if( aBuf[i].n==0 ){ fts5BufferSwap(&doclist, &aBuf[i]); fts5BufferZero(&doclist); }else{ fts5MergePrefixLists(p, &doclist, &aBuf[i]); fts5BufferZero(&aBuf[i]); } } iLastRowid = 0; } if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ iLastRowid = iRowid; } } for(i=0; i<nBuf; i++){ if( p->rc==SQLITE_OK ){ fts5MergePrefixLists(p, &doclist, &aBuf[i]); } fts5BufferFree(&aBuf[i]); } fts5MultiIterFree(p, p1); pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); if( pData ){ pData->p = (u8*)&pData[1]; pData->nn = pData->szLeaf = doclist.n; memcpy(pData->p, doclist.p, doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); ................................................................................ ** to the document with rowid iRowid. */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); } /* Flush the hash table to disk if required */ if( iRowid<p->iWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) || (p->nPendingData > p->pConfig->nHashSize) ){ ................................................................................ ** to the database. Additionally, assume that the contents of the %_data ** table may have changed on disk. So any in-memory caches of %_data ** records must be invalidated. */ int sqlite3Fts5IndexRollback(Fts5Index *p){ fts5CloseReader(p); fts5IndexDiscardData(p); assert( p->rc==SQLITE_OK ); return SQLITE_OK; } /* ** The %_data table is completely empty when this function is called. This ** function populates it with the initial structure objects for each index, ** and the initial version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; memset(&s, 0, sizeof(Fts5Structure)); fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); return fts5IndexReturn(p); } /* ................................................................................ /* ** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen(). */ int sqlite3Fts5IndexClose(Fts5Index *p){ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); } return rc; } /* ** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ static int fts5IndexCharlenToBytelen(const char *p, int nByte, int nChar){ int n = 0; int i; for(i=0; i<nChar; i++){ if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */ if( (unsigned char)p[n++]>=0xc0 ){ while( (p[n] & 0xc0)==0x80 ) n++; } ................................................................................ /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); if( nByte ){ rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, nByte ); } } ................................................................................ Fts5Index *p, /* FTS index to query */ const char *pToken, int nToken, /* Token (or prefix) to query for */ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ Fts5Colset *pColset, /* Match these columns only */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5IndexIter *pRet = 0; int iIdx = 0; Fts5Buffer buf = {0, 0, 0}; /* If the QUERY_SCAN flag is set, all other flags must be clear. */ assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); #ifdef SQLITE_DEBUG /* If the QUERY_TEST_NOIDX flag was specified, then this must be a ** prefix-query. Instead of using a prefix-index (if one exists), ** evaluate the prefix query using the main FTS index. This is used ** for internal sanity checking by the integrity-check in debug ** mode only. */ if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; }else #endif if( flags & FTS5INDEX_QUERY_PREFIX ){ int nChar = fts5IndexCharlen(pToken, nToken); for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ if( pConfig->aPrefix[iIdx-1]==nChar ) break; } } if( iIdx<=pConfig->nPrefix ){ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); if( pStruct ){ fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet); fts5StructureRelease(pStruct); } }else{ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); } if( p->rc ){ sqlite3Fts5IterClose(pRet); pRet = 0; fts5CloseReader(p); } *ppIter = pRet; sqlite3Fts5BufferFree(&buf); } return fts5IndexReturn(p); } /* ** Return true if the iterator passed as the only argument is at EOF. */ int sqlite3Fts5IterEof(Fts5IndexIter *pIter){ assert( pIter->pIndex->rc==SQLITE_OK ); return pIter->bEof; } /* ** Move to the next matching rowid. */ int sqlite3Fts5IterNext(Fts5IndexIter *pIter){ assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); return fts5IndexReturn(pIter->pIndex); } /* ** Move to the next matching term/rowid. Used by the fts5vocab module. */ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){ Fts5Index *p = pIter->pIndex; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(p, pIter, 0, 0); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ fts5DataRelease(pSeg->pLeaf); pSeg->pLeaf = 0; pIter->bEof = 1; } } return fts5IndexReturn(pIter->pIndex); } /* ** Move to the next matching rowid that occurs at or after iMatch. The ** definition of "at or after" depends on whether this iterator iterates ** in ascending or descending rowid order. */ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){ fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); return fts5IndexReturn(pIter->pIndex); } /* ** Return the current rowid. */ i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){ return fts5MultiIterRowid(pIter); } /* ** Return the current term. */ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){ int n; const char *z = (const char*)fts5MultiIterTerm(pIter, &n); *pn = n-1; return &z[1]; } static int fts5IndexExtractColset ( Fts5Colset *pColset, /* Colset to filter on */ const u8 *pPos, int nPos, /* Position list */ Fts5Buffer *pBuf /* Output buffer */ ){ int rc = SQLITE_OK; int i; fts5BufferZero(pBuf); for(i=0; i<pColset->nCol; i++){ const u8 *pSub = pPos; int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); if( nSub ){ fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); } } return rc; } /* ** Return a pointer to a buffer containing a copy of the position list for ** the current entry. Output variable *pn is set to the size of the buffer ** in bytes before returning. ** ** The returned position list does not include the "number of bytes" varint ** field that starts the position list on disk. */ int sqlite3Fts5IterPoslist( Fts5IndexIter *pIter, Fts5Colset *pColset, /* Column filter (or NULL) */ const u8 **pp, /* OUT: Pointer to position-list data */ int *pn, /* OUT: Size of position-list in bytes */ i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; if( pColset==0 || pIter->bFiltered ){ *pn = pSeg->nPos; *pp = pPos; }else if( pColset->nCol==1 ){ *pp = pPos; *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]); }else{ fts5BufferZero(&pIter->poslist); fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist); *pp = pIter->poslist.p; *pn = pIter->poslist.n; } }else{ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); *pp = pIter->poslist.p; *pn = pIter->poslist.n; } return fts5IndexReturn(pIter->pIndex); } /* ** This function is similar to sqlite3Fts5IterPoslist(), except that it ** copies the position list into the buffer supplied as the second ** argument. */ int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){ Fts5Index *p = pIter->pIndex; Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( p->rc==SQLITE_OK ); fts5BufferZero(pBuf); fts5SegiterPoslist(p, pSeg, 0, pBuf); return fts5IndexReturn(p); } /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ if( pIter ){ Fts5Index *pIndex = pIter->pIndex; fts5MultiIterFree(pIter->pIndex, pIter); fts5CloseReader(pIndex); } } /* ** Read and decode the "averages" record from the database. ** ................................................................................ ** Below this point is the implementation of the integrity-check ** functionality. */ /* ** Return a simple checksum value based on the arguments. */ static u64 fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, int iIdx, const char *pTerm, int nTerm ){ ................................................................................ Fts5Index *p, /* Fts5 index object */ int iIdx, const char *z, /* Index key to query for */ int n, /* Size of index key in bytes */ int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ i64 dummy; const u8 *pPos; int nPos; i64 rowid = sqlite3Fts5IterRowid(pIdxIter); rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); if( rc==SQLITE_OK ){ Fts5PoslistReader sReader; for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); } rc = sqlite3Fts5IterNext(pIdxIter); } } sqlite3Fts5IterClose(pIdxIter); *pCksum = cksum; return rc; } /* ................................................................................ #endif } /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums ** as calculated by fts5IndexEntryCksum() is cksum. ** ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the ** checksum does not match. Return SQLITE_OK if all checks pass without ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5IndexIter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ #ifdef SQLITE_DEBUG /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ #endif /* Load the FTS index structure */ pStruct = fts5StructureRead(p); /* Check that the internal nodes of each segment match the leaves */ if( pStruct ){ int iLvl, iSeg; ................................................................................ ** variable cksum2) based on entries extracted from the full-text index ** while doing a linear scan of each individual index in turn. ** ** As each term visited by the linear scans, a separate query for the ** same term is performed. cksum3 is calculated based on the entries ** extracted by these queries. */ for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ int n; /* Size of term in bytes */ i64 iPos = 0; /* Position read from poslist */ int iOff = 0; /* Offset within poslist */ i64 iRowid = fts5MultiIterRowid(pIter); char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); poslist.n = 0; fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist); while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); } } fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); fts5MultiIterFree(p, pIter); if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; fts5StructureRelease(pStruct); #ifdef SQLITE_DEBUG fts5BufferFree(&term); #endif fts5BufferFree(&poslist); return fts5IndexReturn(p); } /* ** Calculate and return a checksum that is the XOR of the index entry ** checksum of all entries that would be generated by the token specified ** by the final 5 arguments. */ u64 sqlite3Fts5IndexCksum( Fts5Config *pConfig, /* Configuration object */ i64 iRowid, /* Document term appears in */ int iCol, /* Column term appears in */ int iPos, /* Position term appears in */ const char *pTerm, int nTerm /* Term at iPos */ ){ u64 ret = 0; /* Return value */ int iIdx; /* For iterating through indexes */ ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm); for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){ int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]); if( nByte ){ ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte); } } return ret; } /************************************************************************* ************************************************************************** ** Below this point is the implementation of the fts5_decode() scalar ** function only. */ /* ................................................................................ iDocid += iDelta; sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); } } return iOff; } /* ** The implementation of user-defined scalar function fts5_decode(). */ static void fts5DecodeFunction( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args (always 2) */ ................................................................................ i64 iRowid; /* Rowid for record being decoded */ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ int rc = SQLITE_OK; /* Return code */ int nSpace = 0; assert( nArg==2 ); memset(&s, 0, sizeof(Fts5Buffer)); iRowid = sqlite3_value_int64(apVal[0]); /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[] ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents ** buffer overreads even if the record is corrupt. */ n = sqlite3_value_bytes(apVal[1]); ................................................................................ } }else if( iSegid==0 ){ if( iRowid==FTS5_AVERAGES_ROWID ){ fts5DecodeAverages(&rc, &s, a, n); }else{ fts5DecodeStructure(&rc, &s, a, n); } }else{ Fts5Buffer term; /* Current term read from page */ int szLeaf; /* Offset of pgidx in a[] */ int iPgidxOff; int iPgidxPrev = 0; /* Previous value read from pgidx */ int iTermOff = 0; int iRowidOff = 0; ................................................................................ ** If successful, SQLITE_OK is returned. If an error occurs, some other ** SQLite error code is returned instead. */ int sqlite3Fts5IndexInit(sqlite3 *db){ int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } return rc; } |
> > > > > > > > | < | > > > > > > < < < < < < < < < < < < < > | > > > > > > > > | > > < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < < < < < < < > > | < > | | < > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > | | < < > > > > > > > > > > > > > > > > < | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | < | | > > | | | < < | < > < < | | > > | | | < < | | | < > > > | > | | < | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | > | | > > > > | > | > > > | < > > > > > > > | < < < < < > > > > > > > > > > > > > > > > < | < < | < < < < < < < | | | | | > > | > > | | | | | | | | > > | | | > | | | < | < > | | > > | > | > > > > | | < | > | | > > | > | | | > < < < < < < < < < < < | < < < < < < < < > > > < < < < < < > > > > | < | < < < < < < < < | > > > > > | < < < < < < < < < < < < < > > > > > > | < < < < < < < < | > > > > > > > | > | < < < < | < < < < < < < < < > > > > > > > > > > > | < < < > > > > > > < < | < < | < < < | < < < < < < < | < < < < > > > < < < < > > > > > > > > < < < < < < < < < < < > > > > > > < < | < < | | < | > > > > | < > > | > > > > > > > > > > > > | < < < < < < < < < | | < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > | < < > > > | | | > > > > > > > > > | | > > > > | | > | > > > > > > > > > > > | | > > > | < < < < > > > | | | < < < > | | > | | > > | < | > > > > > > > > | > > | | > | > > | | > > < | < > > | > | < < < < < > > > > > > > > > > > > > > > > > > < > | | > > | > < < > > > > > > > > > > > > > > | > | < < < < < > > > | | | < | | > > > > > > > > > > > > > > > > > > > > > | > < > > > > | > > > > > > > > > | < | | | | > | | | | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < > > | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < < < < > | | < < < < < < < < < < < < < | | | < < < < | < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | < < | < > > | > | > > > < > > > < < | | | | | | > > > > | > > > | > > > > > > < > > > > > > > > > > > > > > > > > > > > > < < < < < < | > > > > > > > > > > | > > | | > > | | | > > > > | | | | < | | | > | > > > | > > > > > | | < > < > > > > > | > > > | > > > > > > > > | > | < < < < < | > | > | | > < < < < < < < | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | > | | > | | | < | < > | | < > | | > > | | | > | > | > > > > > | > | < < < < < < < < < < < < < < < < < < < < < < < < < < < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 ... 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 ... 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 ... 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 ... 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 ... 607 608 609 610 611 612 613 614 615 616 617 618 619 620 ... 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 ... 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 ... 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 .... 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 .... 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 .... 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 .... 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 .... 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 .... 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 .... 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 .... 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 .... 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 .... 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 .... 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 .... 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 .... 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 .... 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 .... 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 .... 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 .... 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 .... 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 .... 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 .... 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 .... 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 .... 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 .... 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 .... 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 .... 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 .... 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 .... 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 .... 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 .... 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 .... 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 .... 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 .... 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 .... 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 .... 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 .... 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 .... 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 .... 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 .... 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 .... 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 .... 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 .... 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 .... 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 .... 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 .... 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 .... 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 .... 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 .... 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 .... 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 .... 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 .... 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 .... 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 .... 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 .... 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 .... 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 .... 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 |
#define FTS5_DATA_ZERO_PADDING 8 #define FTS5_DATA_PADDING 20 typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; typedef struct Fts5Iter Fts5Iter; typedef struct Fts5PageWriter Fts5PageWriter; typedef struct Fts5SegIter Fts5SegIter; typedef struct Fts5DoclistIter Fts5DoclistIter; typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; ................................................................................ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ sqlite3_stmt *pDataVersion; i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ }; struct Fts5DoclistIter { u8 *aEof; /* Pointer to 1 byte past end of doclist */ /* Output variables. aPoslist==0 at EOF */ i64 iRowid; ................................................................................ Fts5StructureSegment *pSeg; /* Segment to iterate through */ int flags; /* Mask of configuration flags */ int iLeafPgno; /* Current leaf page number */ Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ int iLeafOffset; /* Byte offset within current leaf */ /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); /* The page and offset from which the current term was read. The offset ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; int iTermLeafOffset; int iPgidxOff; /* Next offset in pgidx */ int iEndofDoclist; ................................................................................ Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ /* Variables populated based on current entry. */ Fts5Buffer term; /* Current term */ i64 iRowid; /* Current rowid */ int nPos; /* Number of bytes in current position list */ u8 bDel; /* True if the delete flag is set */ }; /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. */ #define ASSERT_SZLEAF_OK(x) assert( \ (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ ) #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 /* ** Argument is a pointer to an Fts5Data structure that contains a leaf ** page. This macro evaluates to true if the leaf contains no terms, or ** false if it contains at least one term. */ #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) ................................................................................ ** aFirst[1] contains the index in aSeg[] of the iterator that points to ** the smallest key overall. aFirst[0] is unused. ** ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. */ struct Fts5Iter { Fts5IndexIter base; /* Base class containing output vars */ Fts5Index *pIndex; /* Index that owns this iterator */ Fts5Structure *pStruct; /* Database structure for this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ Fts5Colset *pColset; /* Restrict matches to these columns */ /* Invoked to set output variables. */ void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*); int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ u8 bSkipEmpty; /* True to skip deleted entries */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; ................................................................................ */ static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ int nCmp = MIN(pLeft->n, pRight->n); int res = memcmp(pLeft->p, pRight->p, nCmp); return (res==0 ? (pLeft->n - pRight->n) : res); } static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ int ret; fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); return ret; } /* ................................................................................ p->rc = rc; p->nRead++; } assert( (pRet==0)==(p->rc!=SQLITE_OK) ); return pRet; } /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). */ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); ................................................................................ pRet->nRef = 1; pRet->nLevel = nLevel; pRet->nSegment = nSegment; i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter); for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){ Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl]; int nTotal = 0; int iSeg; if( i>=nData ){ rc = FTS5_CORRUPT; }else{ i += fts5GetVarint32(&pData[i], pLvl->nMerge); i += fts5GetVarint32(&pData[i], nTotal); assert( nTotal>=pLvl->nMerge ); pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, nTotal * sizeof(Fts5StructureSegment) ); } if( rc==SQLITE_OK ){ pLvl->nSeg = nTotal; for(iSeg=0; iSeg<nTotal; iSeg++){ if( i>=nData ){ rc = FTS5_CORRUPT; break; } i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); } } } if( rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; } } *ppOut = pRet; return rc; } /* ** ................................................................................ } pLvl->aSeg = aNew; }else{ *pRc = SQLITE_NOMEM; } } } static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ Fts5Structure *pRet = 0; Fts5Config *pConfig = p->pConfig; int iCookie; /* Configuration cookie */ Fts5Data *pData; pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); if( p->rc==SQLITE_OK ){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; } } return pRet; } static i64 fts5IndexDataVersion(Fts5Index *p){ i64 iVersion = 0; if( p->rc==SQLITE_OK ){ if( p->pDataVersion==0 ){ p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) ); if( p->rc ) return 0; } if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ iVersion = sqlite3_column_int64(p->pDataVersion, 0); } p->rc = sqlite3_reset(p->pDataVersion); } return iVersion; } /* ** Read, deserialize and return the structure record. ** ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array ** are over-allocated as described for function fts5StructureDecode() ** above. ** ** If an error occurs, NULL is returned and an error code left in the ** Fts5Index handle. If an error has already occurred when this function ** is called, it is a no-op. */ static Fts5Structure *fts5StructureRead(Fts5Index *p){ if( p->pStruct==0 ){ p->iStructVersion = fts5IndexDataVersion(p); if( p->rc==SQLITE_OK ){ p->pStruct = fts5StructureReadUncached(p); } } #if 0 else{ Fts5Structure *pTest = fts5StructureReadUncached(p); if( pTest ){ int i, j; assert_nc( p->pStruct->nSegment==pTest->nSegment ); assert_nc( p->pStruct->nLevel==pTest->nLevel ); for(i=0; i<pTest->nLevel; i++){ assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); for(j=0; j<pTest->aLevel[i].nSeg; j++){ Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; assert_nc( p1->iSegid==p2->iSegid ); assert_nc( p1->pgnoFirst==p2->pgnoFirst ); assert_nc( p1->pgnoLast==p2->pgnoLast ); } } fts5StructureRelease(pTest); } } #endif if( p->rc!=SQLITE_OK ) return 0; assert( p->iStructVersion!=0 ); assert( p->pStruct!=0 ); fts5StructureRef(p->pStruct); return p->pStruct; } static void fts5StructureInvalidate(Fts5Index *p){ if( p->pStruct ){ fts5StructureRelease(p->pStruct); p->pStruct = 0; } } /* ** Return the total number of segments in index structure pStruct. This ** function is only ever used as part of assert() conditions. */ #ifdef SQLITE_DEBUG ................................................................................ ** ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ ASSERT_SZLEAF_OK(pIter->pLeaf); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); pIter->bDel = 0; pIter->nPos = 1; if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ pIter->bDel = 1; iOff++; if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ pIter->nPos = 1; iOff++; }else{ pIter->nPos = 0; } } }else{ int nSz; fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; assert_nc( pIter->nPos>=0 ); } pIter->iLeafOffset = iOff; } } static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); ................................................................................ */ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; /* Offset to read at */ int nNew; /* Bytes of new data */ iOff += fts5GetVarint32(&a[iOff], nNew); if( iOff+nNew>pIter->pLeaf->nn ){ p->rc = FTS5_CORRUPT; return; } pIter->term.n = nKeep; fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); iOff += nNew; pIter->iTermLeafOffset = iOff; pIter->iTermLeafPgno = pIter->iLeafPgno; pIter->iLeafOffset = iOff; ................................................................................ int nExtra; pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); pIter->iEndofDoclist += nExtra; } fts5SegIterLoadRowid(p, pIter); } static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ if( pIter->flags & FTS5_SEGITER_REVERSE ){ pIter->xNext = fts5SegIterNext_Reverse; }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ pIter->xNext = fts5SegIterNext_None; }else{ pIter->xNext = fts5SegIterNext; } } /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when ** this function returns. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ................................................................................ ** at EOF already. */ assert( pIter->pLeaf==0 ); return; } if( p->rc==SQLITE_OK ){ memset(pIter, 0, sizeof(*pIter)); fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; fts5SegIterNextPage(p, pIter); } if( p->rc==SQLITE_OK ){ pIter->iLeafOffset = 4; ................................................................................ ** This function advances the iterator so that it points to the last ** relevant rowid on the page and, if necessary, initializes the ** aRowidOffset[] and iRowidOffset variables. At this point the iterator ** is in its regular state - Fts5SegIter.iLeafOffset points to the first ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ int eDetail = p->pConfig->eDetail; int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; int iRowidOffset = 0; if( n>pIter->iEndofDoclist ){ n = pIter->iEndofDoclist; } ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ if( i<n && a[i]==0 ){ i++; if( i<n && a[i]==0 ) i++; } }else{ int nPos; int bDummy; i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); i += nPos; } if( i>=n ) break; i += fts5GetVarint(&a[i], (u64*)&iDelta); pIter->iRowid += iDelta; /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ int nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; break; } ................................................................................ } /* ** Return true if the iterator passed as the second argument currently ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } /* ** Advance iterator pIter to the next entry. ** ** This version of fts5SegIterNext() is only used by reverse iterators. */ static void fts5SegIterNext_Reverse( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbUnused /* Unused */ ){ assert( pIter->flags & FTS5_SEGITER_REVERSE ); assert( pIter->pNextLeaf==0 ); UNUSED_PARAM(pbUnused); if( pIter->iRowidOffset>0 ){ u8 *a = pIter->pLeaf->p; int iOff; i64 iDelta; pIter->iRowidOffset--; pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; fts5SegIterLoadNPos(p, pIter); iOff = pIter->iLeafOffset; if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ iOff += pIter->nPos; } fts5GetVarint(&a[iOff], (u64*)&iDelta); pIter->iRowid -= iDelta; }else{ fts5SegIterReverseNewPage(p, pIter); } } /* ** Advance iterator pIter to the next entry. ** ** This version of fts5SegIterNext() is only used if detail=none and the ** iterator is not a reverse direction iterator. */ static void fts5SegIterNext_None( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ int iOff; assert( p->rc==SQLITE_OK ); assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); ASSERT_SZLEAF_OK(pIter->pLeaf); iOff = pIter->iLeafOffset; /* Next entry is on the next page */ if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; iOff = 4; } if( iOff<pIter->iEndofDoclist ){ /* Next entry is on the current page */ i64 iDelta; iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); pIter->iLeafOffset = iOff; pIter->iRowid += iDelta; }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ if( pIter->pSeg ){ int nKeep = 0; if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); } pIter->iLeafOffset = iOff; fts5SegIterLoadTerm(p, pIter, nKeep); }else{ const u8 *pList = 0; const char *zTerm = 0; int nList; sqlite3Fts5HashScanNext(p->pHash); sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); if( pList==0 ) goto next_none_eof; pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList; sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } if( pbNewTerm ) *pbNewTerm = 1; }else{ goto next_none_eof; } fts5SegIterLoadNPos(p, pIter); return; next_none_eof: fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; } /* ** Advance iterator pIter to the next entry. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It ** is not considered an error if the iterator reaches EOF. If an error has ** already occurred when this function is called, it is a no-op. */ static void fts5SegIterNext( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter, /* Iterator to advance */ int *pbNewTerm /* OUT: Set for new term */ ){ Fts5Data *pLeaf = pIter->pLeaf; int iOff; int bNewTerm = 0; int nKeep = 0; u8 *a; int n; assert( pbNewTerm==0 || *pbNewTerm==0 ); assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); /* Search for the end of the position list within the current page. */ a = pLeaf->p; n = pLeaf->szLeaf; ASSERT_SZLEAF_OK(pLeaf); iOff = pIter->iLeafOffset + pIter->nPos; if( iOff<n ){ /* The next entry is on the current page. */ assert_nc( iOff<=pIter->iEndofDoclist ); ................................................................................ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], iOff ); pIter->iLeafOffset = iOff; pIter->iEndofDoclist = iOff; bNewTerm = 1; } assert_nc( iOff<pLeaf->szLeaf ); if( iOff>pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; return; } } } /* Check if the iterator is now at EOF. If so, return early. */ ................................................................................ fts5SegIterLoadTerm(p, pIter, nKeep); fts5SegIterLoadNPos(p, pIter); if( pbNewTerm ) *pbNewTerm = 1; } }else{ /* The following could be done by calling fts5SegIterLoadNPos(). But ** this block is particularly performance critical, so equivalent ** code is inlined. ** ** Later: Switched back to fts5SegIterLoadNPos() because it supports ** detail=none mode. Not ideal. */ int nSz; assert( p->rc==SQLITE_OK ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; assert_nc( pIter->nPos>=0 ); } } } #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } #define fts5IndexSkipVarint(a, iOff) { \ int iEnd = iOff+9; \ while( (a[iOff++] & 0x80) && iOff<iEnd ); \ } /* ** Iterator pIter currently points to the first rowid in a doclist. This ** function sets the iterator up so that iterates in reverse order through ** the doclist. */ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ ................................................................................ pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ /* Currently, Fts5SegIter.iLeafOffset points to the first byte of ** position-list content for the current rowid. Back it up so that it ** points to the start of the position-list size field. */ int iPoslist; if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ iPoslist = pIter->iTermLeafOffset; }else{ iPoslist = 4; } fts5IndexSkipVarint(pLeaf->p, iPoslist); pIter->iLeafOffset = iPoslist; /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to ** see where said rowid really is. */ if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ int pgno; Fts5StructureSegment *pSeg = pIter->pSeg; ................................................................................ ){ return; } pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } /* ** The iterator object passed as the second argument currently contains ** no valid values except for the Fts5SegIter.pLeaf member variable. This ** function searches the leaf page for a term matching (pTerm/nTerm). ** ** If the specified term is found on the page, then the iterator is left ** pointing to it. If argument bGe is zero and the term is not found, ................................................................................ int bEndOfPage = 0; assert( p->rc==SQLITE_OK ); iPgidx = szLeaf; iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; if( iOff>n ){ p->rc = FTS5_CORRUPT; return; } while( 1 ){ /* Figure out how many new bytes are in this term */ fts5FastGetVarint32(a, iOff, nNew); if( nKeep<nMatch ){ goto search_failed; ................................................................................ pIter->iEndofDoclist = iTermOff + nExtra; } pIter->iPgidxOff = iPgidx; fts5SegIterLoadRowid(p, pIter); fts5SegIterLoadNPos(p, pIter); } static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ if( p->pIdxSelect==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( "SELECT pgno FROM '%q'.'%q_idx' WHERE " "segid=? AND term<=? ORDER BY term DESC LIMIT 1", pConfig->zDb, pConfig->zName )); } return p->pIdxSelect; } /* ** Initialize the object pIter to point to term pTerm/nTerm within segment ** pSeg. If there is no such term in the index, the iterator is set to EOF. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterSeekInit( Fts5Index *p, /* FTS5 backend */ const u8 *pTerm, int nTerm, /* Term to seek to */ int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5StructureSegment *pSeg, /* Description of segment */ Fts5SegIter *pIter /* Object to populate */ ){ int iPg = 1; int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ sqlite3_stmt *pIdxSelect = 0; assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ pIdxSelect = fts5IdxSelectStmt(p); if( p->rc ) return; sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ i64 val = sqlite3_column_int(pIdxSelect, 0); iPg = (int)(val>>1); bDlidx = (val & 0x0001); } p->rc = sqlite3_reset(pIdxSelect); if( iPg<pSeg->pgnoFirst ){ iPg = pSeg->pgnoFirst; bDlidx = 0; } pIter->iLeafPgno = iPg - 1; ................................................................................ fts5SegIterLoadDlidx(p, pIter); } if( flags & FTS5INDEX_QUERY_DESC ){ fts5SegIterReverse(p, pIter); } } } fts5SegIterSetNext(p, pIter); /* Either: ** ** 1) an error has occurred, or ** 2) the iterator points to EOF, or ** 3) the iterator points to an entry with term (pTerm/nTerm), or ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points ................................................................................ sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf==0 ) return; pLeaf->p = (u8*)pList; pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); pIter->iEndofDoclist = pLeaf->nn; if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; fts5SegIterReverseInitPage(p, pIter); }else{ fts5SegIterLoadNPos(p, pIter); } } fts5SegIterSetNext(p, pIter); } /* ** Zero the iterator passed as the only argument. */ static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); ................................................................................ /* ** This function is used as part of the big assert() procedure implemented by ** fts5AssertMultiIterSetup(). It ensures that the result currently stored ** in *pRes is the correct result of comparing the current positions of the ** two iterators. */ static void fts5AssertComparisonResult( Fts5Iter *pIter, Fts5SegIter *p1, Fts5SegIter *p2, Fts5CResult *pRes ){ int i1 = p1 - pIter->aSeg; int i2 = p2 - pIter->aSeg; ................................................................................ /* ** This function is a no-op unless SQLITE_DEBUG is defined when this module ** is compiled. In that case, this function is essentially an assert() ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int i; assert( (pFirst->pLeaf==0)==pIter->base.bEof ); /* Check that pIter->iSwitchRowid is set correctly. */ for(i=0; i<pIter->nSeg; i++){ Fts5SegIter *p1 = &pIter->aSeg[i]; assert( p1==pFirst || p1->pLeaf==0 || fts5BufferCompare(&pFirst->term, &p1->term) ................................................................................ ** Do the comparison necessary to populate pIter->aFirst[iOut]. ** ** If the returned value is non-zero, then it is the index of an entry ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing ** to a key that is a duplicate of another, higher priority, ** segment-iterator in the pSeg->aSeg[] array. */ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ int i1; /* Index of left-hand Fts5SegIter */ int i2; /* Index of right-hand Fts5SegIter */ int iRes; Fts5SegIter *p1; /* Left-hand Fts5SegIter */ Fts5SegIter *p2; /* Right-hand Fts5SegIter */ Fts5CResult *pRes = &pIter->aFirst[iOut]; ................................................................................ pIter->iLeafPgno = iLeafPgno+1; fts5SegIterReverseNewPage(p, pIter); bMove = 0; } } do{ if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; bMove = 1; }while( p->rc==SQLITE_OK ); } /* ** Free the iterator object passed as the second argument. */ static void fts5MultiIterFree(Fts5Iter *pIter){ if( pIter ){ int i; for(i=0; i<pIter->nSeg; i++){ fts5SegIterClear(&pIter->aSeg[i]); } fts5StructureRelease(pIter->pStruct); fts5BufferFree(&pIter->poslist); sqlite3_free(pIter); } } static void fts5MultiIterAdvanced( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ int iChanged, /* Index of sub-iterator just advanced */ int iMinset /* Minimum entry in aFirst[] to set */ ){ int i; for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ int iEq; if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ Fts5SegIter *pSeg = &pIter->aSeg[iEq]; assert( p->rc==SQLITE_OK ); pSeg->xNext(p, pSeg, 0); i = pIter->nSeg + iEq; } } } /* ** Sub-iterator iChanged of iterator pIter has just been advanced. It still ................................................................................ ** If it does so successfully, 0 is returned. Otherwise 1. ** ** If non-zero is returned, the caller should call fts5MultiIterAdvanced() ** on the iterator instead. That function does the same as this one, except ** that it deals with more complicated cases as well. */ static int fts5MultiIterAdvanceRowid( Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ int iChanged, /* Index of sub-iterator just advanced */ Fts5SegIter **ppFirst ){ Fts5SegIter *pNew = &pIter->aSeg[iChanged]; if( pNew->iRowid==pIter->iSwitchRowid || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev ){ int i; ................................................................................ pRes->iFirst = (u16)(pNew - pIter->aSeg); if( i==1 ) break; pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ]; } } *ppFirst = pNew; return 0; } /* ** Set the pIter->bEof variable based on the state of the sub-iterators. */ static void fts5MultiIterSetEof(Fts5Iter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; pIter->base.bEof = pSeg->pLeaf==0; pIter->iSwitchRowid = pSeg->iRowid; } /* ** Move the iterator to the next entry. ** ** If an error occurs, an error code is left in Fts5Index.rc. It is not ** considered an error if the iterator reaches EOF, or if it is already at ** EOF when this function is called. */ static void fts5MultiIterNext( Fts5Index *p, Fts5Iter *pIter, int bFrom, /* True if argument iFrom is valid */ i64 iFrom /* Advance at least as far as this */ ){ int bUseFrom = bFrom; while( p->rc==SQLITE_OK ){ int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; assert( p->rc==SQLITE_OK ); if( bUseFrom && pSeg->pDlidx ){ fts5SegIterNextFrom(p, pSeg, iFrom); }else{ pSeg->xNext(p, pSeg, &bNewTerm); } if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; if( pSeg->pLeaf==0 ) return; } fts5AssertMultiIterSetup(p, pIter); assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); if( pIter->bSkipEmpty==0 || pSeg->nPos ){ pIter->xSetOutputs(pIter, pSeg); return; } bUseFrom = 0; } } static void fts5MultiIterNext2( Fts5Index *p, Fts5Iter *pIter, int *pbNewTerm /* OUT: True if *might* be new term */ ){ assert( pIter->bSkipEmpty ); if( p->rc==SQLITE_OK ){ do { int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; assert( p->rc==SQLITE_OK ); pSeg->xNext(p, pSeg, &bNewTerm); if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); *pbNewTerm = 1; }else{ *pbNewTerm = 0; } fts5AssertMultiIterSetup(p, pIter); }while( fts5MultiIterIsEmpty(p, pIter) ); } } static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){ UNUSED_PARAM2(pUnused1, pUnused2); } static Fts5Iter *fts5MultiIterAlloc( Fts5Index *p, /* FTS5 backend to iterate within */ int nSeg ){ Fts5Iter *pNew; int nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2); pNew = fts5IdxMalloc(p, sizeof(Fts5Iter) + /* pNew */ sizeof(Fts5SegIter) * (nSlot-1) + /* pNew->aSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); if( pNew ){ pNew->nSeg = nSlot; pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; pNew->pIndex = p; pNew->xSetOutputs = fts5IterSetOutputs_Noop; } return pNew; } static void fts5PoslistCallback( Fts5Index *pUnused, void *pContext, const u8 *pChunk, int nChunk ){ UNUSED_PARAM(pUnused); assert_nc( nChunk>=0 ); if( nChunk>0 ){ fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); } } typedef struct PoslistCallbackCtx PoslistCallbackCtx; struct PoslistCallbackCtx { Fts5Buffer *pBuf; /* Append to this buffer */ Fts5Colset *pColset; /* Restrict matches to this column */ int eState; /* See above */ }; typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; struct PoslistOffsetsCtx { Fts5Buffer *pBuf; /* Append to this buffer */ Fts5Colset *pColset; /* Restrict matches to this column */ int iRead; int iWrite; }; /* ** TODO: Make this more efficient! */ static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ int i; for(i=0; i<pColset->nCol; i++){ if( pColset->aiCol[i]==iCol ) return 1; } return 0; } static void fts5PoslistOffsetsCallback( Fts5Index *pUnused, void *pContext, const u8 *pChunk, int nChunk ){ PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; UNUSED_PARAM(pUnused); assert_nc( nChunk>=0 ); if( nChunk>0 ){ int i = 0; while( i<nChunk ){ int iVal; i += fts5GetVarint32(&pChunk[i], iVal); iVal += pCtx->iRead - 2; pCtx->iRead = iVal; if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); pCtx->iWrite = iVal; } } } } static void fts5PoslistFilterCallback( Fts5Index *pUnused, void *pContext, const u8 *pChunk, int nChunk ){ PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; UNUSED_PARAM(pUnused); assert_nc( nChunk>=0 ); if( nChunk>0 ){ /* Search through to find the first varint with value 1. This is the ** start of the next columns hits. */ int i = 0; int iStart = 0; if( pCtx->eState==2 ){ int iCol; fts5FastGetVarint32(pChunk, i, iCol); if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ pCtx->eState = 1; fts5BufferSafeAppendVarint(pCtx->pBuf, 1); }else{ pCtx->eState = 0; } } do { while( i<nChunk && pChunk[i]!=0x01 ){ while( pChunk[i] & 0x80 ) i++; i++; } if( pCtx->eState ){ fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); } if( i<nChunk ){ int iCol; iStart = i; i++; if( i>=nChunk ){ pCtx->eState = 2; }else{ fts5FastGetVarint32(pChunk, i, iCol); pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); if( pCtx->eState ){ fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); iStart = i; } } } }while( i<nChunk ); } } static void fts5ChunkIterate( Fts5Index *p, /* Index object */ Fts5SegIter *pSeg, /* Poslist of this iterator */ void *pCtx, /* Context pointer for xChunk callback */ void (*xChunk)(Fts5Index*, void*, const u8*, int) ................................................................................ ){ int nRem = pSeg->nPos; /* Number of bytes still to come */ Fts5Data *pData = 0; u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); int pgno = pSeg->iLeafPgno; int pgnoSave = 0; /* This function does notmwork with detail=none databases. */ assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ pgnoSave = pgno+1; } while( 1 ){ xChunk(p, pCtx, pChunk, nChunk); ................................................................................ pSeg->pNextLeaf = pData; pData = 0; } } } } /* ** Iterator pIter currently points to a valid entry (not EOF). This ** function appends the position list data for the current entry to ** buffer pBuf. It does not make a copy of the position-list size ** field. */ static void fts5SegiterPoslist( Fts5Index *p, Fts5SegIter *pSeg, Fts5Colset *pColset, Fts5Buffer *pBuf ){ if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ PoslistCallbackCtx sCtx; sCtx.pBuf = pBuf; sCtx.pColset = pColset; sCtx.eState = fts5IndexColsetTest(pColset, 0); assert( sCtx.eState==0 || sCtx.eState==1 ); fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); }else{ PoslistOffsetsCtx sCtx; memset(&sCtx, 0, sizeof(sCtx)); sCtx.pBuf = pBuf; sCtx.pColset = pColset; fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); } } } } /* ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ** the position list contains entries for column iCol, then (*pa) is set ** to point to the sub-position-list for that column and the number of ** bytes in it returned. Or, if the argument position list does not ** contain any entries for column iCol, return 0. */ static int fts5IndexExtractCol( const u8 **pa, /* IN/OUT: Pointer to poslist */ int n, /* IN: Size of poslist in bytes */ int iCol /* Column to extract from poslist */ ){ int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ const u8 *p = *pa; const u8 *pEnd = &p[n]; /* One byte past end of position list */ while( iCol>iCurrent ){ /* Advance pointer p until it points to pEnd or an 0x01 byte that is ** not part of a varint. Note that it is not possible for a negative ** or extremely large varint to occur within an uncorrupted position ** list. So the last byte of each varint may be assumed to have a clear ** 0x80 bit. */ while( *p!=0x01 ){ while( *p++ & 0x80 ); if( p>=pEnd ) return 0; } *pa = p++; iCurrent = *p++; if( iCurrent & 0x80 ){ p--; p += fts5GetVarint32(p, iCurrent); } } if( iCol!=iCurrent ) return 0; /* Advance pointer p until it points to pEnd or an 0x01 byte that is ** not part of a varint */ while( p<pEnd && *p!=0x01 ){ while( *p++ & 0x80 ); } return p - (*pa); } static int fts5IndexExtractColset ( Fts5Colset *pColset, /* Colset to filter on */ const u8 *pPos, int nPos, /* Position list */ Fts5Buffer *pBuf /* Output buffer */ ){ int rc = SQLITE_OK; int i; fts5BufferZero(pBuf); for(i=0; i<pColset->nCol; i++){ const u8 *pSub = pPos; int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); if( nSub ){ fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); } } return rc; } /* ** xSetOutputs callback used by detail=none tables. */ static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); pIter->base.iRowid = pSeg->iRowid; pIter->base.nData = pSeg->nPos; } /* ** xSetOutputs callback used by detail=full and detail=col tables when no ** column filters are specified. */ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ pIter->base.iRowid = pSeg->iRowid; pIter->base.nData = pSeg->nPos; assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); assert( pIter->pColset==0 ); if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ /* All data is stored on the current page. Populate the output ** variables to point into the body of the page object. */ pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; }else{ /* The data is distributed over two or more pages. Copy it into the ** Fts5Iter.poslist buffer and then set the output pointer to point ** to this buffer. */ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); pIter->base.pData = pIter->poslist.p; } } /* ** xSetOutputs callback used by detail=col when there is a column filter ** and there are 100 or more columns. Also called as a fallback from ** fts5IterSetOutputs_Col100 if the column-list spans more than one page. */ static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); pIter->base.iRowid = pSeg->iRowid; pIter->base.pData = pIter->poslist.p; pIter->base.nData = pIter->poslist.n; } /* ** xSetOutputs callback used when: ** ** * detail=col, ** * there is a column filter, and ** * the table contains 100 or fewer columns. ** ** The last point is to ensure all column numbers are stored as ** single-byte varints. */ static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); assert( pIter->pColset ); if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ fts5IterSetOutputs_Col(pIter, pSeg); }else{ u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; u8 *pEnd = (u8*)&a[pSeg->nPos]; int iPrev = 0; int *aiCol = pIter->pColset->aiCol; int *aiColEnd = &aiCol[pIter->pColset->nCol]; u8 *aOut = pIter->poslist.p; int iPrevOut = 0; pIter->base.iRowid = pSeg->iRowid; while( a<pEnd ){ iPrev += (int)a++[0] - 2; while( *aiCol<iPrev ){ aiCol++; if( aiCol==aiColEnd ) goto setoutputs_col_out; } if( *aiCol==iPrev ){ *aOut++ = (iPrev - iPrevOut) + 2; iPrevOut = iPrev; } } setoutputs_col_out: pIter->base.pData = pIter->poslist.p; pIter->base.nData = aOut - pIter->poslist.p; } } /* ** xSetOutputs callback used by detail=full when there is a column filter. */ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ Fts5Colset *pColset = pIter->pColset; pIter->base.iRowid = pSeg->iRowid; assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); assert( pColset ); if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ /* All data is stored on the current page. Populate the output ** variables to point into the body of the page object. */ const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; if( pColset->nCol==1 ){ pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); pIter->base.pData = a; }else{ fts5BufferZero(&pIter->poslist); fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); pIter->base.pData = pIter->poslist.p; pIter->base.nData = pIter->poslist.n; } }else{ /* The data is distributed over two or more pages. Copy it into the ** Fts5Iter.poslist buffer and then set the output pointer to point ** to this buffer. */ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); pIter->base.pData = pIter->poslist.p; pIter->base.nData = pIter->poslist.n; } } static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ if( *pRc==SQLITE_OK ){ Fts5Config *pConfig = pIter->pIndex->pConfig; if( pConfig->eDetail==FTS5_DETAIL_NONE ){ pIter->xSetOutputs = fts5IterSetOutputs_None; } else if( pIter->pColset==0 ){ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; } else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ pIter->xSetOutputs = fts5IterSetOutputs_Full; } else{ assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); if( pConfig->nCol<=100 ){ pIter->xSetOutputs = fts5IterSetOutputs_Col100; sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); }else{ pIter->xSetOutputs = fts5IterSetOutputs_Col; } } } } /* ** Allocate a new Fts5Iter object. ** ** The new object will be used to iterate through data in structure pStruct. ** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel ** is zero or greater, data from the first nSegment segments on level iLevel ** is merged. ** ** The iterator initially points to the first term/rowid entry in the ** iterated data. */ static void fts5MultiIterNew( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Structure *pStruct, /* Structure of specific index */ int flags, /* FTS5INDEX_QUERY_XXX flags */ Fts5Colset *pColset, /* Colset to filter on (or NULL) */ const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ int iLevel, /* Level to iterate (-1 for all) */ int nSegment, /* Number of segments to merge (iLevel>=0) */ Fts5Iter **ppOut /* New object */ ){ int nSeg = 0; /* Number of segment-iters in use */ int iIter = 0; /* */ int iSeg; /* Used to iterate through segments */ Fts5StructureLevel *pLvl; Fts5Iter *pNew; assert( (pTerm==0 && nTerm==0) || iLevel<0 ); /* Allocate space for the new multi-seg-iterator. */ if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; nSeg += (p->pHash ? 1 : 0); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } } *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); if( pNew==0 ) return; pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); pNew->pStruct = pStruct; pNew->pColset = pColset; fts5StructureRef(pStruct); if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ fts5IterSetOutputCb(&p->rc, pNew); } /* Initialize each of the component segment iterators. */ if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; if( p->pHash ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); } for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){ for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; Fts5SegIter *pIter = &pNew->aSeg[iIter++]; if( pTerm==0 ){ fts5SegIterInit(p, pSeg, pIter); }else{ fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter); } } } }else{ pLvl = &pStruct->aLevel[iLevel]; for(iSeg=nSeg-1; iSeg>=0; iSeg--){ fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); } } assert( iIter==nSeg ); } /* If the above was successful, each component iterators now points ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ for(iIter=pNew->nSeg-1; iIter>0; iIter--){ int iEq; if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ Fts5SegIter *pSeg = &pNew->aSeg[iEq]; if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); fts5MultiIterAdvanced(p, pNew, iEq, iIter); } } fts5MultiIterSetEof(pNew); fts5AssertMultiIterSetup(p, pNew); if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ fts5MultiIterNext(p, pNew, 0, 0); }else if( pNew->base.bEof==0 ){ Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; pNew->xSetOutputs(pNew, pSeg); } }else{ fts5MultiIterFree(pNew); *ppOut = 0; } } /* ** Create an Fts5Iter that iterates through the doclist provided ** as the second argument. */ static void fts5MultiIterNew2( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Data *pData, /* Doclist to iterate through */ int bDesc, /* True for descending rowid order */ Fts5Iter **ppOut /* New object */ ){ Fts5Iter *pNew; pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); pIter->iEndofDoclist = pData->nn; pNew->aFirst[1].iFirst = 1; if( bDesc ){ pNew->bRev = 1; pIter->flags |= FTS5_SEGITER_REVERSE; fts5SegIterReverseInitPage(p, pIter); }else{ fts5SegIterLoadNPos(p, pIter); } pData = 0; }else{ pNew->base.bEof = 1; } fts5SegIterSetNext(p, pIter); *ppOut = pNew; } fts5DataRelease(pData); } /* ** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ assert( p->rc || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof ); return (p->rc || pIter->base.bEof); } /* ** Return the rowid of the entry that the iterator currently points ** to. If the iterator points to EOF when this function is called the ** results are undefined. */ static i64 fts5MultiIterRowid(Fts5Iter *pIter){ assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; } /* ** Move the iterator to the next entry at or following iMatch. */ static void fts5MultiIterNextFrom( Fts5Index *p, Fts5Iter *pIter, i64 iMatch ){ while( 1 ){ i64 iRowid; fts5MultiIterNext(p, pIter, 1, iMatch); if( fts5MultiIterEof(p, pIter) ) break; iRowid = fts5MultiIterRowid(pIter); if( pIter->bRev==0 && iRowid>=iMatch ) break; if( pIter->bRev!=0 && iRowid<=iMatch ) break; } } /* ** Return a pointer to a buffer containing the term associated with the ** entry that the iterator currently points to. */ static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; *pn = p->term.n; return p->term.p; } /* ** Allocate a new segment-id for the structure pStruct. The new segment ** id must be between 1 and 65335 inclusive, and must not be used by ** any currently existing segment. If a free segment id cannot be found, ** SQLITE_FULL is returned. ** ................................................................................ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ int iSegid = 0; if( p->rc==SQLITE_OK ){ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ p->rc = SQLITE_FULL; }else{ /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following ** array is 63 elements, or 252 bytes, in size. */ u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; int iLvl, iSeg; int i; u32 mask; memset(aUsed, 0, sizeof(aUsed)); for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; if( iId<=FTS5_MAX_SEGMENT ){ aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); } } } for(i=0; aUsed[i]==0xFFFFFFFF; i++); mask = aUsed[i]; for(iSegid=0; mask & (1 << iSegid); iSegid++); iSegid += 1 + i*32; #ifdef SQLITE_DEBUG for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); } } assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); { sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); if( p->rc==SQLITE_OK ){ u8 aBlob[2] = {0xff, 0xff}; sqlite3_bind_int(pIdxSelect, 1, iSegid); sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); p->rc = sqlite3_reset(pIdxSelect); } } #endif } } return iSegid; } /* ................................................................................ if( p->pHash ){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; } } /* ** Return the size of the prefix, in bytes, that buffer ** (pNew/<length-unknown>) shares with buffer (pOld/nOld). ** ** Buffer (pNew/<length-unknown>) is guaranteed to be greater ** than buffer (pOld/nOld). */ static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){ int i; for(i=0; i<nOld; i++){ if( pOld[i]!=pNew[i] ) break; } return i; } static void fts5WriteDlidxClear( ................................................................................ } } static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; static int nCall = 0; nCall++; assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); ................................................................................ ** Usually, the previous term is available in pPage->term. The exception ** is if this is the first term written in an incremental-merge step. ** In this case the previous term is not available, so just write a ** copy of (pTerm/nTerm) into the parent node. This is slightly ** inefficient, but still correct. */ int n = nTerm; if( pPage->term.n ){ n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); } fts5WriteBtreeTerm(p, pWriter, n, pTerm); pPage = &pWriter->writer; } }else{ nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); } /* Append the number of bytes of new data, then the term data itself ** to the page. */ fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix); fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix]); ................................................................................ /* ** Append a rowid and position-list size field to the writers output. */ static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, i64 iRowid ){ if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ fts5WriteFlushLeaf(p, pWriter); } ................................................................................ }else{ assert( p->rc || iRowid>pWriter->iPrevRowid ); fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; pWriter->bFirstRowidInPage = 0; } } static void fts5WriteAppendPoslistData( Fts5Index *p, Fts5SegWriter *pWriter, const u8 *aData, ................................................................................ Fts5PageWriter *pLeaf = &pWriter->writer; if( p->rc==SQLITE_OK ){ assert( pLeaf->pgno>=1 ); if( pLeaf->buf.n>4 ){ fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; if( pLeaf->pgno>1 ){ fts5WriteFlushBtree(p, pWriter); } } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); fts5BufferFree(&pLeaf->pgidx); fts5BufferFree(&pWriter->btterm); for(i=0; i<pWriter->nDlidx; i++){ sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); ................................................................................ } /* ** Iterator pIter was used to iterate through the input segments of on an ** incremental merge operation. This function is called if the incremental ** merge step has finished but the input has not been completely exhausted. */ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ int i; Fts5Buffer buf; memset(&buf, 0, sizeof(Fts5Buffer)); for(i=0; i<pIter->nSeg; i++){ Fts5SegIter *pSeg = &pIter->aSeg[i]; if( pSeg->pSeg==0 ){ /* no-op */ ................................................................................ Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ int iLvl, /* Level to read input from */ int *pnRem /* Write up to this many output leaves */ ){ Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; Fts5StructureLevel *pLvlOut; Fts5Iter *pIter = 0; /* Iterator to read input data */ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ int eDetail = p->pConfig->eDetail; const int flags = FTS5INDEX_QUERY_NOOUTPUT; assert( iLvl<pStruct->nLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); memset(&writer, 0, sizeof(Fts5SegWriter)); memset(&term, 0, sizeof(Fts5Buffer)); if( pLvl->nMerge ){ ................................................................................ /* Read input from all segments in the input level */ nInput = pLvl->nSeg; } bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); assert( iLvl>=0 ); for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int nPos; /* position-list size field value */ int nTerm; const u8 *pTerm; ................................................................................ /* This is a new term. Append a term to the output segment. */ fts5WriteAppendTerm(p, &writer, nTerm, pTerm); fts5BufferSet(&p->rc, &term, nTerm, pTerm); } /* Append the rowid to the output */ /* WRITEPOSLISTSIZE */ fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); if( eDetail==FTS5_DETAIL_NONE ){ if( pSegIter->bDel ){ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); if( pSegIter->nPos>0 ){ fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); } } }else{ /* Append the position-list data to the output */ nPos = pSegIter->nPos*2 + pSegIter->bDel; fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); } } /* Flush the last leaf page to disk. Set the output segment b-tree height ** and last leaf page number at the same time. */ fts5WriteFinish(p, &writer, &pSeg->pgnoLast); if( fts5MultiIterEof(p, pIter) ){ int i; ................................................................................ } }else{ assert( pSeg->pgnoLast>0 ); fts5TrimSegments(p, pIter); pLvl->nMerge = nInput; } fts5MultiIterFree(pIter); fts5BufferFree(&term); if( pnRem ) *pnRem -= writer.nLeafWritten; } /* ** Do up to nPg pages of automerge work on the index. ** ** Return true if any changes were actually made, or false otherwise. */ static int fts5IndexMerge( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ int nPg, /* Pages of work to do */ int nMin /* Minimum number of segments to merge */ ){ int nRem = nPg; int bRet = 0; Fts5Structure *pStruct = *ppStruct; while( nRem>0 && p->rc==SQLITE_OK ){ int iLvl; /* To iterate through levels */ int iBestLvl = 0; /* Level offering the most input segments */ int nBest = 0; /* Number of input segments on best level */ /* Set iBestLvl to the level to read input segments from. */ ................................................................................ /* If nBest is still 0, then the index must be empty. */ #ifdef SQLITE_DEBUG for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){ assert( pStruct->aLevel[iLvl].nSeg==0 ); } #endif if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){ break; } bRet = 1; fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } } *ppStruct = pStruct; return bRet; } /* ** A total of nLeaf leaf pages of data has just been flushed to a level-0 ** segment. This function updates the write-counter accordingly and, if ** necessary, performs incremental merge work. ** ................................................................................ /* Update the write-counter. While doing so, set nWork. */ nWrite = pStruct->nWriteCounter; nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit)); pStruct->nWriteCounter += nLeaf; nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); } } static void fts5IndexCrisismerge( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ ){ ................................................................................ int iSegid; int pgnoLast = 0; /* Last leaf page number in segment */ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); fts5StructureInvalidate(p); if( iSegid ){ const int pgsz = p->pConfig->pgsz; int eDetail = p->pConfig->eDetail; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); ................................................................................ i64 iDelta = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOff<nDoclist ){ iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta); iRowid += iDelta; if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); }else{ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); } assert( pBuf->n<=pBuf->nSpace ); if( eDetail==FTS5_DETAIL_NONE ){ if( iOff<nDoclist && pDoclist[iOff]==0 ){ pBuf->p[pBuf->n++] = 0; iOff++; if( iOff<nDoclist && pDoclist[iOff]==0 ){ pBuf->p[pBuf->n++] = 0; iOff++; } } if( (pBuf->n + pPgidx->n)>=pgsz ){ fts5WriteFlushLeaf(p, &writer); } }else{ int bDummy; int nPos; int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); nCopy += nPos; if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy ** it in one go. */ fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ /* The entire poslist will not fit on this leaf. So it needs ** to be broken into sections. The only qualification being ................................................................................ } if( iPos>=nCopy ) break; } } iOff += nCopy; } } } /* TODO2: Doclist terminator written here. */ /* pBuf->p[pBuf->n++] = '\0'; */ assert( pBuf->n<=pBuf->nSpace ); sqlite3Fts5HashScanNext(pHash); } sqlite3Fts5HashClear(pHash); ................................................................................ if( p->nPendingData ){ assert( p->pHash ); p->nPendingData = 0; fts5FlushOneHash(p); } } static Fts5Structure *fts5IndexOptimizeStruct( Fts5Index *p, Fts5Structure *pStruct ){ Fts5Structure *pNew = 0; int nByte = sizeof(Fts5Structure); int nSeg = pStruct->nSegment; int i; /* Figure out if this structure requires optimization. A structure does ** not require optimization if either: ** ** + it consists of fewer than two segments, or ** + all segments are on the same level, or ** + all segments except one are currently inputs to a merge operation. ** ** In the first case, return NULL. In the second, increment the ref-count ** on *pStruct and return a copy of the pointer to it. */ if( nSeg<2 ) return 0; for(i=0; i<pStruct->nLevel; i++){ int nThis = pStruct->aLevel[i].nSeg; if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ fts5StructureRef(pStruct); return pStruct; } assert( pStruct->aLevel[i].nMerge<=nThis ); } nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ Fts5StructureLevel *pLvl; nByte = nSeg * sizeof(Fts5StructureSegment); pNew->nLevel = pStruct->nLevel+1; pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; pLvl = &pNew->aLevel[pStruct->nLevel]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ int iLvl, iSeg; int iSegOut = 0; /* Iterate through all segments, from oldest to newest. Add them to ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest ** segment in the data structure. */ for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; iSegOut++; } } pNew->nSegment = pLvl->nSeg = nSeg; }else{ sqlite3_free(pNew); pNew = 0; } } return pNew; } int sqlite3Fts5IndexOptimize(Fts5Index *p){ Fts5Structure *pStruct; Fts5Structure *pNew = 0; assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); pStruct = fts5StructureRead(p); fts5StructureInvalidate(p); if( pStruct ){ pNew = fts5IndexOptimizeStruct(p, pStruct); } fts5StructureRelease(pStruct); assert( pNew==0 || pNew->nSegment>0 ); if( pNew ){ int iLvl; for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){} while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ int nRem = FTS5_OPT_WORK_UNIT; fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); } fts5StructureWrite(p, pNew); fts5StructureRelease(pNew); } return fts5IndexReturn(p); } /* ** This is called to implement the special "VALUES('merge', $nMerge)" ** INSERT command. */ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pStruct = fts5StructureRead(p); if( pStruct ){ int nMin = p->pConfig->nUsermerge; fts5StructureInvalidate(p); if( nMerge<0 ){ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); fts5StructureRelease(pStruct); pStruct = pNew; nMin = 2; nMerge = nMerge*-1; } if( pStruct && pStruct->nLevel ){ if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ fts5StructureWrite(p, pStruct); } } fts5StructureRelease(pStruct); } return fts5IndexReturn(p); } static void fts5AppendRowid( Fts5Index *p, i64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ UNUSED_PARAM(pUnused); fts5BufferAppendVarint(&p->rc, pBuf, iDelta); } static void fts5AppendPoslist( Fts5Index *p, i64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ int nData = pMulti->base.nData; assert( nData>0 ); if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ fts5BufferSafeAppendVarint(pBuf, iDelta); fts5BufferSafeAppendVarint(pBuf, nData*2); fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); } } static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; assert( pIter->aPoslist ); if( p>=pIter->aEof ){ pIter->aPoslist = 0; ................................................................................ #endif #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ (iLastRowid) = (iRowid); \ } /* ** Swap the contents of buffer *p1 with that of *p2. */ static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ Fts5Buffer tmp = *p1; *p1 = *p2; *p2 = tmp; } static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ int i = *piOff; if( i>=pBuf->n ){ *piOff = -1; }else{ u64 iVal; *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); *piRowid += iVal; } } /* ** This is the equivalent of fts5MergePrefixLists() for detail=none mode. ** In this case the buffers consist of a delta-encoded list of rowids only. */ static void fts5MergeRowidLists( Fts5Index *p, /* FTS5 backend object */ Fts5Buffer *p1, /* First list to merge */ Fts5Buffer *p2 /* Second list to merge */ ){ int i1 = 0; int i2 = 0; i64 iRowid1 = 0; i64 iRowid2 = 0; i64 iOut = 0; Fts5Buffer out; memset(&out, 0, sizeof(out)); sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); if( p->rc ) return; fts5NextRowid(p1, &i1, &iRowid1); fts5NextRowid(p2, &i2, &iRowid2); while( i1>=0 || i2>=0 ){ if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){ assert( iOut==0 || iRowid1>iOut ); fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); iOut = iRowid1; fts5NextRowid(p1, &i1, &iRowid1); }else{ assert( iOut==0 || iRowid2>iOut ); fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); iOut = iRowid2; if( i1>=0 && iRowid1==iRowid2 ){ fts5NextRowid(p1, &i1, &iRowid1); } fts5NextRowid(p2, &i2, &iRowid2); } } fts5BufferSwap(&out, p1); fts5BufferFree(&out); } /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. ** ** If an error occurs, an error code is left in p->rc. If an error has ................................................................................ Fts5Buffer *p1, /* First list to merge */ Fts5Buffer *p2 /* Second list to merge */ ){ if( p2->n ){ i64 iLastRowid = 0; Fts5DoclistIter i1; Fts5DoclistIter i2; Fts5Buffer out = {0, 0, 0}; Fts5Buffer tmp = {0, 0, 0}; if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return; fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); while( 1 ){ if( i1.iRowid<i2.iRowid ){ /* Copy entry from i1 */ fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize); fts5DoclistIterNext(&i1); if( i1.aPoslist==0 ) break; } else if( i2.iRowid!=i1.iRowid ){ /* Copy entry from i2 */ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize); fts5DoclistIterNext(&i2); if( i2.aPoslist==0 ) break; } else{ /* Merge the two position lists. */ i64 iPos1 = 0; i64 iPos2 = 0; int iOff1 = 0; int iOff2 = 0; u8 *a1 = &i1.aPoslist[i1.nSize]; u8 *a2 = &i2.aPoslist[i2.nSize]; i64 iPrev = 0; Fts5PoslistWriter writer; memset(&writer, 0, sizeof(writer)); fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferZero(&tmp); sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist); if( p->rc ) break; sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); assert( iPos1>=0 && iPos2>=0 ); if( iPos1<iPos2 ){ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); }else{ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); } if( iPos1>=0 && iPos2>=0 ){ while( 1 ){ if( iPos1<iPos2 ){ if( iPos1!=iPrev ){ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); } sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); if( iPos1<0 ) break; }else{ assert( iPos2!=iPrev ); sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); if( iPos2<0 ) break; } } } if( iPos1>=0 ){ if( iPos1!=iPrev ){ sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); } fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1); }else{ assert( iPos2>=0 && iPos2!=iPrev ); sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2); } /* WRITEPOSLISTSIZE */ fts5BufferSafeAppendVarint(&out, tmp.n * 2); fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); fts5DoclistIterNext(&i1); fts5DoclistIterNext(&i2); if( i1.aPoslist==0 || i2.aPoslist==0 ) break; } } if( i1.aPoslist ){ fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist); } else if( i2.aPoslist ){ fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); } fts5BufferSet(&p->rc, p1, out.n, out.p); fts5BufferFree(&tmp); fts5BufferFree(&out); } } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; }else{ xMerge = fts5MergePrefixLists; xAppend = fts5AppendPoslist; } aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ const int flags = FTS5INDEX_QUERY_SCAN | FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_NOOUTPUT; int i; i64 iLastRowid = 0; Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; Fts5Buffer doclist; int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); fts5IterSetOutputCb(&p->rc, p1); for( /* no-op */ ; fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) ){ Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; int nTerm = pSeg->term.n; const u8 *pTerm = pSeg->term.p; p1->xSetOutputs(p1, pSeg); assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break; } if( p1->base.nData==0 ) continue; if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ assert( i<nBuf ); if( aBuf[i].n==0 ){ fts5BufferSwap(&doclist, &aBuf[i]); fts5BufferZero(&doclist); }else{ xMerge(p, &doclist, &aBuf[i]); fts5BufferZero(&aBuf[i]); } } iLastRowid = 0; } xAppend(p, p1->base.iRowid-iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } for(i=0; i<nBuf; i++){ if( p->rc==SQLITE_OK ){ xMerge(p, &doclist, &aBuf[i]); } fts5BufferFree(&aBuf[i]); } fts5MultiIterFree(p1); pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); if( pData ){ pData->p = (u8*)&pData[1]; pData->nn = pData->szLeaf = doclist.n; memcpy(pData->p, doclist.p, doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); ................................................................................ ** to the document with rowid iRowid. */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); } /* Flush the hash table to disk if required */ if( iRowid<p->iWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) || (p->nPendingData > p->pConfig->nHashSize) ){ ................................................................................ ** to the database. Additionally, assume that the contents of the %_data ** table may have changed on disk. So any in-memory caches of %_data ** records must be invalidated. */ int sqlite3Fts5IndexRollback(Fts5Index *p){ fts5CloseReader(p); fts5IndexDiscardData(p); fts5StructureInvalidate(p); /* assert( p->rc==SQLITE_OK ); */ return SQLITE_OK; } /* ** The %_data table is completely empty when this function is called. This ** function populates it with the initial structure objects for each index, ** and the initial version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; fts5StructureInvalidate(p); memset(&s, 0, sizeof(Fts5Structure)); fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); return fts5IndexReturn(p); } /* ................................................................................ /* ** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen(). */ int sqlite3Fts5IndexClose(Fts5Index *p){ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); fts5StructureInvalidate(p); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3_finalize(p->pDataVersion); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); } return rc; } /* ** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ int sqlite3Fts5IndexCharlenToBytelen( const char *p, int nByte, int nChar ){ int n = 0; int i; for(i=0; i<nChar; i++){ if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */ if( (unsigned char)p[n++]>=0xc0 ){ while( (p[n] & 0xc0)==0x80 ) n++; } ................................................................................ /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ const int nChar = pConfig->aPrefix[i]; int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, nByte ); } } ................................................................................ Fts5Index *p, /* FTS index to query */ const char *pToken, int nToken, /* Token (or prefix) to query for */ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ Fts5Colset *pColset, /* Match these columns only */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5Iter *pRet = 0; Fts5Buffer buf = {0, 0, 0}; /* If the QUERY_SCAN flag is set, all other flags must be clear. */ assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ memcpy(&buf.p[1], pToken, nToken); /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to ** greater than pConfig->nPrefix to indicate that the query will be ** satisfied by scanning multiple terms in the main index. ** ** If the QUERY_TEST_NOIDX flag was specified, then this must be a ** prefix-query. Instead of using a prefix-index (if one exists), ** evaluate the prefix query using the main FTS index. This is used ** for internal sanity checking by the integrity-check in debug ** mode only. */ #ifdef SQLITE_DEBUG if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; }else #endif if( flags & FTS5INDEX_QUERY_PREFIX ){ int nChar = fts5IndexCharlen(pToken, nToken); for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ if( pConfig->aPrefix[iIdx-1]==nChar ) break; } } if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); if( pStruct ){ fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, pColset, buf.p, nToken+1, -1, 0, &pRet ); fts5StructureRelease(pStruct); } }else{ /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); assert( p->rc!=SQLITE_OK || pRet->pColset==0 ); fts5IterSetOutputCb(&p->rc, pRet); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); } } if( p->rc ){ sqlite3Fts5IterClose(&pRet->base); pRet = 0; fts5CloseReader(p); } *ppIter = &pRet->base; sqlite3Fts5BufferFree(&buf); } return fts5IndexReturn(p); } /* ** Return true if the iterator passed as the only argument is at EOF. */ /* ** Move to the next matching rowid. */ int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); return fts5IndexReturn(pIter->pIndex); } /* ** Move to the next matching term/rowid. Used by the fts5vocab module. */ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *p = pIter->pIndex; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(p, pIter, 0, 0); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ fts5DataRelease(pSeg->pLeaf); pSeg->pLeaf = 0; pIter->base.bEof = 1; } } return fts5IndexReturn(pIter->pIndex); } /* ** Move to the next matching rowid that occurs at or after iMatch. The ** definition of "at or after" depends on whether this iterator iterates ** in ascending or descending rowid order. */ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); return fts5IndexReturn(pIter->pIndex); } /* ** Return the current term. */ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ int n; const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); *pn = n-1; return &z[1]; } /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ if( pIndexIter ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *pIndex = pIter->pIndex; fts5MultiIterFree(pIter); fts5CloseReader(pIndex); } } /* ** Read and decode the "averages" record from the database. ** ................................................................................ ** Below this point is the implementation of the integrity-check ** functionality. */ /* ** Return a simple checksum value based on the arguments. */ u64 sqlite3Fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, int iIdx, const char *pTerm, int nTerm ){ ................................................................................ Fts5Index *p, /* Fts5 index object */ int iIdx, const char *z, /* Index key to query for */ int n, /* Size of index key in bytes */ int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIter = 0; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){ i64 rowid = pIter->iRowid; if( eDetail==FTS5_DETAIL_NONE ){ cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); }else{ Fts5PoslistReader sReader; for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); } } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNext(pIter); } } sqlite3Fts5IterClose(pIter); *pCksum = cksum; return rc; } /* ................................................................................ #endif } /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums ** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. ** ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the ** checksum does not match. Return SQLITE_OK if all checks pass without ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ int eDetail = p->pConfig->eDetail; u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5Iter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ #ifdef SQLITE_DEBUG /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ #endif const int flags = FTS5INDEX_QUERY_NOOUTPUT; /* Load the FTS index structure */ pStruct = fts5StructureRead(p); /* Check that the internal nodes of each segment match the leaves */ if( pStruct ){ int iLvl, iSeg; ................................................................................ ** variable cksum2) based on entries extracted from the full-text index ** while doing a linear scan of each individual index in turn. ** ** As each term visited by the linear scans, a separate query for the ** same term is performed. cksum3 is calculated based on the entries ** extracted by these queries. */ for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ int n; /* Size of term in bytes */ i64 iPos = 0; /* Position read from poslist */ int iOff = 0; /* Offset within poslist */ i64 iRowid = fts5MultiIterRowid(pIter); char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); if( eDetail==FTS5_DETAIL_NONE ){ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); } }else{ poslist.n = 0; fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); } } } fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); fts5MultiIterFree(pIter); if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; fts5StructureRelease(pStruct); #ifdef SQLITE_DEBUG fts5BufferFree(&term); #endif fts5BufferFree(&poslist); return fts5IndexReturn(p); } /************************************************************************* ************************************************************************** ** Below this point is the implementation of the fts5_decode() scalar ** function only. */ /* ................................................................................ iDocid += iDelta; sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); } } return iOff; } /* ** This function is part of the fts5_decode() debugging function. It is ** only ever used with detail=none tables. ** ** Buffer (pData/nData) contains a doclist in the format used by detail=none ** tables. This function appends a human-readable version of that list to ** buffer pBuf. ** ** If *pRc is other than SQLITE_OK when this function is called, it is a ** no-op. If an OOM or other error occurs within this function, *pRc is ** set to an SQLite error code before returning. The final state of buffer ** pBuf is undefined in this case. */ static void fts5DecodeRowidList( int *pRc, /* IN/OUT: Error code */ Fts5Buffer *pBuf, /* Buffer to append text to */ const u8 *pData, int nData /* Data to decode list-of-rowids from */ ){ int i = 0; i64 iRowid = 0; while( i<nData ){ const char *zApp = ""; u64 iVal; i += sqlite3Fts5GetVarint(&pData[i], &iVal); iRowid += iVal; if( i<nData && pData[i]==0x00 ){ i++; if( i<nData && pData[i]==0x00 ){ i++; zApp = "+"; }else{ zApp = "*"; } } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); } } /* ** The implementation of user-defined scalar function fts5_decode(). */ static void fts5DecodeFunction( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args (always 2) */ ................................................................................ i64 iRowid; /* Rowid for record being decoded */ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ int rc = SQLITE_OK; /* Return code */ int nSpace = 0; int eDetailNone = (sqlite3_user_data(pCtx)!=0); assert( nArg==2 ); UNUSED_PARAM(nArg); memset(&s, 0, sizeof(Fts5Buffer)); iRowid = sqlite3_value_int64(apVal[0]); /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[] ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents ** buffer overreads even if the record is corrupt. */ n = sqlite3_value_bytes(apVal[1]); ................................................................................ } }else if( iSegid==0 ){ if( iRowid==FTS5_AVERAGES_ROWID ){ fts5DecodeAverages(&rc, &s, a, n); }else{ fts5DecodeStructure(&rc, &s, a, n); } }else if( eDetailNone ){ Fts5Buffer term; /* Current term read from page */ int szLeaf; int iPgidxOff = szLeaf = fts5GetU16(&a[2]); int iTermOff; int nKeep = 0; int iOff; memset(&term, 0, sizeof(Fts5Buffer)); /* Decode any entries that occur before the first term. */ if( szLeaf<n ){ iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff); }else{ iTermOff = szLeaf; } fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4); iOff = iTermOff; while( iOff<szLeaf ){ int nAppend; /* Read the term data for the next term*/ iOff += fts5GetVarint32(&a[iOff], nAppend); term.n = nKeep; fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]); sqlite3Fts5BufferAppendPrintf( &rc, &s, " term=%.*s", term.n, (const char*)term.p ); iOff += nAppend; /* Figure out where the doclist for this term ends */ if( iPgidxOff<n ){ int nIncr; iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr); iTermOff += nIncr; }else{ iTermOff = szLeaf; } fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff); iOff = iTermOff; if( iOff<szLeaf ){ iOff += fts5GetVarint32(&a[iOff], nKeep); } } fts5BufferFree(&term); }else{ Fts5Buffer term; /* Current term read from page */ int szLeaf; /* Offset of pgidx in a[] */ int iPgidxOff; int iPgidxPrev = 0; /* Previous value read from pgidx */ int iTermOff = 0; int iRowidOff = 0; ................................................................................ ** If successful, SQLITE_OK is returned. If an error occurs, some other ** SQLite error code is returned instead. */ int sqlite3Fts5IndexInit(sqlite3 *db){ int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_decode_none", 2, SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0 ); } if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } return rc; } int sqlite3Fts5IndexReset(Fts5Index *p){ assert( p->pStruct==0 || p->iStructVersion!=0 ); if( fts5IndexDataVersion(p)!=p->iStructVersion ){ fts5StructureInvalidate(p); } return fts5IndexReturn(p); } |
Changes to ext/fts5/fts5_main.c.
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 ... 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 ... 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 ... 635 636 637 638 639 640 641 642 643 644 645 646 647 648 ... 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 ... 777 778 779 780 781 782 783 784 785 786 787 788 789 790 ... 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 ... 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 ... 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 .... 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 .... 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 .... 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 .... 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 .... 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 .... 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 .... 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 .... 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 .... 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 .... 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 .... 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 .... 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 .... 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 .... 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 .... 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 .... 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 .... 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 .... 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 .... 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 .... 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 .... 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 .... 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 |
#define FTS5_BI_ORDER_RANK 0x0020 #define FTS5_BI_ORDER_ROWID 0x0040 #define FTS5_BI_ORDER_DESC 0x0080 /* ** Values for Fts5Cursor.csrflags */ #define FTS5CSR_REQUIRE_CONTENT 0x01 #define FTS5CSR_REQUIRE_DOCSIZE 0x02 #define FTS5CSR_REQUIRE_INST 0x04 #define FTS5CSR_EOF 0x08 #define FTS5CSR_FREE_ZRANK 0x10 #define FTS5CSR_REQUIRE_RESEEK 0x20 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) /* ** Macros to Set(), Clear() and Test() cursor flags. ................................................................................ aColMap[1] = pConfig->nCol; aColMap[2] = pConfig->nCol+1; /* Set idxFlags flags for all WHERE clause terms that will be used. */ for(i=0; i<pInfo->nConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; int j; for(j=0; j<(int)ArraySize(aConstraint); j++){ struct Constraint *pC = &aConstraint[j]; if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){ if( p->usable ){ pC->iConsIndex = i; idxFlags |= pC->fts5op; }else if( j==0 ){ /* As there exists an unusable MATCH constraint this is an ................................................................................ pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; }else{ pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; } /* Assign argvIndex values to each constraint in use. */ iNext = 1; for(i=0; i<(int)ArraySize(aConstraint); i++){ struct Constraint *pC = &aConstraint[i]; if( pC->iConsIndex>=0 ){ pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; } } pInfo->idxNum = idxFlags; return SQLITE_OK; } /* ** Implementation of xOpen method. */ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5Table *pTab = (Fts5Table*)pVTab; Fts5Config *pConfig = pTab->pConfig; Fts5Cursor *pCsr; /* New cursor object */ int nByte; /* Bytes of space to allocate */ int rc = SQLITE_OK; /* Return code */ nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); if( pCsr ){ Fts5Global *pGlobal = pTab->pGlobal; memset(pCsr, 0, nByte); pCsr->aColumnSize = (int*)&pCsr[1]; pCsr->pNext = pGlobal->pCsr; pGlobal->pCsr = pCsr; pCsr->iCsrId = ++pGlobal->iNextId; }else{ rc = SQLITE_NOMEM; } *ppCsr = (sqlite3_vtab_cursor*)pCsr; return rc; } static int fts5StmtType(Fts5Cursor *pCsr){ if( pCsr->ePlan==FTS5_PLAN_SCAN ){ return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; ................................................................................ ** specific to the previous row stored by the cursor object. */ static void fts5CsrNewrow(Fts5Cursor *pCsr){ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE | FTS5CSR_REQUIRE_INST ); } static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); Fts5Auxdata *pData; Fts5Auxdata *pNext; ................................................................................ int iOff = 0; rc = SQLITE_OK; pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); for(i=0; i<(pSorter->nIdx-1); i++){ int iVal; a += fts5GetVarint32(a, iVal); iOff += iVal; pSorter->aIdx[i] = iOff; } pSorter->aIdx[i] = &aBlob[nBlob] - a; pSorter->aPoslist = a; fts5CsrNewrow(pCsr); } return rc; } ................................................................................ *pbSkip = 1; } CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK); fts5CsrNewrow(pCsr); if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } } return rc; } /* ................................................................................ ** ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned ** even if we reach end-of-file. The fts5EofMethod() will be called ** subsequently to determine whether or not an EOF was hit. */ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; assert( (pCsr->ePlan<3)== (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) ); if( pCsr->ePlan<3 ){ int bSkip = 0; if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } fts5CsrNewrow(pCsr); }else{ switch( pCsr->ePlan ){ case FTS5_PLAN_SPECIAL: { CsrFlagSet(pCsr, FTS5CSR_EOF); break; } case FTS5_PLAN_SORTED_MATCH: { rc = fts5SorterNext(pCsr); break; } ................................................................................ } } return rc; } static sqlite3_stmt *fts5PrepareStatement( int *pRc, Fts5Config *pConfig, const char *zFmt, ... ){ sqlite3_stmt *pRet = 0; va_list ap; va_start(ap, zFmt); if( *pRc==SQLITE_OK ){ int rc; char *zSql = sqlite3_vmprintf(zFmt, ap); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0); if( rc!=SQLITE_OK ){ *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); } sqlite3_free(zSql); } *pRc = rc; } va_end(ap); return pRet; } static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ Fts5Config *pConfig = pTab->pConfig; Fts5Sorter *pSorter; int nPhrase; int nByte; int rc = SQLITE_OK; const char *zRank = pCsr->zRank; const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); if( pSorter==0 ) return SQLITE_NOMEM; ................................................................................ /* TODO: It would be better to have some system for reusing statement ** handles here, rather than preparing a new one for each query. But that ** is not possible as SQLite reference counts the virtual table objects. ** And since the statement required here reads from this very virtual ** table, saving it creates a circular reference. ** ** If SQLite a built-in statement cache, this wouldn't be a problem. */ pSorter->pStmt = fts5PrepareStatement(&rc, pConfig, "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", pConfig->zDb, pConfig->zName, zRank, pConfig->zName, (zRankArgs ? ", " : ""), (zRankArgs ? zRankArgs : ""), bDesc ? "DESC" : "ASC" ); ................................................................................ ** 1. Full-text search using a MATCH operator. ** 2. A by-rowid lookup. ** 3. A full-table scan. */ static int fts5FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); Fts5Config *pConfig = pTab->pConfig; Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; /* Error code */ ................................................................................ int bOrderByRank; /* True if ORDER BY rank */ sqlite3_value *pMatch = 0; /* <tbl> MATCH ? expression (or NULL) */ sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ char **pzErrmsg = pConfig->pzErrmsg; if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); } assert( pCsr->pStmt==0 ); ................................................................................ assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); assert( pCsr->iLastRowid==LARGEST_INT64 ); assert( pCsr->iFirstRowid==SMALLEST_INT64 ); pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ if( zExpr[0]=='*' ){ ................................................................................ } } return rc; } static int fts5SpecialDelete( Fts5Table *pTab, sqlite3_value **apVal, sqlite3_int64 *piRowid ){ int rc = SQLITE_OK; int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]); } return rc; } static void fts5StorageInsert( int *pRc, Fts5Table *pTab, ................................................................................ && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL ){ /* A "special" INSERT op. These are handled separately. */ const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); if( pConfig->eContent!=FTS5_CONTENT_NORMAL && 0==sqlite3_stricmp("delete", z) ){ rc = fts5SpecialDelete(pTab, apVal, pRowid); }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } }else{ /* A regular INSERT, UPDATE or DELETE statement. The trick here is that ** any conflict on the rowid value must be detected before any ** modifications are made to the database file. There are 4 cases: ................................................................................ pTab->base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName ); rc = SQLITE_ERROR; } /* Case 1: DELETE */ else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); } /* Case 2: INSERT */ else if( eType0!=SQLITE_INTEGER ){ /* If this is a REPLACE, first remove the current entry (if any) */ if( eConflict==SQLITE_REPLACE && sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew); } fts5StorageInsert(&rc, pTab, apVal, pRowid); } /* Case 2: UPDATE */ else{ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ if( iOld!=iNew ){ if( eConflict==SQLITE_REPLACE ){ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew); } fts5StorageInsert(&rc, pTab, apVal, pRowid); }else{ rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid); } } }else{ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); fts5StorageInsert(&rc, pTab, apVal, pRowid); } } } pTab->pConfig->pzErrmsg = 0; return rc; ................................................................................ } /* ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); return SQLITE_OK; } /* ** Implementation of xCommit() method. This is a no-op. The contents of ** the pending-terms hash-table have already been flushed into the database ** by fts5SyncMethod(). */ static int fts5CommitMethod(sqlite3_vtab *pVtab){ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0); return SQLITE_OK; } /* ** Implementation of xRollback(). Discard the contents of the pending-terms ** hash-table. Any changes made to the database are reverted by SQLite. ................................................................................ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ int rc; Fts5Table *pTab = (Fts5Table*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); rc = sqlite3Fts5StorageRollback(pTab->pStorage); return rc; } static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return pCsr->pAux->pUserData; } static int fts5ApiColumnCount(Fts5Context *pCtx){ ................................................................................ } static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ int n; if( pCsr->pSorter ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); n = pSorter->aIdx[iPhrase] - i1; *pa = &pSorter->aPoslist[i1]; }else{ n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); } return n; } /* ** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated ** correctly for the current view. Return SQLITE_OK if successful, or an ** SQLite error code otherwise. */ ................................................................................ aIter = pCsr->aInstIter; if( aIter ){ int nInst = 0; /* Number instances seen so far */ int i; /* Initialize all iterators */ for(i=0; i<nIter; i++){ const u8 *a; int n = fts5CsrPoslist(pCsr, i, &a); sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); } while( 1 ){ int *aInst; int iBest = -1; for(i=0; i<nIter; i++){ if( (aIter[i].bEof==0) && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) ){ ................................................................................ aInst = &pCsr->aInst[3 * (nInst-1)]; aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); sqlite3Fts5PoslistReaderNext(&aIter[iBest]); } pCsr->nInstCount = nInst; CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); } return rc; } ................................................................................ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; *piOff = pCsr->aInst[iIdx*3 + 2]; } } return rc; } static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ return fts5CursorRowid((Fts5Cursor*)pCtx); } static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, const char **pz, int *pn ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ *pz = 0; *pn = 0; }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); } } return rc; } static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ int *pCnt = (int*)pContext; if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ (*pCnt)++; } return SQLITE_OK; } static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ ................................................................................ } } return pRet; } static void fts5ApiPhraseNext( Fts5Context *pCtx, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; }else{ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); if( iVal==1 ){ ................................................................................ *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } *piOff += (iVal-2); } } static void fts5ApiPhraseFirst( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); pIter->b = &pIter->a[n]; *piCol = 0; *piOff = 0; fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); } static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { 2, /* iVersion */ ................................................................................ fts5ApiColumnText, fts5ApiColumnSize, fts5ApiQueryPhrase, fts5ApiSetAuxdata, fts5ApiGetAuxdata, fts5ApiPhraseFirst, fts5ApiPhraseNext, }; /* ** Implementation of API function xQueryPhrase(). */ static int fts5ApiQueryPhrase( Fts5Context *pCtx, int iPhrase, ................................................................................ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc; Fts5Cursor *pNew = 0; rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); if( rc==SQLITE_OK ){ Fts5Config *pConf = pTab->pConfig; pNew->ePlan = FTS5_PLAN_MATCH; pNew->iFirstRowid = SMALLEST_INT64; pNew->iLastRowid = LARGEST_INT64; pNew->base.pVtab = (sqlite3_vtab*)pTab; rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); } if( rc==SQLITE_OK ){ for(rc = fts5CursorFirst(pTab, pNew, 0); rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0; rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew) ){ ................................................................................ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ int i; int rc = SQLITE_OK; int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); Fts5Buffer val; memset(&val, 0, sizeof(Fts5Buffer)); /* Append the varints */ for(i=0; i<(nPhrase-1); i++){ const u8 *dummy; int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); } ................................................................................ /* Append the position lists */ for(i=0; i<nPhrase; i++){ const u8 *pPoslist; int nPoslist; nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); } sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); return rc; } /* ** This is the xColumn method, called by SQLite to request a value from ................................................................................ /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ static int fts5FindFunctionMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ int nArg, /* Number of SQL function arguments */ const char *zName, /* Name of SQL function */ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ Fts5Table *pTab = (Fts5Table*)pVtab; Fts5Auxiliary *pAux; pAux = fts5FindAuxiliary(pTab, zName); if( pAux ){ *pxFunc = fts5ApiCallback; *ppArg = (void*)pAux; return 1; } ................................................................................ /* ** The xSavepoint() method. ** ** Flush the contents of the pending-terms table to disk. */ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5Table *pTab = (Fts5Table*)pVtab; fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); fts5TripCursors(pTab); return sqlite3Fts5StorageSync(pTab->pStorage, 0); } /* ** The xRelease() method. ** ** This is a no-op. */ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5Table *pTab = (Fts5Table*)pVtab; fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); fts5TripCursors(pTab); return sqlite3Fts5StorageSync(pTab->pStorage, 0); } /* ** The xRollbackTo() method. ** ** Discard the contents of the pending terms table. */ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5Table *pTab = (Fts5Table*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); return sqlite3Fts5StorageRollback(pTab->pStorage); } /* ** Register a new auxiliary function with global context pGlobal. ................................................................................ sqlite3_free(pGlobal); } static void fts5Fts5Func( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ sqlite3_value **apVal /* Function arguments */ ){ Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); char buf[8]; assert( nArg==0 ); assert( sizeof(buf)>=sizeof(pGlobal) ); memcpy(buf, (void*)&pGlobal, sizeof(pGlobal)); sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT); } /* ** Implementation of fts5_source_id() function. */ static void fts5SourceIdFunc( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ sqlite3_value **apVal /* Function arguments */ ){ assert( nArg==0 ); sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT); } static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { /* iVersion */ 2, /* xCreate */ fts5CreateMethod, ................................................................................ } if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 ); } } return rc; } /* ** The following functions are used to register the module with SQLite. If ** this module is being built as part of the SQLite core (SQLITE_CORE is ** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. |
> | | | < > | | > > > > > > > > | | > > > > > > < | > > > | > | < < > | | > > < < | | < | < > | | | | > > > > | < | | | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > | > > > | > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > | | > | | > | > | | | > > > > > > > > > < < < < < < < < < < < < < < < < < < < < < | | | | > > | > | > | > | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < | > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > | > | > > > > > > > > > > > > |
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 ... 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 ... 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 ... 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 ... 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 ... 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 ... 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 ... 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 ... 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 .... 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 .... 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 .... 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 .... 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 .... 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 .... 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 .... 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 .... 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 .... 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 .... 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 .... 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 .... 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 .... 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 .... 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 .... 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 .... 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 .... 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 .... 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 .... 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 .... 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 .... 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 .... 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 |
#define FTS5_BI_ORDER_RANK 0x0020 #define FTS5_BI_ORDER_ROWID 0x0040 #define FTS5_BI_ORDER_DESC 0x0080 /* ** Values for Fts5Cursor.csrflags */ #define FTS5CSR_EOF 0x01 #define FTS5CSR_REQUIRE_CONTENT 0x02 #define FTS5CSR_REQUIRE_DOCSIZE 0x04 #define FTS5CSR_REQUIRE_INST 0x08 #define FTS5CSR_FREE_ZRANK 0x10 #define FTS5CSR_REQUIRE_RESEEK 0x20 #define FTS5CSR_REQUIRE_POSLIST 0x40 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) /* ** Macros to Set(), Clear() and Test() cursor flags. ................................................................................ aColMap[1] = pConfig->nCol; aColMap[2] = pConfig->nCol+1; /* Set idxFlags flags for all WHERE clause terms that will be used. */ for(i=0; i<pInfo->nConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; int j; for(j=0; j<ArraySize(aConstraint); j++){ struct Constraint *pC = &aConstraint[j]; if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){ if( p->usable ){ pC->iConsIndex = i; idxFlags |= pC->fts5op; }else if( j==0 ){ /* As there exists an unusable MATCH constraint this is an ................................................................................ pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; }else{ pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; } /* Assign argvIndex values to each constraint in use. */ iNext = 1; for(i=0; i<ArraySize(aConstraint); i++){ struct Constraint *pC = &aConstraint[i]; if( pC->iConsIndex>=0 ){ pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; } } pInfo->idxNum = idxFlags; return SQLITE_OK; } static int fts5NewTransaction(Fts5Table *pTab){ Fts5Cursor *pCsr; for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; } return sqlite3Fts5StorageReset(pTab->pStorage); } /* ** Implementation of xOpen method. */ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5Table *pTab = (Fts5Table*)pVTab; Fts5Config *pConfig = pTab->pConfig; Fts5Cursor *pCsr = 0; /* New cursor object */ int nByte; /* Bytes of space to allocate */ int rc; /* Return code */ rc = fts5NewTransaction(pTab); if( rc==SQLITE_OK ){ nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); if( pCsr ){ Fts5Global *pGlobal = pTab->pGlobal; memset(pCsr, 0, nByte); pCsr->aColumnSize = (int*)&pCsr[1]; pCsr->pNext = pGlobal->pCsr; pGlobal->pCsr = pCsr; pCsr->iCsrId = ++pGlobal->iNextId; }else{ rc = SQLITE_NOMEM; } } *ppCsr = (sqlite3_vtab_cursor*)pCsr; return rc; } static int fts5StmtType(Fts5Cursor *pCsr){ if( pCsr->ePlan==FTS5_PLAN_SCAN ){ return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; ................................................................................ ** specific to the previous row stored by the cursor object. */ static void fts5CsrNewrow(Fts5Cursor *pCsr){ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE | FTS5CSR_REQUIRE_INST | FTS5CSR_REQUIRE_POSLIST ); } static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); Fts5Auxdata *pData; Fts5Auxdata *pNext; ................................................................................ int iOff = 0; rc = SQLITE_OK; pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); /* nBlob==0 in detail=none mode. */ if( nBlob>0 ){ for(i=0; i<(pSorter->nIdx-1); i++){ int iVal; a += fts5GetVarint32(a, iVal); iOff += iVal; pSorter->aIdx[i] = iOff; } pSorter->aIdx[i] = &aBlob[nBlob] - a; pSorter->aPoslist = a; } fts5CsrNewrow(pCsr); } return rc; } ................................................................................ *pbSkip = 1; } CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK); fts5CsrNewrow(pCsr); if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); *pbSkip = 1; } } return rc; } /* ................................................................................ ** ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned ** even if we reach end-of-file. The fts5EofMethod() will be called ** subsequently to determine whether or not an EOF was hit. */ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc; assert( (pCsr->ePlan<3)== (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) ); assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); if( pCsr->ePlan<3 ){ int bSkip = 0; if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr)); fts5CsrNewrow(pCsr); }else{ switch( pCsr->ePlan ){ case FTS5_PLAN_SPECIAL: { CsrFlagSet(pCsr, FTS5CSR_EOF); rc = SQLITE_OK; break; } case FTS5_PLAN_SORTED_MATCH: { rc = fts5SorterNext(pCsr); break; } ................................................................................ } } return rc; } static int fts5PrepareStatement( sqlite3_stmt **ppStmt, Fts5Config *pConfig, const char *zFmt, ... ){ sqlite3_stmt *pRet = 0; int rc; char *zSql; va_list ap; va_start(ap, zFmt); zSql = sqlite3_vmprintf(zFmt, ap); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0); if( rc!=SQLITE_OK ){ *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); } sqlite3_free(zSql); } va_end(ap); *ppStmt = pRet; return rc; } static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ Fts5Config *pConfig = pTab->pConfig; Fts5Sorter *pSorter; int nPhrase; int nByte; int rc; const char *zRank = pCsr->zRank; const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); if( pSorter==0 ) return SQLITE_NOMEM; ................................................................................ /* TODO: It would be better to have some system for reusing statement ** handles here, rather than preparing a new one for each query. But that ** is not possible as SQLite reference counts the virtual table objects. ** And since the statement required here reads from this very virtual ** table, saving it creates a circular reference. ** ** If SQLite a built-in statement cache, this wouldn't be a problem. */ rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", pConfig->zDb, pConfig->zName, zRank, pConfig->zName, (zRankArgs ? ", " : ""), (zRankArgs ? zRankArgs : ""), bDesc ? "DESC" : "ASC" ); ................................................................................ ** 1. Full-text search using a MATCH operator. ** 2. A by-rowid lookup. ** 3. A full-table scan. */ static int fts5FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *zUnused, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); Fts5Config *pConfig = pTab->pConfig; Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; /* Error code */ ................................................................................ int bOrderByRank; /* True if ORDER BY rank */ sqlite3_value *pMatch = 0; /* <tbl> MATCH ? expression (or NULL) */ sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ char **pzErrmsg = pConfig->pzErrmsg; UNUSED_PARAM(zUnused); UNUSED_PARAM(nVal); if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); } assert( pCsr->pStmt==0 ); ................................................................................ assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); assert( pCsr->iLastRowid==LARGEST_INT64 ); assert( pCsr->iFirstRowid==SMALLEST_INT64 ); pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); sqlite3Fts5ExprClearEof(pCsr->pExpr); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ if( zExpr[0]=='*' ){ ................................................................................ } } return rc; } static int fts5SpecialDelete( Fts5Table *pTab, sqlite3_value **apVal ){ int rc = SQLITE_OK; int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); } return rc; } static void fts5StorageInsert( int *pRc, Fts5Table *pTab, ................................................................................ && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL ){ /* A "special" INSERT op. These are handled separately. */ const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); if( pConfig->eContent!=FTS5_CONTENT_NORMAL && 0==sqlite3_stricmp("delete", z) ){ rc = fts5SpecialDelete(pTab, apVal); }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } }else{ /* A regular INSERT, UPDATE or DELETE statement. The trick here is that ** any conflict on the rowid value must be detected before any ** modifications are made to the database file. There are 4 cases: ................................................................................ pTab->base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName ); rc = SQLITE_ERROR; } /* DELETE */ else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); } /* INSERT */ else if( eType0!=SQLITE_INTEGER ){ /* If this is a REPLACE, first remove the current entry (if any) */ if( eConflict==SQLITE_REPLACE && sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); } /* UPDATE */ else{ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ if( iOld!=iNew ){ if( eConflict==SQLITE_REPLACE ){ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); }else{ rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid); } } }else{ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); fts5StorageInsert(&rc, pTab, apVal, pRowid); } } } pTab->pConfig->pzErrmsg = 0; return rc; ................................................................................ } /* ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); fts5NewTransaction((Fts5Table*)pVtab); return SQLITE_OK; } /* ** Implementation of xCommit() method. This is a no-op. The contents of ** the pending-terms hash-table have already been flushed into the database ** by fts5SyncMethod(). */ static int fts5CommitMethod(sqlite3_vtab *pVtab){ UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0); return SQLITE_OK; } /* ** Implementation of xRollback(). Discard the contents of the pending-terms ** hash-table. Any changes made to the database are reverted by SQLite. ................................................................................ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ int rc; Fts5Table *pTab = (Fts5Table*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); rc = sqlite3Fts5StorageRollback(pTab->pStorage); return rc; } static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return pCsr->pAux->pUserData; } static int fts5ApiColumnCount(Fts5Context *pCtx){ ................................................................................ } static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, const char **pz, int *pn ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ *pz = 0; *pn = 0; }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); } } return rc; } static int fts5CsrPoslist( Fts5Cursor *pCsr, int iPhrase, const u8 **pa, int *pn ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; int bLive = (pCsr->pSorter==0); if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){ int n; const char *z; rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } } sqlite3_free(aPopulator); if( pCsr->pSorter ){ sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); } } CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); *pn = pSorter->aIdx[iPhrase] - i1; *pa = &pSorter->aPoslist[i1]; }else{ *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); } return rc; } /* ** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated ** correctly for the current view. Return SQLITE_OK if successful, or an ** SQLite error code otherwise. */ ................................................................................ aIter = pCsr->aInstIter; if( aIter ){ int nInst = 0; /* Number instances seen so far */ int i; /* Initialize all iterators */ for(i=0; i<nIter && rc==SQLITE_OK; i++){ const u8 *a; int n; rc = fts5CsrPoslist(pCsr, i, &a, &n); if( rc==SQLITE_OK ){ sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); } } if( rc==SQLITE_OK ){ while( 1 ){ int *aInst; int iBest = -1; for(i=0; i<nIter; i++){ if( (aIter[i].bEof==0) && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) ){ ................................................................................ aInst = &pCsr->aInst[3 * (nInst-1)]; aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); sqlite3Fts5PoslistReaderNext(&aIter[iBest]); } } pCsr->nInstCount = nInst; CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); } return rc; } ................................................................................ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; #if 0 }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 2]; *piOff = -1; #endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; *piOff = pCsr->aInst[iIdx*3 + 2]; } } return rc; } static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ return fts5CursorRowid((Fts5Cursor*)pCtx); } static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ int tflags, const char *pUnused, /* Buffer containing token */ int nUnused, /* Size of token in bytes */ int iUnused1, /* Start offset of token */ int iUnused2 /* End offset of token */ ){ int *pCnt = (int*)pContext; UNUSED_PARAM2(pUnused, nUnused); UNUSED_PARAM2(iUnused1, iUnused2); if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ (*pCnt)++; } return SQLITE_OK; } static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ ................................................................................ } } return pRet; } static void fts5ApiPhraseNext( Fts5Context *pUnused, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ UNUSED_PARAM(pUnused); if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; }else{ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); if( iVal==1 ){ ................................................................................ *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } *piOff += (iVal-2); } } static int fts5ApiPhraseFirst( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int n; int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); if( rc==SQLITE_OK ){ pIter->b = &pIter->a[n]; *piCol = 0; *piOff = 0; fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); } return rc; } static void fts5ApiPhraseNextColumn( Fts5Context *pCtx, Fts5PhraseIter *pIter, int *piCol ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ if( pIter->a>=pIter->b ){ *piCol = -1; }else{ int iIncr; pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); *piCol += (iIncr-2); } }else{ while( 1 ){ int dummy; if( pIter->a>=pIter->b ){ *piCol = -1; return; } if( pIter->a[0]==0x01 ) break; pIter->a += fts5GetVarint32(pIter->a, dummy); } pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); } } static int fts5ApiPhraseFirstColumn( Fts5Context *pCtx, int iPhrase, Fts5PhraseIter *pIter, int *piCol ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ Fts5Sorter *pSorter = pCsr->pSorter; int n; if( pSorter ){ int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); n = pSorter->aIdx[iPhrase] - i1; pIter->a = &pSorter->aPoslist[i1]; }else{ rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); } if( rc==SQLITE_OK ){ pIter->b = &pIter->a[n]; *piCol = 0; fts5ApiPhraseNextColumn(pCtx, pIter, piCol); } }else{ int n; rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); if( rc==SQLITE_OK ){ pIter->b = &pIter->a[n]; if( n<=0 ){ *piCol = -1; }else if( pIter->a[0]==0x01 ){ pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); }else{ *piCol = 0; } } } return rc; } static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { 2, /* iVersion */ ................................................................................ fts5ApiColumnText, fts5ApiColumnSize, fts5ApiQueryPhrase, fts5ApiSetAuxdata, fts5ApiGetAuxdata, fts5ApiPhraseFirst, fts5ApiPhraseNext, fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, }; /* ** Implementation of API function xQueryPhrase(). */ static int fts5ApiQueryPhrase( Fts5Context *pCtx, int iPhrase, ................................................................................ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc; Fts5Cursor *pNew = 0; rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); if( rc==SQLITE_OK ){ pNew->ePlan = FTS5_PLAN_MATCH; pNew->iFirstRowid = SMALLEST_INT64; pNew->iLastRowid = LARGEST_INT64; pNew->base.pVtab = (sqlite3_vtab*)pTab; rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr); } if( rc==SQLITE_OK ){ for(rc = fts5CursorFirst(pTab, pNew, 0); rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0; rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew) ){ ................................................................................ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ int i; int rc = SQLITE_OK; int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); Fts5Buffer val; memset(&val, 0, sizeof(Fts5Buffer)); switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ case FTS5_DETAIL_FULL: /* Append the varints */ for(i=0; i<(nPhrase-1); i++){ const u8 *dummy; int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); } ................................................................................ /* Append the position lists */ for(i=0; i<nPhrase; i++){ const u8 *pPoslist; int nPoslist; nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); } break; case FTS5_DETAIL_COLUMNS: /* Append the varints */ for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ const u8 *dummy; int nByte; rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); } /* Append the position lists */ for(i=0; rc==SQLITE_OK && i<nPhrase; i++){ const u8 *pPoslist; int nPoslist; rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist); sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); } break; default: break; } sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); return rc; } /* ** This is the xColumn method, called by SQLite to request a value from ................................................................................ /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ static int fts5FindFunctionMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ int nUnused, /* Number of SQL function arguments */ const char *zName, /* Name of SQL function */ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ Fts5Table *pTab = (Fts5Table*)pVtab; Fts5Auxiliary *pAux; UNUSED_PARAM(nUnused); pAux = fts5FindAuxiliary(pTab, zName); if( pAux ){ *pxFunc = fts5ApiCallback; *ppArg = (void*)pAux; return 1; } ................................................................................ /* ** The xSavepoint() method. ** ** Flush the contents of the pending-terms table to disk. */ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5Table *pTab = (Fts5Table*)pVtab; UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); fts5TripCursors(pTab); return sqlite3Fts5StorageSync(pTab->pStorage, 0); } /* ** The xRelease() method. ** ** This is a no-op. */ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5Table *pTab = (Fts5Table*)pVtab; UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); fts5TripCursors(pTab); return sqlite3Fts5StorageSync(pTab->pStorage, 0); } /* ** The xRollbackTo() method. ** ** Discard the contents of the pending terms table. */ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5Table *pTab = (Fts5Table*)pVtab; UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); return sqlite3Fts5StorageRollback(pTab->pStorage); } /* ** Register a new auxiliary function with global context pGlobal. ................................................................................ sqlite3_free(pGlobal); } static void fts5Fts5Func( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ sqlite3_value **apUnused /* Function arguments */ ){ Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); char buf[8]; UNUSED_PARAM2(nArg, apUnused); assert( nArg==0 ); assert( sizeof(buf)>=sizeof(pGlobal) ); memcpy(buf, (void*)&pGlobal, sizeof(pGlobal)); sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT); } /* ** Implementation of fts5_source_id() function. */ static void fts5SourceIdFunc( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ sqlite3_value **apUnused /* Function arguments */ ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT); } static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { /* iVersion */ 2, /* xCreate */ fts5CreateMethod, ................................................................................ } if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 ); } } /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file ** fts5_test_mi.c is compiled and linked into the executable. And call ** its entry point to enable the matchinfo() demo. */ #ifdef SQLITE_FTS5_ENABLE_TEST_MI if( rc==SQLITE_OK ){ extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); rc = sqlite3Fts5TestRegisterMatchinfo(db); } #endif return rc; } /* ** The following functions are used to register the module with SQLite. If ** this module is being built as part of the SQLite core (SQLITE_CORE is ** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. |
Changes to ext/fts5/fts5_storage.c.
141 142 143 144 145 146 147 148 149 150 151 152 153 154 ... 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 ... 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 ... 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 ... 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 ... 674 675 676 677 678 679 680 681 682 683 684 685 686 687 ... 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 ... 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 .... 1095 1096 1097 1098 1099 1100 1101 1102 1103 |
if( rc!=SQLITE_OK && pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); } } } *ppStmt = p->aStmt[eStmt]; return rc; } static int fts5ExecPrintf( sqlite3 *db, char **pzErr, ................................................................................ */ int sqlite3Fts5StorageClose(Fts5Storage *p){ int rc = SQLITE_OK; if( p ){ int i; /* Finalize all SQL statements */ for(i=0; i<(int)ArraySize(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p); } return rc; } ................................................................................ ** Tokenization callback used when inserting tokens into the FTS index. */ static int fts5StorageInsertCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } /* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. */ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek; /* SELECT to read row iDel from %_data */ int rc; /* Return code */ rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); if( rc==SQLITE_OK ){ int rc2; sqlite3_bind_int64(pSeek, 1, iDel); if( sqlite3_step(pSeek)==SQLITE_ROW ){ int iCol; Fts5InsertCtx ctx; ctx.pStorage = p; ctx.iCol = -1; rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pSeek, iCol), sqlite3_column_bytes(pSeek, iCol), (void*)&ctx, fts5StorageInsertCallback ); p->aTotalSize[iCol-1] -= (i64)ctx.szCol; } p->nTotalRow--; } rc2 = sqlite3_reset(pSeek); if( rc==SQLITE_OK ) rc = rc2; } return rc; } /* ** Insert a record into the %_docsize table. Specifically, do: ** ................................................................................ return rc; } /* ** Remove a row from the FTS table. */ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; rc = fts5StorageLoadTotals(p, 1); /* Delete the index records */ if( rc==SQLITE_OK ){ rc = fts5StorageDeleteFromIndex(p, iDel); } /* Delete the %_docsize record */ if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDel, 1, iDel); ................................................................................ } /* Delete the %_content record */ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); } if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDel, 1, iDel); sqlite3_step(pDel); rc = sqlite3_reset(pDel); } } /* Write the averages record */ if( rc==SQLITE_OK ){ rc = fts5StorageSaveTotals(p); } return rc; } int sqlite3Fts5StorageSpecialDelete( Fts5Storage *p, i64 iDel, sqlite3_value **apVal ){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; assert( pConfig->eContent!=FTS5_CONTENT_NORMAL ); rc = fts5StorageLoadTotals(p, 1); /* Delete the index records */ if( rc==SQLITE_OK ){ int iCol; Fts5InsertCtx ctx; ctx.pStorage = p; ctx.iCol = -1; rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_value_text(apVal[iCol]), sqlite3_value_bytes(apVal[iCol]), (void*)&ctx, fts5StorageInsertCallback ); p->aTotalSize[iCol] -= (i64)ctx.szCol; } p->nTotalRow--; } /* Delete the %_docsize record */ if( pConfig->bColumnsize ){ if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); } if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDel, 1, iDel); sqlite3_step(pDel); rc = sqlite3_reset(pDel); } } ................................................................................ int sqlite3Fts5StorageOptimize(Fts5Storage *p){ return sqlite3Fts5IndexOptimize(p->pIndex); } int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ return sqlite3Fts5IndexMerge(p->pIndex, nMerge); } /* ** Allocate a new rowid. This is used for "external content" tables when ** a NULL value is inserted into the rowid column. The new rowid is allocated ** by inserting a dummy row into the %_docsize table. The dummy will be ** overwritten later. ** ................................................................................ */ typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; struct Fts5IntegrityCtx { i64 iRowid; int iCol; int szCol; u64 cksum; Fts5Config *pConfig; }; /* ** Tokenization callback used by integrity check. */ static int fts5StorageIntegrityCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } pCtx->cksum ^= sqlite3Fts5IndexCksum( pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken ); return SQLITE_OK; } /* ** Check that the contents of the FTS index match that of the %_content ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return ** some other SQLite error code if an error occurs while attempting to ** determine this. ................................................................................ int rc2; while( SQLITE_ROW==sqlite3_step(pScan) ){ int i; ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; if( pConfig->bColumnsize ){ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); } for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, i+1), sqlite3_column_bytes(pScan, i+1), (void*)&ctx, fts5StorageIntegrityCallback ); if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ rc = FTS5_CORRUPT; } aTotalSize[i] += ctx.szCol; } if( rc!=SQLITE_OK ) break; } rc2 = sqlite3_reset(pScan); if( rc==SQLITE_OK ) rc = rc2; } /* Test that the "totals" (sometimes called "averages") record looks Ok */ ................................................................................ rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); if( rc==SQLITE_OK ){ p->pConfig->iCookie = iNew; } } return rc; } |
> | | | > > | > > > > | > > > > | | < | < < > > > > | | | | | | < > > > > > > | > | | | | < < | > | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < > > > > > > | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | < > | > > > > > > > > > > > > > > > | > > > > > > > > | > > > | > > > > < < |
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 ... 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 ... 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 ... 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 ... 530 531 532 533 534 535 536 537 538 539 540 541 542 543 ... 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 ... 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 ... 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 .... 1124 1125 1126 1127 1128 1129 1130 |
if( rc!=SQLITE_OK && pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); } } } *ppStmt = p->aStmt[eStmt]; sqlite3_reset(*ppStmt); return rc; } static int fts5ExecPrintf( sqlite3 *db, char **pzErr, ................................................................................ */ int sqlite3Fts5StorageClose(Fts5Storage *p){ int rc = SQLITE_OK; if( p ){ int i; /* Finalize all SQL statements */ for(i=0; i<ArraySize(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p); } return rc; } ................................................................................ ** Tokenization callback used when inserting tokens into the FTS index. */ static int fts5StorageInsertCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iUnused1, /* Start offset of token */ int iUnused2 /* End offset of token */ ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; UNUSED_PARAM2(iUnused1, iUnused2); if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } /* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. */ static int fts5StorageDeleteFromIndex( Fts5Storage *p, i64 iDel, sqlite3_value **apVal ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ int rc; /* Return code */ int rc2; /* sqlite3_reset() return code */ int iCol; Fts5InsertCtx ctx; if( apVal==0 ){ rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int64(pSeek, 1, iDel); if( sqlite3_step(pSeek)!=SQLITE_ROW ){ return sqlite3_reset(pSeek); } } ctx.pStorage = p; ctx.iCol = -1; rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ const char *zText; int nText; if( pSeek ){ zText = (const char*)sqlite3_column_text(pSeek, iCol); nText = sqlite3_column_bytes(pSeek, iCol); }else{ zText = (const char*)sqlite3_value_text(apVal[iCol-1]); nText = sqlite3_value_bytes(apVal[iCol-1]); } ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, zText, nText, (void*)&ctx, fts5StorageInsertCallback ); p->aTotalSize[iCol-1] -= (i64)ctx.szCol; } } p->nTotalRow--; rc2 = sqlite3_reset(pSeek); if( rc==SQLITE_OK ) rc = rc2; return rc; } /* ** Insert a record into the %_docsize table. Specifically, do: ** ................................................................................ return rc; } /* ** Remove a row from the FTS table. */ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 ); rc = fts5StorageLoadTotals(p, 1); /* Delete the index records */ if( rc==SQLITE_OK ){ rc = fts5StorageDeleteFromIndex(p, iDel, apVal); } /* Delete the %_docsize record */ if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDel, 1, iDel); ................................................................................ } /* Delete the %_content record */ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); } if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDel, 1, iDel); sqlite3_step(pDel); rc = sqlite3_reset(pDel); } } ................................................................................ int sqlite3Fts5StorageOptimize(Fts5Storage *p){ return sqlite3Fts5IndexOptimize(p->pIndex); } int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ return sqlite3Fts5IndexMerge(p->pIndex, nMerge); } int sqlite3Fts5StorageReset(Fts5Storage *p){ return sqlite3Fts5IndexReset(p->pIndex); } /* ** Allocate a new rowid. This is used for "external content" tables when ** a NULL value is inserted into the rowid column. The new rowid is allocated ** by inserting a dummy row into the %_docsize table. The dummy will be ** overwritten later. ** ................................................................................ */ typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; struct Fts5IntegrityCtx { i64 iRowid; int iCol; int szCol; u64 cksum; Fts5Termset *pTermset; Fts5Config *pConfig; }; /* ** Tokenization callback used by integrity check. */ static int fts5StorageIntegrityCallback( void *pContext, /* Pointer to Fts5IntegrityCtx object */ int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iUnused1, /* Start offset of token */ int iUnused2 /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; Fts5Termset *pTermset = pCtx->pTermset; int bPresent; int ii; int rc = SQLITE_OK; int iPos; int iCol; UNUSED_PARAM2(iUnused1, iUnused2); if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } switch( pCtx->pConfig->eDetail ){ case FTS5_DETAIL_FULL: iPos = pCtx->szCol-1; iCol = pCtx->iCol; break; case FTS5_DETAIL_COLUMNS: iPos = pCtx->iCol; iCol = 0; break; default: assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); iPos = 0; iCol = 0; break; } rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); if( rc==SQLITE_OK && bPresent==0 ){ pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( pCtx->iRowid, iCol, iPos, 0, pToken, nToken ); } for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){ const int nChar = pCtx->pConfig->aPrefix[ii]; int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); if( bPresent==0 ){ pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte ); } } } return rc; } /* ** Check that the contents of the FTS index match that of the %_content ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return ** some other SQLite error code if an error occurs while attempting to ** determine this. ................................................................................ int rc2; while( SQLITE_ROW==sqlite3_step(pScan) ){ int i; ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; if( pConfig->bColumnsize ){ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); } if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, i+1), sqlite3_column_bytes(pScan, i+1), (void*)&ctx, fts5StorageIntegrityCallback ); } if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ rc = FTS5_CORRUPT; } aTotalSize[i] += ctx.szCol; if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ sqlite3Fts5TermsetFree(ctx.pTermset); ctx.pTermset = 0; } } sqlite3Fts5TermsetFree(ctx.pTermset); ctx.pTermset = 0; if( rc!=SQLITE_OK ) break; } rc2 = sqlite3_reset(pScan); if( rc==SQLITE_OK ) rc = rc2; } /* Test that the "totals" (sometimes called "averages") record looks Ok */ ................................................................................ rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); if( rc==SQLITE_OK ){ p->pConfig->iCookie = iNew; } } return rc; } |
Changes to ext/fts5/fts5_tcl.c.
20 21 22 23 24 25 26 27 28 29 30 31 32 33 ... 231 232 233 234 235 236 237 238 239 240 241 242 243 244 ... 424 425 426 427 428 429 430 431 432 433 434 435 436 437 .... 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 .... 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 |
#include "fts5.h" #include <string.h> #include <assert.h> extern int sqlite3_fts5_may_be_corrupt; extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *); /************************************************************************* ** This is a copy of the first part of the SqliteDb structure in ** tclsqlite.c. We need it here so that the get_sqlite_pointer routine ** can extract the sqlite3* pointer from an existing Tcl SQLite ** connection. */ ................................................................................ { "xColumnText", 1, "COL" }, /* 9 */ { "xColumnSize", 1, "COL" }, /* 10 */ { "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */ { "xSetAuxdata", 1, "VALUE" }, /* 12 */ { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { 0, 0, 0} }; int rc; int iSub = 0; F5tApi *p = (F5tApi*)clientData; ................................................................................ } CASE(15, "xGetAuxdataInt") { int iVal; int bClear; if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR; iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0); Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); break; } default: assert( 0 ); break; } ................................................................................ rc = sqlite3Fts5TestRegisterMatchinfo(db); if( rc!=SQLITE_OK ){ Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_ERROR; } return TCL_OK; } /* ** Entry point. */ int Fts5tcl_Init(Tcl_Interp *interp){ static struct Cmd { char *zName; ................................................................................ } aCmd[] = { { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 } }; int i; F5tTokenizerContext *pContext; pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext)); memset(pContext, 0, sizeof(*pContext)); |
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > |
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 ... 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 ... 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 .... 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 .... 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 |
#include "fts5.h" #include <string.h> #include <assert.h> extern int sqlite3_fts5_may_be_corrupt; extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*); /************************************************************************* ** This is a copy of the first part of the SqliteDb structure in ** tclsqlite.c. We need it here so that the get_sqlite_pointer routine ** can extract the sqlite3* pointer from an existing Tcl SQLite ** connection. */ ................................................................................ { "xColumnText", 1, "COL" }, /* 9 */ { "xColumnSize", 1, "COL" }, /* 10 */ { "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */ { "xSetAuxdata", 1, "VALUE" }, /* 12 */ { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ { 0, 0, 0} }; int rc; int iSub = 0; F5tApi *p = (F5tApi*)clientData; ................................................................................ } CASE(15, "xGetAuxdataInt") { int iVal; int bClear; if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR; iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0); Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); break; } CASE(16, "xPhraseForeach") { int iPhrase; int iCol; int iOff; const char *zColvar; const char *zOffvar; Tcl_Obj *pScript = objv[5]; Fts5PhraseIter iter; if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; zColvar = Tcl_GetString(objv[3]); zOffvar = Tcl_GetString(objv[4]); rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){ Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); rc = Tcl_EvalObjEx(interp, pScript, 0); if( rc==TCL_CONTINUE ) rc = TCL_OK; if( rc!=TCL_OK ){ if( rc==TCL_BREAK ) rc = TCL_OK; break; } } break; } CASE(17, "xPhraseColumnForeach") { int iPhrase; int iCol; const char *zColvar; Tcl_Obj *pScript = objv[4]; Fts5PhraseIter iter; if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; zColvar = Tcl_GetString(objv[3]); rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){ Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); rc = Tcl_EvalObjEx(interp, pScript, 0); if( rc==TCL_CONTINUE ) rc = TCL_OK; if( rc!=TCL_OK ){ if( rc==TCL_BREAK ) rc = TCL_OK; break; } } break; } default: assert( 0 ); break; } ................................................................................ rc = sqlite3Fts5TestRegisterMatchinfo(db); if( rc!=SQLITE_OK ){ Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_ERROR; } return TCL_OK; } static int f5tRegisterTok( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ int rc; sqlite3 *db = 0; fts5_api *pApi = 0; if( objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB"); return TCL_ERROR; } if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){ return TCL_ERROR; } rc = sqlite3Fts5TestRegisterTok(db, pApi); if( rc!=SQLITE_OK ){ Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_ERROR; } return TCL_OK; } /* ** Entry point. */ int Fts5tcl_Init(Tcl_Interp *interp){ static struct Cmd { char *zName; ................................................................................ } aCmd[] = { { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }, { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 } }; int i; F5tTokenizerContext *pContext; pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext)); memset(pContext, 0, sizeof(*pContext)); |
Changes to ext/fts5/fts5_test_mi.c.
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 .. 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 ... 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 ... 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 ... 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 |
** This file exports a single function that may be called to register the ** matchinfo() implementation with a database handle: ** ** int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db); */ #ifdef SQLITE_TEST #ifdef SQLITE_ENABLE_FTS5 #include "fts5.h" #include <tcl.h> #include <assert.h> #include <string.h> typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx; typedef unsigned int u32; struct Fts5MatchinfoCtx { int nCol; /* Number of cols in FTS5 table */ int nPhrase; /* Number of phrases in FTS5 query */ char *zArg; /* nul-term'd copy of 2nd arg */ int nRet; /* Number of elements in aRet[] */ u32 *aRet; /* Array of 32-bit unsigned ints to return */ ................................................................................ /* ** Return a pointer to the fts5_api pointer for database connection db. ** If an error occurs, return NULL and leave an error in the database ** handle (accessible using sqlite3_errcode()/errmsg()). */ static fts5_api *fts5_api_from_db(sqlite3 *db){ fts5_api *pRet = 0; sqlite3_stmt *pStmt = 0; if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0) && SQLITE_ROW==sqlite3_step(pStmt) && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0) ){ memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet)); } sqlite3_finalize(pStmt); return pRet; } /* ** Argument f should be a flag accepted by matchinfo() (a valid character ** in the string passed as the second argument). If it is not, -1 is ** returned. Otherwise, if f is a valid matchinfo flag, the value returned ................................................................................ ){ Fts5PhraseIter iter; int iCol, iOff; u32 *aOut = (u32*)pUserData; int iPrev = -1; for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); iOff>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ aOut[iCol*3+1]++; if( iCol!=iPrev ) aOut[iCol*3 + 2]++; iPrev = iCol; } ................................................................................ char f, u32 *aOut ){ int i; int rc = SQLITE_OK; switch( f ){ case 'b': case 'x': case 'y': { int nMul = (f=='x' ? 3 : 1); int iPhrase; if( f=='b' ){ int nInt = ((p->nCol + 31) / 32) * p->nPhrase; for(i=0; i<nInt; i++) aOut[i] = 0; }else{ for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; } for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ Fts5PhraseIter iter; int iOff, iCol; for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); iOff>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ if( f=='b' ){ aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); }else{ aOut[nMul * (iCol + iPhrase * p->nCol)]++; } } } break; } case 'l': { for(i=0; rc==SQLITE_OK && i<p->nCol; i++){ int nToken; ................................................................................ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ int rc; /* Return code */ fts5_api *pApi; /* FTS5 API functions */ /* Extract the FTS5 API pointer from the database handle. The ** fts5_api_from_db() function above is copied verbatim from the ** FTS5 documentation. Refer there for details. */ pApi = fts5_api_from_db(db); /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered ** with this database handle, or an error (OOM perhaps?) has occurred. ** ** Also check that the fts5_api object is version 2 or newer. */ if( pApi==0 || pApi->iVersion<1 ){ return SQLITE_ERROR; } /* Register the implementation of matchinfo() */ rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); return rc; } #endif /* SQLITE_ENABLE_FTS5 */ #endif /* SQLITE_TEST */ |
< < > > > | < > > | > | | | | | | > > | | | > > > > > > > > > > > > > > > > > > < < < < | < < < < < | > | < |
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 .. 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 ... 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 ... 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 ... 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 |
** This file exports a single function that may be called to register the ** matchinfo() implementation with a database handle: ** ** int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db); */ #ifdef SQLITE_ENABLE_FTS5 #include "fts5.h" #include <assert.h> #include <string.h> typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx; #ifndef SQLITE_AMALGAMATION typedef unsigned int u32; #endif struct Fts5MatchinfoCtx { int nCol; /* Number of cols in FTS5 table */ int nPhrase; /* Number of phrases in FTS5 query */ char *zArg; /* nul-term'd copy of 2nd arg */ int nRet; /* Number of elements in aRet[] */ u32 *aRet; /* Array of 32-bit unsigned ints to return */ ................................................................................ /* ** Return a pointer to the fts5_api pointer for database connection db. ** If an error occurs, return NULL and leave an error in the database ** handle (accessible using sqlite3_errcode()/errmsg()). */ static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){ sqlite3_stmt *pStmt = 0; int rc; *ppApi = 0; rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0) ){ memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*)); } rc = sqlite3_finalize(pStmt); } return rc; } /* ** Argument f should be a flag accepted by matchinfo() (a valid character ** in the string passed as the second argument). If it is not, -1 is ** returned. Otherwise, if f is a valid matchinfo flag, the value returned ................................................................................ ){ Fts5PhraseIter iter; int iCol, iOff; u32 *aOut = (u32*)pUserData; int iPrev = -1; for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); iCol>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ aOut[iCol*3+1]++; if( iCol!=iPrev ) aOut[iCol*3 + 2]++; iPrev = iCol; } ................................................................................ char f, u32 *aOut ){ int i; int rc = SQLITE_OK; switch( f ){ case 'b': { int iPhrase; int nInt = ((p->nCol + 31) / 32) * p->nPhrase; for(i=0; i<nInt; i++) aOut[i] = 0; for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ Fts5PhraseIter iter; int iCol; for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); iCol>=0; pApi->xPhraseNextColumn(pFts, &iter, &iCol) ){ aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); } } break; } case 'x': case 'y': { int nMul = (f=='x' ? 3 : 1); int iPhrase; for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ Fts5PhraseIter iter; int iOff, iCol; for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); iOff>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ aOut[nMul * (iCol + iPhrase * p->nCol)]++; } } break; } case 'l': { for(i=0; rc==SQLITE_OK && i<p->nCol; i++){ int nToken; ................................................................................ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ int rc; /* Return code */ fts5_api *pApi; /* FTS5 API functions */ /* Extract the FTS5 API pointer from the database handle. The ** fts5_api_from_db() function above is copied verbatim from the ** FTS5 documentation. Refer there for details. */ rc = fts5_api_from_db(db, &pApi); if( rc!=SQLITE_OK ) return rc; /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered ** with this database handle, or an error (OOM perhaps?) has occurred. ** ** Also check that the fts5_api object is version 2 or newer. */ if( pApi==0 || pApi->iVersion<2 ){ return SQLITE_ERROR; } /* Register the implementation of matchinfo() */ rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0); return rc; } #endif /* SQLITE_ENABLE_FTS5 */ |
Added ext/fts5/fts5_test_tok.c.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |