/ Check-in [1c78d892]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge latest trunk changes with this branch.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 1c78d8920fb59da3cb97dd2eb09b3e08dfd14259
User & Date: dan 2015-04-24 20:18:21
Context
2015-04-25
18:56
Add tests for fts5. check-in: e748651c user: dan tags: fts5
2015-04-24
20:18
Merge latest trunk changes with this branch. check-in: 1c78d892 user: dan tags: fts5
19:41
Add the "unindexed" column option to fts5. check-in: 86309961 user: dan tags: fts5
18:31
Fix fuzzershell so that it works with SQLITE_OMIT_TRACE. check-in: dc88fe7e user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.in.

947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964




965
966
967
968
969
970
971
972
973
974

975
976
977
978
979
980
981
982
983
984
985
986
987
988
TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))

testfixture$(TEXE):	$(TESTFIXTURE_SRC)
	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not aeverything.
fulltestonly:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/full.test





# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	testfixture$(TEXE) sqlite3$(TEXE)

	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	testfixture$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/main.test

sqlite3_analyzer.c: sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
	echo "#define TCLSH 2" > $@
	cat sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c >> $@
	echo "static const char *tclsh_main_loop(void){" >> $@
	echo "static const char *zMainloop = " >> $@







|










>
>
>
>



|





|
>






|







947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))

testfixture$(TEXE):	$(TESTFIXTURE_SRC)
	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	testfixture$(TEXE) sqlite3$(TEXE) fuzztest
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not aeverything.
fulltestonly:	testfixture$(TEXE) sqlite3$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzershell$(TEXE)
	./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt

# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	testfixture$(TEXE) sqlite3$(TEXE) fuzztest
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	testfixture$(TEXE) sqlite3$(TEXE) fuzzershell$(TEXE)
	valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt
	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	testfixture$(TEXE) fuzzershell$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/main.test

sqlite3_analyzer.c: sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
	echo "#define TCLSH 2" > $@
	cat sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c >> $@
	echo "static const char *tclsh_main_loop(void){" >> $@
	echo "static const char *zMainloop = " >> $@

Changes to Makefile.msc.

1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638



1639
1640
1641
1642
1643
1644
1645
1646
		-DBUILD_sqlite -I$(TCLINCDIR) \
		$(TESTFIXTURE_SRC) \
		/link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

extensiontest: testfixture.exe testloadext.dll
	.\testfixture.exe $(TOP)\test\loadext.test

fulltest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\all.test

soaktest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\all.test -soak=1

fulltestonly:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner




test:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\veryquick.test

smoketest:	testfixture.exe
	.\testfixture.exe $(TOP)\test\main.test

sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\test_stat.c $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
	copy $(SQLITE3C) + $(TOP)\src\test_stat.c + $(TOP)\src\tclsqlite.c $@







|





|





>
>
>
|







1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
		-DBUILD_sqlite -I$(TCLINCDIR) \
		$(TESTFIXTURE_SRC) \
		/link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)

extensiontest: testfixture.exe testloadext.dll
	.\testfixture.exe $(TOP)\test\loadext.test

fulltest:	testfixture.exe sqlite3.exe fuzztest
	.\testfixture.exe $(TOP)\test\all.test

soaktest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\all.test -soak=1

fulltestonly:	testfixture.exe sqlite3.exe fuzztest
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner

fuzztest:	fuzzershell.exe
	.\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt

test:	testfixture.exe sqlite3.exe fuzztest
	.\testfixture.exe $(TOP)\test\veryquick.test

smoketest:	testfixture.exe
	.\testfixture.exe $(TOP)\test\main.test

sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\test_stat.c $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
	copy $(SQLITE3C) + $(TOP)\src\test_stat.c + $(TOP)\src\tclsqlite.c $@

Changes to ext/fts3/fts3.c.

2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
  ** for the pending-terms. If this is a scan, then this call must be being
  ** made by an fts4aux module, not an FTS table. In this case calling
  ** Fts3SegReaderPending might segfault, as the data structures used by 
  ** fts4aux are not completely populated. So it's easiest to filter these
  ** calls out here.  */
  if( iLevel<0 && p->aIndex ){
    Fts3SegReader *pSeg = 0;
    rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
    if( rc==SQLITE_OK && pSeg ){
      rc = fts3SegReaderCursorAppend(pCsr, pSeg);
    }
  }

  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
    if( rc==SQLITE_OK ){







|







2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
  ** for the pending-terms. If this is a scan, then this call must be being
  ** made by an fts4aux module, not an FTS table. In this case calling
  ** Fts3SegReaderPending might segfault, as the data structures used by 
  ** fts4aux are not completely populated. So it's easiest to filter these
  ** calls out here.  */
  if( iLevel<0 && p->aIndex ){
    Fts3SegReader *pSeg = 0;
    rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg);
    if( rc==SQLITE_OK && pSeg ){
      rc = fts3SegReaderCursorAppend(pCsr, pSeg);
    }
  }

  if( iLevel!=FTS3_SEGCURSOR_PENDING ){
    if( rc==SQLITE_OK ){

Changes to ext/fts3/fts3_write.c.

322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
....
3440
3441
3442
3443
3444
3445
3446
3447

3448
3449
3450
3451
3452
3453
3454
....
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
....
5140
5141
5142
5143
5144
5145
5146
5147

5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
....
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
/* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=?",
/* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(?,?)",
/* 24 */  "",
/* 25 */  "",

/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",

/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
** of the oldest level in the db that contains at least ? segments. Or,
** if no level in the FTS index contains more than ? segments, the statement
** returns zero rows.  */
/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
................................................................................
  int bSeenDone = 0;
  int rc;
  sqlite3_stmt *pAllLangid = 0;

  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pAllLangid, 1, p->nIndex);

    while( sqlite3_step(pAllLangid)==SQLITE_ROW ){
      int i;
      int iLangid = sqlite3_column_int(pAllLangid, 0);
      for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
        rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL);
        if( rc==SQLITE_DONE ){
          bSeenDone = 1;
................................................................................
  i = pHint->n-2;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;

  pHint->n = i;
  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
  i += fts3GetVarint32(&pHint->a[i], pnInput);
  if( i!=nHint ) return SQLITE_CORRUPT_VTAB;

  return SQLITE_OK;
}


/*
** Attempt an incremental merge that writes nMerge leaf blocks.
................................................................................
  u64 cksum2 = 0;                 /* Checksum based on %_content contents */
  sqlite3_stmt *pAllLangid = 0;   /* Statement to return all language-ids */

  /* This block calculates the checksum according to the FTS index. */
  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pAllLangid, 1, p->nIndex);

    while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
      int iLangid = sqlite3_column_int(pAllLangid, 0);
      int i;
      for(i=0; i<p->nIndex; i++){
        cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc);
      }
    }
    rc2 = sqlite3_reset(pAllLangid);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  /* This block calculates the checksum according to the %_content table */
  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
    sqlite3_stmt *pStmt = 0;
    char *zSql;
   
    zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
    if( !zSql ){
................................................................................
*/
static int fts3DoIntegrityCheck(
  Fts3Table *p                    /* FTS3 table handle */
){
  int rc;
  int bOk = 0;
  rc = fts3IntegrityCheck(p, &bOk);
  if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
  return rc;
}

/*
** Handle a 'special' INSERT of the form:
**
**   "INSERT INTO tbl(tbl) VALUES(<expr>)"







|







 







|
>







 







|







 







|
>












<







 







|







322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
....
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
....
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
....
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161

5162
5163
5164
5165
5166
5167
5168
....
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
/* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=?",
/* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(?,?)",
/* 24 */  "",
/* 25 */  "",

/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'",

/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
** of the oldest level in the db that contains at least ? segments. Or,
** if no level in the FTS index contains more than ? segments, the statement
** returns zero rows.  */
/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
................................................................................
  int bSeenDone = 0;
  int rc;
  sqlite3_stmt *pAllLangid = 0;

  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
    sqlite3_bind_int(pAllLangid, 2, p->nIndex);
    while( sqlite3_step(pAllLangid)==SQLITE_ROW ){
      int i;
      int iLangid = sqlite3_column_int(pAllLangid, 0);
      for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
        rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL);
        if( rc==SQLITE_DONE ){
          bSeenDone = 1;
................................................................................
  i = pHint->n-2;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;

  pHint->n = i;
  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
  i += fts3GetVarint32(&pHint->a[i], pnInput);
  if( i!=nHint ) return FTS_CORRUPT_VTAB;

  return SQLITE_OK;
}


/*
** Attempt an incremental merge that writes nMerge leaf blocks.
................................................................................
  u64 cksum2 = 0;                 /* Checksum based on %_content contents */
  sqlite3_stmt *pAllLangid = 0;   /* Statement to return all language-ids */

  /* This block calculates the checksum according to the FTS index. */
  rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
    sqlite3_bind_int(pAllLangid, 2, p->nIndex);
    while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
      int iLangid = sqlite3_column_int(pAllLangid, 0);
      int i;
      for(i=0; i<p->nIndex; i++){
        cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc);
      }
    }
    rc2 = sqlite3_reset(pAllLangid);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  /* This block calculates the checksum according to the %_content table */

  if( rc==SQLITE_OK ){
    sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
    sqlite3_stmt *pStmt = 0;
    char *zSql;
   
    zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
    if( !zSql ){
................................................................................
*/
static int fts3DoIntegrityCheck(
  Fts3Table *p                    /* FTS3 table handle */
){
  int rc;
  int bOk = 0;
  rc = fts3IntegrityCheck(p, &bOk);
  if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
  return rc;
}

/*
** Handle a 'special' INSERT of the form:
**
**   "INSERT INTO tbl(tbl) VALUES(<expr>)"

Changes to ext/fts5/fts5_config.c.

403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
**
** If *pRc is other than SQLITE_OK when this function is called, it is
** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
** set if a parse error (failed to find close quote) occurs.
*/
static const char *fts5ConfigGobbleWord(
  int *pRc, 
  const char *zIn, 
  char **pzOut, 
  int *pbQuoted
){
  const char *zRet = 0;
  *pbQuoted = 0;
  *pzOut = 0;

  if( *pRc==SQLITE_OK ){
    int nIn = strlen(zIn);







|
|
|
|







403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
**
** If *pRc is other than SQLITE_OK when this function is called, it is
** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
** set if a parse error (failed to find close quote) occurs.
*/
static const char *fts5ConfigGobbleWord(
  int *pRc,                       /* IN/OUT: Error code */
  const char *zIn,                /* Buffer to gobble string/bareword from */
  char **pzOut,                   /* OUT: malloc'd buffer containing str/bw */
  int *pbQuoted                   /* OUT: Set to true if dequoting required */
){
  const char *zRet = 0;
  *pbQuoted = 0;
  *pzOut = 0;

  if( *pRc==SQLITE_OK ){
    int nIn = strlen(zIn);

Changes to main.mk.

707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725



726
727
728







729
730
731
732
733
734
735

fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner




test:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/veryquick.test








# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                  $(TOP)/test/tt3_checkpoint.c \
                  $(TOP)/test/tt3_index.c      \







|





|





>
>
>
|


>
>
>
>
>
>
>







707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745

fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
	$(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS)                  \
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	testfixture$(EXE) sqlite3$(EXE) fuzztest
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	testfixture$(EXE) sqlite3$(EXE) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner

fuzztest:	fuzzershell$(EXE)
	./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt

test:	testfixture$(EXE) sqlite3$(EXE) fuzztest
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	testfixture$(EXE) sqlite3$(EXE) fuzzershell$(EXE)
	valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt
	OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                  $(TOP)/test/tt3_checkpoint.c \
                  $(TOP)/test/tt3_index.c      \

Changes to src/expr.c.

3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
    }
    case TK_AS: {
      sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken);
      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
      break;
    }
    case TK_ID: {
      sqlite3TreeViewLine(pView,"ID %Q", pExpr->u.zToken);
      break;
    }
#ifndef SQLITE_OMIT_CAST
    case TK_CAST: {
      /* Expressions of the form:   CAST(pLeft AS token) */
      sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken);
      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);







|







3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
    }
    case TK_AS: {
      sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken);
      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
      break;
    }
    case TK_ID: {
      sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken);
      break;
    }
#ifndef SQLITE_OMIT_CAST
    case TK_CAST: {
      /* Expressions of the form:   CAST(pLeft AS token) */
      sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken);
      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);

Changes to src/resolve.c.

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
..
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
**     SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase;
**
** Should be transformed into:
**
**     SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase;
**
** The nSubquery parameter specifies how many levels of subquery the
** alias is removed from the original expression.  The usually value is
** zero but it might be more if the alias is contained within a subquery
** of the original expression.  The Expr.op2 field of TK_AGG_FUNCTION
** structures must be increased by the nSubquery amount.
*/
static void resolveAlias(
  Parse *pParse,         /* Parsing context */
  ExprList *pEList,      /* A result set */
................................................................................
  Expr *pOrig;           /* The iCol-th column of the result set */
  Expr *pDup;            /* Copy of pOrig */
  sqlite3 *db;           /* The database connection */

  assert( iCol>=0 && iCol<pEList->nExpr );
  pOrig = pEList->a[iCol].pExpr;
  assert( pOrig!=0 );
  assert( pOrig->flags & EP_Resolved );
  db = pParse->db;
  pDup = sqlite3ExprDup(db, pOrig, 0);
  if( pDup==0 ) return;
  if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){
    incrAggFunctionDepth(pDup, nSubquery);
    pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0);
    if( pDup==0 ) return;







|







 







|







75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
..
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
**     SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase;
**
** Should be transformed into:
**
**     SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase;
**
** The nSubquery parameter specifies how many levels of subquery the
** alias is removed from the original expression.  The usual value is
** zero but it might be more if the alias is contained within a subquery
** of the original expression.  The Expr.op2 field of TK_AGG_FUNCTION
** structures must be increased by the nSubquery amount.
*/
static void resolveAlias(
  Parse *pParse,         /* Parsing context */
  ExprList *pEList,      /* A result set */
................................................................................
  Expr *pOrig;           /* The iCol-th column of the result set */
  Expr *pDup;            /* Copy of pOrig */
  sqlite3 *db;           /* The database connection */

  assert( iCol>=0 && iCol<pEList->nExpr );
  pOrig = pEList->a[iCol].pExpr;
  assert( pOrig!=0 );
  assert( (pOrig->flags & EP_Resolved)!=0 || zType[0]==0 );
  db = pParse->db;
  pDup = sqlite3ExprDup(db, pOrig, 0);
  if( pDup==0 ) return;
  if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){
    incrAggFunctionDepth(pDup, nSubquery);
    pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0);
    if( pDup==0 ) return;

Changes to test/fts4check.test.

174
175
176
177
178
179
180
181






























182
183
  db close
  sqlite3 db test.db
  catchsql {
    INSERT INTO t4(t4) VALUES('integrity-check');
  }
} {1 {database disk image is malformed}}
reset_db































finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
  db close
  sqlite3 db test.db
  catchsql {
    INSERT INTO t4(t4) VALUES('integrity-check');
  }
} {1 {database disk image is malformed}}
reset_db

#--------------------------------------------------------------------------
# Test case 5.*
#
# Test that the integrity-check works if there is uncommitted data.
#
do_execsql_test 5.0 {
  BEGIN;
  CREATE VIRTUAL TABLE t5 USING fts4(a, prefix="1,2,3");
  INSERT INTO t5 VALUES('And down by Kosiosko, where the reed-banks sweep');
  INSERT INTO t5 VALUES('and sway, and the rolling plains are wide, the');
  INSERT INTO t5 VALUES('man from snowy river is a household name today,');
  INSERT INTO t5 VALUES('and the stockmen tell the story of his ride');
}

do_execsql_test 5.1 {
  INSERT INTO t5(t5) VALUES('integrity-check');
} {}

do_catchsql_test 5.2 {
  INSERT INTO t5_content VALUES(5, 'his hardy mountain pony');
  INSERT INTO t5(t5) VALUES('integrity-check');
} {1 {database disk image is malformed}}

do_execsql_test 5.3 ROLLBACK

do_execsql_test 5.4 {
  CREATE VIRTUAL TABLE t5 USING fts4(a, prefix="1,2,3");
  INSERT INTO t5(t5) VALUES('integrity-check');
} {}

finish_test

Added test/fuzzdata1.txt.

cannot compute difference between binary files

Changes to test/releasetest.tcl.

14
15
16
17
18
19
20

21
22
23
24
25
26
27
...
195
196
197
198
199
200
201


202
203
204
205
206
207
208
...
251
252
253
254
255
256
257


258
259
260
261
262
263
264
...
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
...
478
479
480
481
482
483
484

485
486
487
488
489
490
491
...
551
552
553
554
555
556
557




558
559
560
561
562
563
564
    --config   CONFIGNAME              (Run only CONFIGNAME)
    --quick                            (Run "veryquick.test" only)
    --veryquick                        (Run "make smoketest" only)
    --msvc                             (Use MSVC as the compiler)
    --buildonly                        (Just build testfixture - do not run)
    --dryrun                           (Print what would have happened)
    --info                             (Show diagnostic info)


The default value for --srcdir is the parent of the directory holding
this script.

The script determines the default value for --platform using the
$tcl_platform(os) and $tcl_platform(machine) variables.  Supported
platforms are "Linux-x86", "Linux-x86_64", "Darwin-i386",
................................................................................
  # different names for them all so that they results appear in separate
  # subdirectories.
  #
  Fail0 {-O0}
  Fail2 {-O0}
  Fail3 {-O0}
  Fail4 {-O0}


}]

array set ::Platforms [strip_comments {
  Linux-x86_64 {
    "Check-Symbols"           checksymbols
    "Debug-One"               "mptest test"
    "Have-Not"                test
................................................................................
  #
  Failure-Detection {
    Fail0     "TEST_FAILURE=0 test"
    Sanitize  "TEST_FAILURE=1 test"
    Fail2     "TEST_FAILURE=2 valgrindtest"
    Fail3     "TEST_FAILURE=3 valgrindtest"
    Fail4     "TEST_FAILURE=4 test"


  }
}]


# End of configuration section.
#########################################################################
#########################################################################
................................................................................
  # OPTS Makefile variable. Variable $cflags holds the value for
  # CFLAGS. The makefile will pass OPTS to both gcc and lemon, but
  # CFLAGS is only passed to gcc.
  #
  set cflags [expr {$::MSVC ? "-Zi" : "-g"}]
  set opts ""
  set title ${name}($testtarget)
  set configOpts ""

  regsub -all {#[^\n]*\n} $config \n config
  foreach arg $config {
    if {[regexp {^-[UD]} $arg]} {
      lappend opts $arg
    } elseif {[regexp {^[A-Z]+=} $arg]} {
      lappend testtarget $arg
................................................................................
  set ::SRCDIR    [file normalize [file dirname [file dirname $::argv0]]]
  set ::QUICK     0
  set ::MSVC      0
  set ::BUILDONLY 0
  set ::DRYRUN    0
  set ::EXEC      exec
  set ::TRACE     0

  set config {}
  set platform $::tcl_platform(os)-$::tcl_platform(machine)

  for {set i 0} {$i < [llength $argv]} {incr i} {
    set x [lindex $argv $i]
    if {[regexp {^--[a-z]} $x]} {set x [string range $x 1 end]}
    switch -glob -- $x {
................................................................................
      -g {
        if {$::MSVC} {
          lappend ::EXTRACONFIG -Zi
        } else {
          lappend ::EXTRACONFIG [lindex $argv $i]
        }
      }





      -D* -
      -O* -
      -enable-* -
      -disable-* -
      *=* {
        lappend ::EXTRACONFIG [lindex $argv $i]







>







 







>
>







 







>
>







 







|







 







>







 







>
>
>
>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
...
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
...
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
...
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
...
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
    --config   CONFIGNAME              (Run only CONFIGNAME)
    --quick                            (Run "veryquick.test" only)
    --veryquick                        (Run "make smoketest" only)
    --msvc                             (Use MSVC as the compiler)
    --buildonly                        (Just build testfixture - do not run)
    --dryrun                           (Print what would have happened)
    --info                             (Show diagnostic info)
    --with-tcl=DIR                     (Use TCL build at DIR)

The default value for --srcdir is the parent of the directory holding
this script.

The script determines the default value for --platform using the
$tcl_platform(os) and $tcl_platform(machine) variables.  Supported
platforms are "Linux-x86", "Linux-x86_64", "Darwin-i386",
................................................................................
  # different names for them all so that they results appear in separate
  # subdirectories.
  #
  Fail0 {-O0}
  Fail2 {-O0}
  Fail3 {-O0}
  Fail4 {-O0}
  FuzzFail1 {-O0}
  FuzzFail2 {-O0}
}]

array set ::Platforms [strip_comments {
  Linux-x86_64 {
    "Check-Symbols"           checksymbols
    "Debug-One"               "mptest test"
    "Have-Not"                test
................................................................................
  #
  Failure-Detection {
    Fail0     "TEST_FAILURE=0 test"
    Sanitize  "TEST_FAILURE=1 test"
    Fail2     "TEST_FAILURE=2 valgrindtest"
    Fail3     "TEST_FAILURE=3 valgrindtest"
    Fail4     "TEST_FAILURE=4 test"
    FuzzFail1 "TEST_FAILURE=5 test"
    FuzzFail2 "TEST_FAILURE=5 valgrindtest"
  }
}]


# End of configuration section.
#########################################################################
#########################################################################
................................................................................
  # OPTS Makefile variable. Variable $cflags holds the value for
  # CFLAGS. The makefile will pass OPTS to both gcc and lemon, but
  # CFLAGS is only passed to gcc.
  #
  set cflags [expr {$::MSVC ? "-Zi" : "-g"}]
  set opts ""
  set title ${name}($testtarget)
  set configOpts $::WITHTCL

  regsub -all {#[^\n]*\n} $config \n config
  foreach arg $config {
    if {[regexp {^-[UD]} $arg]} {
      lappend opts $arg
    } elseif {[regexp {^[A-Z]+=} $arg]} {
      lappend testtarget $arg
................................................................................
  set ::SRCDIR    [file normalize [file dirname [file dirname $::argv0]]]
  set ::QUICK     0
  set ::MSVC      0
  set ::BUILDONLY 0
  set ::DRYRUN    0
  set ::EXEC      exec
  set ::TRACE     0
  set ::WITHTCL   {}
  set config {}
  set platform $::tcl_platform(os)-$::tcl_platform(machine)

  for {set i 0} {$i < [llength $argv]} {incr i} {
    set x [lindex $argv $i]
    if {[regexp {^--[a-z]} $x]} {set x [string range $x 1 end]}
    switch -glob -- $x {
................................................................................
      -g {
        if {$::MSVC} {
          lappend ::EXTRACONFIG -Zi
        } else {
          lappend ::EXTRACONFIG [lindex $argv $i]
        }
      }

      -with-tcl=* {
        set ::WITHTCL -$x
      }

      -D* -
      -O* -
      -enable-* -
      -disable-* -
      *=* {
        lappend ::EXTRACONFIG [lindex $argv $i]

Changes to test/resolver01.test.

7
8
9
10
11
12
13
14
15
16


17
18
19
20
21
22
23
...
197
198
199
200
201
202
203






204
205
206
207
208
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests features of the name resolver (the component that
# figures out what identifiers in the SQL statement refer to) that
# were fixed by ticket [2500cdb9be]
#
# See also tickets [1c69be2daf] and [f617ea3125] from 2013-08-14.


#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# "ORDER BY y" binds to the output result-set column named "y"
# if available.  If no output column is named "y", then try to
................................................................................
  INSERT INTO t63 VALUES ('abc');
  SELECT count(),
       NULLIF(name,'abc') AS name
    FROM t63
   GROUP BY lower(name);
} {1 {} 1 {}}











finish_test







|


>
>







 







>
>
>
>
>
>





7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
...
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests features of the name resolver (the component that
# figures out what identifiers in the SQL statement refer to) that
# were fixed by ticket [2500cdb9be].
#
# See also tickets [1c69be2daf] and [f617ea3125] from 2013-08-14.
#
# Also a fuzzer-discovered problem on 2015-04-23.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# "ORDER BY y" binds to the output result-set column named "y"
# if available.  If no output column is named "y", then try to
................................................................................
  INSERT INTO t63 VALUES ('abc');
  SELECT count(),
       NULLIF(name,'abc') AS name
    FROM t63
   GROUP BY lower(name);
} {1 {} 1 {}}

do_execsql_test resolver01-7.1 {
  SELECT 2 AS x WHERE (SELECT x AS y WHERE 3>y);
} {2}
do_execsql_test resolver01-7.2 {
  SELECT 2 AS x WHERE (SELECT x AS y WHERE 1>y);
} {}




finish_test

Changes to test/trace2.test.

139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC"
  }

  do_trace_test 2.3 {
    INSERT INTO x1(x1) VALUES('optimize');
  } {
    "INSERT INTO x1(x1) VALUES('optimize');"
    "-- SELECT DISTINCT level / (1024 * ?) FROM 'main'.'x1_segdir'"
    "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC"
    "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
    "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
    "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
    "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
  }
}

finish_test







|









139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC"
  }

  do_trace_test 2.3 {
    INSERT INTO x1(x1) VALUES('optimize');
  } {
    "INSERT INTO x1(x1) VALUES('optimize');"
    "-- SELECT ? UNION SELECT level / (1024 * ?) FROM 'main'.'x1_segdir'"
    "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC"
    "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
    "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
    "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
    "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
  }
}

finish_test

Changes to tool/fuzzershell.c.

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56






57
58
59
60
61
62
63
...
134
135
136
137
138
139
140


141


142
143
144
145
146
147
148

149
150
151
152
153
154
155
...
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253


254

255
256


257
258
259
260
261
262
263
...
347
348
349
350
351
352
353










354
355
356
357
358
359
360
...
388
389
390
391
392
393
394

395
396

397
398

399
400

401
402
403
404
405
406
407
408
409
410
411
412
413




414
415
416
417
418
419





420
421
422
423
424
425




426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
...
464
465
466
467
468
469
470











471
472
473
474
475
476





477
478
479
480
481
482
483
484

485
486
487






488






489
490
491
492
493
494
495
...
503
504
505
506
507
508
509

510

511
512
513
514
515
516
517
518
519
520
521








522
523
524
525
526
527
528
529
530
531
532

533
534
535
536
537
538
539

540
541
542
543
544


545
546
547
548
549
550
551



































552
553
554
555
556
557
558
559
** 2015-04-20: The input text can be divided into separate SQL chunks using
** lines of the form:
**
**       |****<...>****|
**
** where the "..." is arbitrary text, except the "|" should really be "/".
** ("|" is used here to avoid compiler warnings about nested comments.)
** Each such SQL comment is printed as it is encountered.  A separate 
** in-memory SQLite database is created to run each chunk of SQL.  This
** feature allows the "queue" of AFL to be captured into a single big
** file using a command like this:
**
**    (for i in id:*; do echo '|****<'$i'>****|'; cat $i; done) >~/all-queue.txt
**
** (Once again, change the "|" to "/") Then all elements of the AFL queue
** can be run in a single go (for regression testing, for example) by typing:
**
**    fuzzershell -f ~/all-queue.txt >out.txt
**
** After running each chunk of SQL, the database connection is closed.  The
** program aborts if the close fails or if there is any unfreed memory after
** the close.






*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <ctype.h>
#include "sqlite3.h"
................................................................................
      printf("[%s]\n", argv[i]);
    }else{
      printf("NULL\n");
    }
  }
  return 0;
}





/*
** This callback is invoked by sqlite3_trace() as each SQL statement
** starts.
*/
static void traceCallback(void *NotUsed, const char *zMsg){
  printf("TRACE: %s\n", zMsg);
}


/***************************************************************************
** eval() implementation copied from ../ext/misc/eval.c
*/
/*
** Structure used to accumulate the output
*/
................................................................................
** Print sketchy documentation for this utility program
*/
static void showHelp(void){
  printf("Usage: %s [options]\n", g.zArgv0);
  printf(
"Read SQL text from standard input and evaluate it.\n"
"Options:\n"
"  --autovacuum        Enable AUTOVACUUM mode\n"
"  -f FILE             Read SQL text from FILE instead of standard input\n"
"  --heap SZ MIN       Memory allocator uses SZ bytes & min allocation MIN\n"
"  --help              Show this help text\n"    
"  --initdb DBFILE     Initialize the in-memory database using template DBFILE\n"
"  --lookaside N SZ    Configure lookaside for N slots of SZ bytes each\n"
"  --pagesize N        Set the page size to N\n"
"  --pcache N SZ       Configure N pages of pagecache each of size SZ bytes\n"


"  --scratch N SZ      Configure scratch memory for N slots of SZ bytes each\n"

"  --utf16be           Set text encoding to UTF-16BE\n"
"  --utf16le           Set text encoding to UTF-16LE\n"


  );
}

/*
** Return the value of a hexadecimal digit.  Return -1 if the input
** is not a hex digit.
*/
................................................................................
  void *pLook = 0;              /* Allocated lookaside space */
  void *pPCache = 0;            /* Allocated storage for pcache */
  void *pScratch = 0;           /* Allocated storage for scratch */
  int doAutovac = 0;            /* True for --autovacuum */
  char *zSql;                   /* SQL to run */
  char *zToFree = 0;            /* Call sqlite3_free() on this afte running zSql */
  int iMode = FZMODE_Generic;   /* Operating mode */












  g.zArgv0 = argv[0];
  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
................................................................................
        i += 2;
      }else
      if( strcmp(z,"mode")==0 ){
        if( i>=argc-1 ) abendError("missing argument on %s", argv[i]);
        z = argv[++i];
        if( strcmp(z,"generic")==0 ){
          iMode = FZMODE_Printf;

        }else if( strcmp(z, "glob")==0 ){
          iMode = FZMODE_Glob;

        }else if( strcmp(z, "printf")==0 ){
          iMode = FZMODE_Printf;

        }else if( strcmp(z, "strftime")==0 ){
          iMode = FZMODE_Strftime;

        }else{
          abendError("unknown --mode: %s", z);
        }
      }else
      if( strcmp(z,"pagesize")==0 ){
        if( i>=argc-1 ) abendError("missing argument on %s", argv[i]);
        pageSize = integerValue(argv[++i]);
      }else
      if( strcmp(z,"pcache")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);
        nPCache = integerValue(argv[i+1]);
        szPCache = integerValue(argv[i+2]);
        i += 2;




      }else
      if( strcmp(z,"scratch")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);
        nScratch = integerValue(argv[i+1]);
        szScratch = integerValue(argv[i+2]);
        i += 2;





      }else
      if( strcmp(z,"utf16le")==0 ){
        zEncoding = "utf16le";
      }else
      if( strcmp(z,"utf16be")==0 ){
        zEncoding = "utf16be";




      }else
      {
        abendError("unknown option: %s", argv[i]);
      }
    }else{
      abendError("unknown argument: %s", argv[i]);
    }
  }
  sqlite3_config(SQLITE_CONFIG_LOG, shellLog, 0);
  if( nHeap>0 ){
    pHeap = malloc( nHeap );
    if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap);
    rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap);
    if( rc ) abendError("heap configuration failed: %d\n", rc);
  }
  if( nLook>0 ){
................................................................................
    zIn = realloc(zIn, nAlloc);
    if( zIn==0 ) fatalError("out of memory");
    got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); 
    nIn += (int)got;
    zIn[nIn] = 0;
    if( got==0 ) break;
  }











  if( zInitDb ){
    rc = sqlite3_open_v2(zInitDb, &dbInit, SQLITE_OPEN_READONLY, 0);
    if( rc!=SQLITE_OK ){
      abendError("unable to open initialization database \"%s\"", zInitDb);
    }
  }





  for(i=0; i<nIn; i=iNext){
    char cSaved;
    if( strncmp(&zIn[i], "/****<",6)==0 ){
      char *z = strstr(&zIn[i], ">****/");
      if( z ){
        z += 6;
        printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]);
        i += (int)(z-&zIn[i]);

      }
    }
    for(iNext=i; iNext<nIn && strncmp(&zIn[iNext],"/****<",6)!=0; iNext++){}






    






    rc = sqlite3_open_v2(
      "main.db", &db,
      SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY,
      0);
    if( rc!=SQLITE_OK ){
      abendError("Unable to open the in-memory database");
    }
................................................................................
      rc = sqlite3_backup_step(pBackup, -1);
      if( rc!=SQLITE_DONE ){
        abendError("attempt to initialize the in-memory database failed (rc=%d)",
                   rc);
      }
      sqlite3_backup_finish(pBackup);
    }

    sqlite3_trace(db, traceCallback, 0);

    sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000);
    if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding);
    if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);
    if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL");
    cSaved = zIn[iNext];
    zIn[iNext] = 0;
    printf("INPUT (offset: %d, size: %d): [%s]\n",
            i, (int)strlen(&zIn[i]), &zIn[i]);
    zSql = &zIn[i];








    switch( iMode ){
      case FZMODE_Glob:
        zSql = zToFree = sqlite3_mprintf("SELECT glob(%s);", zSql);
        break;
      case FZMODE_Printf:
        zSql = zToFree = sqlite3_mprintf("SELECT printf(%s);", zSql);
        break;
      case FZMODE_Strftime:
        zSql = zToFree = sqlite3_mprintf("SELECT strftime(%s);", zSql);
        break;
    }

    rc = sqlite3_exec(db, zSql, execCallback, 0, &zErrMsg);
    if( zToFree ){
      sqlite3_free(zToFree);
      zToFree = 0;
    }
    zIn[iNext] = cSaved;


    printf("RESULT-CODE: %d\n", rc);
    if( zErrMsg ){
      printf("ERROR-MSG: [%s]\n", zErrMsg);
      sqlite3_free(zErrMsg);
    }


    rc = sqlite3_close(db);
    if( rc ){
      abendError("sqlite3_close() failed with rc=%d", rc);
    }
    if( sqlite3_memory_used()>0 ){
      abendError("memory in use after close: %lld bytes", sqlite3_memory_used());
    }



































  }
  free(zIn);
  free(pHeap);
  free(pLook);
  free(pScratch);
  free(pPCache);
  return 0;
}







<
|
|







|




>
>
>
>
>
>







 







>
>
|
>
>







>







 







|
|
|
|
|
|
|
|
>
>
|
>
|
|
>
>







 







>
>
>
>
>
>
>
>
>
>







 







>


>


>


>













>
>
>
>






>
>
>
>
>






>
>
>
>








|







 







>
>
>
>
>
>
>
>
>
>
>






>
>
>
>
>
|





|

>



>
>
>
>
>
>
|
>
>
>
>
>
>







 







>
|
>






|
|
|
|
|
>
>
>
>
>
>
>
>











>
|





<
>
|
|
|
<
|
>
>







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>








35
36
37
38
39
40
41

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
...
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
...
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
...
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
...
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
...
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
...
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620

621
622
623
624

625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
** 2015-04-20: The input text can be divided into separate SQL chunks using
** lines of the form:
**
**       |****<...>****|
**
** where the "..." is arbitrary text, except the "|" should really be "/".
** ("|" is used here to avoid compiler warnings about nested comments.)

** A separate in-memory SQLite database is created to run each chunk of SQL.
** This feature allows the "queue" of AFL to be captured into a single big
** file using a command like this:
**
**    (for i in id:*; do echo '|****<'$i'>****|'; cat $i; done) >~/all-queue.txt
**
** (Once again, change the "|" to "/") Then all elements of the AFL queue
** can be run in a single go (for regression testing, for example) by typing:
**
**    fuzzershell -f ~/all-queue.txt
**
** After running each chunk of SQL, the database connection is closed.  The
** program aborts if the close fails or if there is any unfreed memory after
** the close.
**
** New cases can be appended to all-queue.txt at any time.  If redundant cases
** are added, that can be eliminated by running:
**
**    fuzzershell -f ~/all-queue.txt --unique-cases ~/unique-cases.txt
**
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <ctype.h>
#include "sqlite3.h"
................................................................................
      printf("[%s]\n", argv[i]);
    }else{
      printf("NULL\n");
    }
  }
  return 0;
}
static int execNoop(void *NotUsed, int argc, char **argv, char **colv){
  return 0;
}

#ifndef SQLITE_OMIT_TRACE
/*
** This callback is invoked by sqlite3_trace() as each SQL statement
** starts.
*/
static void traceCallback(void *NotUsed, const char *zMsg){
  printf("TRACE: %s\n", zMsg);
}
#endif

/***************************************************************************
** eval() implementation copied from ../ext/misc/eval.c
*/
/*
** Structure used to accumulate the output
*/
................................................................................
** Print sketchy documentation for this utility program
*/
static void showHelp(void){
  printf("Usage: %s [options]\n", g.zArgv0);
  printf(
"Read SQL text from standard input and evaluate it.\n"
"Options:\n"
"  --autovacuum          Enable AUTOVACUUM mode\n"
"  -f FILE               Read SQL text from FILE instead of standard input\n"
"  --heap SZ MIN         Memory allocator uses SZ bytes & min allocation MIN\n"
"  --help                Show this help text\n"    
"  --initdb DBFILE       Initialize the in-memory database using template DBFILE\n"
"  --lookaside N SZ      Configure lookaside for N slots of SZ bytes each\n"
"  --pagesize N          Set the page size to N\n"
"  --pcache N SZ         Configure N pages of pagecache each of size SZ bytes\n"
"  -q                    Reduced output\n"
"  --quiet               Reduced output\n"
"  --scratch N SZ        Configure scratch memory for N slots of SZ bytes each\n"
"  --unique-cases FILE   Write all unique test cases to FILE\n"
"  --utf16be             Set text encoding to UTF-16BE\n"
"  --utf16le             Set text encoding to UTF-16LE\n"
"  -v                    Increased output\n"
"  --verbose             Increased output\n"
  );
}

/*
** Return the value of a hexadecimal digit.  Return -1 if the input
** is not a hex digit.
*/
................................................................................
  void *pLook = 0;              /* Allocated lookaside space */
  void *pPCache = 0;            /* Allocated storage for pcache */
  void *pScratch = 0;           /* Allocated storage for scratch */
  int doAutovac = 0;            /* True for --autovacuum */
  char *zSql;                   /* SQL to run */
  char *zToFree = 0;            /* Call sqlite3_free() on this afte running zSql */
  int iMode = FZMODE_Generic;   /* Operating mode */
  const char *zCkGlob = 0;      /* Inputs must match this glob */
  int verboseFlag = 0;          /* --verbose or -v flag */
  int quietFlag = 0;            /* --quiet or -q flag */
  int nTest = 0;                /* Number of test cases run */
  int multiTest = 0;            /* True if there will be multiple test cases */
  int lastPct = -1;             /* Previous percentage done output */
  sqlite3 *dataDb = 0;          /* Database holding compacted input data */
  sqlite3_stmt *pStmt = 0;      /* Statement to insert testcase into dataDb */
  const char *zDataOut = 0;     /* Write compacted data to this output file */
  int nHeader = 0;              /* Bytes of header comment text on input file */


  g.zArgv0 = argv[0];
  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
................................................................................
        i += 2;
      }else
      if( strcmp(z,"mode")==0 ){
        if( i>=argc-1 ) abendError("missing argument on %s", argv[i]);
        z = argv[++i];
        if( strcmp(z,"generic")==0 ){
          iMode = FZMODE_Printf;
          zCkGlob = 0;
        }else if( strcmp(z, "glob")==0 ){
          iMode = FZMODE_Glob;
          zCkGlob = "'*','*'";
        }else if( strcmp(z, "printf")==0 ){
          iMode = FZMODE_Printf;
          zCkGlob = "'*',*";
        }else if( strcmp(z, "strftime")==0 ){
          iMode = FZMODE_Strftime;
          zCkGlob = "'*',*";
        }else{
          abendError("unknown --mode: %s", z);
        }
      }else
      if( strcmp(z,"pagesize")==0 ){
        if( i>=argc-1 ) abendError("missing argument on %s", argv[i]);
        pageSize = integerValue(argv[++i]);
      }else
      if( strcmp(z,"pcache")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);
        nPCache = integerValue(argv[i+1]);
        szPCache = integerValue(argv[i+2]);
        i += 2;
      }else
      if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){
        quietFlag = 1;
        verboseFlag = 0;
      }else
      if( strcmp(z,"scratch")==0 ){
        if( i>=argc-2 ) abendError("missing arguments on %s", argv[i]);
        nScratch = integerValue(argv[i+1]);
        szScratch = integerValue(argv[i+2]);
        i += 2;
      }else
      if( strcmp(z, "unique-cases")==0 ){
        if( i>=argc-1 ) abendError("missing arguments on %s", argv[i]);
        if( zDataOut ) abendError("only one --minimize allowed");
        zDataOut = argv[++i];
      }else
      if( strcmp(z,"utf16le")==0 ){
        zEncoding = "utf16le";
      }else
      if( strcmp(z,"utf16be")==0 ){
        zEncoding = "utf16be";
      }else
      if( strcmp(z,"verbose")==0 || strcmp(z,"v")==0 ){
        quietFlag = 0;
        verboseFlag = 1;
      }else
      {
        abendError("unknown option: %s", argv[i]);
      }
    }else{
      abendError("unknown argument: %s", argv[i]);
    }
  }
  if( verboseFlag ) sqlite3_config(SQLITE_CONFIG_LOG, shellLog, 0);
  if( nHeap>0 ){
    pHeap = malloc( nHeap );
    if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap);
    rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap);
    if( rc ) abendError("heap configuration failed: %d\n", rc);
  }
  if( nLook>0 ){
................................................................................
    zIn = realloc(zIn, nAlloc);
    if( zIn==0 ) fatalError("out of memory");
    got = fread(zIn+nIn, 1, nAlloc-nIn-1, in); 
    nIn += (int)got;
    zIn[nIn] = 0;
    if( got==0 ) break;
  }
  if( in!=stdin ) fclose(in);
  if( zDataOut ){
    rc = sqlite3_open(":memory:", &dataDb);
    if( rc ) abendError("cannot open :memory: database");
    rc = sqlite3_exec(dataDb,
          "CREATE TABLE testcase(sql BLOB PRIMARY KEY) WITHOUT ROWID;",0,0,0);
    if( rc ) abendError("%s", sqlite3_errmsg(dataDb));
    rc = sqlite3_prepare_v2(dataDb, "INSERT OR IGNORE INTO testcase(sql)VALUES(?1)",
                            -1, &pStmt, 0);
    if( rc ) abendError("%s", sqlite3_errmsg(dataDb));
  }
  if( zInitDb ){
    rc = sqlite3_open_v2(zInitDb, &dbInit, SQLITE_OPEN_READONLY, 0);
    if( rc!=SQLITE_OK ){
      abendError("unable to open initialization database \"%s\"", zInitDb);
    }
  }
  for(i=0; i<nIn; i=iNext+1){   /* Skip initial lines beginning with '#' */
    if( zIn[i]!='#' ) break;
    for(iNext=i+1; iNext<nIn && zIn[iNext]!='\n'; iNext++){}
  }
  nHeader = i;
  for(nTest=0; i<nIn; i=iNext, nTest++){
    char cSaved;
    if( strncmp(&zIn[i], "/****<",6)==0 ){
      char *z = strstr(&zIn[i], ">****/");
      if( z ){
        z += 6;
        if( verboseFlag ) printf("%.*s\n", (int)(z-&zIn[i]), &zIn[i]);
        i += (int)(z-&zIn[i]);
        multiTest = 1;
      }
    }
    for(iNext=i; iNext<nIn && strncmp(&zIn[iNext],"/****<",6)!=0; iNext++){}
    if( zDataOut ){
      sqlite3_bind_blob(pStmt, 1, &zIn[i], iNext-i, SQLITE_STATIC);
      rc = sqlite3_step(pStmt);
      if( rc!=SQLITE_DONE ) abendError("%s", sqlite3_errmsg(dataDb));
      sqlite3_reset(pStmt);
      continue;
    }
    cSaved = zIn[iNext];
    zIn[iNext] = 0;
    if( zCkGlob && sqlite3_strglob(zCkGlob,&zIn[i])!=0 ){
      zIn[iNext] = cSaved;
      continue;
    }
    rc = sqlite3_open_v2(
      "main.db", &db,
      SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY,
      0);
    if( rc!=SQLITE_OK ){
      abendError("Unable to open the in-memory database");
    }
................................................................................
      rc = sqlite3_backup_step(pBackup, -1);
      if( rc!=SQLITE_DONE ){
        abendError("attempt to initialize the in-memory database failed (rc=%d)",
                   rc);
      }
      sqlite3_backup_finish(pBackup);
    }
#ifndef SQLITE_OMIT_TRACE
    if( verboseFlag ) sqlite3_trace(db, traceCallback, 0);
#endif
    sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
    sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000);
    if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding);
    if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);
    if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL");
    zSql = &zIn[i];
    if( verboseFlag ){
      printf("INPUT (offset: %d, size: %d): [%s]\n",
              i, (int)strlen(&zIn[i]), &zIn[i]);
    }else if( multiTest && !quietFlag ){
      int pct = 10*iNext/nIn;
      if( pct!=lastPct ){
        if( lastPct<0 ) printf("fuzz test:");
        printf(" %d%%", pct*10);
        fflush(stdout);
        lastPct = pct;
      }
    }
    switch( iMode ){
      case FZMODE_Glob:
        zSql = zToFree = sqlite3_mprintf("SELECT glob(%s);", zSql);
        break;
      case FZMODE_Printf:
        zSql = zToFree = sqlite3_mprintf("SELECT printf(%s);", zSql);
        break;
      case FZMODE_Strftime:
        zSql = zToFree = sqlite3_mprintf("SELECT strftime(%s);", zSql);
        break;
    }
    zErrMsg = 0;
    rc = sqlite3_exec(db, zSql, verboseFlag ? execCallback : execNoop, 0, &zErrMsg);
    if( zToFree ){
      sqlite3_free(zToFree);
      zToFree = 0;
    }
    zIn[iNext] = cSaved;

    if( verboseFlag ){
      printf("RESULT-CODE: %d\n", rc);
      if( zErrMsg ){
        printf("ERROR-MSG: [%s]\n", zErrMsg);

      }
    }
    sqlite3_free(zErrMsg);
    rc = sqlite3_close(db);
    if( rc ){
      abendError("sqlite3_close() failed with rc=%d", rc);
    }
    if( sqlite3_memory_used()>0 ){
      abendError("memory in use after close: %lld bytes", sqlite3_memory_used());
    }
    if( nTest==1 ){
      /* Simulate an error if the TEST_FAILURE environment variable is "5" */
      char *zFailCode = getenv("TEST_FAILURE");
      if( zFailCode ){
        if( zFailCode[0]=='5' && zFailCode[1]==0 ){
          abendError("simulated failure");
        }else if( zFailCode[0]!=0 ){
          /* If TEST_FAILURE is something other than 5, just exit the test
          ** early */
          printf("\nExit early due to TEST_FAILURE being set");
          break;
        }
      }
    }
  }
  if( !verboseFlag && multiTest && !quietFlag ) printf("\n");
  if( nTest>1 && !quietFlag ){
    printf("%d fuzz tests with no errors\nSQLite %s %s\n",
           nTest, sqlite3_libversion(), sqlite3_sourceid());
  }
  if( zDataOut ){
    int n = 0;
    FILE *out = fopen(zDataOut, "wb");
    if( out==0 ) abendError("cannot open %s for writing", zDataOut);
    if( nHeader>0 ) fwrite(zIn, nHeader, 1, out);
    sqlite3_finalize(pStmt);
    rc = sqlite3_prepare_v2(dataDb, "SELECT sql FROM testcase", -1, &pStmt, 0);
    if( rc ) abendError("%s", sqlite3_errmsg(dataDb));
    while( sqlite3_step(pStmt)==SQLITE_ROW ){
      fprintf(out,"/****<%d>****/", ++n);
      fwrite(sqlite3_column_blob(pStmt,0),sqlite3_column_bytes(pStmt,0),1,out);
    }
    fclose(out);
    sqlite3_finalize(pStmt);
    sqlite3_close(dataDb);
  }
  free(zIn);
  free(pHeap);
  free(pLook);
  free(pScratch);
  free(pPCache);
  return 0;
}