SQLite: Check-in [4e3e516a]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	More robust handling of zeroblob() with oversized arguments. Fix fuzzcheck so that it can be run with limited heap memory.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| trunk
Files:	files \| file ages \| folders
SHA1:	4e3e516a42059c97f42a7eb59bdf5cded0ff843a
User & Date:	drh 2015-07-24 15:49:23

Context

2015-07-24
16:24		Add the sqlite3_result_zeroblob64() API. Use it in the SQL zeroblob() function. (check-in: c6445b9f user: dan tags: trunk)
15:49		More robust handling of zeroblob() with oversized arguments. Fix fuzzcheck so that it can be run with limited heap memory. (check-in: 4e3e516a user: drh tags: trunk)
14:17		Modify a few test scripts to avoid leaving the sqlite3Config structure in a non-default state. (check-in: 562687d9 user: dan tags: trunk)

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Makefile.in.


# Do extra testing but not everything.
fulltestonly:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
	./fuzzcheck$(TEXE) $(FUZZDATA)

valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
	valgrind ./fuzzcheck$(TEXE) --cell-size-check --quiet $(FUZZDATA)

# Minimal testing that runs in less than 3 minutes
#
quicktest:	./testfixture$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)

# This is the common case.  Run many tests that do not take too long,








|


|


# Do extra testing but not everything.
fulltestonly:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA)
	./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)

valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA)
	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 100M $(FUZZDATA)

# Minimal testing that runs in less than 3 minutes
#
quicktest:	./testfixture$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)

# This is the common case.  Run many tests that do not take too long,

Changes to Makefile.msc.

fulltestonly:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS)

fuzztest:	fuzzcheck.exe
	.\fuzzcheck.exe $(FUZZDATA)

# Minimal testing that runs in less than 3 minutes (on a fast machine)
#
quicktest:	testfixture.exe
	.\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS)

# This is the common case.  Run many tests that do not take too long,

fulltestonly:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS)

fuzztest:	fuzzcheck.exe
	.\fuzzcheck.exe --limit-mem 100M $(FUZZDATA)

# Minimal testing that runs in less than 3 minutes (on a fast machine)
#
quicktest:	testfixture.exe
	.\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS)

# This is the common case.  Run many tests that do not take too long,

Changes to main.mk.

		libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)

sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)

fuzzershell$(EXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
	$(TCCX) -o fuzzershell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION\
		$(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) $(THREADLIB)

fuzzcheck$(EXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
	$(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION\

		$(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) $(THREADLIB)

mptester$(EXE):	sqlite3.c $(TOP)/mptest/mptest.c
	$(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
		$(TLIBS) $(THREADLIB)

MPTEST1=./mptester$(EXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
................................................................................
fulltestonly:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test $(TESTOPTS)

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner $(TESTOPTS)

fuzztest:	fuzzcheck$(EXE) $(FUZZDATA)
	./fuzzcheck$(EXE) $(FUZZDATA)

valgrindfuzz:	fuzzcheck$(EXE) $(FUZZDATA)
	valgrind ./fuzzcheck$(EXE) --cell-size-check --quiet $(FUZZDATA)

# A very quick test using only testfixture and omitting all the slower
# tests.  Designed to run in under 3 minutes on a workstation.
#
quicktest:	./testfixture$(EXE)
	./testfixture$(EXE) $(TOP)/test/extraquick.test $(TESTOPTS)

		libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB)

sqldiff$(EXE):	$(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
	$(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \
		$(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB)

fuzzershell$(EXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
	$(TCCX) -o fuzzershell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		$(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) $(THREADLIB)

fuzzcheck$(EXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
	$(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		-DSQLITE_ENABLE_MEMSYS5 \
		$(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) $(THREADLIB)

mptester$(EXE):	sqlite3.c $(TOP)/mptest/mptest.c
	$(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
		$(TLIBS) $(THREADLIB)

MPTEST1=./mptester$(EXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20
................................................................................
fulltestonly:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test $(TESTOPTS)

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner $(TESTOPTS)

fuzztest:	fuzzcheck$(EXE) $(FUZZDATA)
	./fuzzcheck$(EXE) --limit-mem 100M $(FUZZDATA)

valgrindfuzz:	fuzzcheck$(EXE) $(FUZZDATA)
	valgrind ./fuzzcheck$(EXE) --cell-size-check --limit-mem 100M $(FUZZDATA)

# A very quick test using only testfixture and omitting all the slower
# tests.  Designed to run in under 3 minutes on a workstation.
#
quicktest:	./testfixture$(EXE)
	./testfixture$(EXE) $(TOP)/test/extraquick.test $(TESTOPTS)

Changes to src/func.c.

  UNUSED_PARAMETER(argc);
  n = sqlite3_value_int64(argv[0]);
  testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] );
  testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 );
  if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){
    sqlite3_result_error_toobig(context);
  }else{

    sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */
  }
}

/*
** The replace() function.  Three arguments are all strings: call
** them A, B, and C. The result is also a string which is derived

  UNUSED_PARAMETER(argc);
  n = sqlite3_value_int64(argv[0]);
  testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH] );
  testcase( n==db->aLimit[SQLITE_LIMIT_LENGTH]+1 );
  if( n>db->aLimit[SQLITE_LIMIT_LENGTH] ){
    sqlite3_result_error_toobig(context);
  }else{
    if( n<0 ) n = 0;
    sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */
  }
}

/*
** The replace() function.  Three arguments are all strings: call
** them A, B, and C. The result is also a string which is derived

Changes to src/vdbe.c.

  pRec = pLast;
  do{
    assert( memIsValid(pRec) );
    pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format);
    len = sqlite3VdbeSerialTypeLen(serial_type);
    if( pRec->flags & MEM_Zero ){
      if( nData ){
        sqlite3VdbeMemExpandBlob(pRec);
      }else{
        nZero += pRec->u.nZero;
        len -= pRec->u.nZero;
      }
    }
    nData += len;
    testcase( serial_type==127 );

  pRec = pLast;
  do{
    assert( memIsValid(pRec) );
    pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format);
    len = sqlite3VdbeSerialTypeLen(serial_type);
    if( pRec->flags & MEM_Zero ){
      if( nData ){
        if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem;
      }else{
        nZero += pRec->u.nZero;
        len -= pRec->u.nZero;
      }
    }
    nData += len;
    testcase( serial_type==127 );

Changes to src/vdbeapi.c.

/**************************** sqlite3_value_  *******************************
** The following routines extract information from a Mem or sqlite3_value
** structure.
*/
const void *sqlite3_value_blob(sqlite3_value *pVal){
  Mem *p = (Mem*)pVal;
  if( p->flags & (MEM_Blob|MEM_Str) ){
    sqlite3VdbeMemExpandBlob(p);
    p->flags |= MEM_Blob;
    return p->n ? p->z : 0;
  }else{
    return sqlite3_value_text(pVal);
  }
}
int sqlite3_value_bytes(sqlite3_value *pVal){

/**************************** sqlite3_value_  *******************************
** The following routines extract information from a Mem or sqlite3_value
** structure.
*/
const void *sqlite3_value_blob(sqlite3_value *pVal){
  Mem *p = (Mem*)pVal;
  if( p->flags & (MEM_Blob|MEM_Str) ){
    if( sqlite3VdbeMemExpandBlob(p)!=SQLITE_OK ) return 0;
    p->flags |= MEM_Blob;
    return p->n ? p->z : 0;
  }else{
    return sqlite3_value_text(pVal);
  }
}
int sqlite3_value_bytes(sqlite3_value *pVal){

Changes to test/fuzzcheck.c.

     "INSERT INTO xsql(sqlid,sqltext) SELECT NULL, sqltext FROM sx ORDER BY 2;\n"
     "DROP TABLE sx;\n"
     "COMMIT;\n"
     "PRAGMA page_size=1024;\n"
     "VACUUM;\n", 0, 0, 0);
  if( rc ) fatalError("cannot rebuild: %s", sqlite3_errmsg(db));
}



























































/*
** Print sketchy documentation for this utility program
*/
static void showHelp(void){
  printf("Usage: %s [options] SOURCE-DB ?ARGS...?\n", g.zArgv0);
  printf(
................................................................................
"each database, checking for crashes and memory leaks.\n"
"Options:\n"
"  --cell-size-check     Set the PRAGMA cell_size_check=ON\n"
"  --dbid N              Use only the database where dbid=N\n"
"  --help                Show this help text\n"
"  -q                    Reduced output\n"
"  --quiet               Reduced output\n"

"  --limit-vdbe          Panic if an sync SQL runs for more than 100,000 cycles\n"
"  --load-sql ARGS...    Load SQL scripts fro files into SOURCE-DB\n"
"  --load-db ARGS...     Load template databases from files into SOURCE_DB\n"
"  -m TEXT               Add a description to the database\n"
"  --native-vfs          Use the native VFS for initially empty database files\n"
"  --rebuild             Rebuild and vacuum the database file\n"
"  --result-trace        Show the results of each SQL command\n"
................................................................................
  int iSrcDb;                  /* Loop over all source databases */
  int nTest = 0;               /* Total number of tests performed */
  char *zDbName = "";          /* Appreviated name of a source database */
  const char *zFailCode = 0;   /* Value of the TEST_FAILURE environment variable */
  int cellSzCkFlag = 0;        /* --cell-size-check */
  int sqlFuzz = 0;             /* True for SQL fuzz testing. False for DB fuzz */
  int iTimeout = 120;          /* Default 120-second timeout */


  iBegin = timeOfDay();
#ifdef __unix__
  signal(SIGALRM, timeoutHandler);
#endif
  g.zArgv0 = argv[0];
  zFailCode = getenv("TEST_FAILURE");
................................................................................
      z++;
      if( z[0]=='-' ) z++;
      if( strcmp(z,"cell-size-check")==0 ){
        cellSzCkFlag = 1;
      }else
      if( strcmp(z,"dbid")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
        onlyDbid = atoi(argv[++i]);
      }else
      if( strcmp(z,"help")==0 ){
        showHelp();
        return 0;
      }else




      if( strcmp(z,"limit-vdbe")==0 ){
        vdbeLimitFlag = 1;
      }else
      if( strcmp(z,"load-sql")==0 ){
        zInsSql = "INSERT INTO xsql(sqltext) VALUES(CAST(readfile(?1) AS text))";
        iFirstInsArg = i+1;
        break;
................................................................................
        rebuildFlag = 1;
      }else
      if( strcmp(z,"result-trace")==0 ){
        runFlags |= SQL_OUTPUT;
      }else
      if( strcmp(z,"sqlid")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
        onlySqlid = atoi(argv[++i]);
      }else
      if( strcmp(z,"timeout")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
        iTimeout = atoi(argv[++i]);
      }else
      if( strcmp(z,"timeout-test")==0 ){
        timeoutTest = 1;
#ifndef __unix__
        fatalError("timeout is not available on non-unix systems");
#endif
      }else
................................................................................
    /* Close the source database.  Verify that no SQLite memory allocations are
    ** outstanding.
    */
    sqlite3_close(db);
    if( sqlite3_memory_used()>0 ){
      fatalError("SQLite has memory in use before the start of testing");
    }











  
    /* Register the in-memory virtual filesystem
    */
    formatVfs();
    inmemVfsRegister();
    
    /* Run a test using each SQL script against each database.








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







>







 







|





>
>
>
>







 







|



|







 







>
>
>
>
>
>
>
>
>
>
>

     "INSERT INTO xsql(sqlid,sqltext) SELECT NULL, sqltext FROM sx ORDER BY 2;\n"
     "DROP TABLE sx;\n"
     "COMMIT;\n"
     "PRAGMA page_size=1024;\n"
     "VACUUM;\n", 0, 0, 0);
  if( rc ) fatalError("cannot rebuild: %s", sqlite3_errmsg(db));
}

/*
** Return the value of a hexadecimal digit.  Return -1 if the input
** is not a hex digit.
*/
static int hexDigitValue(char c){
  if( c>='0' && c<='9' ) return c - '0';
  if( c>='a' && c<='f' ) return c - 'a' + 10;
  if( c>='A' && c<='F' ) return c - 'A' + 10;
  return -1;
}

/*
** Interpret zArg as an integer value, possibly with suffixes.
*/
static int integerValue(const char *zArg){
  sqlite3_int64 v = 0;
  static const struct { char *zSuffix; int iMult; } aMult[] = {
    { "KiB", 1024 },
    { "MiB", 1024*1024 },
    { "GiB", 1024*1024*1024 },
    { "KB",  1000 },
    { "MB",  1000000 },
    { "GB",  1000000000 },
    { "K",   1000 },
    { "M",   1000000 },
    { "G",   1000000000 },
  };
  int i;
  int isNeg = 0;
  if( zArg[0]=='-' ){
    isNeg = 1;
    zArg++;
  }else if( zArg[0]=='+' ){
    zArg++;
  }
  if( zArg[0]=='0' && zArg[1]=='x' ){
    int x;
    zArg += 2;
    while( (x = hexDigitValue(zArg[0]))>=0 ){
      v = (v<<4) + x;
      zArg++;
    }
  }else{
    while( isdigit(zArg[0]) ){
      v = v*10 + zArg[0] - '0';
      zArg++;
    }
  }
  for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){
    if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){
      v *= aMult[i].iMult;
      break;
    }
  }
  if( v>0x7fffffff ) fatalError("parameter too large - max 2147483648");
  return (int)(isNeg? -v : v);
}

/*
** Print sketchy documentation for this utility program
*/
static void showHelp(void){
  printf("Usage: %s [options] SOURCE-DB ?ARGS...?\n", g.zArgv0);
  printf(
................................................................................
"each database, checking for crashes and memory leaks.\n"
"Options:\n"
"  --cell-size-check     Set the PRAGMA cell_size_check=ON\n"
"  --dbid N              Use only the database where dbid=N\n"
"  --help                Show this help text\n"
"  -q                    Reduced output\n"
"  --quiet               Reduced output\n"
"  --limit-mem N         Limit memory used by test SQLite instance to N bytes\n"
"  --limit-vdbe          Panic if an sync SQL runs for more than 100,000 cycles\n"
"  --load-sql ARGS...    Load SQL scripts fro files into SOURCE-DB\n"
"  --load-db ARGS...     Load template databases from files into SOURCE_DB\n"
"  -m TEXT               Add a description to the database\n"
"  --native-vfs          Use the native VFS for initially empty database files\n"
"  --rebuild             Rebuild and vacuum the database file\n"
"  --result-trace        Show the results of each SQL command\n"
................................................................................
  int iSrcDb;                  /* Loop over all source databases */
  int nTest = 0;               /* Total number of tests performed */
  char *zDbName = "";          /* Appreviated name of a source database */
  const char *zFailCode = 0;   /* Value of the TEST_FAILURE environment variable */
  int cellSzCkFlag = 0;        /* --cell-size-check */
  int sqlFuzz = 0;             /* True for SQL fuzz testing. False for DB fuzz */
  int iTimeout = 120;          /* Default 120-second timeout */
  int nMem = 0;                /* Memory limit */

  iBegin = timeOfDay();
#ifdef __unix__
  signal(SIGALRM, timeoutHandler);
#endif
  g.zArgv0 = argv[0];
  zFailCode = getenv("TEST_FAILURE");
................................................................................
      z++;
      if( z[0]=='-' ) z++;
      if( strcmp(z,"cell-size-check")==0 ){
        cellSzCkFlag = 1;
      }else
      if( strcmp(z,"dbid")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
        onlyDbid = integerValue(argv[++i]);
      }else
      if( strcmp(z,"help")==0 ){
        showHelp();
        return 0;
      }else
      if( strcmp(z,"limit-mem")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
        nMem = integerValue(argv[++i]);
      }else
      if( strcmp(z,"limit-vdbe")==0 ){
        vdbeLimitFlag = 1;
      }else
      if( strcmp(z,"load-sql")==0 ){
        zInsSql = "INSERT INTO xsql(sqltext) VALUES(CAST(readfile(?1) AS text))";
        iFirstInsArg = i+1;
        break;
................................................................................
        rebuildFlag = 1;
      }else
      if( strcmp(z,"result-trace")==0 ){
        runFlags |= SQL_OUTPUT;
      }else
      if( strcmp(z,"sqlid")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
        onlySqlid = integerValue(argv[++i]);
      }else
      if( strcmp(z,"timeout")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
        iTimeout = integerValue(argv[++i]);
      }else
      if( strcmp(z,"timeout-test")==0 ){
        timeoutTest = 1;
#ifndef __unix__
        fatalError("timeout is not available on non-unix systems");
#endif
      }else
................................................................................
    /* Close the source database.  Verify that no SQLite memory allocations are
    ** outstanding.
    */
    sqlite3_close(db);
    if( sqlite3_memory_used()>0 ){
      fatalError("SQLite has memory in use before the start of testing");
    }

    /* Limit available memory, if requested */
    if( nMem>0 ){
      void *pHeap;
      sqlite3_shutdown();
      pHeap = malloc(nMem);
      if( pHeap==0 ){
        fatalError("failed to allocate %d bytes of heap memory", nMem);
      }
      sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMem, 128);
    }
  
    /* Register the in-memory virtual filesystem
    */
    formatVfs();
    inmemVfsRegister();
    
    /* Run a test using each SQL script against each database.

Changes to test/zeroblob.test.

  execsql {
    INSERT INTO t1 VALUES(2,3,4,zeroblob(1000000));
  }
  set ::sqlite3_max_blobsize
} {10}

do_test zeroblob-1.1.1 {
  expr {[sqlite3_memory_highwater]<$::memused+25000}
} {1}
do_test zeroblob-1.2 {
  execsql {
    SELECT length(d) FROM t1
  }
} {1000000}

  execsql {
    INSERT INTO t1 VALUES(2,3,4,zeroblob(1000000));
  }
  set ::sqlite3_max_blobsize
} {10}

do_test zeroblob-1.1.1 {
  expr {[sqlite3_memory_highwater]<$::memused+35000}
} {1}
do_test zeroblob-1.2 {
  execsql {
    SELECT length(d) FROM t1
  }
} {1000000}