SQLite

	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)

ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)

sessionfuzz$(TEXE):	$(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h
	$(LTLINK) -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS)

dbfuzz$(TEXE):	$(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h
	$(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS)

DBFUZZ2_OPTS = \
  -DSQLITE_THREADSAFE=0 \
  -DSQLITE_OMIT_LOAD_EXTENSION \
  -DSQLITE_ENABLE_DESERIALIZE \
  -DSQLITE_DEBUG \
  -DSQLITE_ENABLE_DBSTAT_VTAB \
  -DSQLITE_ENABLE_RTREE \
  -DSQLITE_ENABLE_FTS4 \
  -DSQLITE_EANBLE_FTS5

dbfuzz2:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
		-fsanitize=fuzzer,undefined,address -o dbfuzz2 \
		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c
	mkdir -p dbfuzz2-dir
	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir

mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
		$(TLIBS) -rpath "$(libdir)"

sqlite3_checker$(TEXE):	sqlite3_checker.c
	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)

dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
	$(LTLINK) -DDBDUMP_STANDALONE -o $@ \
           $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS)

dbtotxt$(TEXE): $(TOP)/tool/dbtotxt.c
	$(LTLINK)-o $@ $(TOP)/tool/dbtotxt.c

showdb$(TEXE):	$(TOP)/tool/showdb.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS)

showstat4$(TEXE):	$(TOP)/tool/showstat4.c sqlite3.lo
	$(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS)

testloadext.lo:	$(TOP)\src\test_loadext.c $(SQLITE3H)
	$(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c

testloadext.dll:	testloadext.lo
	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo

dbtotxt.exe:	$(TOP)\tool\dbtotxt.c
	$(LTLINK) $(NO_WARN)	$(TOP)\tool\dbtotxt.c /link $(LDFLAGS) $(LTLINKOPTS)

showdb.exe:	$(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		$(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

showstat4.exe:	$(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H)
	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
		$(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
$(TEST_EXTENSION): $(TOP)/src/test_loadext.c
	$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)

extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
	./testfixture$(EXE) $(TOP)/test/loadext.test

dbtotxt$(EXE):	$(TOP)/tool/dbtotxt.c
	$(TCC) -o dbtotxt$(EXE) $(TOP)/tool/dbtotxt.c

showdb$(EXE):	$(TOP)/tool/showdb.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \
		$(TOP)/tool/showdb.c sqlite3.o $(THREADLIB)

showstat4$(EXE):	$(TOP)/tool/showstat4.c sqlite3.o
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showstat4$(EXE) \
		$(TOP)/tool/showstat4.c sqlite3.o $(THREADLIB)

*/
#define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
#define SHELL_OPEN_NORMAL      1      /* Normal database file */
#define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
#define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
#define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
#define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
#define SHELL_OPEN_HEXDB       6      /* Use "dbtotxt" output as data source */

/* Allowed values for ShellState.eTraceType
*/
#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */

  "       -e    Invoke system text editor",
  "       -x    Open in a spreadsheet",
  ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  "     Options:",
  "        --append        Use appendvfs to append database to the end of FILE",
#ifdef SQLITE_ENABLE_DESERIALIZE
  "        --deserialize   Load into memory useing sqlite3_deserialize()",
  "        --hexdb         Load the output of \"dbtotxt\" as an in-memory database",
#endif
  "        --new           Initialize FILE to an empty database",
  "        --readonly      Open FILE readonly",
  "        --zip           FILE is a ZIP archive",
  ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  "     If FILE begins with '|' then open it as a pipe.",
  ".print STRING...         Print literal STRING",

      rc = SHELL_OPEN_ZIPFILE;
    }
  }
  fclose(f);
  return rc;  
}

#ifdef SQLITE_ENABLE_DESERIALIZE
/*
** Reconstruct an in-memory database using the output from the "dbtotxt"
** program.  Read content from the file in p->zDbFilename.  If p->zDbFilename
** is 0, then read from standard input.
*/
static unsigned char *readHexDb(ShellState *p, int *pnData){
  unsigned char *a = 0;
  int nLine = 1;
  int n = 0;
  int pgsz = 0;
  int iOffset = 0;
  int j, k;
  int rc;
  FILE *in;
  unsigned char x[16];
  char zLine[100];
  if( p->zDbFilename ){
    in = fopen(p->zDbFilename, "r");
    if( in==0 ){
      utf8_printf(stderr, "cannot open \"%s\" for reading\n", p->zDbFilename);
      return 0;
    }
  }else{
    in = stdin;
  }
  *pnData = 0;
  if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
  rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz);
  if( rc!=2 ) goto readHexDb_error;
  if( n<=0 ) goto readHexDb_error;
  a = sqlite3_malloc( n );
  if( a==0 ){
    utf8_printf(stderr, "Out of memory!\n");
    goto readHexDb_error;
  }
  memset(a, 0, n);
  if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){
    utf8_printf(stderr, "invalid pagesize\n");
    goto readHexDb_error;
  }
  for(nLine=2; fgets(zLine, sizeof(zLine), in)!=0; nLine++){
    rc = sscanf(zLine, "| page %d offset %d", &j, &k);
    if( rc==2 ){
      iOffset = k;
      continue;
    }
    if( strncmp(zLine, "| end ", 6)==0 ){
      break;
    }
    rc = sscanf(zLine,"| %d: %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx"
                      "  %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx",
                &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7],
                &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]);
    if( rc==17 ){
      k = iOffset+j;
      if( k+16>n ){
        utf8_printf(stderr, "continue exceeds file size\n");
        goto readHexDb_error;
      }
      memcpy(a+k, x, 16);
    }
  }
  *pnData = n;
  if( in!=stdin ) fclose(in);
  return a;

readHexDb_error:
  if( in!=stdin ){
    fclose(in);
  }else{
    while( fgets(zLine, sizeof(zLine), in)!=0 ){
      if(strncmp(zLine, "| end ", 6)==0 ) break;
    }
  }
  sqlite3_free(a);
  utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine);
  return 0;
}
#endif /* SQLITE_ENABLE_DESERIALIZE */

/* Flags for open_db().
**
** The default behavior of open_db() is to exit(1) if the database fails to
** open.  The OPEN_DB_KEEPALIVE flag changes that so that it prints an error
** but still returns without calling exit.
**
** The OPEN_DB_ZIPFILE flag causes open_db() to prefer to open files as a

    }
    switch( p->openMode ){
      case SHELL_OPEN_APPENDVFS: {
        sqlite3_open_v2(p->zDbFilename, &p->db, 
           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs");
        break;
      }
      case SHELL_OPEN_HEXDB:
      case SHELL_OPEN_DESERIALIZE: {
        sqlite3_open(0, &p->db);
        break;
      }
      case SHELL_OPEN_ZIPFILE: {
        sqlite3_open(":memory:", &p->db);
        break;

    if( p->openMode==SHELL_OPEN_ZIPFILE ){
      char *zSql = sqlite3_mprintf(
         "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename);
      sqlite3_exec(p->db, zSql, 0, 0, 0);
      sqlite3_free(zSql);
    }
#ifdef SQLITE_ENABLE_DESERIALIZE
    else
    if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){
      int nData = 0;
      unsigned char *aData;
      if( p->openMode==SHELL_OPEN_DESERIALIZE ){
        aData = (unsigned char*)readFile(p->zDbFilename, &nData);
      }else{
        aData = readHexDb(p, &nData);
        if( aData==0 ){
          utf8_printf(stderr, "Error in hexdb input\n");
          return;
        }
      }
      int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
                   SQLITE_DESERIALIZE_RESIZEABLE |
                   SQLITE_DESERIALIZE_FREEONCLOSE);
      if( rc ){
        utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
      }
    }

      }else if( optionMatch(z, "append") ){
        p->openMode = SHELL_OPEN_APPENDVFS;
      }else if( optionMatch(z, "readonly") ){
        p->openMode = SHELL_OPEN_READONLY;
#ifdef SQLITE_ENABLE_DESERIALIZE
      }else if( optionMatch(z, "deserialize") ){
        p->openMode = SHELL_OPEN_DESERIALIZE;
      }else if( optionMatch(z, "hexdb") ){
        p->openMode = SHELL_OPEN_HEXDB;
#endif /* SQLITE_ENABLE_DESERIALIZE */
      }else if( z[0]=='-' ){
        utf8_printf(stderr, "unknown option: %s\n", z);
        rc = 1;
        goto meta_command_exit;
      }
    }
    /* If a filename is specified, try to open it first */
    zNewFilename = nArg>iName ? sqlite3_mprintf("%s", azArg[iName]) : 0;
    if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){
      if( newFlag ) shellDeleteFile(zNewFilename);
      p->zDbFilename = zNewFilename;
      open_db(p, OPEN_DB_KEEPALIVE);
      if( p->db==0 ){
        utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename);
        sqlite3_free(zNewFilename);
      }else{

/*
** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc.
** All Rights Reserved
**
******************************************************************************
**
** This file implements a stand-alone utility program that converts
** a binary file (usually an SQLite database) into a text format that
** is compact and friendly to human-readers.
**
** Usage:
**
**         dbtotxt [--pagesize N] FILENAME
**
** The translation of the database appears on standard output.  If the
** --pagesize command-line option is omitted, then the page size is taken
** from the database header.
**
** Compactness is achieved by suppressing lines of all zero bytes.  This
** works well at compressing test databases that are mostly empty.  But
** the output will probably be lengthy for a real database containing lots
** of real content.  For maximum compactness, it is suggested that test
** databases be constructed with "zeroblob()" rather than "randomblob()"
** used for filler content and with "PRAGMA secure_delete=ON" selected to
** zero-out deleted content.
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
 
/* Return true if the line is all zeros */
static int allZero(unsigned char *aLine){
  int i;
  for(i=0; i<16 && aLine[i]==0; i++){}
  return i==16;
}

int main(int argc, char **argv){
  int pgsz = 0;               /* page size */
  long szFile;                /* Size of the input file in bytes */
  FILE *in;                   /* Input file */
  int i, j;                   /* Loop counters */
  int nErr = 0;               /* Number of errors */
  const char *zInputFile = 0; /* Name of the input file */
  const char *zBaseName = 0;  /* Base name of the file */
  int lastPage = 0;           /* Last page number shown */
  int iPage;                  /* Current page number */
  unsigned char aLine[16];    /* A single line of the file */
  unsigned char aHdr[100];    /* File header */
  for(i=1; i<argc; i++){
    if( argv[i][0]=='-' ){
      const char *z = argv[i];
      z++;
      if( z[0]=='-' ) z++;
      if( strcmp(z,"pagesize")==0 ){
        i++;
        pgsz = atoi(argv[i]);
        if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ){
          fprintf(stderr, "Page size must be a power of two between"
                          " 512 and 65536.\n");
          nErr++;
        }
        continue;
      }
      fprintf(stderr, "Unknown option: %s\n", argv[i]);
      nErr++;
    }else if( zInputFile ){
      fprintf(stderr, "Already using a different input file: [%s]\n", argv[i]);
      nErr++;
    }else{
      zInputFile = argv[i];
    }
  }
  if( zInputFile==0 ){
    fprintf(stderr, "No input file specified.\n");
    nErr++;
  }
  if( nErr ){
    fprintf(stderr, "Usage: %s [--pagesize N] FILENAME\n", argv[0]);
    exit(1);
  }
  in = fopen(zInputFile, "rb");
  if( in==0 ){
    fprintf(stderr, "Cannot open input file [%s]\n", zInputFile);
    exit(1);
  }
  fseek(in, 0, SEEK_END);
  szFile = ftell(in);
  rewind(in);
  if( szFile<512 ){
    fprintf(stderr, "File too short. Minimum size is 512 bytes.\n");
    exit(1);
  }
  if( fread(aHdr, 100, 1, in)!=1 ){
    fprintf(stderr, "Cannot read file header\n");
    exit(1);
  }
  rewind(in);
  if( pgsz==0 ){
    pgsz = (aHdr[16]<<8) | aHdr[17];
    if( pgsz==1 ) pgsz = 65536;
    if( pgsz<512 || (pgsz&(pgsz-1))!=0 ){
      fprintf(stderr, "Invalid page size in header: %d\n", pgsz);
      exit(1);
    }
  }
  zBaseName = zInputFile;
  for(i=0; zInputFile[i]; i++){
    if( zInputFile[i]=='/' && zInputFile[i+1]!=0 ) zBaseName = zInputFile+1;
  }
  printf("| size %d pagesize %d filename %s\n",(int)szFile,pgsz,zBaseName);
  for(i=0; i<szFile; i+=16){
    int got = (int)fread(aLine, 1, 16, in);
    if( got!=16 ){
      static int once = 1;
      if( once ){
        fprintf(stderr, "Could not read input file starting at byte %d\n",
                         i+got);
      }
      memset(aLine+got, 0, 16-got);
    }
    if( allZero(aLine) ) continue;
    iPage = i/pgsz + 1;
    if( lastPage!=iPage ){
      printf("| page %d offset %d\n", iPage, (iPage-1)*pgsz);
      lastPage = iPage;
    }
    printf("|  %5d:", i-(iPage-1)*pgsz);
    for(j=0; j<16; j++) printf(" %02x", aLine[j]);
    printf("   ");
    for(j=0; j<16; j++){
      char c = aLine[j];
      fputc(c>=0x20 && c<=0x7e ? c : '.', stdout);
    }
    fputc('\n', stdout);
  }
  fclose(in);
  printf("| end %s\n", zBaseName);
  return 0;
}

<h1 align="center">The dbtotxt Tool</h1>

The dbtotxt utility program reads an SQLite database file and writes its
raw binary content to screen as a hex dump for testing and debugging
purposes.

The hex-dump output is formatted in such a way as to be easily readable
both by humans and by software.  The dbtotxt utility has long been a part
of the TH3 test suite.  The output of dbtotxt can be embedded in TH3 test
scripts and used to generate very specific database files, perhaps with
deliberately introduced corruption.  The cov1/corrupt*.test modules in
TH3 make extensive use of dbtotxt.

More recently (2018-12-13) the dbtotxt utility has been added to the SQLite 
core and the command-line shell (CLI) has been augmented to be able to read 
dbtotxt output.  The CLI dot-command is:

>     .open --hexdb  ?OPTIONAL-FILENAME?

If the OPTIONAL-FILENAME is included, then content is read from that file.
If OPTIONAL-FILENAME is omitted, then the text is taken from the input stream,
terminated by the "| end" line of the dbtotxt text.  This allows small test
databases to be embedded directly in scripts.  Consider this example:

>
    .open --hexdb
    | size 8192 pagesize 4096 filename x9.db
    | page 1 offset 0
    |      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
    |     16: 10 00 01 01 00 40 20 20 00 00 00 04 00 00 00 02   .....@  ........
    |     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
    |     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
    |     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04   ................
    |     96: 00 2e 30 38 0d 00 00 00 01 0f c0 00 0f c0 00 00   ..08............
    |   4032: 3e 01 06 17 11 11 01 69 74 61 62 6c 65 74 31 74   >......itablet1t
    |   4048: 31 02 43 52 45 41 54 45 20 54 41 42 4c 45 20 74   1.CREATE TABLE t
    |   4064: 31 28 78 2c 79 20 44 45 46 41 55 4c 54 20 78 27   1(x,y DEFAULT x'
    |   4080: 66 66 27 2c 7a 20 44 45 46 41 55 4c 54 20 30 29   ff',z DEFAULT 0)
    | page 2 offset 4096
    |      0: 0d 08 14 00 04 00 10 00 0e 05 0a 0f 04 15 00 10   ................
    |     16: 88 02 03 05 90 04 0e 08 00 00 00 00 00 00 00 00   ................
    |   1040: 00 00 00 00 ff 87 7c 02 05 8f 78 0e 08 00 00 00   ......|...x.....
    |   2064: 00 00 00 ff 0c 0a 01 fb 00 00 00 00 00 00 00 00   ................
    |   2560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 83   ................
    |   2576: 78 01 05 87 70 0e 08 00 00 00 00 00 00 00 00 00   x...p...........
    |   3072: 00 00 00 00 00 00 00 00 00 ff 00 00 01 fb 00 00   ................
    |   3584: 00 00 00 00 00 83 78 00 05 87 70 0e 08 00 00 00   ......x...p.....
    |   4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff   ................
    | end x9.db
    SELECT rowid FROM t1;
    PRAGMA integrity_check;

You can run this script to see that the database file is correctly decoded 
and loaded.  Furthermore, you can make subtle corruptions to the input
database simply by editing the hexadecimal description, then rerun the
script to verify that SQLite correctly handles the corruption.