SQLite

"\n"
"        Generate a new test database file named DBFILE containing N\n"
"        BLOBs each of size M bytes.  The page size of the new database\n"
"        file will be X.  Additional options:\n"
"\n"
"           --variance V           Randomly vary M by plus or minus V\n"
"\n"
"   kvtest export DBFILE DIRECTORY [--tree]\n"
"\n"
"        Export all the blobs in the kv table of DBFILE into separate\n"
"        files in DIRECTORY.  DIRECTORY is created if it does not previously\n"
"        exist.  If the --tree option is used, then the blobs are written\n"
"        into a hierarchy of directories, using names like 00/00/00,\n"
"        00/00/01, 00/00/02, and so forth.  Without the --tree option, all\n"
"        files are in the top-level directory with names like 000000, 000001,\n"
"        000002, and so forth.\n"
"\n"
"   kvtest stat DBFILE\n"
"\n"
"        Display summary information about DBFILE\n"
"\n"
"   kvtest run DBFILE [options]\n"
"\n"

#include <string.h>
#include "sqlite3.h"

#ifndef _WIN32
# include <unistd.h>
#else
  /* Provide Windows equivalent for the needed parts of unistd.h */
# include <direct.h>
# include <io.h>
# define R_OK 2
# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
# define access _access
#endif

  return isNeg? -v : v;
}


/*
** Check the filesystem object zPath.  Determine what it is:
**
**    PATH_DIR     A single directory holding many files
**    PATH_TREE    A directory hierarchy with files at the leaves
**    PATH_DB      An SQLite database
**    PATH_NEXIST  Does not exist
**    PATH_OTHER   Something else
**
** PATH_DIR means all of the separate files are grouped together
** into a single directory with names like 000000, 000001, 000002, and
** so forth.  PATH_TREE means there is a hierarchy of directories so
** that no single directory has too many entries.  The files have names
** like 00/00/00, 00/00/01, 00/00/02 and so forth.  The decision between
** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory
** named "00" at the top-level.
*/
#define PATH_DIR     1
#define PATH_TREE    2
#define PATH_DB      3
#define PATH_NEXIST  0
#define PATH_OTHER   99
static int pathType(const char *zPath){
  struct stat x;
  int rc;
  if( access(zPath,R_OK) ) return PATH_NEXIST;
  memset(&x, 0, sizeof(x));
  rc = stat(zPath, &x);
  if( rc<0 ) return PATH_OTHER;
  if( S_ISDIR(x.st_mode) ){
    char *zLayer1 = sqlite3_mprintf("%s/00", zPath);
    memset(&x, 0, sizeof(x));
    rc = stat(zLayer1, &x);
    sqlite3_free(zLayer1);
    if( rc<0 ) return PATH_DIR;
    if( S_ISDIR(x.st_mode) ) return PATH_TREE;
    return PATH_DIR;
  }
  if( (x.st_size%512)==0 ) return PATH_DB;
  return PATH_OTHER;
}

/*
** Return the size of a file in bytes.  Or return -1 if the
** named object is not a regular file or does not exist.

    printf("Page-count:         %8d\n", sqlite3_column_int(pStmt, 0));
  }
  sqlite3_finalize(pStmt);
  sqlite3_close(db);
  return 0;
}
































/*
**      remember(V,PTR)
**
** Return the integer value V.  Also save the value of V in a
** C-language variable whose address is PTR.
*/
static void rememberFunc(
  sqlite3_context *pCtx,
  int argc,
  sqlite3_value **argv
){
  sqlite3_int64 v;
  sqlite3_int64 ptr;
  assert( argc==2 );
  v = sqlite3_value_int64(argv[0]);
  ptr = sqlite3_value_int64(argv[1]);
  *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v;
  sqlite3_result_int64(pCtx, v);
}

/*
** Make sure a directory named zDir exists.
*/
static void kvtest_mkdir(const char *zDir){
#if defined(_WIN32)
  (void)mkdir(zDir);
#else
  (void)mkdir(zDir, 0755);
#endif
}

/*
** Export the kv table to individual files in the filesystem
*/
static int exportMain(int argc, char **argv){
  char *zDb;
  char *zDir;
  sqlite3 *db;
  sqlite3_stmt *pStmt;
  int rc;
  int ePathType;
  int nFN;
  char *zFN;
  char *zTail;
  size_t nWrote;
  int i;

  assert( strcmp(argv[1],"export")==0 );
  assert( argc>=3 );
  if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]");
  zDb = argv[2];

  zDir = argv[3];
  kvtest_mkdir(zDir);
  for(i=4; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' && z[1]=='-' ) z++;
    if( strcmp(z,"-tree")==0 ){
      zFN = sqlite3_mprintf("%s/00", zDir);
      kvtest_mkdir(zFN);
      sqlite3_free(zFN);
      continue;
    }
    fatalError("unknown argument: \"%s\"\n", argv[i]);
  }
  ePathType = pathType(zDir);
  if( ePathType!=PATH_DIR && ePathType!=PATH_TREE ){
    fatalError("object \"%s\" is not a directory", zDir);
  }
  rc = sqlite3_open(zDb, &db);
  if( rc ){
    fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
  }
  rc = sqlite3_prepare_v2(db, "SELECT k, v FROM kv ORDER BY k", -1, &pStmt, 0);
  if( rc ){
    fatalError("prepare_v2 failed: %s\n", sqlite3_errmsg(db));
  }
  nFN = (int)strlen(zDir);
  zFN = sqlite3_mprintf("%s/00/00/00.extra---------------------", zDir);
  if( zFN==0 ){
    fatalError("malloc failed\n");
  }
  zTail = zFN + nFN + 1;
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    int iKey = sqlite3_column_int(pStmt, 0);
    int nData = sqlite3_column_bytes(pStmt, 1);
    const void *pData = sqlite3_column_blob(pStmt, 1);
    FILE *out;
    if( ePathType==PATH_DIR ){
      sqlite3_snprintf(20, zTail, "%06d", iKey);
    }else{
      sqlite3_snprintf(20, zTail, "%02d", iKey/10000);
      kvtest_mkdir(zFN);
      sqlite3_snprintf(20, zTail, "%02d/%02d", iKey/10000, (iKey/100)%100);
      kvtest_mkdir(zFN);
      sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",
                       iKey/10000, (iKey/100)%100, iKey%100);
    }
    out = fopen(zFN, "wb");      
    nWrote = fwrite(pData, 1, nData, out);
    fclose(out);
    printf("\r%s   ", zTail); fflush(stdout);
    if( nWrote!=nData ){
      fatalError("Wrote only %d of %d bytes to %s\n",
                  (int)nWrote, nData, zFN);
    }
  }
  sqlite3_finalize(pStmt);
  sqlite3_close(db);
  sqlite3_free(zFN);
  printf("\n");
  return 0;
}

/*
** Read the content of file zName into memory obtained from sqlite3_malloc64()
** and return a pointer to the buffer. The caller is responsible for freeing 

      sqlite3_finalize(pStmt);
    }
    pStmt = 0;
    if( !doMultiTrans ) sqlite3_exec(db, "BEGIN", 0, 0, 0);
  }
  if( iMax<=0 ) iMax = 1000;
  for(i=0; i<nCount; i++){
    if( eType==PATH_DIR || eType==PATH_TREE ){
      /* CASE 1: Reading or writing blobs out of separate files */
      char *zKey;
      if( eType==PATH_DIR ){
        zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
      }else{
        zKey = sqlite3_mprintf("%s/%02d/%02d/%02d", zDb, iKey/10000,
                               (iKey/100)%100, iKey%100);
      }
      nData = 0;
      if( isUpdateTest ){
        updateFile(zKey, &nData, doFsync);
      }else{
        pData = readFile(zKey, &nData);
        sqlite3_free(pData);
      }