/ Check-in [f8bbb608]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge in trunk enhancements.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | prepare_v3
Files: files | file ages | folders
SHA3-256: f8bbb608cbf6c245628e3d362e9181fb3dc402b1d8241bcb687caf395f63c916
User & Date: drh 2017-06-07 16:25:25
Context
2017-06-09
15:14
Merge enhancements and fixes from trunk. check-in: 3fd050c3 user: drh tags: prepare_v3
2017-06-07
16:25
Merge in trunk enhancements. check-in: f8bbb608 user: drh tags: prepare_v3
2017-06-06
18:20
Add the SQLITE_DEFAULT_ROWEST compile-time option for changing the estimated number of rows in tables that lack sqlite_stat1 entries. check-in: 234ede26 user: drh tags: trunk
2017-06-01
00:54
Add interfaces sqlite3_prepare_v3() and sqlite3_prepare16_v3() with the extra prepFlags argument. Add the SQLITE_PREPARE_PERSISTENT option as one bit in that argument. check-in: 4a25c588 user: drh tags: prepare_v3
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to doc/lemon.html.

    18     18   Lemon also implements features that can be used
    19     19   to eliminate resource leaks, making is suitable for use
    20     20   in long-running programs such as graphical user interfaces
    21     21   or embedded controllers.</p>
    22     22   
    23     23   <p>This document is an introduction to the Lemon
    24     24   parser generator.</p>
           25  +
           26  +<h2>Security Note</h2>
           27  +
           28  +<p>The language parser code created by Lemon is very robust and
           29  +is well-suited for use in internet-facing applications that need to
           30  +safely process maliciously crafted inputs.
           31  +
           32  +<p>The "lemon.exe" command-line tool itself works great when given a valid
           33  +input grammar file and almost always gives helpful
           34  +error messages for malformed inputs.  However,  it is possible for
           35  +a malicious user to craft a grammar file that will cause 
           36  +lemon.exe to crash.
           37  +We do not see this as a problem, as lemon.exe is not intended to be used
           38  +with hostile inputs.
           39  +To summarize:</p>
           40  +
           41  +<ul>
           42  +<li>Parser code generated by lemon &rarr; Robust and secure
           43  +<li>The "lemon.exe" command line tool itself &rarr; Not so much
           44  +</ul>
    25     45   
    26     46   <h2>Theory of Operation</h2>
    27     47   
    28     48   <p>The main goal of Lemon is to translate a context free grammar (CFG)
    29     49   for a particular language into C code that implements a parser for
    30     50   that language.
    31     51   The program has two inputs:

Changes to ext/misc/series.c.

    29     29   ** Integers 20 through 29.
    30     30   **
    31     31   ** HOW IT WORKS
    32     32   **
    33     33   ** The generate_series "function" is really a virtual table with the
    34     34   ** following schema:
    35     35   **
    36         -**     CREATE FUNCTION generate_series(
           36  +**     CREATE TABLE generate_series(
    37     37   **       value,
    38     38   **       start HIDDEN,
    39     39   **       stop HIDDEN,
    40     40   **       step HIDDEN
    41     41   **     );
    42     42   **
    43     43   ** Function arguments in queries against this virtual table are translated

Changes to src/build.c.

   935    935       pParse->nErr++;
   936    936       goto begin_table_error;
   937    937     }
   938    938     pTable->zName = zName;
   939    939     pTable->iPKey = -1;
   940    940     pTable->pSchema = db->aDb[iDb].pSchema;
   941    941     pTable->nTabRef = 1;
          942  +#ifdef SQLITE_DEFAULT_ROWEST
          943  +  pTable->nRowLogEst = sqlite3LogEst(SQLITE_DEFAULT_ROWEST);
          944  +#else
   942    945     pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
          946  +#endif
   943    947     assert( pParse->pNewTable==0 );
   944    948     pParse->pNewTable = pTable;
   945    949   
   946    950     /* If this is the magic sqlite_sequence table used by autoincrement,
   947    951     ** then record a pointer to this table in the main database structure
   948    952     ** so that INSERT can find the table easily.
   949    953     */

Changes to src/delete.c.

   455    455           iKey = ++pParse->nMem;
   456    456           nKey = 0;   /* Zero tells OP_Found to use a composite key */
   457    457           sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey,
   458    458               sqlite3IndexAffinityStr(pParse->db, pPk), nPk);
   459    459           sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEphCur, iKey, iPk, nPk);
   460    460         }else{
   461    461           /* Add the rowid of the row to be deleted to the RowSet */
   462         -        nKey = 1;  /* OP_Seek always uses a single rowid */
          462  +        nKey = 1;  /* OP_DeferredSeek always uses a single rowid */
   463    463           sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey);
   464    464         }
   465    465       }
   466    466     
   467    467       /* If this DELETE cannot use the ONEPASS strategy, this is the 
   468    468       ** end of the WHERE loop */
   469    469       if( eOnePass!=ONEPASS_OFF ){

Changes to src/insert.c.

   520    520     int tmask;                  /* Mask of trigger times */
   521    521   #endif
   522    522   
   523    523     db = pParse->db;
   524    524     if( pParse->nErr || db->mallocFailed ){
   525    525       goto insert_cleanup;
   526    526     }
          527  +  dest.iSDParm = 0;  /* Suppress a harmless compiler warning */
   527    528   
   528    529     /* If the Select object is really just a simple VALUES() list with a
   529    530     ** single row (the common case) then keep that one row of values
   530    531     ** and discard the other (unused) parts of the pSelect object
   531    532     */
   532    533     if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){
   533    534       pList = pSelect->pEList;

Changes to src/shell.c.

   504    504   ** lower 30 bits of a 32-bit signed integer.
   505    505   */
   506    506   static int strlen30(const char *z){
   507    507     const char *z2 = z;
   508    508     while( *z2 ){ z2++; }
   509    509     return 0x3fffffff & (int)(z2 - z);
   510    510   }
          511  +
          512  +/*
          513  +** Return the length of a string in characters.  Multibyte UTF8 characters
          514  +** count as a single character.
          515  +*/
          516  +static int strlenChar(const char *z){
          517  +  int n = 0;
          518  +  while( *z ){
          519  +    if( (0xc0&*(z++))!=0x80 ) n++;
          520  +  }
          521  +  return n;
          522  +}
   511    523   
   512    524   /*
   513    525   ** This routine reads a line of text from FILE in, stores
   514    526   ** the text in memory obtained from malloc() and returns a pointer
   515    527   ** to the text.  NULL is returned at end of file, or if malloc()
   516    528   ** fails.
   517    529   **
................................................................................
  1913   1925             int w, n;
  1914   1926             if( i<ArraySize(p->colWidth) ){
  1915   1927               w = colWidth[i];
  1916   1928             }else{
  1917   1929               w = 0;
  1918   1930             }
  1919   1931             if( w==0 ){
  1920         -            w = strlen30(azCol[i] ? azCol[i] : "");
         1932  +            w = strlenChar(azCol[i] ? azCol[i] : "");
  1921   1933               if( w<10 ) w = 10;
  1922         -            n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullValue);
         1934  +            n = strlenChar(azArg && azArg[i] ? azArg[i] : p->nullValue);
  1923   1935               if( w<n ) w = n;
  1924   1936             }
  1925   1937             if( i<ArraySize(p->actualWidth) ){
  1926   1938               p->actualWidth[i] = w;
  1927   1939             }
  1928   1940             if( showHdr ){
  1929   1941               utf8_width_print(p->out, w, azCol[i]);
................................................................................
  1950   1962         for(i=0; i<nArg; i++){
  1951   1963           int w;
  1952   1964           if( i<ArraySize(p->actualWidth) ){
  1953   1965              w = p->actualWidth[i];
  1954   1966           }else{
  1955   1967              w = 10;
  1956   1968           }
  1957         -        if( p->cMode==MODE_Explain && azArg[i] && strlen30(azArg[i])>w ){
  1958         -          w = strlen30(azArg[i]);
         1969  +        if( p->cMode==MODE_Explain && azArg[i] && strlenChar(azArg[i])>w ){
         1970  +          w = strlenChar(azArg[i]);
  1959   1971           }
  1960   1972           if( i==1 && p->aiIndent && p->pStmt ){
  1961   1973             if( p->iIndent<p->nIndent ){
  1962   1974               utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], "");
  1963   1975             }
  1964   1976             p->iIndent++;
  1965   1977           }

Changes to src/test_fs.c.

   541    541         for(i=nPrefix; zQuery[i]; i++){
   542    542           if( zQuery[i]==aWild[0] || zQuery[i]==aWild[1] ) break;
   543    543           if( zQuery[i]=='/' ) nDir = i;
   544    544         }
   545    545         zDir = zQuery;
   546    546       }
   547    547     }
          548  +  if( nDir==0 ) nDir = 1;
   548    549   
   549    550     sqlite3_bind_text(pCsr->pStmt, 1, zDir, nDir, SQLITE_TRANSIENT);
   550    551     sqlite3_bind_text(pCsr->pStmt, 2, zRoot, nRoot, SQLITE_TRANSIENT);
   551    552     sqlite3_bind_text(pCsr->pStmt, 3, zPrefix, nPrefix, SQLITE_TRANSIENT);
   552    553   
   553    554   #if SQLITE_OS_WIN
   554    555     sqlite3_free(zPrefix);

Changes to src/vdbe.c.

  2493   2493     u32 avail;         /* Number of bytes of available data */
  2494   2494     u32 t;             /* A type code from the record header */
  2495   2495     Mem *pReg;         /* PseudoTable input register */
  2496   2496   
  2497   2497     pC = p->apCsr[pOp->p1];
  2498   2498     p2 = pOp->p2;
  2499   2499   
  2500         -  /* If the cursor cache is stale, bring it up-to-date */
         2500  +  /* If the cursor cache is stale (meaning it is not currently point at
         2501  +  ** the correct row) then bring it up-to-date by doing the necessary 
         2502  +  ** B-Tree seek. */
  2501   2503     rc = sqlite3VdbeCursorMoveto(&pC, &p2);
  2502   2504     if( rc ) goto abort_due_to_error;
  2503   2505   
  2504   2506     assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
  2505   2507     pDest = &aMem[pOp->p3];
  2506   2508     memAboutToChange(p, pDest);
  2507   2509     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
................................................................................
  5261   5263     }
  5262   5264     assert( pC->deferredMoveto==0 );
  5263   5265     pC->cacheStatus = CACHE_STALE;
  5264   5266     pC->seekResult = 0;
  5265   5267     break;
  5266   5268   }
  5267   5269   
  5268         -/* Opcode: Seek P1 * P3 P4 *
  5269         -** Synopsis: Move P3 to P1.rowid
         5270  +/* Opcode: DeferredSeek P1 * P3 P4 *
         5271  +** Synopsis: Move P3 to P1.rowid if needed
  5270   5272   **
  5271   5273   ** P1 is an open index cursor and P3 is a cursor on the corresponding
  5272   5274   ** table.  This opcode does a deferred seek of the P3 table cursor
  5273   5275   ** to the row that corresponds to the current row of P1.
  5274   5276   **
  5275   5277   ** This is a deferred seek.  Nothing actually happens until
  5276   5278   ** the cursor is used to read a record.  That way, if no reads
................................................................................
  5289   5291   **
  5290   5292   ** Write into register P2 an integer which is the last entry in the record at
  5291   5293   ** the end of the index key pointed to by cursor P1.  This integer should be
  5292   5294   ** the rowid of the table entry to which this index entry points.
  5293   5295   **
  5294   5296   ** See also: Rowid, MakeRecord.
  5295   5297   */
  5296         -case OP_Seek:
  5297         -case OP_IdxRowid: {              /* out2 */
  5298         -  VdbeCursor *pC;                /* The P1 index cursor */
  5299         -  VdbeCursor *pTabCur;           /* The P2 table cursor (OP_Seek only) */
  5300         -  i64 rowid;                     /* Rowid that P1 current points to */
         5298  +case OP_DeferredSeek:
         5299  +case OP_IdxRowid: {           /* out2 */
         5300  +  VdbeCursor *pC;             /* The P1 index cursor */
         5301  +  VdbeCursor *pTabCur;        /* The P2 table cursor (OP_DeferredSeek only) */
         5302  +  i64 rowid;                  /* Rowid that P1 current points to */
  5301   5303   
  5302   5304     assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  5303   5305     pC = p->apCsr[pOp->p1];
  5304   5306     assert( pC!=0 );
  5305   5307     assert( pC->eCurType==CURTYPE_BTREE );
  5306   5308     assert( pC->uc.pCursor!=0 );
  5307   5309     assert( pC->isTable==0 );
................................................................................
  5319   5321   
  5320   5322     if( !pC->nullRow ){
  5321   5323       rowid = 0;  /* Not needed.  Only used to silence a warning. */
  5322   5324       rc = sqlite3VdbeIdxRowid(db, pC->uc.pCursor, &rowid);
  5323   5325       if( rc!=SQLITE_OK ){
  5324   5326         goto abort_due_to_error;
  5325   5327       }
  5326         -    if( pOp->opcode==OP_Seek ){
         5328  +    if( pOp->opcode==OP_DeferredSeek ){
  5327   5329         assert( pOp->p3>=0 && pOp->p3<p->nCursor );
  5328   5330         pTabCur = p->apCsr[pOp->p3];
  5329   5331         assert( pTabCur!=0 );
  5330   5332         assert( pTabCur->eCurType==CURTYPE_BTREE );
  5331   5333         assert( pTabCur->uc.pCursor!=0 );
  5332   5334         assert( pTabCur->isTable );
  5333   5335         pTabCur->nullRow = 0;

Changes to src/wherecode.c.

   962    962   ** Cursor iCur is open on an intkey b-tree (a table). Register iRowid contains
   963    963   ** a rowid value just read from cursor iIdxCur, open on index pIdx. This
   964    964   ** function generates code to do a deferred seek of cursor iCur to the 
   965    965   ** rowid stored in register iRowid.
   966    966   **
   967    967   ** Normally, this is just:
   968    968   **
   969         -**   OP_Seek $iCur $iRowid
          969  +**   OP_DeferredSeek $iCur $iRowid
   970    970   **
   971    971   ** However, if the scan currently being coded is a branch of an OR-loop and
   972         -** the statement currently being coded is a SELECT, then P3 of the OP_Seek
          972  +** the statement currently being coded is a SELECT, then P3 of OP_DeferredSeek
   973    973   ** is set to iIdxCur and P4 is set to point to an array of integers
   974    974   ** containing one entry for each column of the table cursor iCur is open 
   975    975   ** on. For each table column, if the column is the i'th column of the 
   976    976   ** index, then the corresponding array entry is set to (i+1). If the column
   977    977   ** does not appear in the index at all, the array entry is set to 0.
   978    978   */
   979    979   static void codeDeferredSeek(
................................................................................
   984    984   ){
   985    985     Parse *pParse = pWInfo->pParse; /* Parse context */
   986    986     Vdbe *v = pParse->pVdbe;        /* Vdbe to generate code within */
   987    987   
   988    988     assert( iIdxCur>0 );
   989    989     assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 );
   990    990     
   991         -  sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur);
          991  +  sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur);
   992    992     if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
   993    993      && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask)
   994    994     ){
   995    995       int i;
   996    996       Table *pTab = pIdx->pTable;
   997    997       int *ai = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*(pTab->nCol+1));
   998    998       if( ai ){

Changes to test/kvtest.c.

    67     67   "\n"
    68     68   "        Generate a new test database file named DBFILE containing N\n"
    69     69   "        BLOBs each of size M bytes.  The page size of the new database\n"
    70     70   "        file will be X.  Additional options:\n"
    71     71   "\n"
    72     72   "           --variance V           Randomly vary M by plus or minus V\n"
    73     73   "\n"
    74         -"   kvtest export DBFILE DIRECTORY\n"
           74  +"   kvtest export DBFILE DIRECTORY [--tree]\n"
    75     75   "\n"
    76     76   "        Export all the blobs in the kv table of DBFILE into separate\n"
    77         -"        files in DIRECTORY.\n"
           77  +"        files in DIRECTORY.  DIRECTORY is created if it does not previously\n"
           78  +"        exist.  If the --tree option is used, then the blobs are written\n"
           79  +"        into a hierarchy of directories, using names like 00/00/00,\n"
           80  +"        00/00/01, 00/00/02, and so forth.  Without the --tree option, all\n"
           81  +"        files are in the top-level directory with names like 000000, 000001,\n"
           82  +"        000002, and so forth.\n"
           83  +"\n"
           84  +"   kvtest stat DBFILE [options]\n"
    78     85   "\n"
    79         -"   kvtest stat DBFILE\n"
           86  +"        Display summary information about DBFILE.  Options:\n"
    80     87   "\n"
    81         -"        Display summary information about DBFILE\n"
           88  +"           --vacuum               Run VACUUM on the database file\n"
    82     89   "\n"
    83     90   "   kvtest run DBFILE [options]\n"
    84     91   "\n"
    85     92   "        Run a performance test.  DBFILE can be either the name of a\n"
    86     93   "        database or a directory containing sample files.  Options:\n"
    87     94   "\n"
    88     95   "           --asc                  Read blobs in ascending order\n"
    89     96   "           --blob-api             Use the BLOB API\n"
    90     97   "           --cache-size N         Database cache size\n"
    91     98   "           --count N              Read N blobs\n"
    92     99   "           --desc                 Read blobs in descending order\n"
          100  +"           --fsync                Synchronous file writes\n"
          101  +"           --integrity-check      Run \"PRAGMA integrity_check\" after test\n"
    93    102   "           --max-id N             Maximum blob key to use\n"
    94    103   "           --mmap N               Mmap as much as N bytes of DBFILE\n"
          104  +"           --multitrans           Each read or write in its own transaction\n"
          105  +"           --nocheckpoint         Omit the checkpoint on WAL mode writes\n"
          106  +"           --nosync               Set \"PRAGMA synchronous=OFF\"\n"
    95    107   "           --jmode MODE           Set MODE journal mode prior to starting\n"
    96    108   "           --random               Read blobs in a random order\n"
    97    109   "           --start N              Start reading with this blob key\n"
    98    110   "           --stats                Output operating stats before exiting\n"
          111  +"           --update               Do an overwrite test\n"
    99    112   ;
   100    113   
   101    114   /* Reference resources used */
   102    115   #include <stdio.h>
   103    116   #include <stdlib.h>
   104    117   #include <sys/types.h>
   105    118   #include <sys/stat.h>
................................................................................
   107    120   #include <string.h>
   108    121   #include "sqlite3.h"
   109    122   
   110    123   #ifndef _WIN32
   111    124   # include <unistd.h>
   112    125   #else
   113    126     /* Provide Windows equivalent for the needed parts of unistd.h */
          127  +# include <direct.h>
   114    128   # include <io.h>
   115    129   # define R_OK 2
   116    130   # define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
   117    131   # define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
   118    132   # define access _access
   119    133   #endif
   120    134   
          135  +#include <stdint.h>
          136  +
          137  +/*
          138  +** The following macros are used to cast pointers to integers and
          139  +** integers to pointers.  The way you do this varies from one compiler
          140  +** to the next, so we have developed the following set of #if statements
          141  +** to generate appropriate macros for a wide range of compilers.
          142  +**
          143  +** The correct "ANSI" way to do this is to use the intptr_t type.
          144  +** Unfortunately, that typedef is not available on all compilers, or
          145  +** if it is available, it requires an #include of specific headers
          146  +** that vary from one machine to the next.
          147  +**
          148  +** Ticket #3860:  The llvm-gcc-4.2 compiler from Apple chokes on
          149  +** the ((void*)&((char*)0)[X]) construct.  But MSVC chokes on ((void*)(X)).
          150  +** So we have to define the macros in different ways depending on the
          151  +** compiler.
          152  +*/
          153  +#if defined(__PTRDIFF_TYPE__)  /* This case should work for GCC */
          154  +# define SQLITE_INT_TO_PTR(X)  ((void*)(__PTRDIFF_TYPE__)(X))
          155  +# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(__PTRDIFF_TYPE__)(X))
          156  +#else
          157  +# define SQLITE_INT_TO_PTR(X)  ((void*)(intptr_t)(X))
          158  +# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(intptr_t)(X))
          159  +#endif
   121    160   
   122    161   /*
   123    162   ** Show thqe help text and quit.
   124    163   */
   125    164   static void showHelp(void){
   126    165     fprintf(stdout, "%s", zHelp);
   127    166     exit(1);
................................................................................
   197    236     return isNeg? -v : v;
   198    237   }
   199    238   
   200    239   
   201    240   /*
   202    241   ** Check the filesystem object zPath.  Determine what it is:
   203    242   **
   204         -**    PATH_DIR     A directory
          243  +**    PATH_DIR     A single directory holding many files
          244  +**    PATH_TREE    A directory hierarchy with files at the leaves
   205    245   **    PATH_DB      An SQLite database
   206    246   **    PATH_NEXIST  Does not exist
   207    247   **    PATH_OTHER   Something else
          248  +**
          249  +** PATH_DIR means all of the separate files are grouped together
          250  +** into a single directory with names like 000000, 000001, 000002, and
          251  +** so forth.  PATH_TREE means there is a hierarchy of directories so
          252  +** that no single directory has too many entries.  The files have names
          253  +** like 00/00/00, 00/00/01, 00/00/02 and so forth.  The decision between
          254  +** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory
          255  +** named "00" at the top-level.
   208    256   */
   209    257   #define PATH_DIR     1
   210         -#define PATH_DB      2
          258  +#define PATH_TREE    2
          259  +#define PATH_DB      3
   211    260   #define PATH_NEXIST  0
   212    261   #define PATH_OTHER   99
   213    262   static int pathType(const char *zPath){
   214    263     struct stat x;
   215    264     int rc;
   216    265     if( access(zPath,R_OK) ) return PATH_NEXIST;
   217    266     memset(&x, 0, sizeof(x));
   218    267     rc = stat(zPath, &x);
   219    268     if( rc<0 ) return PATH_OTHER;
   220         -  if( S_ISDIR(x.st_mode) ) return PATH_DIR;
          269  +  if( S_ISDIR(x.st_mode) ){
          270  +    char *zLayer1 = sqlite3_mprintf("%s/00", zPath);
          271  +    memset(&x, 0, sizeof(x));
          272  +    rc = stat(zLayer1, &x);
          273  +    sqlite3_free(zLayer1);
          274  +    if( rc<0 ) return PATH_DIR;
          275  +    if( S_ISDIR(x.st_mode) ) return PATH_TREE;
          276  +    return PATH_DIR;
          277  +  }
   221    278     if( (x.st_size%512)==0 ) return PATH_DB;
   222    279     return PATH_OTHER;
   223    280   }
   224    281   
   225    282   /*
   226    283   ** Return the size of a file in bytes.  Or return -1 if the
   227    284   ** named object is not a regular file or does not exist.
................................................................................
   324    381   */
   325    382   static int statMain(int argc, char **argv){
   326    383     char *zDb;
   327    384     int i, rc;
   328    385     sqlite3 *db;
   329    386     char *zSql;
   330    387     sqlite3_stmt *pStmt;
          388  +  int doVacuum = 0;
   331    389   
   332    390     assert( strcmp(argv[1],"stat")==0 );
   333    391     assert( argc>=3 );
   334    392     zDb = argv[2];
   335    393     for(i=3; i<argc; i++){
   336    394       char *z = argv[i];
   337    395       if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
   338    396       if( z[1]=='-' ) z++;
          397  +    if( strcmp(z, "-vacuum")==0 ){
          398  +      doVacuum = 1;
          399  +      continue;
          400  +    }
   339    401       fatalError("unknown option: \"%s\"", argv[i]);
   340    402     }
   341    403     rc = sqlite3_open(zDb, &db);
   342    404     if( rc ){
   343    405       fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
   344    406     }
          407  +  if( doVacuum ){
          408  +    printf("Vacuuming...."); fflush(stdout);
          409  +    sqlite3_exec(db, "VACUUM", 0, 0, 0);
          410  +    printf("       done\n");
          411  +  }
   345    412     zSql = sqlite3_mprintf(
   346    413       "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))"
   347    414       "  FROM kv"
   348    415     );
   349    416     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
   350    417     if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
   351    418     sqlite3_free(zSql);
................................................................................
   369    436     zSql = sqlite3_mprintf("PRAGMA page_count");
   370    437     rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
   371    438     if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
   372    439     sqlite3_free(zSql);
   373    440     if( sqlite3_step(pStmt)==SQLITE_ROW ){
   374    441       printf("Page-count:         %8d\n", sqlite3_column_int(pStmt, 0));
   375    442     }
          443  +  sqlite3_finalize(pStmt);
          444  +  zSql = sqlite3_mprintf("PRAGMA freelist_count");
          445  +  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
          446  +  if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
          447  +  sqlite3_free(zSql);
          448  +  if( sqlite3_step(pStmt)==SQLITE_ROW ){
          449  +    printf("Freelist-count:     %8d\n", sqlite3_column_int(pStmt, 0));
          450  +  }
          451  +  sqlite3_finalize(pStmt);
          452  +  rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check(10)", -1, &pStmt, 0);
          453  +  if( rc ) fatalError("cannot prepare integrity check: %s", sqlite3_errmsg(db));
          454  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          455  +    printf("Integrity-check:    %s\n", sqlite3_column_text(pStmt, 0));
          456  +  }
   376    457     sqlite3_finalize(pStmt);
   377    458     sqlite3_close(db);
   378    459     return 0;
   379    460   }
   380    461   
   381    462   /*
   382         -** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
   383         -** is written into file X.  The number of bytes written is returned.  Or
   384         -** NULL is returned if something goes wrong, such as being unable to open
   385         -** file X for writing.
          463  +**      remember(V,PTR)
          464  +**
          465  +** Return the integer value V.  Also save the value of V in a
          466  +** C-language variable whose address is PTR.
   386    467   */
   387         -static void writefileFunc(
   388         -  sqlite3_context *context,
          468  +static void rememberFunc(
          469  +  sqlite3_context *pCtx,
   389    470     int argc,
   390    471     sqlite3_value **argv
   391    472   ){
   392         -  FILE *out;
   393         -  const char *z;
   394         -  sqlite3_int64 rc;
   395         -  const char *zFile;
          473  +  sqlite3_int64 v;
          474  +  sqlite3_int64 ptr;
          475  +  assert( argc==2 );
          476  +  v = sqlite3_value_int64(argv[0]);
          477  +  ptr = sqlite3_value_int64(argv[1]);
          478  +  *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v;
          479  +  sqlite3_result_int64(pCtx, v);
          480  +}
   396    481   
   397         -  zFile = (const char*)sqlite3_value_text(argv[0]);
   398         -  if( zFile==0 ) return;
   399         -  out = fopen(zFile, "wb");
   400         -  if( out==0 ) return;
   401         -  z = (const char*)sqlite3_value_blob(argv[1]);
   402         -  if( z==0 ){
   403         -    rc = 0;
   404         -  }else{
   405         -    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
   406         -  }
   407         -  fclose(out);
   408         -  printf("\r%s   ", zFile); fflush(stdout);
   409         -  sqlite3_result_int64(context, rc);
          482  +/*
          483  +** Make sure a directory named zDir exists.
          484  +*/
          485  +static void kvtest_mkdir(const char *zDir){
          486  +#if defined(_WIN32)
          487  +  (void)mkdir(zDir);
          488  +#else
          489  +  (void)mkdir(zDir, 0755);
          490  +#endif
   410    491   }
   411    492   
   412    493   /*
   413    494   ** Export the kv table to individual files in the filesystem
   414    495   */
   415    496   static int exportMain(int argc, char **argv){
   416    497     char *zDb;
   417    498     char *zDir;
   418    499     sqlite3 *db;
   419         -  char *zSql;
          500  +  sqlite3_stmt *pStmt;
   420    501     int rc;
   421         -  char *zErrMsg = 0;
          502  +  int ePathType;
          503  +  int nFN;
          504  +  char *zFN;
          505  +  char *zTail;
          506  +  size_t nWrote;
          507  +  int i;
   422    508   
   423    509     assert( strcmp(argv[1],"export")==0 );
   424    510     assert( argc>=3 );
          511  +  if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]");
   425    512     zDb = argv[2];
   426         -  if( argc!=4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY");
   427    513     zDir = argv[3];
   428         -  if( pathType(zDir)!=PATH_DIR ){
          514  +  kvtest_mkdir(zDir);
          515  +  for(i=4; i<argc; i++){
          516  +    const char *z = argv[i];
          517  +    if( z[0]=='-' && z[1]=='-' ) z++;
          518  +    if( strcmp(z,"-tree")==0 ){
          519  +      zFN = sqlite3_mprintf("%s/00", zDir);
          520  +      kvtest_mkdir(zFN);
          521  +      sqlite3_free(zFN);
          522  +      continue;
          523  +    }
          524  +    fatalError("unknown argument: \"%s\"\n", argv[i]);
          525  +  }
          526  +  ePathType = pathType(zDir);
          527  +  if( ePathType!=PATH_DIR && ePathType!=PATH_TREE ){
   429    528       fatalError("object \"%s\" is not a directory", zDir);
   430    529     }
   431    530     rc = sqlite3_open(zDb, &db);
   432    531     if( rc ){
   433    532       fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
   434    533     }
   435         -  sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
   436         -                          writefileFunc, 0, 0);
   437         -  zSql = sqlite3_mprintf(
   438         -    "SELECT writefile(printf('%s/%%06d',k),v) FROM kv;",
   439         -    zDir
   440         -  );
   441         -  rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
   442         -  if( rc ) fatalError("database create failed: %s", zErrMsg);
   443         -  sqlite3_free(zSql);
          534  +  rc = sqlite3_prepare_v2(db, "SELECT k, v FROM kv ORDER BY k", -1, &pStmt, 0);
          535  +  if( rc ){
          536  +    fatalError("prepare_v2 failed: %s\n", sqlite3_errmsg(db));
          537  +  }
          538  +  nFN = (int)strlen(zDir);
          539  +  zFN = sqlite3_mprintf("%s/00/00/00.extra---------------------", zDir);
          540  +  if( zFN==0 ){
          541  +    fatalError("malloc failed\n");
          542  +  }
          543  +  zTail = zFN + nFN + 1;
          544  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          545  +    int iKey = sqlite3_column_int(pStmt, 0);
          546  +    sqlite3_int64 nData = sqlite3_column_bytes(pStmt, 1);
          547  +    const void *pData = sqlite3_column_blob(pStmt, 1);
          548  +    FILE *out;
          549  +    if( ePathType==PATH_DIR ){
          550  +      sqlite3_snprintf(20, zTail, "%06d", iKey);
          551  +    }else{
          552  +      sqlite3_snprintf(20, zTail, "%02d", iKey/10000);
          553  +      kvtest_mkdir(zFN);
          554  +      sqlite3_snprintf(20, zTail, "%02d/%02d", iKey/10000, (iKey/100)%100);
          555  +      kvtest_mkdir(zFN);
          556  +      sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",
          557  +                       iKey/10000, (iKey/100)%100, iKey%100);
          558  +    }
          559  +    out = fopen(zFN, "wb");      
          560  +    nWrote = fwrite(pData, 1, nData, out);
          561  +    fclose(out);
          562  +    printf("\r%s   ", zTail); fflush(stdout);
          563  +    if( nWrote!=nData ){
          564  +      fatalError("Wrote only %d of %d bytes to %s\n",
          565  +                  (int)nWrote, nData, zFN);
          566  +    }
          567  +  }
          568  +  sqlite3_finalize(pStmt);
   444    569     sqlite3_close(db);
          570  +  sqlite3_free(zFN);
   445    571     printf("\n");
   446    572     return 0;
   447    573   }
   448    574   
   449    575   /*
   450    576   ** Read the content of file zName into memory obtained from sqlite3_malloc64()
   451    577   ** and return a pointer to the buffer. The caller is responsible for freeing 
................................................................................
   457    583   ** For convenience, a nul-terminator byte is always appended to the data read
   458    584   ** from the file before the buffer is returned. This byte is not included in
   459    585   ** the final value of (*pnByte), if applicable.
   460    586   **
   461    587   ** NULL is returned if any error is encountered. The final value of *pnByte
   462    588   ** is undefined in this case.
   463    589   */
   464         -static unsigned char *readFile(const char *zName, int *pnByte){
          590  +static unsigned char *readFile(const char *zName, sqlite3_int64 *pnByte){
   465    591     FILE *in;               /* FILE from which to read content of zName */
   466    592     sqlite3_int64 nIn;      /* Size of zName in bytes */
   467    593     size_t nRead;           /* Number of bytes actually read */
   468    594     unsigned char *pBuf;    /* Content read from disk */
   469    595   
   470    596     nIn = fileSize(zName);
   471    597     if( nIn<0 ) return 0;
................................................................................
   475    601     if( pBuf==0 ) return 0;
   476    602     nRead = fread(pBuf, (size_t)nIn, 1, in);
   477    603     fclose(in);
   478    604     if( nRead!=1 ){
   479    605       sqlite3_free(pBuf);
   480    606       return 0;
   481    607     }
   482         -  if( pnByte ) *pnByte = (int)nIn;
          608  +  if( pnByte ) *pnByte = nIn;
   483    609     return pBuf;
   484    610   }
          611  +
          612  +/*
          613  +** Overwrite a file with randomness.  Do not change the size of the
          614  +** file.
          615  +*/
          616  +static void updateFile(const char *zName, sqlite3_int64 *pnByte, int doFsync){
          617  +  FILE *out;              /* FILE from which to read content of zName */
          618  +  sqlite3_int64 sz;       /* Size of zName in bytes */
          619  +  size_t nWritten;        /* Number of bytes actually read */
          620  +  unsigned char *pBuf;    /* Content to store on disk */
          621  +  const char *zMode = "wb";   /* Mode for fopen() */
          622  +
          623  +  sz = fileSize(zName);
          624  +  if( sz<0 ){
          625  +    fatalError("No such file: \"%s\"", zName);
          626  +  }
          627  +  *pnByte = sz;
          628  +  if( sz==0 ) return;
          629  +  pBuf = sqlite3_malloc64( sz );
          630  +  if( pBuf==0 ){
          631  +    fatalError("Cannot allocate %lld bytes\n", sz);
          632  +  }
          633  +  sqlite3_randomness((int)sz, pBuf); 
          634  +#if defined(_WIN32)
          635  +  if( doFsync ) zMode = "wbc";
          636  +#endif
          637  +  out = fopen(zName, zMode);
          638  +  if( out==0 ){
          639  +    fatalError("Cannot open \"%s\" for writing\n", zName);
          640  +  }
          641  +  nWritten = fwrite(pBuf, 1, (size_t)sz, out);
          642  +  if( doFsync ){
          643  +#if defined(_WIN32)
          644  +    fflush(out);
          645  +#else
          646  +    fsync(fileno(out));
          647  +#endif
          648  +  }
          649  +  fclose(out);
          650  +  if( nWritten!=(size_t)sz ){
          651  +    fatalError("Wrote only %d of %d bytes to \"%s\"\n",
          652  +               (int)nWritten, (int)sz, zName);
          653  +  }
          654  +  sqlite3_free(pBuf);
          655  +}
   485    656   
   486    657   /*
   487    658   ** Return the current time in milliseconds since the beginning of
   488    659   ** the Julian epoch.
   489    660   */
   490    661   static sqlite3_int64 timeOfDay(void){
   491    662     static sqlite3_vfs *clockVfs = 0;
................................................................................
   633    804     int iKey = 1;               /* Next blob key */
   634    805     int iMax = 0;               /* Largest allowed key */
   635    806     int iPagesize = 0;          /* Database page size */
   636    807     int iCache = 1000;          /* Database cache size in kibibytes */
   637    808     int bBlobApi = 0;           /* Use the incremental blob I/O API */
   638    809     int bStats = 0;             /* Print stats before exiting */
   639    810     int eOrder = ORDER_ASC;     /* Access order */
          811  +  int isUpdateTest = 0;       /* Do in-place updates rather than reads */
          812  +  int doIntegrityCk = 0;      /* Run PRAGMA integrity_check after the test */
          813  +  int noSync = 0;             /* Disable synchronous mode */
          814  +  int doFsync = 0;            /* Update disk files synchronously */
          815  +  int doMultiTrans = 0;       /* Each operation in its own transaction */
          816  +  int noCheckpoint = 0;       /* Omit the checkpoint in WAL mode */
   640    817     sqlite3 *db = 0;            /* Database connection */
   641    818     sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
   642    819     sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
   643    820     sqlite3_int64 tmStart;      /* Start time */
   644    821     sqlite3_int64 tmElapsed;    /* Elapsed time */
   645    822     int mmapSize = 0;           /* --mmap N argument */
   646         -  int nData = 0;              /* Bytes of data */
          823  +  sqlite3_int64 nData = 0;    /* Bytes of data */
   647    824     sqlite3_int64 nTotal = 0;   /* Total data read */
   648    825     unsigned char *pData = 0;   /* Content of the blob */
   649         -  int nAlloc = 0;             /* Space allocated for pData[] */
          826  +  sqlite3_int64 nAlloc = 0;   /* Space allocated for pData[] */
   650    827     const char *zJMode = 0;     /* Journal mode */
   651    828     
   652    829   
   653    830     assert( strcmp(argv[1],"run")==0 );
   654    831     assert( argc>=3 );
   655    832     zDb = argv[2];
   656    833     eType = pathType(zDb);
   657    834     if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb);
   658    835     if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb);
   659    836     for(i=3; i<argc; i++){
   660    837       char *z = argv[i];
   661    838       if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
   662    839       if( z[1]=='-' ) z++;
          840  +    if( strcmp(z, "-asc")==0 ){
          841  +      eOrder = ORDER_ASC;
          842  +      continue;
          843  +    }
          844  +    if( strcmp(z, "-blob-api")==0 ){
          845  +      bBlobApi = 1;
          846  +      continue;
          847  +    }
          848  +    if( strcmp(z, "-cache-size")==0 ){
          849  +      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
          850  +      iCache = integerValue(argv[++i]);
          851  +      continue;
          852  +    }
   663    853       if( strcmp(z, "-count")==0 ){
   664    854         if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
   665    855         nCount = integerValue(argv[++i]);
   666    856         if( nCount<1 ) fatalError("the --count must be positive");
   667    857         continue;
          858  +    }
          859  +    if( strcmp(z, "-desc")==0 ){
          860  +      eOrder = ORDER_DESC;
          861  +      continue;
          862  +    }
          863  +    if( strcmp(z, "-fsync")==0 ){
          864  +      doFsync = 1;
          865  +      continue;
          866  +    }
          867  +    if( strcmp(z, "-integrity-check")==0 ){
          868  +      doIntegrityCk = 1;
          869  +      continue;
          870  +    }
          871  +    if( strcmp(z, "-jmode")==0 ){
          872  +      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
          873  +      zJMode = argv[++i];
          874  +      continue;
   668    875       }
   669    876       if( strcmp(z, "-mmap")==0 ){
   670    877         if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
   671    878         mmapSize = integerValue(argv[++i]);
   672    879         if( nCount<0 ) fatalError("the --mmap must be non-negative");
   673    880         continue;
   674    881       }
   675    882       if( strcmp(z, "-max-id")==0 ){
   676    883         if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
   677    884         iMax = integerValue(argv[++i]);
   678    885         continue;
   679    886       }
   680         -    if( strcmp(z, "-start")==0 ){
   681         -      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
   682         -      iKey = integerValue(argv[++i]);
   683         -      if( iKey<1 ) fatalError("the --start must be positive");
          887  +    if( strcmp(z, "-multitrans")==0 ){
          888  +      doMultiTrans = 1;
          889  +      continue;
          890  +    }
          891  +    if( strcmp(z, "-nocheckpoint")==0 ){
          892  +      noCheckpoint = 1;
   684    893         continue;
   685    894       }
   686         -    if( strcmp(z, "-cache-size")==0 ){
   687         -      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
   688         -      iCache = integerValue(argv[++i]);
   689         -      continue;
   690         -    }
   691         -    if( strcmp(z, "-jmode")==0 ){
   692         -      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
   693         -      zJMode = argv[++i];
          895  +    if( strcmp(z, "-nosync")==0 ){
          896  +      noSync = 1;
   694    897         continue;
   695    898       }
   696    899       if( strcmp(z, "-random")==0 ){
   697    900         eOrder = ORDER_RANDOM;
   698    901         continue;
   699    902       }
   700         -    if( strcmp(z, "-asc")==0 ){
   701         -      eOrder = ORDER_ASC;
   702         -      continue;
   703         -    }
   704         -    if( strcmp(z, "-desc")==0 ){
   705         -      eOrder = ORDER_DESC;
   706         -      continue;
   707         -    }
   708         -    if( strcmp(z, "-blob-api")==0 ){
   709         -      bBlobApi = 1;
          903  +    if( strcmp(z, "-start")==0 ){
          904  +      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
          905  +      iKey = integerValue(argv[++i]);
          906  +      if( iKey<1 ) fatalError("the --start must be positive");
   710    907         continue;
   711    908       }
   712    909       if( strcmp(z, "-stats")==0 ){
   713    910         bStats = 1;
   714    911         continue;
          912  +    }
          913  +    if( strcmp(z, "-update")==0 ){
          914  +      isUpdateTest = 1;
          915  +      continue;
   715    916       }
   716    917       fatalError("unknown option: \"%s\"", argv[i]);
          918  +  }
          919  +  if( eType==PATH_DB ){
          920  +    /* Recover any prior crashes prior to starting the timer */
          921  +    sqlite3_open(zDb, &db);
          922  +    sqlite3_exec(db, "SELECT rowid FROM sqlite_master LIMIT 1", 0, 0, 0);
          923  +    sqlite3_close(db);
          924  +    db = 0;
   717    925     }
   718    926     tmStart = timeOfDay();
   719    927     if( eType==PATH_DB ){
   720    928       char *zSql;
   721    929       rc = sqlite3_open(zDb, &db);
   722    930       if( rc ){
   723    931         fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
   724    932       }
   725    933       zSql = sqlite3_mprintf("PRAGMA mmap_size=%d", mmapSize);
   726    934       sqlite3_exec(db, zSql, 0, 0, 0);
          935  +    sqlite3_free(zSql);
   727    936       zSql = sqlite3_mprintf("PRAGMA cache_size=%d", iCache);
   728    937       sqlite3_exec(db, zSql, 0, 0, 0);
   729    938       sqlite3_free(zSql);
          939  +    if( noSync ){
          940  +      sqlite3_exec(db, "PRAGMA synchronous=OFF", 0, 0, 0);
          941  +    }
   730    942       pStmt = 0;
   731    943       sqlite3_prepare_v2(db, "PRAGMA page_size", -1, &pStmt, 0);
   732    944       if( sqlite3_step(pStmt)==SQLITE_ROW ){
   733    945         iPagesize = sqlite3_column_int(pStmt, 0);
   734    946       }
   735    947       sqlite3_finalize(pStmt);
   736    948       sqlite3_prepare_v2(db, "PRAGMA cache_size", -1, &pStmt, 0);
................................................................................
   741    953       }
   742    954       sqlite3_finalize(pStmt);
   743    955       pStmt = 0;
   744    956       if( zJMode ){
   745    957         zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode);
   746    958         sqlite3_exec(db, zSql, 0, 0, 0);
   747    959         sqlite3_free(zSql);
          960  +      if( noCheckpoint ){
          961  +        sqlite3_exec(db, "PRAGMA wal_autocheckpoint=0", 0, 0, 0);
          962  +      }
   748    963       }
   749    964       sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0);
   750    965       if( sqlite3_step(pStmt)==SQLITE_ROW ){
   751    966         zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
   752    967       }else{
   753    968         zJMode = "???";
   754    969       }
................................................................................
   757    972         sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0);
   758    973         if( sqlite3_step(pStmt)==SQLITE_ROW ){
   759    974           iMax = sqlite3_column_int(pStmt, 0);
   760    975         }
   761    976         sqlite3_finalize(pStmt);
   762    977       }
   763    978       pStmt = 0;
   764         -    sqlite3_exec(db, "BEGIN", 0, 0, 0);
          979  +    if( !doMultiTrans ) sqlite3_exec(db, "BEGIN", 0, 0, 0);
   765    980     }
   766    981     if( iMax<=0 ) iMax = 1000;
   767    982     for(i=0; i<nCount; i++){
   768         -    if( eType==PATH_DIR ){
   769         -      /* CASE 1: Reading blobs out of separate files */
          983  +    if( eType==PATH_DIR || eType==PATH_TREE ){
          984  +      /* CASE 1: Reading or writing blobs out of separate files */
   770    985         char *zKey;
   771         -      zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
          986  +      if( eType==PATH_DIR ){
          987  +        zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
          988  +      }else{
          989  +        zKey = sqlite3_mprintf("%s/%02d/%02d/%02d", zDb, iKey/10000,
          990  +                               (iKey/100)%100, iKey%100);
          991  +      }
   772    992         nData = 0;
   773         -      pData = readFile(zKey, &nData);
          993  +      if( isUpdateTest ){
          994  +        updateFile(zKey, &nData, doFsync);
          995  +      }else{
          996  +        pData = readFile(zKey, &nData);
          997  +        sqlite3_free(pData);
          998  +      }
   774    999         sqlite3_free(zKey);
   775         -      sqlite3_free(pData);
   776   1000       }else if( bBlobApi ){
   777   1001         /* CASE 2: Reading from database using the incremental BLOB I/O API */
   778   1002         if( pBlob==0 ){
   779         -        rc = sqlite3_blob_open(db, "main", "kv", "v", iKey, 0, &pBlob);
         1003  +        rc = sqlite3_blob_open(db, "main", "kv", "v", iKey,
         1004  +                               isUpdateTest, &pBlob);
   780   1005           if( rc ){
   781   1006             fatalError("could not open sqlite3_blob handle: %s",
   782   1007                        sqlite3_errmsg(db));
   783   1008           }
   784   1009         }else{
   785   1010           rc = sqlite3_blob_reopen(pBlob, iKey);
   786   1011         }
   787   1012         if( rc==SQLITE_OK ){
   788   1013           nData = sqlite3_blob_bytes(pBlob);
   789   1014           if( nAlloc<nData+1 ){
   790   1015             nAlloc = nData+100;
   791         -          pData = sqlite3_realloc(pData, nAlloc);
         1016  +          pData = sqlite3_realloc64(pData, nAlloc);
   792   1017           }
   793   1018           if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1);
   794         -        rc = sqlite3_blob_read(pBlob, pData, nData, 0);
   795         -        if( rc!=SQLITE_OK ){
   796         -          fatalError("could not read the blob at %d: %s", iKey,
   797         -                     sqlite3_errmsg(db));
         1019  +        if( isUpdateTest ){
         1020  +          sqlite3_randomness((int)nData, pData);
         1021  +          rc = sqlite3_blob_write(pBlob, pData, (int)nData, 0);
         1022  +          if( rc!=SQLITE_OK ){
         1023  +            fatalError("could not write the blob at %d: %s", iKey,
         1024  +                      sqlite3_errmsg(db));
         1025  +          }
         1026  +        }else{
         1027  +          rc = sqlite3_blob_read(pBlob, pData, (int)nData, 0);
         1028  +          if( rc!=SQLITE_OK ){
         1029  +            fatalError("could not read the blob at %d: %s", iKey,
         1030  +                      sqlite3_errmsg(db));
         1031  +          }
   798   1032           }
   799   1033         }
   800   1034       }else{
   801   1035         /* CASE 3: Reading from database using SQL */
   802   1036         if( pStmt==0 ){
   803         -        rc = sqlite3_prepare_v2(db, 
   804         -               "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0);
         1037  +        if( isUpdateTest ){
         1038  +          sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0,
         1039  +                                  rememberFunc, 0, 0);
         1040  +
         1041  +          rc = sqlite3_prepare_v2(db, 
         1042  +            "UPDATE kv SET v=randomblob(remember(length(v),?2))"
         1043  +            " WHERE k=?1", -1, &pStmt, 0);
         1044  +          sqlite3_bind_int64(pStmt, 2, SQLITE_PTR_TO_INT(&nData));
         1045  +        }else{
         1046  +          rc = sqlite3_prepare_v2(db, 
         1047  +                 "SELECT v FROM kv WHERE k=?1", -1, &pStmt, 0);
         1048  +        }
   805   1049           if( rc ){
   806   1050             fatalError("cannot prepare query: %s", sqlite3_errmsg(db));
   807   1051           }
   808   1052         }else{
   809   1053           sqlite3_reset(pStmt);
   810   1054         }
   811   1055         sqlite3_bind_int(pStmt, 1, iKey);
         1056  +      nData = 0;
   812   1057         rc = sqlite3_step(pStmt);
   813   1058         if( rc==SQLITE_ROW ){
   814   1059           nData = sqlite3_column_bytes(pStmt, 0);
   815   1060           pData = (unsigned char*)sqlite3_column_blob(pStmt, 0);
   816         -      }else{
   817         -        nData = 0;
   818   1061         }
   819   1062       }
   820   1063       if( eOrder==ORDER_ASC ){
   821   1064         iKey++;
   822   1065         if( iKey>iMax ) iKey = 1;
   823   1066       }else if( eOrder==ORDER_DESC ){
   824   1067         iKey--;
................................................................................
   831   1074     }
   832   1075     if( nAlloc ) sqlite3_free(pData);
   833   1076     if( pStmt ) sqlite3_finalize(pStmt);
   834   1077     if( pBlob ) sqlite3_blob_close(pBlob);
   835   1078     if( bStats ){
   836   1079       display_stats(db, 0);
   837   1080     }
   838         -  if( db ) sqlite3_close(db);
         1081  +  if( db ){
         1082  +    if( !doMultiTrans ) sqlite3_exec(db, "COMMIT", 0, 0, 0);
         1083  +    if( !noCheckpoint ){
         1084  +      sqlite3_close(db);
         1085  +      db = 0;
         1086  +    }
         1087  +  }
   839   1088     tmElapsed = timeOfDay() - tmStart;
         1089  +  if( db && noCheckpoint ){
         1090  +    sqlite3_close(db);
         1091  +    db = 0;
         1092  +  }
   840   1093     if( nExtra ){
   841   1094       printf("%d cycles due to %d misses\n", nCount, nExtra);
   842   1095     }
   843   1096     if( eType==PATH_DB ){
   844   1097       printf("SQLite version: %s\n", sqlite3_libversion());
         1098  +    if( doIntegrityCk ){
         1099  +      sqlite3_open(zDb, &db);
         1100  +      sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pStmt, 0);
         1101  +      while( sqlite3_step(pStmt)==SQLITE_ROW ){
         1102  +        printf("integrity-check: %s\n", sqlite3_column_text(pStmt, 0));
         1103  +      }
         1104  +      sqlite3_finalize(pStmt);
         1105  +      sqlite3_close(db);
         1106  +      db = 0;
         1107  +    }
   845   1108     }
   846   1109     printf("--count %d --max-id %d", nCount-nExtra, iMax);
   847   1110     switch( eOrder ){
   848   1111       case ORDER_RANDOM:  printf(" --random\n");  break;
   849   1112       case ORDER_DESC:    printf(" --desc\n");    break;
   850   1113       default:            printf(" --asc\n");     break;
   851   1114     }
   852   1115     if( eType==PATH_DB ){
   853   1116       printf("--cache-size %d --jmode %s\n", iCache, zJMode);
   854   1117       printf("--mmap %d%s\n", mmapSize, bBlobApi ? " --blob-api" : "");
         1118  +    if( noSync ) printf("--nosync\n");
   855   1119     }
   856   1120     if( iPagesize ) printf("Database page size: %d\n", iPagesize);
   857   1121     printf("Total elapsed time: %.3f\n", tmElapsed/1000.0);
   858         -  printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);
   859         -  printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
         1122  +  if( isUpdateTest ){
         1123  +    printf("Microseconds per BLOB write: %.3f\n", tmElapsed*1000.0/nCount);
         1124  +    printf("Content write rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
         1125  +  }else{
         1126  +    printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);
         1127  +    printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
         1128  +  }
   860   1129     return 0;
   861   1130   }
   862   1131   
   863   1132   
   864   1133   int main(int argc, char **argv){
   865   1134     if( argc<3 ) showHelp();
   866   1135     if( strcmp(argv[1],"init")==0 ){

Changes to test/vtabH.test.

   212    212         set fd [open $path w]
   213    213         puts -nonewline $fd [string repeat 1 $sz]
   214    214         close $fd
   215    215       }
   216    216     } {}
   217    217   
   218    218     set pwd [pwd]
   219         -  do_execsql_test 3.5 {
   220         -    SELECT path, size FROM fstree WHERE path GLOB $pwd || '/subdir/*' ORDER BY 1
   221         -  } [list \
   222         -    "$pwd/subdir/x1.txt" 143 \
   223         -    "$pwd/subdir/x2.txt" 153 \
   224         -  ]
   225         -  do_execsql_test 3.6 {
   226         -    SELECT path, size FROM fstree WHERE path LIKE $pwd || '/subdir/%' ORDER BY 1
   227         -  } [list \
   228         -    "$pwd/subdir/x1.txt" 143 \
   229         -    "$pwd/subdir/x2.txt" 153 \
   230         -  ]
   231         -  do_execsql_test 3.7 {
   232         -    SELECT sum(size) FROM fstree WHERE path LIKE $pwd || '/subdir/%'
   233         -  } 296
   234         -  do_execsql_test 3.8 {
   235         -    SELECT size FROM fstree WHERE path = $pwd || '/subdir/x1.txt'
   236         -  } 143
          219  +  if {![string match {*[_%]*} $pwd]} {
          220  +    do_execsql_test 3.5 {
          221  +      SELECT path, size FROM fstree 
          222  +       WHERE path GLOB $pwd || '/subdir/*' ORDER BY 1
          223  +    } [list \
          224  +      "$pwd/subdir/x1.txt" 143 \
          225  +      "$pwd/subdir/x2.txt" 153 \
          226  +    ]
          227  +    do_execsql_test 3.6 {
          228  +      SELECT path, size FROM fstree
          229  +       WHERE path LIKE $pwd || '/subdir/%' ORDER BY 1
          230  +    } [list \
          231  +      "$pwd/subdir/x1.txt" 143 \
          232  +      "$pwd/subdir/x2.txt" 153 \
          233  +    ]
          234  +    do_execsql_test 3.7 {
          235  +      SELECT sum(size) FROM fstree WHERE path LIKE $pwd || '/subdir/%'
          236  +    } 296
          237  +    do_execsql_test 3.8 {
          238  +      SELECT size FROM fstree WHERE path = $pwd || '/subdir/x1.txt'
          239  +    } 143
          240  +  }
   237    241   
   238    242   }
   239    243   
   240    244   
   241    245   finish_test