/ Check-in [30f08d58]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enhancements to deserialize: (1) Add the SQLITE_FCNTL_SIZE_LIMIT file control to set a maximum size for an in-memory database, defaulting to SQLITE_MEMDB_DEFAULT_MAXSIZE or 1GiB. (2) Honor the SQLITE_DESERIALIZE_READONLY flag. (3) Enhance the TCL interface to support -maxsize N and -readonly BOOLEAN. (4) Add the --maxsize option to the ".open" command and on the command-line for the CLI.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 30f08d58882819a69e353bcc1b6b349664bbfbe00aa1c115ba44a9fd899fcc5b
User & Date: drh 2019-01-22 16:06:20
Context
2019-01-22
16:11
Update dbfuzz2 to set a maximum database size of 100MiB by default, but with the new --max-db-size N option to change that limit. check-in: 21d6bb78 user: drh tags: trunk
16:06
Enhancements to deserialize: (1) Add the SQLITE_FCNTL_SIZE_LIMIT file control to set a maximum size for an in-memory database, defaulting to SQLITE_MEMDB_DEFAULT_MAXSIZE or 1GiB. (2) Honor the SQLITE_DESERIALIZE_READONLY flag. (3) Enhance the TCL interface to support -maxsize N and -readonly BOOLEAN. (4) Add the --maxsize option to the ".open" command and on the command-line for the CLI. check-in: 30f08d58 user: drh tags: trunk
13:45
Make sure cursors are opened on all indexes for an UPDATE OR REPLACE regardless of whether or not the indexes are partial or contain columns that might need to be updated. check-in: e148cdad user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/memdb.c.

30
31
32
33
34
35
36

37
38
39
40
41
42





43
44
45
46
47
48
49
...
156
157
158
159
160
161
162





163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179

180
181
182
183
184
185
186
187
188
189
190
...
246
247
248
249
250
251
252













253
254
255
256
257
258
259
...
307
308
309
310
311
312
313

314
315
316
317
318
319
320
...
556
557
558
559
560
561
562

563



564
565
566
567
568
569
570
*/
#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))

/* An open file */
struct MemFile {
  sqlite3_file base;              /* IO methods */
  sqlite3_int64 sz;               /* Size of the file */

  sqlite3_int64 szMax;            /* Space allocated to aData */
  unsigned char *aData;           /* content of the file */
  int nMmap;                      /* Number of memory mapped pages */
  unsigned mFlags;                /* Flags */
  int eLock;                      /* Most recent lock against this file */
};






/*
** Methods for MemFile
*/
static int memdbClose(sqlite3_file*);
static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
................................................................................
** Try to enlarge the memory allocation to hold at least sz bytes
*/
static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){
  unsigned char *pNew;
  if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){
    return SQLITE_FULL;
  }





  pNew = sqlite3_realloc64(p->aData, newSz);
  if( pNew==0 ) return SQLITE_NOMEM;
  p->aData = pNew;
  p->szMax = newSz;
  return SQLITE_OK;
}

/*
** Write data to an memdb-file.
*/
static int memdbWrite(
  sqlite3_file *pFile,
  const void *z,
  int iAmt,
  sqlite_int64 iOfst
){
  MemFile *p = (MemFile *)pFile;

  if( iOfst+iAmt>p->sz ){
    int rc;
    if( iOfst+iAmt>p->szMax
     && (rc = memdbEnlarge(p, (iOfst+iAmt)*2))!=SQLITE_OK
    ){
      return rc;
    }
    if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
    p->sz = iOfst+iAmt;
  }
  memcpy(p->aData+iOfst, z, iAmt);
................................................................................
static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){
  MemFile *p = (MemFile *)pFile;
  int rc = SQLITE_NOTFOUND;
  if( op==SQLITE_FCNTL_VFSNAME ){
    *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz);
    rc = SQLITE_OK;
  }













  return rc;
}

#if 0  /* Not used because of SQLITE_IOCAP_POWERSAFE_OVERWRITE */
/*
** Return the sector-size in bytes for an memdb-file.
*/
................................................................................
    return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags);
  }
  memset(p, 0, sizeof(*p));
  p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
  assert( pOutFlags!=0 );  /* True because flags==SQLITE_OPEN_MAIN_DB */
  *pOutFlags = flags | SQLITE_OPEN_MEMORY;
  p->base.pMethods = &memdb_io_methods;

  return SQLITE_OK;
}

#if 0 /* Only used to delete rollback journals, master journals, and WAL
      ** files, none of which exist in memdb.  So this routine is never used */
/*
** Delete the file located at zPath. If the dirSync argument is true,
................................................................................
  }
  p = memdbFromDbSchema(db, zSchema);
  if( p==0 ){
    rc = SQLITE_ERROR;
  }else{
    p->aData = pData;
    p->sz = szDb;

    p->szMax = szBuf;



    p->mFlags = mFlags;
    rc = SQLITE_OK;
  }

end_deserialize:
  sqlite3_finalize(pStmt);
  sqlite3_mutex_leave(db->mutex);







>
|





>
>
>
>
>







 







>
>
>
>
>



|













>


|
|







 







>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







>

>
>
>







30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
...
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
...
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
...
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
...
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
*/
#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))

/* An open file */
struct MemFile {
  sqlite3_file base;              /* IO methods */
  sqlite3_int64 sz;               /* Size of the file */
  sqlite3_int64 szAlloc;          /* Space allocated to aData */
  sqlite3_int64 szMax;            /* Maximum allowed size of the file */
  unsigned char *aData;           /* content of the file */
  int nMmap;                      /* Number of memory mapped pages */
  unsigned mFlags;                /* Flags */
  int eLock;                      /* Most recent lock against this file */
};

/* The default maximum size of an in-memory database */
#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE
# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824
#endif

/*
** Methods for MemFile
*/
static int memdbClose(sqlite3_file*);
static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
................................................................................
** Try to enlarge the memory allocation to hold at least sz bytes
*/
static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){
  unsigned char *pNew;
  if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){
    return SQLITE_FULL;
  }
  if( newSz>p->szMax ){
    return SQLITE_FULL;
  }
  newSz *= 2;
  if( newSz>p->szMax ) newSz = p->szMax;
  pNew = sqlite3_realloc64(p->aData, newSz);
  if( pNew==0 ) return SQLITE_NOMEM;
  p->aData = pNew;
  p->szAlloc = newSz;
  return SQLITE_OK;
}

/*
** Write data to an memdb-file.
*/
static int memdbWrite(
  sqlite3_file *pFile,
  const void *z,
  int iAmt,
  sqlite_int64 iOfst
){
  MemFile *p = (MemFile *)pFile;
  if( p->mFlags & SQLITE_DESERIALIZE_READONLY ) return SQLITE_READONLY;
  if( iOfst+iAmt>p->sz ){
    int rc;
    if( iOfst+iAmt>p->szAlloc
     && (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK
    ){
      return rc;
    }
    if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
    p->sz = iOfst+iAmt;
  }
  memcpy(p->aData+iOfst, z, iAmt);
................................................................................
static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){
  MemFile *p = (MemFile *)pFile;
  int rc = SQLITE_NOTFOUND;
  if( op==SQLITE_FCNTL_VFSNAME ){
    *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz);
    rc = SQLITE_OK;
  }
  if( op==SQLITE_FCNTL_SIZE_LIMIT ){
    sqlite3_int64 iLimit = *(sqlite3_int64*)pArg;
    if( iLimit<p->sz ){
      if( iLimit<0 ){
        iLimit = p->szMax;
      }else{
        iLimit = p->sz;
      }
    }
    p->szMax = iLimit;
    *(sqlite3_int64*)pArg = iLimit;
    rc = SQLITE_OK;
  }
  return rc;
}

#if 0  /* Not used because of SQLITE_IOCAP_POWERSAFE_OVERWRITE */
/*
** Return the sector-size in bytes for an memdb-file.
*/
................................................................................
    return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFile, flags, pOutFlags);
  }
  memset(p, 0, sizeof(*p));
  p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE;
  assert( pOutFlags!=0 );  /* True because flags==SQLITE_OPEN_MAIN_DB */
  *pOutFlags = flags | SQLITE_OPEN_MEMORY;
  p->base.pMethods = &memdb_io_methods;
  p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
  return SQLITE_OK;
}

#if 0 /* Only used to delete rollback journals, master journals, and WAL
      ** files, none of which exist in memdb.  So this routine is never used */
/*
** Delete the file located at zPath. If the dirSync argument is true,
................................................................................
  }
  p = memdbFromDbSchema(db, zSchema);
  if( p==0 ){
    rc = SQLITE_ERROR;
  }else{
    p->aData = pData;
    p->sz = szDb;
    p->szAlloc = szBuf;
    p->szMax = szBuf;
    if( p->szMax<SQLITE_MEMDB_DEFAULT_MAXSIZE ){
      p->szMax = SQLITE_MEMDB_DEFAULT_MAXSIZE;
    }
    p->mFlags = mFlags;
    rc = SQLITE_OK;
  }

end_deserialize:
  sqlite3_finalize(pStmt);
  sqlite3_mutex_leave(db->mutex);

Changes to src/shell.c.in.

1021
1022
1023
1024
1025
1026
1027

1028
1029
1030
1031
1032
1033
1034
....
3445
3446
3447
3448
3449
3450
3451

3452
3453
3454
3455
3456
3457
3458
....
3923
3924
3925
3926
3927
3928
3929



3930
3931
3932
3933
3934
3935
3936
....
6837
6838
6839
6840
6841
6842
6843

6844
6845
6846
6847
6848
6849
6850
....
6855
6856
6857
6858
6859
6860
6861


6862
6863
6864
6865
6866
6867
6868
....
8545
8546
8547
8548
8549
8550
8551



8552
8553
8554
8555
8556
8557
8558
8559
8560
8561
8562
8563



8564
8565
8566
8567
8568
8569
8570
....
8867
8868
8869
8870
8871
8872
8873


8874
8875
8876
8877
8878
8879
8880
....
8968
8969
8970
8971
8972
8973
8974


8975
8976
8977
8978
8979
8980
8981
  int modePrior;         /* Saved mode */
  int cMode;             /* temporary output mode for the current query */
  int normalMode;        /* Output mode before ".explain on" */
  int writableSchema;    /* True if PRAGMA writable_schema=ON */
  int showHeader;        /* True to show column names in List or Column mode */
  int nCheck;            /* Number of ".check" commands run */
  unsigned shellFlgs;    /* Various flags */

  char *zDestTable;      /* Name of destination table when MODE_Insert */
  char *zTempFile;       /* Temporary file that might need deleting */
  char zTestcase[30];    /* Name of current test case */
  char colSeparator[20]; /* Column separator character for several modes */
  char rowSeparator[20]; /* Row separator character for MODE_Ascii */
  char colSepPrior[20];  /* Saved column separator */
  char rowSepPrior[20];  /* Saved row separator */
................................................................................
  "       -x    Open in a spreadsheet",
  ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  "     Options:",
  "        --append        Use appendvfs to append database to the end of FILE",
#ifdef SQLITE_ENABLE_DESERIALIZE
  "        --deserialize   Load into memory useing sqlite3_deserialize()",
  "        --hexdb         Load the output of \"dbtotxt\" as an in-memory database",

#endif
  "        --new           Initialize FILE to an empty database",
  "        --readonly      Open FILE readonly",
  "        --zip           FILE is a ZIP archive",
  ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  "     If FILE begins with '|' then open it as a pipe.",
  ".print STRING...         Print literal STRING",
................................................................................
      }
      rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
                   SQLITE_DESERIALIZE_RESIZEABLE |
                   SQLITE_DESERIALIZE_FREEONCLOSE);
      if( rc ){
        utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
      }



    }
#endif
  }
}

/*
** Attempt to close the databaes connection.  Report errors.
................................................................................
    session_close_all(p);
    close_db(p->db);
    p->db = 0;
    p->zDbFilename = 0;
    sqlite3_free(p->zFreeOnClose);
    p->zFreeOnClose = 0;
    p->openMode = SHELL_OPEN_UNSPEC;

    /* Check for command-line arguments */
    for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){
      const char *z = azArg[iName];
      if( optionMatch(z,"new") ){
        newFlag = 1;
#ifdef SQLITE_HAVE_ZLIB
      }else if( optionMatch(z, "zip") ){
................................................................................
      }else if( optionMatch(z, "readonly") ){
        p->openMode = SHELL_OPEN_READONLY;
#ifdef SQLITE_ENABLE_DESERIALIZE
      }else if( optionMatch(z, "deserialize") ){
        p->openMode = SHELL_OPEN_DESERIALIZE;
      }else if( optionMatch(z, "hexdb") ){
        p->openMode = SHELL_OPEN_HEXDB;


#endif /* SQLITE_ENABLE_DESERIALIZE */
      }else if( z[0]=='-' ){
        utf8_printf(stderr, "unknown option: %s\n", z);
        rc = 1;
        goto meta_command_exit;
      }
    }
................................................................................
  "   -append              append the database to the end of the file\n"
  "   -ascii               set output mode to 'ascii'\n"
  "   -bail                stop after hitting an error\n"
  "   -batch               force batch I/O\n"
  "   -column              set output mode to 'column'\n"
  "   -cmd COMMAND         run \"COMMAND\" before reading stdin\n"
  "   -csv                 set output mode to 'csv'\n"



  "   -echo                print commands before execution\n"
  "   -init FILENAME       read/process named file\n"
  "   -[no]header          turn headers on or off\n"
#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
  "   -heap SIZE           Size of heap for memsys3 or memsys5\n"
#endif
  "   -help                show this message\n"
  "   -html                set output mode to HTML\n"
  "   -interactive         force interactive I/O\n"
  "   -line                set output mode to 'line'\n"
  "   -list                set output mode to 'list'\n"
  "   -lookaside SIZE N    use N entries of SZ bytes for lookaside memory\n"



  "   -mmap N              default mmap size set to N\n"
#ifdef SQLITE_ENABLE_MULTIPLEX
  "   -multiplex           enable the multiplexor VFS\n"
#endif
  "   -newline SEP         set output row separator. Default: '\\n'\n"
  "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
  "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
................................................................................
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;
#ifdef SQLITE_ENABLE_DESERIALIZE
    }else if( strcmp(z,"-deserialize")==0 ){
      data.openMode = SHELL_OPEN_DESERIALIZE;


#endif
    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
    }else if( strncmp(z, "-A",2)==0 ){
      /* All remaining command-line arguments are passed to the ".archive"
      ** command, so ignore them */
................................................................................
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;
#ifdef SQLITE_ENABLE_DESERIALIZE
    }else if( strcmp(z,"-deserialize")==0 ){
      data.openMode = SHELL_OPEN_DESERIALIZE;


#endif
    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
    }else if( strcmp(z,"-ascii")==0 ){
      data.mode = MODE_Ascii;
      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
                       SEP_Unit);







>







 







>







 







>
>
>







 







>







 







>
>







 







>
>
>












>
>
>







 







>
>







 







>
>







1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
....
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
....
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
....
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
....
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
....
8553
8554
8555
8556
8557
8558
8559
8560
8561
8562
8563
8564
8565
8566
8567
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580
8581
8582
8583
8584
....
8881
8882
8883
8884
8885
8886
8887
8888
8889
8890
8891
8892
8893
8894
8895
8896
....
8984
8985
8986
8987
8988
8989
8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
  int modePrior;         /* Saved mode */
  int cMode;             /* temporary output mode for the current query */
  int normalMode;        /* Output mode before ".explain on" */
  int writableSchema;    /* True if PRAGMA writable_schema=ON */
  int showHeader;        /* True to show column names in List or Column mode */
  int nCheck;            /* Number of ".check" commands run */
  unsigned shellFlgs;    /* Various flags */
  sqlite3_int64 szMax;   /* --maxsize argument to .open */
  char *zDestTable;      /* Name of destination table when MODE_Insert */
  char *zTempFile;       /* Temporary file that might need deleting */
  char zTestcase[30];    /* Name of current test case */
  char colSeparator[20]; /* Column separator character for several modes */
  char rowSeparator[20]; /* Row separator character for MODE_Ascii */
  char colSepPrior[20];  /* Saved column separator */
  char rowSepPrior[20];  /* Saved row separator */
................................................................................
  "       -x    Open in a spreadsheet",
  ".open ?OPTIONS? ?FILE?   Close existing database and reopen FILE",
  "     Options:",
  "        --append        Use appendvfs to append database to the end of FILE",
#ifdef SQLITE_ENABLE_DESERIALIZE
  "        --deserialize   Load into memory useing sqlite3_deserialize()",
  "        --hexdb         Load the output of \"dbtotxt\" as an in-memory database",
  "        --maxsize N     Maximum size for --hexdb or --deserialized database",
#endif
  "        --new           Initialize FILE to an empty database",
  "        --readonly      Open FILE readonly",
  "        --zip           FILE is a ZIP archive",
  ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
  "     If FILE begins with '|' then open it as a pipe.",
  ".print STRING...         Print literal STRING",
................................................................................
      }
      rc = sqlite3_deserialize(p->db, "main", aData, nData, nData,
                   SQLITE_DESERIALIZE_RESIZEABLE |
                   SQLITE_DESERIALIZE_FREEONCLOSE);
      if( rc ){
        utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
      }
      if( p->szMax>0 ){
        sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax);
      }
    }
#endif
  }
}

/*
** Attempt to close the databaes connection.  Report errors.
................................................................................
    session_close_all(p);
    close_db(p->db);
    p->db = 0;
    p->zDbFilename = 0;
    sqlite3_free(p->zFreeOnClose);
    p->zFreeOnClose = 0;
    p->openMode = SHELL_OPEN_UNSPEC;
    p->szMax = 0;
    /* Check for command-line arguments */
    for(iName=1; iName<nArg && azArg[iName][0]=='-'; iName++){
      const char *z = azArg[iName];
      if( optionMatch(z,"new") ){
        newFlag = 1;
#ifdef SQLITE_HAVE_ZLIB
      }else if( optionMatch(z, "zip") ){
................................................................................
      }else if( optionMatch(z, "readonly") ){
        p->openMode = SHELL_OPEN_READONLY;
#ifdef SQLITE_ENABLE_DESERIALIZE
      }else if( optionMatch(z, "deserialize") ){
        p->openMode = SHELL_OPEN_DESERIALIZE;
      }else if( optionMatch(z, "hexdb") ){
        p->openMode = SHELL_OPEN_HEXDB;
      }else if( optionMatch(z, "maxsize") && iName+1<nArg ){
        p->szMax = integerValue(azArg[++iName]);
#endif /* SQLITE_ENABLE_DESERIALIZE */
      }else if( z[0]=='-' ){
        utf8_printf(stderr, "unknown option: %s\n", z);
        rc = 1;
        goto meta_command_exit;
      }
    }
................................................................................
  "   -append              append the database to the end of the file\n"
  "   -ascii               set output mode to 'ascii'\n"
  "   -bail                stop after hitting an error\n"
  "   -batch               force batch I/O\n"
  "   -column              set output mode to 'column'\n"
  "   -cmd COMMAND         run \"COMMAND\" before reading stdin\n"
  "   -csv                 set output mode to 'csv'\n"
#if defined(SQLITE_ENABLE_DESERIALIZE)
  "   -deserialize         open the database using sqlite3_deserialize()\n"
#endif
  "   -echo                print commands before execution\n"
  "   -init FILENAME       read/process named file\n"
  "   -[no]header          turn headers on or off\n"
#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
  "   -heap SIZE           Size of heap for memsys3 or memsys5\n"
#endif
  "   -help                show this message\n"
  "   -html                set output mode to HTML\n"
  "   -interactive         force interactive I/O\n"
  "   -line                set output mode to 'line'\n"
  "   -list                set output mode to 'list'\n"
  "   -lookaside SIZE N    use N entries of SZ bytes for lookaside memory\n"
#if defined(SQLITE_ENABLE_DESERIALIZE)
  "   -maxsize N           maximum size for a --deserialize database\n"
#endif
  "   -mmap N              default mmap size set to N\n"
#ifdef SQLITE_ENABLE_MULTIPLEX
  "   -multiplex           enable the multiplexor VFS\n"
#endif
  "   -newline SEP         set output row separator. Default: '\\n'\n"
  "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
  "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
................................................................................
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;
#ifdef SQLITE_ENABLE_DESERIALIZE
    }else if( strcmp(z,"-deserialize")==0 ){
      data.openMode = SHELL_OPEN_DESERIALIZE;
    }else if( strcmp(z,"-maxsize")==0 && i+1<argc ){
      data.szMax = integerValue(argv[++i]);
#endif
    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
    }else if( strncmp(z, "-A",2)==0 ){
      /* All remaining command-line arguments are passed to the ".archive"
      ** command, so ignore them */
................................................................................
      data.openMode = SHELL_OPEN_ZIPFILE;
#endif
    }else if( strcmp(z,"-append")==0 ){
      data.openMode = SHELL_OPEN_APPENDVFS;
#ifdef SQLITE_ENABLE_DESERIALIZE
    }else if( strcmp(z,"-deserialize")==0 ){
      data.openMode = SHELL_OPEN_DESERIALIZE;
    }else if( strcmp(z,"-maxsize")==0 && i+1<argc ){
      data.szMax = integerValue(argv[++i]);
#endif
    }else if( strcmp(z,"-readonly")==0 ){
      data.openMode = SHELL_OPEN_READONLY;
    }else if( strcmp(z,"-ascii")==0 ){
      data.mode = MODE_Ascii;
      sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,
                       SEP_Unit);

Changes to src/sqlite.h.in.

819
820
821
822
823
824
825









826
827
828
829
830
831
832
....
1127
1128
1129
1130
1131
1132
1133

1134
1135
1136
1137
1138
1139
1140
** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS
** layer a hint of how large the database file will grow to be during the
** current transaction.  This hint is not guaranteed to be accurate but it
** is often close.  The underlying VFS might choose to preallocate database
** file space based on this hint in order to help writes to the database
** file run faster.
**









** <li>[[SQLITE_FCNTL_CHUNK_SIZE]]
** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS
** extends and truncates the database file in chunks of a size specified
** by the user. The fourth argument to [sqlite3_file_control()] should 
** point to an integer (type int) containing the new chunk-size to use
** for the nominated database. Allocating database file space in large
** chunks (say 1MB at a time), may reduce file-system fragmentation and
................................................................................
#define SQLITE_FCNTL_WIN32_GET_HANDLE       29
#define SQLITE_FCNTL_PDB                    30
#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE     31
#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE    32
#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE  33
#define SQLITE_FCNTL_LOCK_TIMEOUT           34
#define SQLITE_FCNTL_DATA_VERSION           35


/* deprecated names */
#define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
#define SQLITE_SET_LOCKPROXYFILE      SQLITE_FCNTL_SET_LOCKPROXYFILE
#define SQLITE_LAST_ERRNO             SQLITE_FCNTL_LAST_ERRNO









>
>
>
>
>
>
>
>
>







 







>







819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
....
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS
** layer a hint of how large the database file will grow to be during the
** current transaction.  This hint is not guaranteed to be accurate but it
** is often close.  The underlying VFS might choose to preallocate database
** file space based on this hint in order to help writes to the database
** file run faster.
**
** <li>[[SQLITE_FCNTL_SIZE_LIMIT]]
** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that
** implements [sqlite3_deserialize()] to set an upper bound on the size
** of the in-memory database.  The argument is a pointer to a [sqlite3_int64].
** If the integer pointed to is negative, then it is filled in with the
** current limit.  Otherwise the limit is set to the larger of the value
** of the integer pointed to and the current database size.  The integer
** pointed to is set to the new limit.
**
** <li>[[SQLITE_FCNTL_CHUNK_SIZE]]
** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS
** extends and truncates the database file in chunks of a size specified
** by the user. The fourth argument to [sqlite3_file_control()] should 
** point to an integer (type int) containing the new chunk-size to use
** for the nominated database. Allocating database file space in large
** chunks (say 1MB at a time), may reduce file-system fragmentation and
................................................................................
#define SQLITE_FCNTL_WIN32_GET_HANDLE       29
#define SQLITE_FCNTL_PDB                    30
#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE     31
#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE    32
#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE  33
#define SQLITE_FCNTL_LOCK_TIMEOUT           34
#define SQLITE_FCNTL_DATA_VERSION           35
#define SQLITE_FCNTL_SIZE_LIMIT             36

/* deprecated names */
#define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
#define SQLITE_SET_LOCKPROXYFILE      SQLITE_FCNTL_SET_LOCKPROXYFILE
#define SQLITE_LAST_ERRNO             SQLITE_FCNTL_LAST_ERRNO


Changes to src/tclsqlite.c.

2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435



2436

2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447





















2448
2449
2450
2451
2452
2453

2454
2455

2456

2457


2458
2459
2460
2461


2462


2463
2464
2465
2466
2467
2468
2469
                       (char*)0);
      rc = TCL_ERROR;
    }
    break;
  }

  /*
  **     $db deserialize ?DATABASE? VALUE
  **
  ** Reopen DATABASE (default "main") using the content in $VALUE
  */
  case DB_DESERIALIZE: {
#ifndef SQLITE_ENABLE_DESERIALIZE
    Tcl_AppendResult(interp, "MEMDB not available in this build",
                     (char*)0);
    rc = TCL_ERROR;
#else
    const char *zSchema;
    Tcl_Obj *pValue;
    unsigned char *pBA;
    unsigned char *pData;
    int len, xrc;



    

    if( objc==3 ){
      zSchema = 0;
      pValue = objv[2];
    }else if( objc==4 ){
      zSchema = Tcl_GetString(objv[2]);
      pValue = objv[3];
    }else{
      Tcl_WrongNumArgs(interp, 2, objv, "?DATABASE? VALUE");
      rc = TCL_ERROR;
      break;
    }





















    pBA = Tcl_GetByteArrayFromObj(pValue, &len);
    pData = sqlite3_malloc64( len );
    if( pData==0 && len>0 ){
      Tcl_AppendResult(interp, "out of memory", (char*)0);
      rc = TCL_ERROR;
    }else{

      if( len>0 ) memcpy(pData, pBA, len);
      xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len,

                SQLITE_DESERIALIZE_FREEONCLOSE |

                SQLITE_DESERIALIZE_RESIZEABLE);


      if( xrc ){
        Tcl_AppendResult(interp, "unable to set MEMDB content", (char*)0);
        rc = TCL_ERROR;
      }


    }


#endif
    break; 
  }

  /*
  **    $db enable_load_extension BOOLEAN
  **







|









|
|



>
>
>
|
>
|
<
<
<
<
<
<




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>






>

<
>
|
>
|
>
>




>
>
|
>
>







2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441






2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474

2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
                       (char*)0);
      rc = TCL_ERROR;
    }
    break;
  }

  /*
  **     $db deserialize ?-maxsize N? ?-readonly BOOL? ?DATABASE? VALUE
  **
  ** Reopen DATABASE (default "main") using the content in $VALUE
  */
  case DB_DESERIALIZE: {
#ifndef SQLITE_ENABLE_DESERIALIZE
    Tcl_AppendResult(interp, "MEMDB not available in this build",
                     (char*)0);
    rc = TCL_ERROR;
#else
    const char *zSchema = 0;
    Tcl_Obj *pValue = 0;
    unsigned char *pBA;
    unsigned char *pData;
    int len, xrc;
    sqlite3_int64 mxSize = 0;
    int i;
    int isReadonly = 0;


    if( objc<3 ){






      Tcl_WrongNumArgs(interp, 2, objv, "?DATABASE? VALUE");
      rc = TCL_ERROR;
      break;
    }
    for(i=2; i<objc-1; i++){
      const char *z = Tcl_GetString(objv[i]);
      if( strcmp(z,"-maxsize")==0 && i<objc-2 ){
        rc = Tcl_GetWideIntFromObj(interp, objv[++i], &mxSize);
        if( rc ) goto deserialize_error;
        continue;
      }
      if( strcmp(z,"-readonly")==0 && i<objc-2 ){
        rc = Tcl_GetBooleanFromObj(interp, objv[++i], &isReadonly);
        if( rc ) goto deserialize_error;
        continue;
      }
      if( zSchema==0 && i==objc-2 && z[0]!='-' ){
        zSchema = z;
        continue;
      }
      Tcl_AppendResult(interp, "unknown option: ", z, (char*)0);
      rc = TCL_ERROR;
      goto deserialize_error;
    }
    pValue = objv[objc-1];
    pBA = Tcl_GetByteArrayFromObj(pValue, &len);
    pData = sqlite3_malloc64( len );
    if( pData==0 && len>0 ){
      Tcl_AppendResult(interp, "out of memory", (char*)0);
      rc = TCL_ERROR;
    }else{
      int flags;
      if( len>0 ) memcpy(pData, pBA, len);

      if( isReadonly ){
        flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY;
      }else{
        flags = SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE;
      }
      xrc = sqlite3_deserialize(pDb->db, zSchema, pData, len, len, flags);
      if( xrc ){
        Tcl_AppendResult(interp, "unable to set MEMDB content", (char*)0);
        rc = TCL_ERROR;
      }
      if( mxSize>0 ){
        sqlite3_file_control(pDb->db, zSchema,SQLITE_FCNTL_SIZE_LIMIT,&mxSize);
      }
    }
deserialize_error:
#endif
    break; 
  }

  /*
  **    $db enable_load_extension BOOLEAN
  **

Changes to test/memdb1.test.

67
68
69
70
71
72
73

























74
75
76
77
78
79
80
...
150
151
152
153
154
155
156
157
158
159
160
161
162
163
  DROP TABLE t2;
  PRAGMA page_count;
} {116}
do_execsql_test 140 {
  VACUUM;
  PRAGMA page_count;
} {2}


























# Build a largish on-disk database and serialize it.  Verify that the
# serialization works.
#
db close
forcedelete test.db
sqlite3 db test.db
................................................................................
do_test 600 {
  set rc [catch {db deserialize} msg]
  lappend rc $msg
} {1 {wrong # args: should be "db deserialize ?DATABASE? VALUE"}}
do_test 610 {
  set rc [catch {db deserialize a b c} msg]
  lappend rc $msg
} {1 {wrong # args: should be "db deserialize ?DATABASE? VALUE"}}
do_test 620 {
  set rc [catch {db serialize a b} msg]
  lappend rc $msg
} {1 {wrong # args: should be "db serialize ?DATABASE?"}}

finish_test







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|






67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
...
175
176
177
178
179
180
181
182
183
184
185
186
187
188
  DROP TABLE t2;
  PRAGMA page_count;
} {116}
do_execsql_test 140 {
  VACUUM;
  PRAGMA page_count;
} {2}

do_test 150 {
  catch {db deserialize -unknown 1 $db1} msg
  set msg
} {unknown option: -unknown}
do_test 151 {
  db deserialize -readonly 1 $db1
  db eval {SELECT * FROM t1}
} {1 2}
do_test 152 {
  catchsql {INSERT INTO t1 VALUES(3,4);}
} {1 {attempt to write a readonly database}}

breakpoint
do_test 160 {
  db deserialize -maxsize 32768 $db1
  db eval {SELECT * FROM t1}
} {1 2}
do_test 161 {
  db eval {INSERT INTO t1 VALUES(3,4); SELECT * FROM t1}
} {1 2 3 4}
do_test 162 {
  catchsql {INSERT INTO t1 VALUES(5,randomblob(100000))}
} {1 {database or disk is full}}


# Build a largish on-disk database and serialize it.  Verify that the
# serialization works.
#
db close
forcedelete test.db
sqlite3 db test.db
................................................................................
do_test 600 {
  set rc [catch {db deserialize} msg]
  lappend rc $msg
} {1 {wrong # args: should be "db deserialize ?DATABASE? VALUE"}}
do_test 610 {
  set rc [catch {db deserialize a b c} msg]
  lappend rc $msg
} {1 {unknown option: a}}
do_test 620 {
  set rc [catch {db serialize a b} msg]
  lappend rc $msg
} {1 {wrong # args: should be "db serialize ?DATABASE?"}}

finish_test