/ Check-in [f07e97ae]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improved documentation for sqlite3_serialize() and sqlite3_deserialize(). Change the name of the compile-time option to enable these interfaces from SQLITE_ENABLE_MEMDB to SQLITE_ENABLE_DESERIALIZE.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | memdb
Files: files | file ages | folders
SHA3-256:f07e97aed435b02e1473053c0257ec5c89bf0b3e46076b7a9382de432bbc2497
User & Date: drh 2018-03-06 21:43:19
Context
2018-03-07
01:37
Mark an unreachable branch using NEVER(). Closed-Leaf check-in: fadbc5e2 user: drh tags: memdb
2018-03-06
21:43
Improved documentation for sqlite3_serialize() and sqlite3_deserialize(). Change the name of the compile-time option to enable these interfaces from SQLITE_ENABLE_MEMDB to SQLITE_ENABLE_DESERIALIZE. check-in: f07e97ae user: drh tags: memdb
20:54
Handle some boundary cases in memdb associated with OOM faults. check-in: b58ca4cb user: drh tags: memdb
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/attach.c.

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

  UNUSED_PARAMETER(NotUsed);
  zFile = (const char *)sqlite3_value_text(argv[0]);
  zName = (const char *)sqlite3_value_text(argv[1]);
  if( zFile==0 ) zFile = "";
  if( zName==0 ) zName = "";

#ifdef SQLITE_ENABLE_MEMDB
# define REOPEN_AS_MEMDB(db)  (db->init.reopenMemdb)
#else
# define REOPEN_AS_MEMDB(db)  (0)
#endif

  if( REOPEN_AS_MEMDB(db) ){
    /* This is not a real ATTACH.  Instead, this routine is being called







|







80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

  UNUSED_PARAMETER(NotUsed);
  zFile = (const char *)sqlite3_value_text(argv[0]);
  zName = (const char *)sqlite3_value_text(argv[1]);
  if( zFile==0 ) zFile = "";
  if( zName==0 ) zName = "";

#ifdef SQLITE_ENABLE_DESERIALIZE
# define REOPEN_AS_MEMDB(db)  (db->init.reopenMemdb)
#else
# define REOPEN_AS_MEMDB(db)  (0)
#endif

  if( REOPEN_AS_MEMDB(db) ){
    /* This is not a real ATTACH.  Instead, this routine is being called

Changes to src/main.c.

235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    if( sqlite3GlobalConfig.isPCacheInit==0 ){
      rc = sqlite3PcacheInitialize();
    }
    if( rc==SQLITE_OK ){
      sqlite3GlobalConfig.isPCacheInit = 1;
      rc = sqlite3OsInit();
    }
#ifdef SQLITE_ENABLE_MEMDB
    if( rc==SQLITE_OK ){
      rc = sqlite3MemdbInit();
    }
#endif
    if( rc==SQLITE_OK ){
      sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, 
          sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage);







|







235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
    if( sqlite3GlobalConfig.isPCacheInit==0 ){
      rc = sqlite3PcacheInitialize();
    }
    if( rc==SQLITE_OK ){
      sqlite3GlobalConfig.isPCacheInit = 1;
      rc = sqlite3OsInit();
    }
#ifdef SQLITE_ENABLE_DESERIALIZE
    if( rc==SQLITE_OK ){
      rc = sqlite3MemdbInit();
    }
#endif
    if( rc==SQLITE_OK ){
      sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, 
          sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage);

Changes to src/memdb.c.

12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
...
564
565
566
567
568
569
570
571
**
** This file implements in-memory VFS.  A database is held as a contiguous
** block of memory.
**
** This file also implements interface sqlite3_serialize() and
** sqlite3_deserialize().
*/
#ifdef SQLITE_ENABLE_MEMDB
#include "sqliteInt.h"

/*
** Forward declaration of objects used by this utility
*/
typedef struct sqlite3_vfs MemVfs;
typedef struct MemFile MemFile;
................................................................................
  sqlite3_vfs *pLower = sqlite3_vfs_find(0);
  int sz = pLower->szOsFile;
  memdb_vfs.pAppData = pLower;
  if( sz<sizeof(MemFile) ) sz = sizeof(MemFile);
  memdb_vfs.szOsFile = sz;
  return sqlite3_vfs_register(&memdb_vfs, 0);
}
#endif /* SQLITE_ENABLE_MEMDB */







|







 







|
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
...
564
565
566
567
568
569
570
571
**
** This file implements in-memory VFS.  A database is held as a contiguous
** block of memory.
**
** This file also implements interface sqlite3_serialize() and
** sqlite3_deserialize().
*/
#ifdef SQLITE_ENABLE_DESERIALIZE
#include "sqliteInt.h"

/*
** Forward declaration of objects used by this utility
*/
typedef struct sqlite3_vfs MemVfs;
typedef struct MemFile MemFile;
................................................................................
  sqlite3_vfs *pLower = sqlite3_vfs_find(0);
  int sz = pLower->szOsFile;
  memdb_vfs.pAppData = pLower;
  if( sz<sizeof(MemFile) ) sz = sizeof(MemFile);
  memdb_vfs.szOsFile = sz;
  return sqlite3_vfs_register(&memdb_vfs, 0);
}
#endif /* SQLITE_ENABLE_DESERIALIZE */

Changes to src/pager.c.

4719
4720
4721
4722
4723
4724
4725

4726



4727
4728
4729
4730
4731
4732
4733
....
4847
4848
4849
4850
4851
4852
4853

4854

4855
4856
4857
4858
4859
4860
4861
  void (*xReinit)(DbPage*) /* Function to reinitialize pages */
){
  u8 *pPtr;
  Pager *pPager = 0;       /* Pager object to allocate and return */
  int rc = SQLITE_OK;      /* Return code */
  int tempFile = 0;        /* True for temp files (incl. in-memory files) */
  int memDb = 0;           /* True if this is an in-memory file */

  int memJM = 0;           /* Memory journal mode */



  int readOnly = 0;        /* True if this is a read-only file */
  int journalFileSize;     /* Bytes to allocate for each journal fd */
  char *zPathname = 0;     /* Full path to database file */
  int nPathname = 0;       /* Number of bytes in zPathname */
  int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */
  int pcacheSize = sqlite3PcacheSize();       /* Bytes to allocate for PCache */
  u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE;  /* Default page size */
................................................................................

  /* Open the pager file.
  */
  if( zFilename && zFilename[0] ){
    int fout = 0;                    /* VFS flags returned by xOpen() */
    rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout);
    assert( !memDb );

    memJM = (fout&SQLITE_OPEN_MEMORY)!=0;

    readOnly = (fout&SQLITE_OPEN_READONLY)!=0;

    /* If the file was successfully opened for read/write access,
    ** choose a default page size in case we have to create the
    ** database file. The default page size is the maximum of:
    **
    **    + SQLITE_DEFAULT_PAGE_SIZE,







>

>
>
>







 







>

>







4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
....
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
  void (*xReinit)(DbPage*) /* Function to reinitialize pages */
){
  u8 *pPtr;
  Pager *pPager = 0;       /* Pager object to allocate and return */
  int rc = SQLITE_OK;      /* Return code */
  int tempFile = 0;        /* True for temp files (incl. in-memory files) */
  int memDb = 0;           /* True if this is an in-memory file */
#ifdef SQLITE_ENABLE_DESERIALIZE
  int memJM = 0;           /* Memory journal mode */
#else
# define memJM 0
#endif
  int readOnly = 0;        /* True if this is a read-only file */
  int journalFileSize;     /* Bytes to allocate for each journal fd */
  char *zPathname = 0;     /* Full path to database file */
  int nPathname = 0;       /* Number of bytes in zPathname */
  int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */
  int pcacheSize = sqlite3PcacheSize();       /* Bytes to allocate for PCache */
  u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE;  /* Default page size */
................................................................................

  /* Open the pager file.
  */
  if( zFilename && zFilename[0] ){
    int fout = 0;                    /* VFS flags returned by xOpen() */
    rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout);
    assert( !memDb );
#ifdef SQLITE_ENABLE_DESERIALIZE
    memJM = (fout&SQLITE_OPEN_MEMORY)!=0;
#endif
    readOnly = (fout&SQLITE_OPEN_READONLY)!=0;

    /* If the file was successfully opened for read/write access,
    ** choose a default page size in case we have to create the
    ** database file. The default page size is the maximum of:
    **
    **    + SQLITE_DEFAULT_PAGE_SIZE,

Changes to src/sqlite.h.in.

8758
8759
8760
8761
8762
8763
8764



8765
8766
8767
8768
8769
8770
8771
8772
8773
8774
8775











8776
8777
8778
8779
8780
8781
8782
....
8798
8799
8800
8801
8802
8803
8804



8805
8806
8807
8808
8809
8810
8811
....
8812
8813
8814
8815
8816
8817
8818
8819
8820















8821
8822
8823
8824
8825
8826
8827
** The size of the database is written into *P even if the 
** SQLITE_SERIALIZE_NOCOPY bit is set but no contigious copy
** of the database exists.
**
** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the
** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory
** allocation error occurs.



*/
unsigned char *sqlite3_serialize(
  sqlite3 *db,           /* The database connection */
  const char *zSchema,   /* Which DB to serialize. ex: "main", "temp", ... */
  sqlite3_int64 *piSize, /* Write size of the DB here, if not NULL */
  unsigned int mFlags    /* Zero or more SQLITE_SERIALIZE_* flags */
);

/*
** CAPI3REF: Flags for sqlite3_serialize
** EXPERIMENTAL











*/
#define SQLITE_SERIALIZE_NOCOPY 0x001   /* Do no memory allocations */

/*
** CAPI3REF: Deserialize a database
** EXPERIMENTAL
**
................................................................................
** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the
** database is currently in a read transaction or is involved in a backup
** operation.
**
** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the 
** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then
** [sqlite3_free()] is invoked on argument P prior to returning.



*/
int sqlite3_deserialize(
  sqlite3 *db,            /* The database connection */
  const char *zSchema,    /* Which DB to reopen with the deserialization */
  unsigned char *pData,   /* The serialized database content */
  sqlite3_int64 szDb,     /* Number bytes in the deserialization */
  sqlite3_int64 szBuf,    /* Total size of buffer pData[] */
................................................................................
  unsigned mFlags         /* Zero or more SQLITE_DESERIALIZE_* flags */
);

/*
** CAPI3REF: Flags for sqlite3_deserialize()
** EXPERIMENTAL
**
** The following are allowed values for the 6th argument (the "flags" or "F"
** argument) of the [sqlite3_deserialize()] interface.















*/
#define SQLITE_DESERIALIZE_FREEONCLOSE 1 /* Call sqlite3_free() on close */
#define SQLITE_DESERIALIZE_RESIZEABLE  2 /* Resize using sqlite3_realloc64() */
#define SQLITE_DESERIALIZE_READONLY    4 /* Database is read-only */

/*
** Undo the hack that converts floating point types to integer for







>
>
>











>
>
>
>
>
>
>
>
>
>
>







 







>
>
>







 







|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







8758
8759
8760
8761
8762
8763
8764
8765
8766
8767
8768
8769
8770
8771
8772
8773
8774
8775
8776
8777
8778
8779
8780
8781
8782
8783
8784
8785
8786
8787
8788
8789
8790
8791
8792
8793
8794
8795
8796
....
8812
8813
8814
8815
8816
8817
8818
8819
8820
8821
8822
8823
8824
8825
8826
8827
8828
....
8829
8830
8831
8832
8833
8834
8835
8836
8837
8838
8839
8840
8841
8842
8843
8844
8845
8846
8847
8848
8849
8850
8851
8852
8853
8854
8855
8856
8857
8858
8859
** The size of the database is written into *P even if the 
** SQLITE_SERIALIZE_NOCOPY bit is set but no contigious copy
** of the database exists.
**
** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the
** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory
** allocation error occurs.
**
** This interface is only available if SQLite is compiled with the
** [SQLITE_ENABLE_DESERIALIZE] option.
*/
unsigned char *sqlite3_serialize(
  sqlite3 *db,           /* The database connection */
  const char *zSchema,   /* Which DB to serialize. ex: "main", "temp", ... */
  sqlite3_int64 *piSize, /* Write size of the DB here, if not NULL */
  unsigned int mFlags    /* Zero or more SQLITE_SERIALIZE_* flags */
);

/*
** CAPI3REF: Flags for sqlite3_serialize
** EXPERIMENTAL
**
** Zero or more of the following constants can be OR-ed together for
** the F argument to [sqlite3_serialize(D,S,P,F)].
**
** SQLITE_SERIALIZE_NOCOPY means that [sqlite3_serialize()] will return
** a pointer to contiguous in-memory database that it is currently using,
** without making a copy of the database.  If SQLite is not currently using
** a contiguous in-memory database, then this option causes
** [sqlite3_serialize()] to return a NULL pointer.  SQLite will only be
** using a contiguous in-memory database if it has been initialized by a
** prior call to [sqlite3_deserialize()].
*/
#define SQLITE_SERIALIZE_NOCOPY 0x001   /* Do no memory allocations */

/*
** CAPI3REF: Deserialize a database
** EXPERIMENTAL
**
................................................................................
** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the
** database is currently in a read transaction or is involved in a backup
** operation.
**
** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the 
** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then
** [sqlite3_free()] is invoked on argument P prior to returning.
**
** This interface is only available if SQLite is compiled with the
** [SQLITE_ENABLE_DESERIALIZE] option.
*/
int sqlite3_deserialize(
  sqlite3 *db,            /* The database connection */
  const char *zSchema,    /* Which DB to reopen with the deserialization */
  unsigned char *pData,   /* The serialized database content */
  sqlite3_int64 szDb,     /* Number bytes in the deserialization */
  sqlite3_int64 szBuf,    /* Total size of buffer pData[] */
................................................................................
  unsigned mFlags         /* Zero or more SQLITE_DESERIALIZE_* flags */
);

/*
** CAPI3REF: Flags for sqlite3_deserialize()
** EXPERIMENTAL
**
** The following are allowed values for 6th argument (the F argument) to
** the [sqlite3_deserialize(D,S,P,N,M,F)] interface.
**
** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization
** in the P argument is held in memory obtained from [sqlite3_malloc64()]
** and that SQLite should take ownership of this memory and automatically
** free it when it has finished using it.  Without this flag, the caller
** is resposible for freeing any dynamically allocated memory.
**
** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to
** grow the size of the database usign calls to [sqlite3_realloc64()].  This
** flag should only be used if SQLITE_DESERIALIZE_FREEONCLOSE is also used.
** Without this flag, the deserialized database cannot increase in size beyond
** the number of bytes specified by the M parameter.
**
** The SQLITE_DESERIALIZE_READONLY flag means that the deserialized database
** should be treated as read-only.
*/
#define SQLITE_DESERIALIZE_FREEONCLOSE 1 /* Call sqlite3_free() on close */
#define SQLITE_DESERIALIZE_RESIZEABLE  2 /* Resize using sqlite3_realloc64() */
#define SQLITE_DESERIALIZE_READONLY    4 /* Database is read-only */

/*
** Undo the hack that converts floating point types to integer for

Changes to src/sqliteInt.h.

4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
u8 sqlite3HexToInt(int h);
int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);

#if defined(SQLITE_NEED_ERR_NAME)
const char *sqlite3ErrName(int);
#endif

#ifdef SQLITE_ENABLE_MEMDB
int sqlite3MemdbInit(void);
#endif

const char *sqlite3ErrStr(int);
int sqlite3ReadSchema(Parse *pParse);
CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int);
CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName);







|







4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
u8 sqlite3HexToInt(int h);
int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);

#if defined(SQLITE_NEED_ERR_NAME)
const char *sqlite3ErrName(int);
#endif

#ifdef SQLITE_ENABLE_DESERIALIZE
int sqlite3MemdbInit(void);
#endif

const char *sqlite3ErrStr(int);
int sqlite3ReadSchema(Parse *pParse);
CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int);
CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName);

Changes to src/tclsqlite.c.

2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
....
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952

  /*
  **     $db deserialize ?DATABASE? VALUE
  **
  ** Reopen DATABASE (default "main") using the content in $VALUE
  */
  case DB_DESERIALIZE: {
#ifndef SQLITE_ENABLE_MEMDB
    Tcl_AppendResult(interp, "MEMDB not available in this build",
                     (char*)0);
    rc = TCL_ERROR;
#else
    const char *zSchema;
    Tcl_Obj *pValue;
    unsigned char *pBA;
................................................................................

  /*
  **     $db serialize ?DATABASE?
  **
  ** Return a serialization of a database.  
  */
  case DB_SERIALIZE: {
#ifndef SQLITE_ENABLE_MEMDB
    Tcl_AppendResult(interp, "MEMDB not available in this build",
                     (char*)0);
    rc = TCL_ERROR;
#else
    const char *zSchema = objc>=3 ? Tcl_GetString(objv[2]) : "main";
    sqlite3_int64 sz = 0;
    unsigned char *pData;







|







 







|







2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
....
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952

  /*
  **     $db deserialize ?DATABASE? VALUE
  **
  ** Reopen DATABASE (default "main") using the content in $VALUE
  */
  case DB_DESERIALIZE: {
#ifndef SQLITE_ENABLE_DESERIALIZE
    Tcl_AppendResult(interp, "MEMDB not available in this build",
                     (char*)0);
    rc = TCL_ERROR;
#else
    const char *zSchema;
    Tcl_Obj *pValue;
    unsigned char *pBA;
................................................................................

  /*
  **     $db serialize ?DATABASE?
  **
  ** Return a serialization of a database.  
  */
  case DB_SERIALIZE: {
#ifndef SQLITE_ENABLE_DESERIALIZE
    Tcl_AppendResult(interp, "MEMDB not available in this build",
                     (char*)0);
    rc = TCL_ERROR;
#else
    const char *zSchema = objc>=3 ? Tcl_GetString(objv[2]) : "main";
    sqlite3_int64 sz = 0;
    unsigned char *pData;

Changes to src/test_config.c.

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS
  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "0", TCL_GLOBAL_ONLY);
#endif

#ifdef SQLITE_ENABLE_MEMDB
  Tcl_SetVar2(interp, "sqlite_options", "memdb", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "memdb", "0", TCL_GLOBAL_ONLY);
#endif

#ifdef SQLITE_ENABLE_MEMSYS3
  Tcl_SetVar2(interp, "sqlite_options", "mem3", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "mem3", "0", TCL_GLOBAL_ONLY);
#endif







|
|

|







144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS
  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "0", TCL_GLOBAL_ONLY);
#endif

#ifdef SQLITE_ENABLE_DESERIALIZE
  Tcl_SetVar2(interp, "sqlite_options", "deserialize", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "deserialize", "0", TCL_GLOBAL_ONLY);
#endif

#ifdef SQLITE_ENABLE_MEMSYS3
  Tcl_SetVar2(interp, "sqlite_options", "mem3", "1", TCL_GLOBAL_ONLY);
#else
  Tcl_SetVar2(interp, "sqlite_options", "mem3", "0", TCL_GLOBAL_ONLY);
#endif

Changes to test/memdb1.test.

13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix memdb1
do_not_use_codec

ifcapable !memdb {
  finish_test
  return
}

# Create a MEMDB and populate it with some dummy data.
# Then extract the database into the $::db1 variable.
# Verify that the size of $::db1 is the same as the size of







|







13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix memdb1
do_not_use_codec

ifcapable !deserialize {
  finish_test
  return
}

# Create a MEMDB and populate it with some dummy data.
# Then extract the database into the $::db1 variable.
# Verify that the size of $::db1 is the same as the size of