/ Check-in [063a03a3]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Initial implementation of the appendvfs extension. Untested.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | appendvfs
Files: files | file ages | folders
SHA3-256: 063a03a3779e8c032dd006712facaaa6d60964425701ea10c753ff981a8f2bd9
User & Date: drh 2017-10-21 12:59:27
Context
2017-11-15
16:29
Merge all the latest changes from trunk. check-in: 1a1a73b8 user: drh tags: appendvfs
2017-10-21
12:59
Initial implementation of the appendvfs extension. Untested. check-in: 063a03a3 user: drh tags: appendvfs
2017-10-19
15:17
Take extra care to avoid an OOB read caused by a corrupt b-tree page. This fixes a problem detected by Natalie Silvanovich of Google Project Zero. check-in: 04925dee user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Added ext/misc/appendvfs.c.

            1  +/*
            2  +** 2017-10-20
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +**
           13  +** This file implements a VFS shim that allows an SQLite database to be
           14  +** appended onto the end of some other file, such as an executable.
           15  +**
           16  +** A special record must appear at the end of the file that identifies the
           17  +** file as an appended database and provides an offset to page 1.  For
           18  +** best performance page 1 should be located at a disk page boundary, though
           19  +** that is not required.
           20  +**
           21  +** An appended database is considered immutable.  It is read-only and no
           22  +** locks are ever taken.
           23  +**
           24  +** If the file being opened is not an appended database, then this shim is
           25  +** a pass-through into the default underlying VFS.
           26  +**/
           27  +#include <sqlite3ext.h>
           28  +SQLITE_EXTENSION_INIT1
           29  +#include <string.h>
           30  +#include <assert.h>
           31  +
           32  +/* The append mark at the end of the database is:
           33  +**
           34  +**     Start-Of-SQLite3-NNNNNNNN
           35  +**     123456789 123456789 12345
           36  +**
           37  +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
           38  +** the offset to page 1.
           39  +*/
           40  +#define APND_MARK_PREFIX     "Start-Of-SQLite3-"
           41  +#define APND_MARK_PREFIX_SZ  17
           42  +#define APND_MARK_SIZE       25
           43  +
           44  +/*
           45  +** Forward declaration of objects used by this utility
           46  +*/
           47  +typedef struct sqlite3_vfs ApndVfs;
           48  +typedef struct ApndFile ApndFile;
           49  +
           50  +/* Access to a lower-level VFS that (might) implement dynamic loading,
           51  +** access to randomness, etc.
           52  +*/
           53  +#define ORIGVFS(p)  ((sqlite3_vfs*)((p)->pAppData))
           54  +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
           55  +
           56  +/* An open file */
           57  +struct ApndFile {
           58  +  sqlite3_file base;              /* IO methods */
           59  +  sqlite3_int64 iPgOne;           /* File offset to page 1 */
           60  +};
           61  +
           62  +/*
           63  +** Methods for ApndFile
           64  +*/
           65  +static int apndClose(sqlite3_file*);
           66  +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
           67  +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
           68  +static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
           69  +static int apndSync(sqlite3_file*, int flags);
           70  +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
           71  +static int apndLock(sqlite3_file*, int);
           72  +static int apndUnlock(sqlite3_file*, int);
           73  +static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
           74  +static int apndFileControl(sqlite3_file*, int op, void *pArg);
           75  +static int apndSectorSize(sqlite3_file*);
           76  +static int apndDeviceCharacteristics(sqlite3_file*);
           77  +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
           78  +static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
           79  +static void apndShmBarrier(sqlite3_file*);
           80  +static int apndShmUnmap(sqlite3_file*, int deleteFlag);
           81  +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
           82  +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
           83  +
           84  +/*
           85  +** Methods for ApndVfs
           86  +*/
           87  +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
           88  +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
           89  +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
           90  +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
           91  +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
           92  +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
           93  +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
           94  +static void apndDlClose(sqlite3_vfs*, void*);
           95  +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
           96  +static int apndSleep(sqlite3_vfs*, int microseconds);
           97  +static int apndCurrentTime(sqlite3_vfs*, double*);
           98  +static int apndGetLastError(sqlite3_vfs*, int, char *);
           99  +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
          100  +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
          101  +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
          102  +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
          103  +
          104  +static sqlite3_vfs apnd_vfs = {
          105  +  3,                            /* iVersion (set when registered) */
          106  +  0,                            /* szOsFile (set when registered) */
          107  +  1024,                         /* mxPathname */
          108  +  0,                            /* pNext */
          109  +  "apndvfs",                    /* zName */
          110  +  0,                            /* pAppData (set when registered) */ 
          111  +  apndOpen,                     /* xOpen */
          112  +  apndDelete,                   /* xDelete */
          113  +  apndAccess,                   /* xAccess */
          114  +  apndFullPathname,             /* xFullPathname */
          115  +  apndDlOpen,                   /* xDlOpen */
          116  +  apndDlError,                  /* xDlError */
          117  +  apndDlSym,                    /* xDlSym */
          118  +  apndDlClose,                  /* xDlClose */
          119  +  apndRandomness,               /* xRandomness */
          120  +  apndSleep,                    /* xSleep */
          121  +  apndCurrentTime,              /* xCurrentTime */
          122  +  apndGetLastError,             /* xGetLastError */
          123  +  apndCurrentTimeInt64,         /* xCurrentTimeInt64 */
          124  +  apndSetSystemCall,            /* xSetSystemCall */
          125  +  apndGetSystemCall,            /* xGetSystemCall */
          126  +  apndNextSystemCall            /* xNextSystemCall */
          127  +};
          128  +
          129  +static const sqlite3_io_methods apnd_io_methods = {
          130  +  3,                              /* iVersion */
          131  +  apndClose,                      /* xClose */
          132  +  apndRead,                       /* xRead */
          133  +  apndWrite,                      /* xWrite */
          134  +  apndTruncate,                   /* xTruncate */
          135  +  apndSync,                       /* xSync */
          136  +  apndFileSize,                   /* xFileSize */
          137  +  apndLock,                       /* xLock */
          138  +  apndUnlock,                     /* xUnlock */
          139  +  apndCheckReservedLock,          /* xCheckReservedLock */
          140  +  apndFileControl,                /* xFileControl */
          141  +  apndSectorSize,                 /* xSectorSize */
          142  +  apndDeviceCharacteristics,      /* xDeviceCharacteristics */
          143  +  apndShmMap,                     /* xShmMap */
          144  +  apndShmLock,                    /* xShmLock */
          145  +  apndShmBarrier,                 /* xShmBarrier */
          146  +  apndShmUnmap,                   /* xShmUnmap */
          147  +  apndFetch,                      /* xFetch */
          148  +  apndUnfetch                     /* xUnfetch */
          149  +};
          150  +
          151  +
          152  +
          153  +/*
          154  +** Close an apnd-file.
          155  +*/
          156  +static int apndClose(sqlite3_file *pFile){
          157  +  pFile = ORIGFILE(pFile);
          158  +  return pFile->pMethods->xClose(pFile);
          159  +}
          160  +
          161  +/*
          162  +** Read data from an apnd-file.
          163  +*/
          164  +static int apndRead(
          165  +  sqlite3_file *pFile, 
          166  +  void *zBuf, 
          167  +  int iAmt, 
          168  +  sqlite_int64 iOfst
          169  +){
          170  +  ApndFile *p = (ApndFile *)pFile;
          171  +  pFile = ORIGFILE(pFile);
          172  +  return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
          173  +}
          174  +
          175  +/*
          176  +** Write data to an apnd-file.
          177  +*/
          178  +static int apndWrite(
          179  +  sqlite3_file *pFile,
          180  +  const void *z,
          181  +  int iAmt,
          182  +  sqlite_int64 iOfst
          183  +){
          184  +  return SQLITE_READONLY;
          185  +}
          186  +
          187  +/*
          188  +** Truncate an apnd-file.
          189  +*/
          190  +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
          191  +  return SQLITE_READONLY;
          192  +}
          193  +
          194  +/*
          195  +** Sync an apnd-file.
          196  +*/
          197  +static int apndSync(sqlite3_file *pFile, int flags){
          198  +  return SQLITE_READONLY;
          199  +}
          200  +
          201  +/*
          202  +** Return the current file-size of an apnd-file.
          203  +*/
          204  +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
          205  +  ApndFile *p = (ApndFile*)pFile;
          206  +  int rc;
          207  +  pFile = ORIGFILE(p);
          208  +  rc = pFile->pMethods->xFileSize(pFile, pSize);
          209  +  if( rc==SQLITE_OK && p->iPgOne ){
          210  +    *pSize -= p->iPgOne + APND_MARK_SIZE;
          211  +  }
          212  +  return rc;
          213  +}
          214  +
          215  +/*
          216  +** Lock an apnd-file.
          217  +*/
          218  +static int apndLock(sqlite3_file *pFile, int eLock){
          219  +  return SQLITE_READONLY;
          220  +}
          221  +
          222  +/*
          223  +** Unlock an apnd-file.
          224  +*/
          225  +static int apndUnlock(sqlite3_file *pFile, int eLock){
          226  +  return SQLITE_OK;
          227  +}
          228  +
          229  +/*
          230  +** Check if another file-handle holds a RESERVED lock on an apnd-file.
          231  +*/
          232  +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
          233  +  *pResOut = 0;
          234  +  return SQLITE_OK;
          235  +}
          236  +
          237  +/*
          238  +** File control method. For custom operations on an apnd-file.
          239  +*/
          240  +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
          241  +  ApndFile *p = (ApndFile *)pFile;
          242  +  int rc;
          243  +  pFile = ORIGFILE(pFile);
          244  +  rc = pFile->pMethods->xFileControl(pFile, op, pArg);
          245  +  if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
          246  +    *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
          247  +  }
          248  +  return rc;
          249  +}
          250  +
          251  +/*
          252  +** Return the sector-size in bytes for an apnd-file.
          253  +*/
          254  +static int apndSectorSize(sqlite3_file *pFile){
          255  +  pFile = ORIGFILE(pFile);
          256  +  return pFile->pMethods->xSectorSize(pFile);
          257  +}
          258  +
          259  +/*
          260  +** Return the device characteristic flags supported by an apnd-file.
          261  +*/
          262  +static int apndDeviceCharacteristics(sqlite3_file *pFile){
          263  +  pFile = ORIGFILE(pFile);
          264  +  return SQLITE_IOCAP_IMMUTABLE |
          265  +           pFile->pMethods->xDeviceCharacteristics(pFile);
          266  +}
          267  +
          268  +/* Create a shared memory file mapping */
          269  +static int apndShmMap(
          270  +  sqlite3_file *pFile,
          271  +  int iPg,
          272  +  int pgsz,
          273  +  int bExtend,
          274  +  void volatile **pp
          275  +){
          276  +  return SQLITE_READONLY;
          277  +}
          278  +
          279  +/* Perform locking on a shared-memory segment */
          280  +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
          281  +  return SQLITE_READONLY;
          282  +}
          283  +
          284  +/* Memory barrier operation on shared memory */
          285  +static void apndShmBarrier(sqlite3_file *pFile){
          286  +  return;
          287  +}
          288  +
          289  +/* Unmap a shared memory segment */
          290  +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
          291  +  return SQLITE_OK;
          292  +}
          293  +
          294  +/* Fetch a page of a memory-mapped file */
          295  +static int apndFetch(
          296  +  sqlite3_file *pFile,
          297  +  sqlite3_int64 iOfst,
          298  +  int iAmt,
          299  +  void **pp
          300  +){
          301  +  ApndFile *p = (ApndFile *)pFile;
          302  +  pFile = ORIGFILE(pFile);
          303  +  return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
          304  +}
          305  +
          306  +/* Release a memory-mapped page */
          307  +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
          308  +  ApndFile *p = (ApndFile *)pFile;
          309  +  pFile = ORIGFILE(pFile);
          310  +  return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
          311  +}
          312  +
          313  +/*
          314  +** Open an apnd file handle.
          315  +*/
          316  +static int apndOpen(
          317  +  sqlite3_vfs *pVfs,
          318  +  const char *zName,
          319  +  sqlite3_file *pFile,
          320  +  int flags,
          321  +  int *pOutFlags
          322  +){
          323  +  ApndFile *p;
          324  +  int rc;
          325  +  sqlite3_int64 sz;
          326  +  pVfs = ORIGVFS(pVfs);
          327  +  if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
          328  +    return pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags);
          329  +  }
          330  +  p = (ApndFile*)pFile;
          331  +  memset(p, 0, sizeof(*p));
          332  +  p->base.pMethods = &apnd_io_methods;
          333  +  pFile = ORIGFILE(pFile);
          334  +  flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE);
          335  +  flags |= SQLITE_OPEN_READONLY;
          336  +  rc = pVfs->xOpen(pVfs, zName, pFile, flags, pOutFlags);
          337  +  if( rc ) return rc;
          338  +  rc = pFile->pMethods->xFileSize(pFile, &sz);
          339  +  if( rc==SQLITE_OK && sz>512 ){
          340  +    unsigned char a[APND_MARK_SIZE];
          341  +    rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
          342  +    if( rc==SQLITE_OK
          343  +     && memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)==0
          344  +    ){
          345  +      p->iPgOne =
          346  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ]<<56) +
          347  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+1]<<48) +
          348  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+2]<<40) +
          349  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+3]<<32) +
          350  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+4]<<24) +
          351  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+5]<<16) +
          352  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+6]<<8) +
          353  +         ((sqlite3_uint64)a[APND_MARK_PREFIX_SZ+7]);
          354  +    }
          355  +  }
          356  +  return SQLITE_OK;
          357  +}
          358  +
          359  +/*
          360  +** All other VFS methods are pass-thrus.
          361  +*/
          362  +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
          363  +  return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
          364  +}
          365  +static int apndAccess(
          366  +  sqlite3_vfs *pVfs, 
          367  +  const char *zPath, 
          368  +  int flags, 
          369  +  int *pResOut
          370  +){
          371  +  return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
          372  +}
          373  +static int apndFullPathname(
          374  +  sqlite3_vfs *pVfs, 
          375  +  const char *zPath, 
          376  +  int nOut, 
          377  +  char *zOut
          378  +){
          379  +  return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
          380  +}
          381  +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
          382  +  return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
          383  +}
          384  +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
          385  +  ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
          386  +}
          387  +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
          388  +  return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
          389  +}
          390  +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
          391  +  ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
          392  +}
          393  +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
          394  +  return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
          395  +}
          396  +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
          397  +  return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
          398  +}
          399  +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
          400  +  return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
          401  +}
          402  +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
          403  +  return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
          404  +}
          405  +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
          406  +  return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
          407  +}
          408  +static int apndSetSystemCall(
          409  +  sqlite3_vfs *pVfs,
          410  +  const char *zName,
          411  +  sqlite3_syscall_ptr pCall
          412  +){
          413  +  return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
          414  +}
          415  +static sqlite3_syscall_ptr apndGetSystemCall(
          416  +  sqlite3_vfs *pVfs,
          417  +  const char *zName
          418  +){
          419  +  return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
          420  +}
          421  +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
          422  +  return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
          423  +}
          424  +
          425  +  
          426  +#ifdef _WIN32
          427  +__declspec(dllexport)
          428  +#endif
          429  +/* 
          430  +** This routine is called when the extension is loaded.
          431  +** Register the new VFS.
          432  +*/
          433  +int sqlite3_appendvfs_init(
          434  +  sqlite3 *db, 
          435  +  char **pzErrMsg, 
          436  +  const sqlite3_api_routines *pApi
          437  +){
          438  +  int rc = SQLITE_OK;
          439  +  sqlite3_vfs *pOrig;
          440  +  SQLITE_EXTENSION_INIT2(pApi);
          441  +  pOrig = sqlite3_vfs_find(0);
          442  +  apnd_vfs.iVersion = pOrig->iVersion;
          443  +  apnd_vfs.pAppData = pOrig;
          444  +  apnd_vfs.szOsFile = sizeof(ApndFile);
          445  +  rc = sqlite3_vfs_register(&apnd_vfs, 1);
          446  +#ifdef APPENDVFS_TEST
          447  +  if( rc==SQLITE_OK ){
          448  +    rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
          449  +  }
          450  +#endif
          451  +  if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
          452  +  return rc;
          453  +}