/ Changes On Branch zonefile
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch zonefile Excluding Merge-Ins

This is equivalent to a diff from 15c587cf to 84e9351b

2018-02-27
20:09
Update zonefile README.md file to mention the frame cache. (Leaf check-in: 84e9351b user: dan tags: zonefile)
19:50
Rationalize some code in zonefile.c. Fix other minor issues in the same. (check-in: f11beb16 user: dan tags: zonefile)
2018-02-26
15:27
Always interpret non-zero floating-point values as true even if their integer part is zero. Fix for ticket [36fae083b450e3af857a459e20]. (check-in: a983fa85 user: drh tags: trunk)
03:20
Experimental implementation of IS TRUE and IS FALSE operators. All TRUE and FALSE to act like constants if the names do not resolve to a column name. (check-in: 40314bc9 user: drh tags: is-true-operator)
2018-02-23
13:45
Merge updates from trunk. (check-in: 53f2100a user: mistachkin tags: zonefile)
13:38
Fix harmless compiler warnings in the zipfile extension seen with MSVC. (check-in: 15c587cf user: mistachkin tags: trunk)
13:38
Enable the JSON1 extension when compiling the test fixture with MSVC. (check-in: e9e9f27b user: mistachkin tags: trunk)

Changes to Makefile.msc.

  1490   1490     $(TOP)\ext\misc\amatch.c \
  1491   1491     $(TOP)\ext\misc\carray.c \
  1492   1492     $(TOP)\ext\misc\closure.c \
  1493   1493     $(TOP)\ext\misc\csv.c \
  1494   1494     $(TOP)\ext\misc\eval.c \
  1495   1495     $(TOP)\ext\misc\fileio.c \
  1496   1496     $(TOP)\ext\misc\fuzzer.c \
         1497  +  $(TOP)\ext\zonefile\zonefile.c \
  1497   1498     $(TOP)\ext\fts5\fts5_tcl.c \
  1498   1499     $(TOP)\ext\fts5\fts5_test_mi.c \
  1499   1500     $(TOP)\ext\fts5\fts5_test_tok.c \
  1500   1501     $(TOP)\ext\misc\ieee754.c \
  1501   1502     $(TOP)\ext\misc\mmapwarm.c \
  1502   1503     $(TOP)\ext\misc\nextchar.c \
  1503   1504     $(TOP)\ext\misc\normalize.c \

Added ext/zonefile/README.md.

            1  +
            2  +# The Zonefile Extension
            3  +
            4  +## Functionality
            5  +
            6  +### Creating Zonefile Files
            7  +
            8  +To create a new zonefile, first create a database table with the following
            9  +schema:
           10  +
           11  +>     CREATE TABLE data(
           12  +>       k INTEGER PRIMARY KEY,
           13  +>       frame INTEGER DEFAULT -1,   -- frame number.  Automatic if -1
           14  +>       idx INTEGER DEFAULT -1,     -- index of entry within frame.  Auto if -1
           15  +>       v BLOB
           16  +>     );
           17  +
           18  +The table may be created in a persistent or temporary database and may
           19  +take any name, but must contain the columns above. The table must be 
           20  +populated with a row for each key intended to appear in the new zonefile
           21  +file.
           22  +
           23  +Once the table is populated, a zonefile is created using the following
           24  +SQL:
           25  +
           26  +>     SELECT zonefile_write(<file>, <table> [, <parameters>]);
           27  +
           28  +where &lt;file&gt; is the name of the file to create on disk, &lt;table&gt; 
           29  +is the name of the database table to read and optional argument 
           30  +&lt;parameters&gt; is a JSON object containing various attributes that
           31  +influence creation of the zonefile file. 
           32  +
           33  +Currently, the following &lt;parameters&gt; attributes are supported:
           34  +
           35  +<table border=1>
           36  +<tr align=left><th>Attribute<th>Default<th>Interpretation
           37  +<tr valign=top><td>maxAutoFrameSize<td>65536
           38  +<td>The maximum uncompressed frame size in bytes for automatically generated
           39  +zonefile frames.
           40  +
           41  +<tr valign=top><td>compressionTypeContent<td>"none"
           42  +<td>The compression type used to compress each frame in the zonefile. 
           43  +Valid values are "none" (no compression), "zstd", "zstd_global_dict",
           44  +"zlib", "brotli", "lz4" and "lz4hc". Not all compression methods are
           45  +supported by all builds. The compression method supported by a build
           46  +depends on the combination of SQLITE_HAVE_ZSTD, SQLITE_HAVE_ZLIB,
           47  +SQLITE_HAVE_BROTLI and SQLITE_HAVE_LZ4 pre-processor symbols defined
           48  +at build time.
           49  +
           50  +<tr valign=top><td>compressionTypeIndexData<td>"none"
           51  +<td>The compression type used to compress the zonefile index structure.
           52  +All values that are valid for the <i>compressionTypeContent</i> parameter,
           53  +except for "zstd_global_dict", are also valid for this option.
           54  +
           55  +<tr valign=top><td>encryptionType<td>"none"
           56  +<td>The encryption type to use. At present the only valid values are
           57  +"none" (no encryption) and "xor" (an insecure mock encryption method
           58  +useful for testing only). Enhanced implementations may support any or
           59  +all of the following encryption schemes:
           60  +<ul>
           61  +  <li> "AES_128_CTR"
           62  +  <li> "AES_128_CBC"
           63  +  <li> "AES_256_CTR"
           64  +  <li> "AES_256_CBC"
           65  +</ul>
           66  +
           67  +<tr valign=top><td>encryptionKey<td>""
           68  +<td>The encryption key to use. The encryption key must be specified as an
           69  +even number of hexadecimal that will be converted to a binary key before
           70  +use. It is the responsibility of the caller to specify a key of the optimal
           71  +length for each encryption algorithm (e.g. 16 bytes (32 hex digits) for
           72  +a 128-bit encryption, or 32 bytes (64 digits) for a 256-bit method).
           73  +This option is ignored if <i>encryptionType</i> is set to "none".
           74  +</table>
           75  +
           76  +For example, to create a zonefile named "test.zonefile" based on the
           77  +contents of database table "test_input", with a maximum automatic
           78  +frame size of 4096 bytes and using "xor" encryption with a 128-bit key:
           79  +
           80  +>     SELECT zonefile_write('test.zonefile', 'test_input',
           81  +>       '{"maxAutoFrameSize":4096,
           82  +>         "encryptionType":"xor",
           83  +>         "encryptionKey":"e6e600bc063aad12f6387beab650c48a"
           84  +>       }'
           85  +>     );
           86  +
           87  +### Using (Reading) Zonefile Files
           88  +
           89  +To create a new zonefile table, one of the following:
           90  +
           91  +>     CREATE VIRTUAL TABLE z1 USING zonefile;
           92  +>     CREATE VIRTUAL TABLE z1 USING zonefile(cachesize=N);
           93  +
           94  +where <i>N</i> is any non-zero positive integer. If the zonefile is used
           95  +to access any files containing compressed or encrypted data, it maintains
           96  +an LRU cache of uncompressed frame data <i>N</i> frames in size. The
           97  +default value of <i>N</i> is 1.
           98  +
           99  +Creating a "zonefile" virtual table actually creates two virtual tables in the
          100  +database schema. One read-only table named "z1", with a schema equivalent to:
          101  +
          102  +>     CREATE TABLE z1(  -- this whole table is read-only
          103  +>       k INTEGER PRIMARY KEY,     -- key value
          104  +>       v BLOB,                    -- associated blob of data
          105  +>       fileid INTEGER,            -- file id (rowid value for z1_files)
          106  +>       sz INTEGER                 -- size of blob of data in bytes
          107  +>     );
          108  +
          109  +And a read-write table named "z1_files" with a schema like:
          110  +
          111  +>     CREATE TABLE z1_files(
          112  +>       filename TEXT PRIMARY KEY,
          113  +>       ekey BLOB,         -- encryption key
          114  +>       header JSON HIDDEN -- read-only
          115  +>     );
          116  +
          117  +Both tables are initially empty. To add a zonefile to the index, insert a
          118  +row into the "z1_files" table:
          119  +
          120  +>     INSERT INTO z1_files(filename) VALUES(<filename>);
          121  +
          122  +If the file is an encrypted file, then the encryption key (a blob) must
          123  +be inserted into the "ekey" column. Encryption keys are not stored in the
          124  +database, they are held in main-memory only. This means that each new
          125  +connection must configure encryption key using UPDATE statements before
          126  +accessing any encrypted files. For example:
          127  +
          128  +>     -- Add new encrypted file to database:
          129  +>     INSERT INTO z1_files(filename, ekey) VALUES(<filename>, <ekey>);
          130  +>
          131  +>     -- Configure encryption key for existing file after opening database:
          132  +>     UPDATE z1_files SET ekey = <ekey> WHERE filename = <filename>;
          133  +
          134  +Currently, values provided for any columns other than "filename" and
          135  +"ekey" are ignored. Files are removed from the index by deleting rows 
          136  +from the z1_files table:
          137  +
          138  +>     DELETE FROM z1_files WHERE filename = <filename>;
          139  +
          140  +Once zonefile files have been added to the index, their contents are 
          141  +visible in table "z1". To retrieve the value associated with a single
          142  +key from one of the zonefile files in the index:
          143  +
          144  +>     SELECT v FROM z1 WHERE k = <key>;
          145  +
          146  +
          147  +## Notes
          148  +
          149  +  *  Contrary to the spec, the implementation uses 32-bit (not 16-bit) frame
          150  +     numbers. So the KeyOffsetPair structure becomes:
          151  +
          152  +     KeyOffsetPair
          153  +     {
          154  +       uint64  key;
          155  +       uint32  frameNo;
          156  +       uint32  frameByteOffset;
          157  +     };
          158  +
          159  +     Also, the ZonefileHeader.numFrames field is now 32-bit. Which makes
          160  +     the ZonefileHeader structure 26 bytes in size. The implementation
          161  +     pads this out to 32 bytes so that the ZoneFileIndex is 8-byte aligned.
          162  +
          163  +  *  Multi-byte integer values are big-endian.
          164  +
          165  +  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are
          166  +     relative to the offset in ZoneFileHeader.byteOffsetFrames. This is
          167  +     necessary as we may not know the offset of the start of the frame data
          168  +     until after the ZoneFileIndex structure is compressed.
          169  +
          170  +  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are the
          171  +     offsets for the first byte past the end of the corresponding frame.
          172  +     For example, byteOffsetZoneFrame[] identifies the first byte of the
          173  +     second frame, and byteOffsetZoneFrame[numFrames-1] is one byte past
          174  +     the end of the last frame in the file.
          175  +
          176  +     This is better as if we store the starting offset of each frame, there
          177  +     is no way to determine the size of the last frame in the file without
          178  +     trusting the filesize itself.
          179  +
          180  +  *  Currently there is no support at all for encryption.
          181  +
          182  +  *  Zonefile currently uses json1 to parse the json argument to
          183  +     zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1
          184  +     or otherwise json1-enabled SQLite.
          185  +
          186  +

Added ext/zonefile/zonefile.c.

            1  +/*
            2  +** 2018-02-10
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +******************************************************************************
           12  +*/
           13  +
           14  +#include "sqlite3ext.h"
           15  +SQLITE_EXTENSION_INIT1
           16  +#ifndef SQLITE_OMIT_VIRTUALTABLE
           17  +
           18  +#include <stdio.h>
           19  +#include <stdlib.h>
           20  +#include <string.h>
           21  +#include <assert.h>
           22  +
           23  +/*
           24  +** Default values for various zonefile_write() parameters.
           25  +*/
           26  +#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
           27  +#define ZONEFILE_DEFAULT_ENCRYPTION       1
           28  +#define ZONEFILE_DEFAULT_COMPRESSION      0
           29  +#define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)
           30  +
           31  +/* 
           32  +** Value to use for the first 4 bytes of a zonefile file header.
           33  +*/
           34  +#define ZONEFILE_MAGIC_NUMBER 0x464B3138
           35  +
           36  +/*
           37  +** Size of a zonefile header. And of each entry in the 
           38  +** ZonefileIndex.keyOffsets array.
           39  +*/
           40  +#define ZONEFILE_SZ_HEADER           32
           41  +#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20
           42  +
           43  +/*
           44  +** Constants for supported compression types. These are copied from the
           45  +** published file format spec.
           46  +*/
           47  +#define ZONEFILE_COMPRESSION_NONE             0
           48  +#define ZONEFILE_COMPRESSION_ZSTD             1
           49  +#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
           50  +#define ZONEFILE_COMPRESSION_ZLIB             3
           51  +#define ZONEFILE_COMPRESSION_BROTLI           4
           52  +#define ZONEFILE_COMPRESSION_LZ4              5
           53  +#define ZONEFILE_COMPRESSION_LZ4HC            6
           54  +
           55  +/*
           56  +** Schema for "zonefile" virtual table.
           57  +*/
           58  +#define ZONEFILE_SCHEMA          \
           59  +  "CREATE TABLE z1("             \
           60  +  "  k INTEGER PRIMARY KEY,"     \
           61  +  "  v BLOB,"                    \
           62  +  "  fileid INTEGER,"            \
           63  +  "  sz INTEGER"                 \
           64  +  ")"
           65  +
           66  +/*
           67  +** Schema for "zonefile_files" virtual table.
           68  +*/
           69  +#define ZONEFILE_FILES_SCHEMA    \
           70  +  "CREATE TABLE z2("             \
           71  +  "  filename TEXT,"             \
           72  +  "  ekey BLOB,"                 \
           73  +  "  header JSON HIDDEN"         \
           74  +  ")"
           75  +
           76  +
           77  +#ifndef SQLITE_AMALGAMATION
           78  +typedef sqlite3_int64 i64;
           79  +typedef sqlite3_uint64 u64;
           80  +typedef unsigned char u8;
           81  +typedef unsigned short u16;
           82  +typedef unsigned long u32;
           83  +#define MIN(a,b) ((a)<(b) ? (a) : (b))
           84  +
           85  +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
           86  +# define ALWAYS(X)      (1)
           87  +# define NEVER(X)       (0)
           88  +#elif !defined(NDEBUG)
           89  +# define ALWAYS(X)      ((X)?1:(assert(0),0))
           90  +# define NEVER(X)       ((X)?(assert(0),1):0)
           91  +#else
           92  +# define ALWAYS(X)      (X)
           93  +# define NEVER(X)       (X)
           94  +#endif
           95  +#endif   /* SQLITE_AMALGAMATION */
           96  +
           97  +/*
           98  +** Forward declarations for encryption/decryption functions.
           99  +**
          100  +** If this module is not compiled with SQLITE_HAVE_ZONEFILE_CODEC, then
          101  +** implementations of the following type and functions that support the
          102  +** mock encryption method "xor" only are provided. Alternatively, the
          103  +** application may append a more functional implementation of the following 
          104  +** type and functions to this file before compiling it with
          105  +** SQLITE_HAVE_ZONEFILE_CODEC defined.
          106  +*/
          107  +typedef struct ZonefileCodec ZonefileCodec;
          108  +static int zonefileCodecCreate(
          109  +    int,int,unsigned char*,int,ZonefileCodec**,char**);
          110  +static int zonefileCodecNonceSize(ZonefileCodec*);
          111  +static void zonefileCodecEncode(ZonefileCodec*, unsigned char*, int);
          112  +static void zonefileCodecDecode(ZonefileCodec*, unsigned char*, int);
          113  +static void zonefileCodecDestroy(ZonefileCodec*);
          114  +
          115  +#ifndef SQLITE_HAVE_ZONEFILE_CODEC
          116  +typedef struct ZonefileCodec ZonefileCodec;
          117  +
          118  +struct ZonefileCodec {
          119  +  u8 aKey[16];
          120  +  int bEncrypt;                   /* Second parameter passed to Create() */
          121  +};
          122  +
          123  +/* Create a new encryption module instance using algorithm iAlg.
          124  +**
          125  +**   iAlg==1   AES128 CTR
          126  +**   iAlg==2   AES128 CBC
          127  +**   iAlg==3   AES256 CTR
          128  +**   iAlg==4   AES256 CBC
          129  +**   iAlg==5   XOR          Testing use only
          130  +**
          131  +** If the requested algorithm is not available, the routine returns
          132  +** a NULL pointer.  NULL is also returned on a OOM error.
          133  +**
          134  +** Use zonefileCodecDestroy() to reclaim memory.
          135  +*/
          136  +static int zonefileCodecCreate(
          137  +  int iAlg, 
          138  +  int bEncrypt,                   /* True for encryption, zero for decryption */
          139  +  unsigned char *pKey, int nKey, 
          140  +  ZonefileCodec **pp, 
          141  +  char **pzErr
          142  +){
          143  +  ZonefileCodec *pRet;
          144  +  int rc = SQLITE_OK;
          145  +  
          146  +  if( iAlg!=5 ){
          147  +    *pzErr = sqlite3_mprintf("unsupported encryption method: %d", iAlg);
          148  +    rc = SQLITE_ERROR;
          149  +  }else{
          150  +    *pp = pRet = (ZonefileCodec*)sqlite3_malloc(sizeof(ZonefileCodec));
          151  +    if( pRet==0 ){
          152  +      rc = SQLITE_NOMEM;
          153  +    }else{
          154  +      int i;
          155  +      for(i=0; i<sizeof(pRet->aKey); i++){
          156  +        pRet->aKey[i] = pKey[i % nKey];
          157  +      }
          158  +      pRet->bEncrypt = bEncrypt;
          159  +    }
          160  +  }
          161  +
          162  +  return rc;
          163  +}
          164  +
          165  +/* Return the size of the nonce used for the given encryption module */
          166  +static int zonefileCodecNonceSize(ZonefileCodec *pCodec){
          167  +  return 16;
          168  +}
          169  +
          170  +/* Encrypt in-place.
          171  +**
          172  +** The size of the content will grow by the nonce size.  Hence, the
          173  +** buffer must have at least nonce bytes of extra space available at
          174  +** the end to accommodate that growth.  When persisting results, be
          175  +** sure to include the extra bytes.
          176  +*/
          177  +static void zonefileCodecEncode(
          178  +  ZonefileCodec *pCodec,
          179  +  unsigned char *pIn, int nIn
          180  +){
          181  +  int i;
          182  +  u8 *aNonce = &pIn[nIn];
          183  +  assert( pCodec->bEncrypt );
          184  +  sqlite3_randomness(16, aNonce);
          185  +  for(i=0; i<nIn; i++){
          186  +    pIn[i] = pIn[i] ^ aNonce[i%16] ^ pCodec->aKey[i%16];
          187  +  }
          188  +}
          189  +
          190  +/* Decrypt in-place.
          191  +**
          192  +** The size of the decrypted text will be less than the input buffer
          193  +** by nonce-size bytes.
          194  +*/
          195  +static void zonefileCodecDecode(
          196  +  ZonefileCodec *pCodec, 
          197  +  unsigned char *pIn, int nIn
          198  +){
          199  +  int i;
          200  +  u8 *aNonce = &pIn[nIn-16];
          201  +  assert( pCodec->bEncrypt==0 );
          202  +  for(i=0; i<nIn-16; i++){
          203  +    pIn[i] = pIn[i] ^ aNonce[i%16] ^ pCodec->aKey[i%16];
          204  +  }
          205  +}
          206  +
          207  +/* Destroy an encryption module.
          208  +** It is harmless to pass in a NULL pointer.
          209  +*/
          210  +static void zonefileCodecDestroy(ZonefileCodec *pCodec){
          211  +  sqlite3_free(pCodec);
          212  +}
          213  +#endif                           /* SQLITE_HAVE_ZONEFILE_CODEC */
          214  +
          215  +/*
          216  +** All zonefile and zonefile_files virtual table instances that belong
          217  +** to the same database handle (sqlite3*) share a single instance of the
          218  +** ZonefileGlobal object. This global object contains a table of 
          219  +** configured encryption keys for the various zonefiles in the system.
          220  +*/
          221  +typedef struct ZonefileGlobal ZonefileGlobal;
          222  +typedef struct ZonefileKey ZonefileKey;
          223  +struct ZonefileGlobal {
          224  +  int nEntry;                     /* Number of entries in the hash table */
          225  +  int nHash;                      /* Size of aHash[] array */
          226  +  ZonefileKey **aHash;            /* Hash buckets */
          227  +};
          228  +struct ZonefileKey {
          229  +  const char *zName;              /* Zonefile table name */
          230  +  const char *zDb;                /* Database name ("main", "temp" etc.) */
          231  +  i64 iFileid;                    /* File id */
          232  +  const u8 *aKey;                 /* Key buffer */
          233  +  int nKey;                       /* Size of zKey in bytes */
          234  +  u32 iHash;                      /* zonefileKeyHash() value */
          235  +  ZonefileKey *pHashNext;         /* Next colliding key in hash table */
          236  +};
          237  +
          238  +/*
          239  +** Return a 32-bit hash value for the three arguments.
          240  +*/
          241  +static u32 zonefileKeyHash(
          242  +  const char *zDb,
          243  +  const char *zTab,
          244  +  i64 iFileid
          245  +){
          246  +  u32 iHash = 0;
          247  +  int i;
          248  +  for(i=0; zDb[i]; i++) iHash += (iHash<<3) + (u8)zDb[i];
          249  +  for(i=0; zTab[i]; i++) iHash += (iHash<<3) + (u8)zTab[i];
          250  +  return (iHash ^ (iFileid & 0xFFFFFFFF));
          251  +}
          252  +
          253  +/* 
          254  +** Store encryption key aKey in the key-store passed as the first argument.
          255  +** Return SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM)
          256  +** otherwise.
          257  +*/
          258  +static int zonefileKeyStore(
          259  +  ZonefileGlobal *pGlobal,
          260  +  const char *zDb,                /* Database containing zonefile table */
          261  +  const char *zTab,               /* Name of zonefile table */
          262  +  i64 iFileid,                    /* File-id to configure key for */
          263  +  const u8 *aKey,                 /* Key to store */
          264  +  int nKey                        /* Size of aKey[] in bytes */
          265  +){
          266  +  ZonefileKey **pp;
          267  +  u32 iHash = zonefileKeyHash(zDb, zTab, iFileid);
          268  +
          269  +  /* Remove any old entry */
          270  +  if( pGlobal->nHash ){
          271  +    for(pp=&pGlobal->aHash[iHash%pGlobal->nHash]; *pp; pp=&((*pp)->pHashNext)){
          272  +      ZonefileKey *pThis = *pp;
          273  +      if( pThis->iFileid==iFileid 
          274  +          && 0==sqlite3_stricmp(zTab, pThis->zName)
          275  +          && 0==sqlite3_stricmp(zDb, pThis->zDb)
          276  +        ){
          277  +        pGlobal->nEntry--;
          278  +        *pp = pThis->pHashNext;
          279  +        sqlite3_free(pThis);
          280  +        break;
          281  +      }
          282  +    }
          283  +  }
          284  +
          285  +  if( aKey ){
          286  +    int nDb = strlen(zDb);
          287  +    int nTab = strlen(zTab);
          288  +    ZonefileKey *pNew;
          289  +
          290  +    /* Resize the hash-table, if necessary */
          291  +    if( pGlobal->nEntry>=pGlobal->nHash ){
          292  +      int i;
          293  +      int n = pGlobal->nHash ? pGlobal->nHash*2 : 16;
          294  +      ZonefileKey **a = (ZonefileKey**)sqlite3_malloc(n*sizeof(ZonefileKey*));
          295  +      if( a==0 ) return SQLITE_NOMEM;
          296  +      memset(a, 0, n*sizeof(ZonefileKey*));
          297  +      for(i=0; i<pGlobal->nHash; i++){
          298  +        ZonefileKey *p;
          299  +        ZonefileKey *pNext;
          300  +        for(p=pGlobal->aHash[i]; p; p=pNext){
          301  +          pNext = p->pHashNext;
          302  +          p->pHashNext = a[p->iHash % n];
          303  +          a[p->iHash % n] = p;
          304  +        }
          305  +      }
          306  +      sqlite3_free(pGlobal->aHash);
          307  +      pGlobal->aHash = a;
          308  +      pGlobal->nHash = n;
          309  +    }
          310  +
          311  +    pNew = (ZonefileKey*)sqlite3_malloc(
          312  +        sizeof(ZonefileKey) + nKey+1 + nDb+1 + nTab+1
          313  +    );
          314  +    if( pNew==0 ) return SQLITE_NOMEM;
          315  +    memset(pNew, 0, sizeof(ZonefileKey));
          316  +    pNew->iFileid = iFileid;
          317  +    pNew->iHash = iHash;
          318  +    pNew->aKey = (const u8*)&pNew[1];
          319  +    pNew->nKey = nKey;
          320  +    pNew->zDb = (const char*)&pNew->aKey[nKey+1];
          321  +    pNew->zName = &pNew->zDb[nDb+1];
          322  +    memcpy((u8*)pNew->aKey, aKey, nKey+1);
          323  +    memcpy((char*)pNew->zDb, zDb, nDb+1);
          324  +    memcpy((char*)pNew->zName, zTab, nTab+1);
          325  +
          326  +    pNew->pHashNext = pGlobal->aHash[iHash % pGlobal->nHash];
          327  +    pGlobal->aHash[iHash % pGlobal->nHash] = pNew;
          328  +    pGlobal->nEntry++;
          329  +  }
          330  +
          331  +  return SQLITE_OK;
          332  +}
          333  +
          334  +/*
          335  +** Search the key-store passed as the first argument for an encryption
          336  +** key to use with the file with file-id iFileid in zonefile table zTab
          337  +** in database zDb. If successful, set (*paKey) to point to the key
          338  +** buffer and return the size of the key in bytes.
          339  +**
          340  +** If no key is found, return 0. The final value of (*paKey) is undefined
          341  +** in this case.
          342  +*/
          343  +static int zonefileKeyFind(
          344  +  ZonefileGlobal *pGlobal,
          345  +  const char *zDb,                /* Database containing zonefile table */
          346  +  const char *zTab,               /* Name of zonefile table */
          347  +  i64 iFileid,                    /* File-id to configure key for */
          348  +  const u8 **paKey                /* OUT: Pointer to key buffer */
          349  +){
          350  +  if( pGlobal->nHash ){
          351  +    ZonefileKey *pKey;
          352  +    u32 iHash = zonefileKeyHash(zDb, zTab, iFileid);
          353  +    for(pKey=pGlobal->aHash[iHash%pGlobal->nHash]; pKey; pKey=pKey->pHashNext){
          354  +      if( pKey->iFileid==iFileid 
          355  +       && 0==sqlite3_stricmp(zTab, pKey->zName)
          356  +       && 0==sqlite3_stricmp(zDb, pKey->zDb)
          357  +      ){
          358  +        *paKey = pKey->aKey;
          359  +        return pKey->nKey;
          360  +      }
          361  +    }
          362  +  }
          363  +
          364  +  return 0;
          365  +}
          366  +
          367  +/*
          368  +** The pointer passed as the only argument must actually point to a
          369  +** ZonefileGlobal structure. This function frees the structure and all
          370  +** of its components.
          371  +*/
          372  +static void zonefileKeyDestroy(void *p){
          373  +  ZonefileGlobal *pGlobal = (ZonefileGlobal*)p;
          374  +  int i;
          375  +  for(i=0; i<pGlobal->nHash; i++){
          376  +    ZonefileKey *pKey;
          377  +    ZonefileKey *pNext;
          378  +    for(pKey=pGlobal->aHash[i]; pKey; pKey=pNext){
          379  +      pNext = pKey->pHashNext;
          380  +      sqlite3_free(pKey);
          381  +    }
          382  +  }
          383  +  sqlite3_free(pGlobal->aHash);
          384  +  sqlite3_free(pGlobal);
          385  +}
          386  +
          387  +/*
          388  +** Write value v to buffer aBuf as an unsigned 32-bit big-endian integer.
          389  +*/
          390  +static void zonefilePut32(u8 *aBuf, u32 v){
          391  +  aBuf[0] = (v >> 24) & 0xFF;
          392  +  aBuf[1] = (v >> 16) & 0xFF;
          393  +  aBuf[2] = (v >>  8) & 0xFF;
          394  +  aBuf[3] = v & 0xFF;
          395  +}
          396  +
          397  +/*
          398  +** Read and return an unsigned 32-bit big-endian integer from buffer aBuf.
          399  +*/
          400  +static u32 zonefileGet32(const u8 *aBuf){
          401  +  return (((u32)aBuf[0]) << 24)
          402  +       + (((u32)aBuf[1]) << 16)
          403  +       + (((u32)aBuf[2]) <<  8)
          404  +       + (((u32)aBuf[3]) <<  0);
          405  +}
          406  +
          407  +/*
          408  +** Generic xOpen, xClose and xUncompressSize methods used by a few
          409  +** different compression method bindings.
          410  +*/
          411  +static int zfGenericOpen(void **pp, u8 *aDict, int nDict){
          412  +  *pp = 0;
          413  +  return SQLITE_OK;
          414  +}
          415  +static void zfGenericClose(void *p){
          416  +}
          417  +static int zfGenericUncompressSize(
          418  +  void *p, 
          419  +  const u8 *aSrc, int nSrc
          420  +){
          421  +  return (int)zonefileGet32(aSrc);
          422  +}
          423  +
          424  +#ifdef SQLITE_HAVE_ZLIB 
          425  +#include <zlib.h>
          426  +static int zfZlibCompressBound(void *p, int nSrc){
          427  +  return (int)compressBound((uLong)nSrc) + 4;
          428  +}
          429  +static int zfZlibCompress(
          430  +  void *p, 
          431  +  u8 *aDest, int *pnDest, 
          432  +  const u8 *aSrc, int nSrc
          433  +){
          434  +  uLongf destLen = (uLongf)(*pnDest)-4;
          435  +  int rc = compress(&aDest[4], &destLen, aSrc, (uLong)nSrc);
          436  +  *pnDest = (int)(destLen+4);
          437  +  zonefilePut32(aDest, nSrc);
          438  +  return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
          439  +}
          440  +static int zfZlibUncompress(
          441  +  void *p, 
          442  +  u8 *aDest, int nDest, 
          443  +  const u8 *aSrc, int nSrc
          444  +){
          445  +  uLongf destLen = (uLongf)nDest;
          446  +  int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4);
          447  +  return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
          448  +}
          449  +#endif
          450  +#ifdef SQLITE_HAVE_ZSTD 
          451  +#include <zstd.h>
          452  +static int zfZstdCompressBound(void *p, int nSrc){
          453  +  return (int)ZSTD_compressBound((size_t)nSrc);
          454  +}
          455  +static int zfZstdCompress(
          456  +  void *p, 
          457  +  u8 *aDest, int *pnDest, 
          458  +  const u8 *aSrc, int nSrc
          459  +){
          460  +  size_t szDest = (size_t)(*pnDest);
          461  +  size_t rc = ZSTD_compress(aDest, szDest, aSrc, (size_t)nSrc, 1);
          462  +  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
          463  +  *pnDest = (int)rc;
          464  +  return SQLITE_OK;
          465  +}
          466  +static int zfZstdUncompressSize(void *p, const u8 *aSrc, int nSrc){
          467  +  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
          468  +}
          469  +static int zfZstdUncompress(
          470  +  void *p, 
          471  +  u8 *aDest, int nDest, 
          472  +  const u8 *aSrc, int nSrc
          473  +){
          474  +  size_t rc = ZSTD_decompress(aDest, (size_t)nDest, aSrc, (size_t)nSrc);
          475  +  if( rc!=(size_t)nDest ) return SQLITE_ERROR;
          476  +  return SQLITE_OK;
          477  +}
          478  +
          479  +#include <zdict.h>
          480  +typedef struct ZfZstddict ZfZstddict;
          481  +struct ZfZstddict {
          482  +  ZSTD_CDict *pCDict;
          483  +  ZSTD_CCtx *pCCtx;
          484  +  ZSTD_DDict *pDDict;
          485  +  ZSTD_DCtx *pDCtx;
          486  +};
          487  +
          488  +static void zfZstddictClose(void *p){
          489  +  if( p ){
          490  +    ZfZstddict *pCmp = (ZfZstddict*)p;
          491  +    if( pCmp->pCDict ) ZSTD_freeCDict(pCmp->pCDict);
          492  +    if( pCmp->pCCtx ) ZSTD_freeCCtx(pCmp->pCCtx);
          493  +    if( pCmp->pDCtx ) ZSTD_freeDCtx(pCmp->pDCtx);
          494  +    sqlite3_free(pCmp);
          495  +  }
          496  +}
          497  +static int zfZstddictOpen(void **pp, u8 *aDict, int nDict){
          498  +  int rc = SQLITE_OK;
          499  +  ZfZstddict *pDict = (ZfZstddict*)sqlite3_malloc(sizeof(ZfZstddict));
          500  +  if( pDict==0 ){
          501  +    rc = SQLITE_NOMEM;
          502  +  }else{
          503  +    memset(pDict, 0, sizeof(ZfZstddict));
          504  +    if( aDict ){
          505  +      pDict->pDDict = ZSTD_createDDict(aDict, nDict);
          506  +      pDict->pDCtx = ZSTD_createDCtx();
          507  +      if( pDict->pDDict==0 || pDict->pDCtx==0 ){
          508  +        zfZstddictClose((void*)pDict);
          509  +        pDict = 0;
          510  +        rc = SQLITE_ERROR;
          511  +      }
          512  +    }
          513  +  }
          514  +  *pp = (void*)pDict;
          515  +  return rc;
          516  +}
          517  +static int zfZstddictTrain(
          518  +  void *p,                        /* Compressor handle */
          519  +  u8 *aDict, int *pnDict,         /* OUT: Dictionary buffer */
          520  +  u8 *aSamp, size_t *aSz, int nSamp  /* IN: Training samples */
          521  +){
          522  +  ZfZstddict *pCmp = (ZfZstddict*)p;
          523  +  size_t sz = ZDICT_trainFromBuffer(aDict, (size_t)*pnDict, aSamp, aSz, nSamp);
          524  +  if( ZDICT_isError(sz) ) return SQLITE_ERROR;
          525  +  pCmp->pCDict = ZSTD_createCDict(aDict, sz, 1);
          526  +  pCmp->pCCtx = ZSTD_createCCtx();
          527  +  if( pCmp->pCDict==0 || pCmp->pCCtx==0 ) return SQLITE_ERROR;
          528  +  *pnDict = (int)sz;
          529  +  return SQLITE_OK;
          530  +}
          531  +static int zfZstddictCompress(
          532  +  void *p, 
          533  +  u8 *aDest, int *pnDest, 
          534  +  const u8 *aSrc, int nSrc
          535  +){
          536  +  ZfZstddict *pCmp = (ZfZstddict*)p;
          537  +  size_t szDest = (size_t)(*pnDest);
          538  +  size_t rc;
          539  +  assert( pCmp && pCmp->pCDict && pCmp->pCCtx );
          540  +  rc = ZSTD_compress_usingCDict(
          541  +      pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict
          542  +  );
          543  +  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
          544  +  *pnDest = (int)rc;
          545  +  return SQLITE_OK;
          546  +}
          547  +static int zfZstddictUncompress(
          548  +  void *p, 
          549  +  u8 *aDest, int nDest, 
          550  +  const u8 *aSrc, int nSrc
          551  +){
          552  +  ZfZstddict *pCmp = (ZfZstddict*)p;
          553  +  size_t rc = ZSTD_decompress_usingDDict(
          554  +      pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict
          555  +  );
          556  +  if( rc!=(size_t)nDest ) return SQLITE_ERROR;
          557  +  return SQLITE_OK;
          558  +}
          559  +#endif
          560  +
          561  +#ifdef SQLITE_HAVE_LZ4 
          562  +#include <lz4.h>
          563  +#include <lz4hc.h>
          564  +static int zfLz4CompressBound(void *p, int nSrc){
          565  +  return (int)LZ4_compressBound(nSrc) + 4;
          566  +}
          567  +static int zfLz4Uncompress(
          568  +  void *p, 
          569  +  u8 *aDest, int nDest, 
          570  +  const u8 *aSrc, int nSrc
          571  +){
          572  +  int rc = LZ4_decompress_safe(
          573  +      (const char*)&aSrc[4], (char*)aDest, nSrc-4, nDest
          574  +  );
          575  +  return rc==nDest ? SQLITE_OK : SQLITE_ERROR;
          576  +}
          577  +static int zfLz4Compress(
          578  +  void *p, 
          579  +  u8 *aDest, int *pnDest, 
          580  +  const u8 *aSrc, int nSrc
          581  +){
          582  +  int rc = LZ4_compress_default(
          583  +      (const char*)aSrc, (char*)&aDest[4], nSrc, (*pnDest - 4)
          584  +  );
          585  +  *pnDest = rc+4;
          586  +  zonefilePut32(aDest, nSrc);
          587  +  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
          588  +}
          589  +static int zfLz4hcCompress(
          590  +  void *p, 
          591  +  u8 *aDest, int *pnDest, 
          592  +  const u8 *aSrc, int nSrc
          593  +){
          594  +  int rc = LZ4_compress_HC(
          595  +      (const char*)aSrc, (char*)&aDest[4], nSrc, *pnDest, 0
          596  +  );
          597  +  *pnDest = rc+4;
          598  +  zonefilePut32(aDest, nSrc);
          599  +  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
          600  +}
          601  +#endif
          602  +
          603  +#ifdef SQLITE_HAVE_BROTLI 
          604  +#include <brotli/encode.h>
          605  +#include <brotli/decode.h>
          606  +
          607  +static int zfBrotliCompressBound(void *p, int nSrc){
          608  +  return (int)BrotliEncoderMaxCompressedSize((size_t)nSrc) + 4;
          609  +}
          610  +static int zfBrotliCompress(
          611  +  void *p, 
          612  +  u8 *aDest, int *pnDest, 
          613  +  const u8 *aSrc, int nSrc
          614  +){
          615  +  size_t nDest = (size_t)*pnDest - 4;
          616  +  BROTLI_BOOL rc = BrotliEncoderCompress(
          617  +      BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE,
          618  +      (size_t)nSrc, aSrc, &nDest, &aDest[4]
          619  +  );
          620  +  *pnDest = (int)nDest + 4;
          621  +  zonefilePut32(aDest, nSrc);
          622  +  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
          623  +}
          624  +static int zfBrotliUncompress(
          625  +  void *p, 
          626  +  u8 *aDest, int nDest, 
          627  +  const u8 *aSrc, int nSrc
          628  +){
          629  +  size_t n = nDest;
          630  +  BrotliDecoderResult rc = BrotliDecoderDecompress(nSrc-4, &aSrc[4], &n, aDest);
          631  +  return rc==BROTLI_DECODER_RESULT_SUCCESS ? SQLITE_OK : SQLITE_ERROR;
          632  +}
          633  +#endif
          634  +
          635  +typedef struct ZonefileCompress ZonefileCompress;
          636  +static struct ZonefileCompress {
          637  +  int eType;
          638  +  const char *zName;
          639  +  int (*xOpen)(void**, u8 *aDict, int nDict);
          640  +  void (*xClose)(void*);
          641  +  int (*xTrain)(void*, u8 *aDict, int *pnDict, u8 *a, size_t *aSz, int n);
          642  +  int (*xCompressBound)(void*, int nSrc);
          643  +  int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc);
          644  +  int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc);
          645  +  int (*xUncompress)(void*, u8 *aDest, int nDest, const u8 *aSrc, int nSrc);
          646  +} aZonefileCompress[] = {
          647  +  { ZONEFILE_COMPRESSION_NONE,             "none",
          648  +    0, 0, 0, 0, 0, 0, 0
          649  +  },
          650  +#ifdef SQLITE_HAVE_ZSTD
          651  +  { ZONEFILE_COMPRESSION_ZSTD,             "zstd",
          652  +    zfGenericOpen, zfGenericClose, 
          653  +    0,
          654  +    zfZstdCompressBound, zfZstdCompress, 
          655  +    zfZstdUncompressSize, zfZstdUncompress
          656  +  },
          657  +  { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict",
          658  +    zfZstddictOpen, zfZstddictClose, 
          659  +    zfZstddictTrain,
          660  +    zfZstdCompressBound, zfZstddictCompress, 
          661  +    zfZstdUncompressSize, zfZstddictUncompress
          662  +  },
          663  +#endif /* SQLITE_HAVE_ZSTD */
          664  +#ifdef SQLITE_HAVE_ZLIB
          665  +  { ZONEFILE_COMPRESSION_ZLIB,             "zlib",
          666  +    zfGenericOpen, zfGenericClose, 
          667  +    0,
          668  +    zfZlibCompressBound, zfZlibCompress, 
          669  +    zfGenericUncompressSize, zfZlibUncompress
          670  +  },
          671  +#endif /* SQLITE_HAVE_ZLIB */
          672  +#ifdef SQLITE_HAVE_BROTLI
          673  +  { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
          674  +    zfGenericOpen, zfGenericClose, 
          675  +    0,
          676  +    zfBrotliCompressBound, zfBrotliCompress, 
          677  +    zfGenericUncompressSize, zfBrotliUncompress
          678  +  },
          679  +#endif /* SQLITE_HAVE_BROTLI */
          680  +#ifdef SQLITE_HAVE_LZ4
          681  +  { ZONEFILE_COMPRESSION_LZ4,              "lz4",
          682  +    zfGenericOpen, zfGenericClose, 
          683  +    0,
          684  +    zfLz4CompressBound, zfLz4Compress, 
          685  +    zfGenericUncompressSize, zfLz4Uncompress
          686  +  },
          687  +  { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",
          688  +    zfGenericOpen, zfGenericClose, 
          689  +    0,
          690  +    zfLz4CompressBound, zfLz4hcCompress, 
          691  +    zfGenericUncompressSize, zfLz4Uncompress
          692  +  },
          693  +#endif /* SQLITE_HAVE_LZ4 */
          694  +};
          695  +
          696  +/*
          697  +** Find the ZonefileCompress object for the compression scheme named
          698  +** by zName. If successful, set output variable (*pp) to point to the
          699  +** object and return SQLITE_OK. Otherwise, return SQLITE_ERROR and
          700  +** leave output variable (*pzErr) pointing to an English language error
          701  +** message. It is the reponsibility of the caller to eventually free
          702  +** the error message buffer using sqlite3_free().
          703  +*/
          704  +static int zonefileCompress(
          705  +  const char *zName,              /* Name of requested compression method */
          706  +  ZonefileCompress **pp,          /* OUT: Pointer to compression object */
          707  +  char **pzErr                    /* OUT: Error message */
          708  +){
          709  +  int i;
          710  +  assert( *pzErr==0 );
          711  +  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
          712  +    if( sqlite3_stricmp(aZonefileCompress[i].zName, zName)==0 ){
          713  +      *pp = &aZonefileCompress[i];
          714  +      return SQLITE_OK;
          715  +    }
          716  +  }
          717  +  *pzErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName);
          718  +  return SQLITE_ERROR;
          719  +}
          720  +
          721  +/*
          722  +** Find a compression routine based on its associated integer constant
          723  +** (defined as part of the zonefile file format). If successful, return
          724  +** a pointer to the associated ZonefileCompression object. Or, if the
          725  +** nominated compression scheme is not supported in this build, return 
          726  +** NULL.
          727  +*/
          728  +static ZonefileCompress *zonefileCompressByValue(int eType){
          729  +  int i;
          730  +  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
          731  +    if( aZonefileCompress[i].eType==eType ){
          732  +      return &aZonefileCompress[i];
          733  +    }
          734  +  }
          735  +  return 0;
          736  +}
          737  +
          738  +/* End of code for compression routines
          739  +**************************************************************************/
          740  +
          741  +
          742  +/*
          743  +** A structure to store the parameters for a single zonefile_write()
          744  +** invocation. An instance of this structure is populated based on
          745  +** the json parameters passed to zonefile_write() by function
          746  +** zonefileGetParams();
          747  +*/
          748  +typedef struct ZonefileParam ZonefileParam;
          749  +struct ZonefileParam {
          750  +  ZonefileCompress *pCmpIdx;      /* For compressing the index */
          751  +  ZonefileCompress *pCmpData;     /* For compressing each frame */
          752  +  int encryptionType;
          753  +  int maxAutoFrameSize;
          754  +  int debugExtendedHeaderSize;    /* Size of extended header */
          755  +  int debugEncryptionKeyText;     /* True to allow text keys */
          756  +  char *encryptionKey;            /* Encryption key */
          757  +};
          758  +
          759  +/*
          760  +** A structure to store a deserialized zonefile header in.
          761  +*/
          762  +typedef struct ZonefileHeader ZonefileHeader;
          763  +struct ZonefileHeader {
          764  +  u32 magicNumber;
          765  +  u8 compressionTypeIndexData;
          766  +  u8 compressionTypeContent;
          767  +  u32 byteOffsetDictionary;
          768  +  u32 byteOffsetFrames;
          769  +  u32 numFrames;
          770  +  u32 numKeys;
          771  +  u8 encryptionType;
          772  +  u8 encryptionKeyIdx;
          773  +  u8 extendedHeaderVersion;
          774  +  u8 extendedHeaderSize;
          775  +};
          776  +
          777  +/*
          778  +** Resizable buffer structure used by the zonefile_write() implementation.
          779  +*/
          780  +typedef struct ZonefileBuffer ZonefileBuffer;
          781  +struct ZonefileBuffer {
          782  +  u8 *a;
          783  +  int n;
          784  +  int nAlloc;
          785  +};
          786  +
          787  +/*
          788  +** Set the error message contained in context ctx to the results of
          789  +** vprintf(zFmt, ...).
          790  +*/
          791  +static void zonefileCtxError(sqlite3_context *ctx, const char *zFmt, ...){
          792  +  char *zMsg = 0;
          793  +  va_list ap;
          794  +  va_start(ap, zFmt);
          795  +  zMsg = sqlite3_vmprintf(zFmt, ap);
          796  +  sqlite3_result_error(ctx, zMsg, -1);
          797  +  sqlite3_free(zMsg);
          798  +  va_end(ap);
          799  +}
          800  +
          801  +/*
          802  +** Retrieve the error message from the database handle associated with
          803  +** context object pCtx and copy it into pCtx itself.
          804  +*/
          805  +static void zonefileTransferError(sqlite3_context *pCtx){
          806  +  sqlite3 *db = sqlite3_context_db_handle(pCtx);
          807  +  const char *zErr = sqlite3_errmsg(db);
          808  +  sqlite3_result_error(pCtx, zErr, -1);
          809  +}
          810  +
          811  +/*
          812  +** Create an SQL statement from the printf() style format string passed
          813  +** as the 4th argument and any trailing arguments. Attempt to prepare
          814  +** it against database handle db. If successful, set output variable
          815  +** (*ppStmt) to point to the new statement handle and return SQLITE_OK.
          816  +** In this case it is the responsibility of the caller to eventually
          817  +** release the statement handle using sqlite3_finalize().
          818  +**
          819  +** Or, if an error occurs, return an SQLite error code and optionally
          820  +** set output parameter (*pzErr) to point to a buffer containing an
          821  +** English language error message. The caller must eventually free any
          822  +** such buffer using sqlite3_free().
          823  +*/
          824  +static int zonefilePrepare(
          825  +  sqlite3 *db,                    /* Database handle */
          826  +  sqlite3_stmt **ppStmt,          /* OUT: New statement handle */
          827  +  char **pzErr,                   /* OUT: Error message */
          828  +  const char *zFmt,               /* printf() style formatting string */
          829  +  ...
          830  +){
          831  +  int rc;
          832  +  va_list ap;
          833  +  char *zSql;
          834  +  va_start(ap, zFmt);
          835  +  zSql = sqlite3_vmprintf(zFmt, ap);
          836  +  *ppStmt = 0;
          837  +  if( zSql ){
          838  +    rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
          839  +    if( rc!=SQLITE_OK ){
          840  +      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
          841  +    }
          842  +    sqlite3_free(zSql);
          843  +  }else{
          844  +    rc = SQLITE_NOMEM;
          845  +  }
          846  +  return rc;
          847  +}
          848  +
          849  +/*
          850  +** Return zero if the two SQL values passed as arguments are equal, or
          851  +** non-zero otherwise. Values with different types are considered unequal,
          852  +** even if they both contain the same numeric value (e.g. 2 and 2.0).
          853  +*/
          854  +static int zonefileCompareValue(sqlite3_value *p1, sqlite3_value *p2){
          855  +  int eType;
          856  +  assert( p1 );
          857  +  eType = sqlite3_value_type(p1);
          858  +  if( sqlite3_value_type(p2)!=eType ) return 1;
          859  +  switch( eType ){
          860  +    case SQLITE_INTEGER:
          861  +      return sqlite3_value_int64(p1)!=sqlite3_value_int64(p2);
          862  +    case SQLITE_FLOAT:
          863  +      return sqlite3_value_double(p1)!=sqlite3_value_double(p2);
          864  +    case SQLITE_TEXT:
          865  +    case SQLITE_BLOB: {
          866  +      int n1 = sqlite3_value_bytes(p1);
          867  +      int n2 = sqlite3_value_bytes(p2);
          868  +      if( n1!=n2 ) return 1;
          869  +      return memcmp(sqlite3_value_blob(p1), sqlite3_value_blob(p2), n1);
          870  +    }
          871  +    default:
          872  +      assert( eType==SQLITE_NULL);
          873  +  }
          874  +
          875  +  return 0;
          876  +}
          877  +
          878  +/*
          879  +** Return true if the value object passed as the only argument contains
          880  +** integer value -1. False otherwise.
          881  +*/
          882  +int zonefileIsAutoFrame(sqlite3_value *pFrame){
          883  +  return (
          884  +      sqlite3_value_type(pFrame)==SQLITE_INTEGER 
          885  +   && sqlite3_value_int64(pFrame)==-1
          886  +  );
          887  +}
          888  +
          889  +#define SQLITE_ZONEFILE_AES_128_CTR 1
          890  +#define SQLITE_ZONEFILE_AES_128_CBC 2
          891  +#define SQLITE_ZONEFILE_AES_256_CTR 3
          892  +#define SQLITE_ZONEFILE_AES_256_CBC 4
          893  +
          894  +/*
          895  +** Argument zName is the name of an encryption scheme. If it is recognized
          896  +** output variable (*peType) is set to the corresponding integer constant
          897  +** (defined as part of the zonefile file format) and SQLITE_OK is returned.
          898  +** Or, if the named encryption method is not recognized, SQLITE_ERROR is
          899  +** returned and (*pzErr) set to point to a buffer containing an English
          900  +** language error message. In this case it is the responsibility of
          901  +** the caller to eventually free the error message buffer using a call
          902  +** to sqlite3_free().
          903  +*/
          904  +static int zonefileEncryption(const char *zName, int *peType, char **pzErr){
          905  +  struct Encryption {
          906  +    const char *zName;
          907  +    int eType;
          908  +  } a[] = {
          909  +    {"NONE", 0}, 
          910  +    {"AES_128_CTR", SQLITE_ZONEFILE_AES_128_CTR}, 
          911  +    {"AES_128_CBC", SQLITE_ZONEFILE_AES_128_CBC}, 
          912  +    {"AES_256_CTR", SQLITE_ZONEFILE_AES_256_CTR}, 
          913  +    {"AES_256_CBC", SQLITE_ZONEFILE_AES_256_CBC}, 
          914  +    {"XOR",         5}, 
          915  +  };
          916  +  int i;
          917  +
          918  +  for(i=0; i<sizeof(a)/sizeof(a[0]); i++){
          919  +    if( 0==sqlite3_stricmp(zName, a[i].zName) ){
          920  +      *peType = a[i].eType;
          921  +      return SQLITE_OK;
          922  +    }
          923  +  }
          924  +
          925  +  *pzErr = sqlite3_mprintf("unknown encryption method: %s", zName);
          926  +  return SQLITE_ERROR;
          927  +}
          928  +
          929  +/*
          930  +** Interpret the "json paramaters" argument passed to an invocation of
          931  +** the zonefile_write() SQL function. Populate the ZonefileParam indicated
          932  +** by the 3rd argument to this function with the results.
          933  +**
          934  +** If successful, return SQLITE_OK. Otherwise, return an SQLite error
          935  +** code and leave an error message in context object pCtx.
          936  +*/
          937  +static int zonefileGetParams(
          938  +  sqlite3_context *pCtx,          /* Leave any error message here */
          939  +  const char *zJson,              /* JSON configuration parameter (or NULL) */
          940  +  ZonefileParam *p                /* Populate this object before returning */
          941  +){
          942  +  sqlite3 *db = sqlite3_context_db_handle(pCtx);
          943  +  sqlite3_stmt *pStmt = 0;
          944  +  char *zErr = 0;
          945  +  int rc = SQLITE_OK;
          946  +  int rc2;
          947  +
          948  +  memset(p, 0, sizeof(ZonefileParam));
          949  +  p->maxAutoFrameSize = ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE;
          950  +  p->encryptionType = ZONEFILE_DEFAULT_ENCRYPTION;
          951  +  p->pCmpData = p->pCmpIdx = zonefileCompressByValue(ZONEFILE_COMPRESSION_NONE);
          952  +
          953  +  rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
          954  +  if( rc==SQLITE_OK ){
          955  +    sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
          956  +  }
          957  +  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
          958  +    const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
          959  +    int iVal = sqlite3_column_int(pStmt, 1);
          960  +    if( sqlite3_stricmp("debugExtendedHeaderSize", zKey)==0 ){
          961  +      if( iVal<0 || iVal>255 ){
          962  +        zErr = sqlite3_mprintf(
          963  +            "debugExtendedHeaderSize value out of range: %d", iVal
          964  +        );
          965  +        rc = SQLITE_ERROR;
          966  +      }
          967  +      p->debugExtendedHeaderSize = iVal;
          968  +    }else
          969  +    if( sqlite3_stricmp("debugEncryptionKeyText", zKey)==0 ){
          970  +      p->debugEncryptionKeyText = iVal;
          971  +    }else
          972  +    if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
          973  +      p->maxAutoFrameSize = iVal;
          974  +    }else
          975  +    if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
          976  +      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
          977  +      rc = zonefileCompress(zName, &p->pCmpIdx, &zErr);
          978  +    }else
          979  +    if( sqlite3_stricmp("compressionTypeContent", zKey)==0 ){
          980  +      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
          981  +      rc = zonefileCompress(zName, &p->pCmpData, &zErr);
          982  +    }else
          983  +    if( sqlite3_stricmp("encryptionKey", zKey)==0 ){
          984  +      const char *zKey = (const char*)sqlite3_column_text(pStmt, 1);
          985  +      p->encryptionKey = sqlite3_mprintf("%s", zKey);
          986  +      if( p->encryptionKey==0 ) rc = SQLITE_NOMEM;
          987  +    }else
          988  +    if( sqlite3_stricmp("encryptionType", zKey)==0 ){
          989  +      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
          990  +      rc = zonefileEncryption(zName, &p->encryptionType, &zErr);
          991  +    }else{
          992  +      rc = SQLITE_ERROR;
          993  +      zErr = sqlite3_mprintf("unknown parameter name: \"%s\"", zKey);
          994  +    }
          995  +  }
          996  +  rc2 = sqlite3_finalize(pStmt);
          997  +  if( rc==SQLITE_OK ) rc = rc2;
          998  +
          999  +  if( zErr ){
         1000  +    sqlite3_result_error(pCtx, zErr, -1);
         1001  +    sqlite3_free(zErr);
         1002  +  }else if( rc ){
         1003  +    sqlite3_result_error_code(pCtx, rc);
         1004  +  }else{
         1005  +    if( p->encryptionKey==0 ){
         1006  +      p->encryptionType = 0;
         1007  +    }
         1008  +  }
         1009  +  return rc;
         1010  +}
         1011  +
         1012  +/*
         1013  +** Check that there is room in buffer pBuf for at least nByte bytes more 
         1014  +** data. If not, attempt to allocate more space. If the allocation attempt
         1015  +** fails, leave an error message in context pCtx and return SQLITE_ERROR.
         1016  +**
         1017  +** If no error occurs, SQLITE_OK is returned.
         1018  +*/
         1019  +static int zonefileBufferGrow(
         1020  +  sqlite3_context *pCtx, 
         1021  +  ZonefileBuffer *pBuf, 
         1022  +  int nByte
         1023  +){
         1024  +  int nReq = pBuf->n + nByte;
         1025  +  if( nReq>pBuf->nAlloc ){
         1026  +    u8 *aNew;
         1027  +    int nNew = pBuf->nAlloc ? pBuf->nAlloc*2 : 128;
         1028  +    while( nNew<nReq ) nNew = nNew*2;
         1029  +    aNew = sqlite3_realloc(pBuf->a, nNew);
         1030  +    if( aNew==0 ){
         1031  +      sqlite3_result_error_nomem(pCtx);
         1032  +      return SQLITE_ERROR;
         1033  +    }
         1034  +    pBuf->a = aNew;
         1035  +    pBuf->nAlloc = nNew;
         1036  +  }
         1037  +  return SQLITE_OK;
         1038  +}
         1039  +
         1040  +/*
         1041  +** Free the memory allocation associated with buffer pBuf.
         1042  +*/
         1043  +static void zonefileBufferFree(ZonefileBuffer *pBuf){
         1044  +  sqlite3_free(pBuf->a);
         1045  +  memset(pBuf, 0, sizeof(ZonefileBuffer));
         1046  +}
         1047  +
         1048  +/*
         1049  +** Read and return a 64-bit unsigned integer in big-endian format from 
         1050  +** buffer aBuf.
         1051  +*/
         1052  +static u64 zonefileGet64(u8 *aBuf){
         1053  +  return (((u64)aBuf[0]) << 56)
         1054  +       + (((u64)aBuf[1]) << 48)
         1055  +       + (((u64)aBuf[2]) << 40)
         1056  +       + (((u64)aBuf[3]) << 32) 
         1057  +       + (((u64)aBuf[4]) << 24)
         1058  +       + (((u64)aBuf[5]) << 16)
         1059  +       + (((u64)aBuf[6]) <<  8)
         1060  +       + (((u64)aBuf[7]) <<  0);
         1061  +}
         1062  +
         1063  +/*
         1064  +** Append a 32-bit big-endian integer with value v to buffer pBuf. Space
         1065  +** must have already been reserved in the buffer using zonefileBufferGrow().
         1066  +*/
         1067  +static void zonefileAppend32(ZonefileBuffer *pBuf, u32 v){
         1068  +  zonefilePut32(&pBuf->a[pBuf->n], v);
         1069  +  pBuf->n += 4;
         1070  +}
         1071  +
         1072  +/*
         1073  +** Append a 64-bit big-endian integer with value v to buffer pBuf. Space
         1074  +** must have already been reserved in the buffer using zonefileBufferGrow().
         1075  +*/
         1076  +static void zonefileAppend64(ZonefileBuffer *pBuf, u64 v){
         1077  +  zonefileAppend32(pBuf, v>>32);
         1078  +  zonefileAppend32(pBuf, v & 0xFFFFFFFF);
         1079  +}
         1080  +
         1081  +/*
         1082  +** Append the n bytse of data in buffer p to buffer pBuf. Space must have
         1083  +** already been reserved in the buffer using zonefileBufferGrow().
         1084  +*/
         1085  +static void zonefileAppendBlob(ZonefileBuffer *pBuf, const u8 *p, int n){
         1086  +  memcpy(&pBuf->a[pBuf->n], p, n);
         1087  +  pBuf->n += n;
         1088  +}
         1089  +
         1090  +/*
         1091  +** Write nBuf bytes of data from buffer aBuf to the file opened by 
         1092  +** file-handle pFd. Return SQLITE_OK if successful, or SQLITE_ERROR
         1093  +** otherwise.
         1094  +*/
         1095  +static int zonefileFileWrite(FILE *pFd, const u8 *aBuf, int nBuf){
         1096  +  size_t res = fwrite(aBuf, 1, nBuf, pFd);
         1097  +  return res!=(size_t)nBuf ? SQLITE_ERROR : SQLITE_OK;
         1098  +}
         1099  +
         1100  +/*
         1101  +** Read nBuf bytes of data from offset iOff of the file opened by 
         1102  +** file-handle pFd into buffer aBuf. Return SQLITE_OK if successful, or
         1103  +** SQLITE_ERROR otherwise.
         1104  +*/
         1105  +static int zonefileFileRead(FILE *pFd, u8 *aBuf, int nBuf, i64 iOff){
         1106  +  int rc = fseek(pFd, (long)iOff, SEEK_SET);
         1107  +  if( rc==0 ){
         1108  +    rc = fread(aBuf, 1, nBuf, pFd);
         1109  +    rc = (rc==nBuf) ? SQLITE_OK : SQLITE_ERROR;
         1110  +  }else{
         1111  +    rc = SQLITE_ERROR;
         1112  +  }
         1113  +  return rc;
         1114  +}
         1115  +
         1116  +/*
         1117  +** Open the file identified by path zFile for writing (if bWrite==1) or
         1118  +** reading (if bWrite==0). If successful, return a pointer to the new
         1119  +** file-handle object. Otherwise, return NULL and set output variable
         1120  +** (*pzErr) to point to a buffer containing an English language error
         1121  +** message. It is the responsibility of the caller to eventually free
         1122  +** any error message buffer using sqlite3_free().
         1123  +*/
         1124  +static FILE *zonefileFileOpen(const char *zFile, int bWrite, char **pzErr){
         1125  +  FILE *pFd = fopen(zFile, bWrite ? "wb" : "rb");
         1126  +  if( pFd==0 ){
         1127  +    *pzErr = sqlite3_mprintf("failed to open file \"%s\" for %s",
         1128  +        zFile, bWrite ? "writing" : "reading"
         1129  +    );
         1130  +  }
         1131  +  return pFd;
         1132  +}
         1133  +
         1134  +/*
         1135  +** Close the file handle passed as the only argument.
         1136  +*/
         1137  +static void zonefileFileClose(FILE *pFd){
         1138  +  if( pFd ) fclose(pFd);
         1139  +}
         1140  +
         1141  +/*
         1142  +** Append the contents of buffer pFrom to buffer pTo. If successful, return
         1143  +** SQLITE_OK. Otherwise, return an SQLite error code and leave an error
         1144  +** message in context object pCtx.
         1145  +**
         1146  +** If argument pMethod is not NULL, then it is used along with pCmp to 
         1147  +** compress the data before appending it to pFrom. Similarly, if argument
         1148  +** pCodec is not NULL, then it is used to encrypt the data before it is
         1149  +** appended.
         1150  +*/
         1151  +static int zonefileAppendData(
         1152  +  sqlite3_context *pCtx,          /* Leave any error message here */
         1153  +  ZonefileCompress *pMethod,      /* Compression method object */
         1154  +  void *pCmp,                     /* Compression handle */
         1155  +  ZonefileCodec *pCodec,          /* Compression method, if any */
         1156  +  ZonefileBuffer *pTo,            /* Append new data here */
         1157  +  ZonefileBuffer *pFrom           /* Input buffer */
         1158  +){
         1159  +  int rc = SQLITE_OK;
         1160  +  int nNonce = pCodec ? zonefileCodecNonceSize(pCodec) : 0;
         1161  +  int nOrig = pTo->n;
         1162  +  if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
         1163  +    if( zonefileBufferGrow(pCtx, pTo, pFrom->n + nNonce) ){
         1164  +      rc = SQLITE_ERROR;
         1165  +    }else{
         1166  +      zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
         1167  +    }
         1168  +  }else{
         1169  +    int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
         1170  +    if( zonefileBufferGrow(pCtx, pTo, nReq + nNonce) ) return SQLITE_ERROR;
         1171  +    rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
         1172  +    pTo->n += nReq;
         1173  +  }
         1174  +
         1175  +  /* Encrypt the data just appended to buffer pTo. */
         1176  +  if( rc==SQLITE_OK && pCodec ){
         1177  +    zonefileCodecEncode(pCodec, &pTo->a[nOrig], pTo->n - nOrig);
         1178  +    pTo->n += nNonce;
         1179  +  }
         1180  +  return rc;
         1181  +}
         1182  +
         1183  +/*
         1184  +** Append nByte bytes of garbage data to the file opened with pFd. Return
         1185  +** SQLITE_OK if successful, or SQLITE_ERROR if an error occurs.
         1186  +**
         1187  +** Parameter nByte is only ever non-zero when running tests. So it doesn't
         1188  +** matter if this function is inefficient in those cases.
         1189  +*/
         1190  +static int zonefilePad(FILE *pFd, int nByte){
         1191  +  assert( nByte>=0 && nByte<256 );
         1192  +  if( nByte ){
         1193  +    int nRem = nByte;
         1194  +    u8 buf[17] = "0123456789ABCDEF";
         1195  +    while( nRem>0 ){
         1196  +      int n = MIN(nRem, sizeof(buf));
         1197  +      if( zonefileFileWrite(pFd, buf, n) ) return SQLITE_ERROR;
         1198  +      nRem -= n;
         1199  +    }
         1200  +  }
         1201  +  return SQLITE_OK;
         1202  +}
         1203  +
         1204  +/*
         1205  +** If character c is not a hexadecimal digit, return -1. Otherwise, return
         1206  +** the value of the hex digit (a value between 0 and 15).
         1207  +*/
         1208  +static int zonefileHexChar(char c){
         1209  +  if( c>='0' && c<='9' ) return c-'0';
         1210  +  c = c & ~0x20;
         1211  +  if( c>='A' && c<='F' ) return c-('A'-10);
         1212  +  return -1;
         1213  +}
         1214  +
         1215  +/*
         1216  +** String ZonefileParam.encryptionKey currently contains a string specified
         1217  +** for the encryptionKey attribute of a JSON object passed to SQL function
         1218  +** zonefile_write(). The string is (*pn) bytes in size.
         1219  +**
         1220  +** If the ZonefileParam.debugEncryptionKeyText flag is true this function
         1221  +** is a no-op. Otherwise, an attempt is made to overwrite the hex string in
         1222  +** ZonefileParam.encryptionKey with the corresponding binary data. If
         1223  +** successful, SQLITE_OK is returned and (*pn) is set to the number of
         1224  +** bytes in the binary key. Otherwise, if an error occurs, an SQLite error
         1225  +** code is returned and (*pzErr) set to point to an English language error
         1226  +** message. It is the responsibility of the caller to eventually free any
         1227  +** error message buffer using sqlite3_free().
         1228  +*/
         1229  +static int zonefileDecodeEncryptionKey(ZonefileParam *p, int *pn, char **pzErr){
         1230  +  if( p->debugEncryptionKeyText==0 ){
         1231  +    u8 *z = (u8*)p->encryptionKey;
         1232  +    int n = *pn;
         1233  +    int i;
         1234  +    if( n&0x01 ) goto bad_format;
         1235  +    for(i=0; i<n; i+=2){
         1236  +      int a = zonefileHexChar(z[i]);
         1237  +      int b = zonefileHexChar(z[i+1]);
         1238  +      if( a<0 || b<0 ) goto bad_format;
         1239  +      z[i/2] = (u8)(a<<4) + (u8)b;
         1240  +    }
         1241  +    *pn = n/2;
         1242  +  }
         1243  +  return SQLITE_OK;
         1244  +
         1245  + bad_format:
         1246  +  *pzErr = sqlite3_mprintf("badly formatted hex string");
         1247  +  return SQLITE_ERROR;
         1248  +}
         1249  +
         1250  +/*
         1251  +** Function:     zonefile_write(F,T[,J])
         1252  +*/
         1253  +static void zonefileWriteFunc(
         1254  +  sqlite3_context *pCtx,       /* Context object */
         1255  +  int objc,                       /* Number of SQL arguments */
         1256  +  sqlite3_value **objv            /* Array of SQL arguments */
         1257  +){
         1258  +  const char *zFile = 0;          /* File to write to */
         1259  +  const char *zTbl = 0;           /* Database object to read from */
         1260  +  const char *zJson = 0;          /* JSON configuration parameters */
         1261  +  ZonefileParam sParam;           /* Decoded JSON parameters */
         1262  +  int nKey = 0;                   /* Number of keys in new zonefile */
         1263  +  int nFrame = 0;                 /* Number of frames in new zonefile */
         1264  +  sqlite3_stmt *pStmt = 0;        /* SQL used to read data from source table */
         1265  +  FILE *pFd = 0;
         1266  +  int rc = SQLITE_OK;
         1267  +  sqlite3_value *pPrev = 0;
         1268  +  char *zErr = 0;
         1269  +  void *pCmp = 0;                 /* Data compressor handle */
         1270  +  u32 iOff;
         1271  +  ZonefileCodec *pCodec = 0;
         1272  +
         1273  +  ZonefileBuffer sFrameIdx = {0, 0, 0};     /* Array of frame offsets */
         1274  +  ZonefileBuffer sKeyIdx = {0, 0, 0};       /* Array of key locations */
         1275  +  ZonefileBuffer sData = {0, 0, 0};         /* All completed frames so far */
         1276  +  ZonefileBuffer sFrame = {0, 0, 0};        /* Current frame (uncompressed) */
         1277  +  ZonefileBuffer sDict = {0, 0, 0};         /* Compression model (if any) */
         1278  +  ZonefileBuffer sSample = {0,0,0};         /* Sample data for compressor */
         1279  +  size_t *aSample = 0;                      /* Array of sample sizes */
         1280  +
         1281  +  u8 aHdr[ZONEFILE_SZ_HEADER];    /* Space to assemble zonefile header */
         1282  +
         1283  +  assert( objc==2 || objc==3 );
         1284  +  zFile = (const char*)sqlite3_value_text(objv[0]);
         1285  +  zTbl = (const char*)sqlite3_value_text(objv[1]);
         1286  +  if( objc==3 ){
         1287  +    zJson = (const char*)sqlite3_value_text(objv[2]);
         1288  +  }
         1289  +  if( zonefileGetParams(pCtx, zJson, &sParam) ) return;
         1290  +
         1291  +  if( sParam.encryptionType!=0 ){
         1292  +    int n = strlen(sParam.encryptionKey);
         1293  +    rc = zonefileDecodeEncryptionKey(&sParam, &n, &zErr);
         1294  +    if( rc==SQLITE_OK ){
         1295  +      rc = zonefileCodecCreate(sParam.encryptionType, 
         1296  +          1, (u8*)sParam.encryptionKey, n, &pCodec, &zErr
         1297  +      );
         1298  +    }
         1299  +    if( rc!=SQLITE_OK ){
         1300  +      if( zErr ){
         1301  +        sqlite3_result_error(pCtx, zErr, -1);
         1302  +      }else{
         1303  +        sqlite3_result_error_code(pCtx, rc);
         1304  +      }
         1305  +      sqlite3_free(zErr);
         1306  +      goto zone_write_out;
         1307  +    }
         1308  +  }
         1309  +
         1310  +  /* Check that the index-data compressor is not one that uses an external
         1311  +  ** dictionary. This is not permitted as there is no sense in using a
         1312  +  ** dictionary to compress a single blob of data, and the index-data
         1313  +  ** compressor only ever compresses a single frame. Also, there is nowhere
         1314  +  ** in the file-format to store such a dictionary for the index-data.  */
         1315  +  if( sParam.pCmpIdx->xTrain ){
         1316  +    zonefileCtxError(pCtx, 
         1317  +        "compressor \"%s\" may not be used to compress the zonefile index", 
         1318  +        sParam.pCmpIdx->zName
         1319  +    );
         1320  +    goto zone_write_out;
         1321  +  }
         1322  +
         1323  +  if( sParam.pCmpData->xOpen ){
         1324  +    rc = sParam.pCmpData->xOpen(&pCmp, 0, 0);
         1325  +    if( rc!=SQLITE_OK ){
         1326  +      zonefileCtxError(pCtx, "error in compressor construction");
         1327  +      goto zone_write_out;
         1328  +    }
         1329  +  }
         1330  +
         1331  +  /* Prepare the SQL statement used to read data from the source table. This
         1332  +  ** also serves to verify the suitability of the source table schema. */
         1333  +  rc = zonefilePrepare(sqlite3_context_db_handle(pCtx), &pStmt, &zErr,
         1334  +      "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl
         1335  +  );
         1336  +
         1337  +  /* Open the file-handle used to write out the zonefile */ 
         1338  +  if( rc==SQLITE_OK ){
         1339  +    pFd = zonefileFileOpen(zFile, 1, &zErr);
         1340  +  }
         1341  +  if( pFd==0 ){
         1342  +    sqlite3_result_error(pCtx, zErr, -1);
         1343  +    sqlite3_free(zErr);
         1344  +    goto zone_write_out;
         1345  +  }
         1346  +
         1347  +  /* If the data compressor uses a global dictionary, create the dictionary
         1348  +  ** and store it in buffer sDict.  */
         1349  +  if( sParam.pCmpData->xTrain ){
         1350  +    int nSample = 0;
         1351  +
         1352  +    while( SQLITE_ROW==sqlite3_step(pStmt) ){
         1353  +      int nByte = sqlite3_column_bytes(pStmt, 2);
         1354  +      const u8 *aByte = (const u8*)sqlite3_column_blob(pStmt, 2);
         1355  +      if( zonefileBufferGrow(pCtx, &sSample, nByte) ){
         1356  +        goto zone_write_out;
         1357  +      }
         1358  +      if( (nSample & (nSample-1))==0 ){
         1359  +        int nNew = nSample ? nSample*2 : 8;
         1360  +        size_t *aNew = (size_t*)sqlite3_realloc(aSample, sizeof(size_t) * nNew);
         1361  +        if( aNew==0 ){
         1362  +          sqlite3_result_error_nomem(pCtx);
         1363  +          goto zone_write_out;
         1364  +        }
         1365  +        aSample = aNew;
         1366  +      }
         1367  +      aSample[nSample] = nByte;
         1368  +      zonefileAppendBlob(&sSample, aByte, nByte);
         1369  +      nSample++;
         1370  +    }
         1371  +    rc = sqlite3_reset(pStmt);
         1372  +    if( rc!=SQLITE_OK ){
         1373  +      zonefileTransferError(pCtx);
         1374  +      goto zone_write_out;
         1375  +    }
         1376  +
         1377  +    if( zonefileBufferGrow(pCtx, &sDict, ZONEFILE_DEFAULT_DICTSIZE) ){
         1378  +      goto zone_write_out;
         1379  +    }
         1380  +    sDict.n = sDict.nAlloc;
         1381  +
         1382  +    rc = sParam.pCmpData->xTrain(
         1383  +        pCmp, sDict.a, &sDict.n, sSample.a, aSample, nSample
         1384  +    );
         1385  +    if( rc!=SQLITE_OK ){
         1386  +      zonefileCtxError(pCtx, "error generating dictionary");
         1387  +      goto zone_write_out;
         1388  +    }
         1389  +  }
         1390  +
         1391  +  while( SQLITE_ROW==sqlite3_step(pStmt) ){
         1392  +    sqlite3_int64 k = sqlite3_column_int64(pStmt, 0);
         1393  +    sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
         1394  +    int nBlob = sqlite3_column_bytes(pStmt, 2);
         1395  +    const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);
         1396  +
         1397  +    int bAuto = zonefileIsAutoFrame(pFrame);
         1398  +    if( sFrame.n>0 ){
         1399  +      if( zonefileCompareValue(pFrame, pPrev) 
         1400  +       || (bAuto && (sFrame.n+nBlob)>sParam.maxAutoFrameSize)
         1401  +      ){
         1402  +        /* Add new entry to sFrame */
         1403  +        if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
         1404  +         || zonefileAppendData(pCtx,sParam.pCmpData,pCmp,pCodec,&sData,&sFrame)
         1405  +        ){
         1406  +          goto zone_write_out;
         1407  +        }
         1408  +        sFrame.n = 0;
         1409  +        zonefileAppend32(&sFrameIdx, sData.n);
         1410  +        sqlite3_value_free(pPrev);
         1411  +        pPrev = 0;
         1412  +        nFrame++;
         1413  +      }
         1414  +    }
         1415  +
         1416  +    if( pPrev==0 ){
         1417  +      pPrev = sqlite3_value_dup(pFrame);
         1418  +      if( pPrev==0 ){
         1419  +        sqlite3_result_error_nomem(pCtx);
         1420  +        goto zone_write_out;
         1421  +      }
         1422  +    }
         1423  +
         1424  +    /* Add new entry to sKeyIdx */
         1425  +    if( zonefileBufferGrow(pCtx, &sKeyIdx, ZONEFILE_SZ_KEYOFFSETS_ENTRY) ){
         1426  +      goto zone_write_out;
         1427  +    }
         1428  +    zonefileAppend64(&sKeyIdx, k);
         1429  +    zonefileAppend32(&sKeyIdx, nFrame);
         1430  +    zonefileAppend32(&sKeyIdx, sFrame.n);
         1431  +    zonefileAppend32(&sKeyIdx, nBlob);
         1432  +
         1433  +    /* Add uncompressed data for new entry to sFrame */
         1434  +    if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out;
         1435  +    zonefileAppendBlob(&sFrame, pBlob, nBlob);
         1436  +    nKey++;
         1437  +  }
         1438  +
         1439  +  if( sFrame.n>0 ){
         1440  +    if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
         1441  +     || zonefileAppendData(pCtx, sParam.pCmpData, pCmp, pCodec, &sData, &sFrame)
         1442  +    ){
         1443  +      goto zone_write_out;
         1444  +    }
         1445  +    zonefileAppend32(&sFrameIdx, sData.n);
         1446  +    nFrame++;
         1447  +  }
         1448  +
         1449  +  /* If a compression method was specified, compress the key-index here */
         1450  +  if( sParam.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
         1451  +    if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out;
         1452  +    zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n);
         1453  +    zonefileBufferFree(&sKeyIdx);
         1454  +    rc = zonefileAppendData(pCtx, sParam.pCmpIdx, 0, 0, &sKeyIdx, &sFrameIdx);
         1455  +    sFrameIdx.n = 0;
         1456  +    if( rc ) goto zone_write_out;
         1457  +  }
         1458  +
         1459  +  /* Create the zonefile header in the in-memory buffer */
         1460  +  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
         1461  +  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
         1462  +  aHdr[4] = (u8)sParam.pCmpIdx->eType;
         1463  +  aHdr[5] = (u8)sParam.pCmpData->eType;
         1464  +  iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
         1465  +  zonefilePut32(&aHdr[6], sDict.n ? iOff+sParam.debugExtendedHeaderSize : 0);
         1466  +  zonefilePut32(&aHdr[10], iOff + sParam.debugExtendedHeaderSize + sDict.n);
         1467  +  zonefilePut32(&aHdr[14], nFrame);
         1468  +  zonefilePut32(&aHdr[18], nKey);
         1469  +  aHdr[22] = (u8)sParam.encryptionType;
         1470  +  aHdr[23] = 0;                   /* Encryption key index */
         1471  +  aHdr[24] = 0;                   /* extended header version */
         1472  +  aHdr[25] = (u8)sParam.debugExtendedHeaderSize;
         1473  +  assert( ZONEFILE_SZ_HEADER>=26 );
         1474  +
         1475  +  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
         1476  +  if( rc==SQLITE_OK ) rc = zonefilePad(pFd, sParam.debugExtendedHeaderSize);
         1477  +  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
         1478  +  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
         1479  +  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
         1480  +  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
         1481  +  if( rc ){
         1482  +    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
         1483  +    goto zone_write_out;
         1484  +  }
         1485  +
         1486  +  if( fclose(pFd) ){
         1487  +    zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
         1488  +  }
         1489  +  pFd = 0;
         1490  +
         1491  + zone_write_out:
         1492  +  if( pCmp ) sParam.pCmpData->xClose(pCmp);
         1493  +  if( pFd ) fclose(pFd);
         1494  +  sqlite3_value_free(pPrev);
         1495  +  sqlite3_finalize(pStmt);
         1496  +  zonefileCodecDestroy(pCodec);
         1497  +  zonefileBufferFree(&sFrameIdx);
         1498  +  zonefileBufferFree(&sKeyIdx);
         1499  +  zonefileBufferFree(&sFrame);
         1500  +  zonefileBufferFree(&sDict);
         1501  +  zonefileBufferFree(&sData);
         1502  +  zonefileBufferFree(&sSample);
         1503  +  sqlite3_free(aSample);
         1504  +  sqlite3_free(sParam.encryptionKey);
         1505  +  if( rc==SQLITE_NOMEM ){
         1506  +    sqlite3_result_error_nomem(pCtx);
         1507  +  }
         1508  +}
         1509  +
         1510  +/*
         1511  +** Virtual table type for zonefile_files virtual tables.
         1512  +*/
         1513  +typedef struct ZonefileFilesTab ZonefileFilesTab;
         1514  +struct ZonefileFilesTab {
         1515  +  sqlite3_vtab base;              /* Base class - must be first */
         1516  +  sqlite3 *db;
         1517  +  char *zBase;                    /* Name of this table */
         1518  +  char *zDb;                      /* Database containing this table */
         1519  +  ZonefileGlobal *pGlobal;        /* Global object */
         1520  +  sqlite3_stmt *pInsert;          /* Insert into the %_shadow_file table */
         1521  +  sqlite3_stmt *pInsertIdx;       /* Insert into the %_shadow_idx table */
         1522  +  sqlite3_stmt *pDeleteIdx;       /* Delete by fileid from %_shadow_idx table */
         1523  +  sqlite3_stmt *pDelete;          /* Delete by rowid from %_shadow_file table */
         1524  +};
         1525  +
         1526  +/*
         1527  +** Virtual table cursor type for zonefile_files virtual tables.
         1528  +*/
         1529  +typedef struct ZonefileFilesCsr ZonefileFilesCsr;
         1530  +struct ZonefileFilesCsr {
         1531  +  sqlite3_vtab_cursor base;       /* Base class - must be first */
         1532  +  sqlite3_stmt *pSelect;
         1533  +};
         1534  +
         1535  +/*
         1536  +** This function does the work of xCreate (if bCreate!=0) or xConnect
         1537  +** (if bCreate==0) for the zonefile_files module.
         1538  +*/
         1539  +static int zffCreateConnect(
         1540  +  int bCreate,
         1541  +  void *pAux,
         1542  +  sqlite3 *db,
         1543  +  int argc, const char *const*argv,
         1544  +  sqlite3_vtab **ppVtab,
         1545  +  char **pzErr
         1546  +){
         1547  +  ZonefileFilesTab *p;
         1548  +  const char *zName = argv[2];
         1549  +  const char *zDb = argv[1];
         1550  +  int nName = strlen(zName);
         1551  +  int nDb = strlen(zDb);
         1552  +  int rc = SQLITE_OK;
         1553  +
         1554  +  if( nName<6 || memcmp(&zName[nName-6], "_files", 6)!=0 ){
         1555  +    *pzErr = sqlite3_mprintf("do not create zonefile_files tables directly!");
         1556  +    *ppVtab = 0;
         1557  +    return SQLITE_ERROR;
         1558  +  }
         1559  +
         1560  +  p = (ZonefileFilesTab*)sqlite3_malloc(sizeof(ZonefileFilesTab)+nName+1+nDb+1);
         1561  +  if( !p ){
         1562  +    rc = SQLITE_NOMEM;
         1563  +  }else{
         1564  +    memset(p, 0, sizeof(ZonefileFilesTab));
         1565  +    p->zBase = (char*)&p[1];
         1566  +    memcpy(p->zBase, zName, nName-6);
         1567  +    p->zBase[nName-6] = '\0';
         1568  +    p->zDb = &p->zBase[nName+1];
         1569  +    memcpy(p->zDb, zDb, nDb+1);
         1570  +    p->db = db;
         1571  +    p->pGlobal = (ZonefileGlobal*)pAux;
         1572  +    rc = sqlite3_declare_vtab(db, ZONEFILE_FILES_SCHEMA);
         1573  +  }
         1574  +
         1575  +  if( rc!=SQLITE_OK ){
         1576  +    sqlite3_free(p);
         1577  +    p = 0;
         1578  +  }
         1579  +  *ppVtab = (sqlite3_vtab*)p;
         1580  +  return rc;
         1581  +}
         1582  +
         1583  +/* 
         1584  +** zonefile_files virtual table module xCreate method.
         1585  +*/
         1586  +static int zffCreate(
         1587  +  sqlite3 *db,
         1588  +  void *pAux,
         1589  +  int argc, const char *const*argv,
         1590  +  sqlite3_vtab **ppVtab,
         1591  +  char **pzErr
         1592  +){
         1593  +  return zffCreateConnect(1, pAux, db, argc, argv, ppVtab, pzErr);
         1594  +}
         1595  +
         1596  +/* 
         1597  +** zonefile_files virtual table module xConnect method.
         1598  +*/
         1599  +static int zffConnect(
         1600  +  sqlite3 *db,
         1601  +  void *pAux,
         1602  +  int argc, const char *const*argv,
         1603  +  sqlite3_vtab **ppVtab,
         1604  +  char **pzErr
         1605  +){
         1606  +  return zffCreateConnect(0, pAux, db, argc, argv, ppVtab, pzErr);
         1607  +}
         1608  +
         1609  +/* 
         1610  +** zonefile_files virtual table module xDisconnect method.
         1611  +*/
         1612  +static int zffDisconnect(sqlite3_vtab *pVtab){
         1613  +  ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab;
         1614  +  sqlite3_finalize(pTab->pInsert);
         1615  +  sqlite3_finalize(pTab->pDelete);
         1616  +  sqlite3_finalize(pTab->pInsertIdx);
         1617  +  sqlite3_finalize(pTab->pDeleteIdx);
         1618  +  sqlite3_free(pTab);
         1619  +  return SQLITE_OK;
         1620  +}
         1621  +
         1622  +/* 
         1623  +** zonefile_files virtual table module xBestIndex method.
         1624  +*/
         1625  +static int zffBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){
         1626  +  return SQLITE_OK;
         1627  +}
         1628  +
         1629  +/* 
         1630  +** zonefile_files virtual table module xOpen method.
         1631  +*/
         1632  +static int zffOpen(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){
         1633  +  ZonefileFilesCsr *pCsr;
         1634  +  pCsr = (ZonefileFilesCsr*)sqlite3_malloc(sizeof(ZonefileFilesCsr));
         1635  +  if( pCsr==0 ){
         1636  +    return SQLITE_NOMEM;
         1637  +  }
         1638  +  memset(pCsr, 0, sizeof(ZonefileFilesCsr));
         1639  +  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
         1640  +  return SQLITE_OK;
         1641  +}
         1642  +
         1643  +/* 
         1644  +** Reset a ZonefileFilesCsr object to the state it is in immediately after
         1645  +** it is allocated by zffOpen().
         1646  +*/
         1647  +static void zffCursorReset(ZonefileFilesCsr *pCsr){
         1648  +  sqlite3_finalize(pCsr->pSelect);
         1649  +  pCsr->pSelect = 0;
         1650  +}
         1651  +
         1652  +/* 
         1653  +** zonefile_files virtual table module xClose method.
         1654  +*/
         1655  +static int zffClose(sqlite3_vtab_cursor *cur){
         1656  +  ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
         1657  +  zffCursorReset(pCsr);
         1658  +  sqlite3_free(pCsr);
         1659  +  return SQLITE_OK;
         1660  +}
         1661  +
         1662  +/* 
         1663  +** zonefile_files virtual table module xNext method.
         1664  +*/
         1665  +static int zffNext(sqlite3_vtab_cursor *cur){
         1666  +  ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
         1667  +  int rc;
         1668  +  if( SQLITE_ROW==sqlite3_step(pCsr->pSelect) ){
         1669  +    rc = SQLITE_OK;
         1670  +  }else{
         1671  +    rc = sqlite3_finalize(pCsr->pSelect);
         1672  +    pCsr->pSelect = 0;
         1673  +  }
         1674  +  return rc;
         1675  +}
         1676  +
         1677  +/* 
         1678  +** zonefile_files virtual table module xFilter method.
         1679  +*/
         1680  +static int zffFilter(
         1681  +  sqlite3_vtab_cursor *cur, 
         1682  +  int idxNum, const char *idxStr,
         1683  +  int argc, sqlite3_value **argv
         1684  +){
         1685  +  ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
         1686  +  ZonefileFilesTab *pTab = (ZonefileFilesTab*)(pCsr->base.pVtab);
         1687  +  int rc;
         1688  +  zffCursorReset(pCsr);
         1689  +
         1690  +  rc = zonefilePrepare(pTab->db, &pCsr->pSelect, &pTab->base.zErrMsg,
         1691  +      "SELECT filename, fileid FROM %Q.'%q_shadow_file'",
         1692  +      pTab->zDb, pTab->zBase
         1693  +  );
         1694  +  if( rc==SQLITE_OK ){
         1695  +    rc = zffNext(cur);
         1696  +  }
         1697  +  return rc;
         1698  +}
         1699  +
         1700  +/*
         1701  +** zonefile_files virtual table module xEof method.
         1702  +*/
         1703  +static int zffEof(sqlite3_vtab_cursor *cur){
         1704  +  ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
         1705  +  return pCsr->pSelect==0;
         1706  +}
         1707  +
         1708  +/*
         1709  +** Deserialize the ZONEFILE_SZ_HEADER byte zonefile header in the
         1710  +** buffer. Populate (*pHdr) with the results.
         1711  +*/
         1712  +static void zonefileHeaderDeserialize(u8 *aBuf, ZonefileHeader *pHdr){
         1713  +  pHdr->magicNumber = zonefileGet32(&aBuf[0]);
         1714  +  pHdr->compressionTypeIndexData = aBuf[4];
         1715  +  pHdr->compressionTypeContent = aBuf[5];
         1716  +  pHdr->byteOffsetDictionary = zonefileGet32(&aBuf[6]);
         1717  +  pHdr->byteOffsetFrames = zonefileGet32(&aBuf[10]);
         1718  +  pHdr->numFrames = zonefileGet32(&aBuf[14]);
         1719  +  pHdr->numKeys = zonefileGet32(&aBuf[18]);
         1720  +  pHdr->encryptionType = aBuf[22];
         1721  +  pHdr->encryptionKeyIdx = aBuf[23];
         1722  +  pHdr->extendedHeaderVersion = aBuf[24];
         1723  +  pHdr->extendedHeaderSize = aBuf[25];
         1724  +}
         1725  +
         1726  +/*
         1727  +** Read and decode a Zonefile header from the start of the file opened
         1728  +** by file-handle pFd. If successful, populate object (*pHdr) before
         1729  +** returning SQLITE_OK. Otherwise, if an error occurs, set output
         1730  +** parameter (*pzErr) to point to an English language error message and
         1731  +** return an SQLite error code.
         1732  +**
         1733  +** It is the responsibility of the caller to eventually free any error
         1734  +** message returned via (*pzErr) using sqlite3_free().
         1735  +*/
         1736  +static int zonefileReadHeader(
         1737  +  FILE *pFd,                      /* File to read from */
         1738  +  const char *zFile,              /* Name of file opened by pFd */
         1739  +  ZonefileHeader *pHdr,           /* Populate this object before returning */
         1740  +  char **pzErr                    /* OUT: Error message */
         1741  +){
         1742  +  u8 aBuf[ZONEFILE_SZ_HEADER];
         1743  +  int rc = zonefileFileRead(pFd, aBuf, ZONEFILE_SZ_HEADER, 0);
         1744  +  if( rc==SQLITE_OK ){
         1745  +    zonefileHeaderDeserialize(aBuf, pHdr);
         1746  +    if( pHdr->magicNumber!=ZONEFILE_MAGIC_NUMBER ){
         1747  +      rc = SQLITE_ERROR;
         1748  +    }
         1749  +  }
         1750  +
         1751  +  if( rc!=SQLITE_OK ){
         1752  +    *pzErr = sqlite3_mprintf(
         1753  +        "failed to read zonefile header from file \"%s\"", zFile
         1754  +    );
         1755  +  }
         1756  +
         1757  +  return rc;
         1758  +}
         1759  +
         1760  +/*
         1761  +** Read the zonefile header from file zFile and set the result of pCtx
         1762  +** to a JSON object that represents the contents. Or, if an error occurs,
         1763  +** leave an error message in pCtx. This function is called whenever the
         1764  +** "header" column of a zonefile_files virtual table is requested.
         1765  +*/
         1766  +static void zonefileJsonHeader(sqlite3_context *pCtx, const char *zFile){
         1767  +  char *zErr = 0;
         1768  +  int rc = SQLITE_OK;
         1769  +  ZonefileHeader hdr = {0};
         1770  +
         1771  +  FILE *pFd = zonefileFileOpen(zFile, 0, &zErr);
         1772  +  if( pFd ){
         1773  +    rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
         1774  +  }else{
         1775  +    rc = SQLITE_ERROR;
         1776  +  }
         1777  +
         1778  +  if( rc==SQLITE_OK ){
         1779  +    char *zJson = sqlite3_mprintf("{"
         1780  +        "\"magicNumber\":%u,"
         1781  +        "\"compressionTypeIndexData\":%u,"
         1782  +        "\"compressionTypeContent\":%u,"
         1783  +        "\"byteOffsetDictionary\":%u,"
         1784  +        "\"byteOffsetFrames\":%u,"
         1785  +        "\"numFrames\":%u,"
         1786  +        "\"numKeys\":%u,"
         1787  +        "\"encryptionType\":%u,"
         1788  +        "\"encryptionKeyIdx\":%u,"
         1789  +        "\"extendedHeaderVersion\":%u,"
         1790  +        "\"extendedHeaderSize\":%u}",
         1791  +        (u32)hdr.magicNumber,
         1792  +        (u32)hdr.compressionTypeIndexData,
         1793  +        (u32)hdr.compressionTypeContent,
         1794  +        (u32)hdr.byteOffsetDictionary,
         1795  +        (u32)hdr.byteOffsetFrames,
         1796  +        (u32)hdr.numFrames,
         1797  +        (u32)hdr.numKeys,
         1798  +        (u32)hdr.encryptionType,
         1799  +        (u32)hdr.encryptionKeyIdx,
         1800  +        (u32)hdr.extendedHeaderVersion,
         1801  +        (u32)hdr.extendedHeaderSize
         1802  +          );
         1803  +    if( zJson ){
         1804  +      sqlite3_result_text(pCtx, zJson, -1, SQLITE_TRANSIENT);
         1805  +      sqlite3_free(zJson);
         1806  +    }else{
         1807  +      sqlite3_result_error_nomem(pCtx);
         1808  +    }
         1809  +  }else{
         1810  +    sqlite3_result_error(pCtx, zErr, -1);
         1811  +    sqlite3_free(zErr);
         1812  +  }
         1813  +
         1814  +  zonefileFileClose(pFd);
         1815  +}
         1816  +
         1817  +/* 
         1818  +** zonefile_files virtual table module xColumn method.
         1819  +*/
         1820  +static int zffColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
         1821  +  ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
         1822  +  if( sqlite3_vtab_nochange(ctx) ){
         1823  +    return SQLITE_OK;
         1824  +  }
         1825  +  switch( i ){
         1826  +    case 0: /* filename */
         1827  +      sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pSelect, 0));
         1828  +      break;
         1829  +    case 1: /* ekey */
         1830  +      break;
         1831  +    default: {
         1832  +      const char *zFile = (const char*)sqlite3_column_text(pCsr->pSelect, 0);
         1833  +      zonefileJsonHeader(ctx, zFile);
         1834  +      assert( i==2 );
         1835  +      break;
         1836  +    }
         1837  +  }
         1838  +  return SQLITE_OK;
         1839  +}
         1840  +
         1841  +/* 
         1842  +** zonefile_files virtual table module xRowid method.
         1843  +*/
         1844  +static int zffRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
         1845  +  ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
         1846  +  *pRowid = sqlite3_column_int64(pCsr->pSelect, 1);
         1847  +  return SQLITE_OK;
         1848  +}
         1849  +
         1850  +/*
         1851  +** Uncompress buffer aIn/nIn using the compression routines pMethod
         1852  +** and the compressor instance pCmp into space obtained from 
         1853  +** sqlite3_malloc(). If successful, return SQLITE_OK and set output
         1854  +** parameters (*paOut) and (*pnOut) to point to the allocated buffer
         1855  +** and its size in bytes respectively. In this case it is the 
         1856  +** responsibility of the caller to eventually free buffer (*paOut)
         1857  +** using sqlite3_free().
         1858  +**
         1859  +** Or, if an error occurs, set both output parameters to zero and
         1860  +** return an SQLite error code.
         1861  +*/
         1862  +static int zonefileUncompress(
         1863  +  ZonefileCompress *pMethod,      /* Compression routines */
         1864  +  void *pCmp,                     /* Compressor instance */
         1865  +  u8 *aIn, int nIn,               /* Input buffer */
         1866  +  u8 **paOut, int *pnOut          /* Output buffer */
         1867  +){
         1868  +  int rc;
         1869  +  int nOut = pMethod->xUncompressSize(pCmp, aIn, nIn);
         1870  +  u8 *aOut = sqlite3_malloc(nOut);
         1871  +
         1872  +  assert( pMethod->eType!=ZONEFILE_COMPRESSION_NONE );
         1873  +  if( aOut==0 ){
         1874  +    rc = SQLITE_NOMEM;
         1875  +  }else{
         1876  +    rc = pMethod->xUncompress(pCmp, aOut, nOut, aIn, nIn);
         1877  +  }
         1878  +  if( rc ){
         1879  +    sqlite3_free(aOut);
         1880  +    aOut = 0;
         1881  +  }
         1882  +
         1883  +  *paOut = aOut;
         1884  +  *pnOut = nOut;
         1885  +  return rc;
         1886  +}
         1887  +
         1888  +/*
         1889  +** Attempt to find the compression methods object for the compression method
         1890  +** identified by integer constant eType (defined as part of the zonefile file
         1891  +** format). If successful, set output parameter (*pp) to point to the object
         1892  +** and return SQLITE_OK. Otherwise, return an SQLite error code and set
         1893  +** (*pzErr) to point to a buffer containing an English language error
         1894  +** message. It is the responsibility of the caller to eventually free any
         1895  +** error message buffer using sqlite3_free().
         1896  +*/
         1897  +static int zonefileFindCompress(int eType, ZonefileCompress **pp, char **pzErr){
         1898  +  int rc = SQLITE_OK;
         1899  +  ZonefileCompress *pCmp;
         1900  +  pCmp = zonefileCompressByValue(eType);
         1901  +  if( pCmp==0 ){
         1902  +    *pzErr = sqlite3_mprintf("unsupported compression method: %d", eType);
         1903  +    rc = SQLITE_ERROR;
         1904  +  }else if( pCmp->eType==ZONEFILE_COMPRESSION_NONE ){
         1905  +    pCmp = 0;
         1906  +  }
         1907  +  *pp = pCmp;
         1908  +  return rc;
         1909  +}
         1910  +
         1911  +/*
         1912  +** Argument pHdr points to a deserialized zonefile header read from the
         1913  +** zonefile opened by file handle pFd. This function attempts to read
         1914  +** the entire zonefile-index structure into space obtained from 
         1915  +** sqlite3_malloc(). If successful, it sets output parameters (*paIdx)
         1916  +** and (*pnIdx) to point to the buffer and its size in bytes respectively
         1917  +** before returning SQLITE_OK.
         1918  +**
         1919  +** Otherwise, if an error occurs, an SQLite error code is returned and
         1920  +** output parameter (*pzErr) may be set to point to a buffer containing an
         1921  +** English language error message. It is the responsibility of the caller to
         1922  +** eventually free any error message buffer using sqlite3_free().
         1923  +*/
         1924  +static int zonefileLoadIndex(
         1925  +  ZonefileHeader *pHdr,           /* Deserialized header read from file */
         1926  +  FILE *pFd,                      /* File to read from */
         1927  +  u8 **paIdx, int *pnIdx,         /* OUT: Buffer containing zonefile index */
         1928  +  char **pzErr                    /* OUT: Error message */
         1929  +){
         1930  +  ZonefileCompress *pCmp = 0;
         1931  +  int rc;
         1932  +  u8 *aIdx = 0;
         1933  +  int nIdx = 0;
         1934  +    
         1935  +  rc = zonefileFindCompress(pHdr->compressionTypeIndexData, &pCmp, pzErr);
         1936  +  if( rc==SQLITE_OK ){
         1937  +    if( pHdr->byteOffsetDictionary ){
         1938  +      nIdx = pHdr->byteOffsetDictionary - ZONEFILE_SZ_HEADER;
         1939  +    }else{
         1940  +      nIdx = pHdr->byteOffsetFrames - ZONEFILE_SZ_HEADER;
         1941  +    }
         1942  +    nIdx -= pHdr->extendedHeaderSize;
         1943  +    aIdx = (u8*)sqlite3_malloc(nIdx);
         1944  +
         1945  +    if( aIdx==0 ){
         1946  +      rc = SQLITE_NOMEM;
         1947  +    }else{
         1948  +      i64 iOff = ZONEFILE_SZ_HEADER + pHdr->extendedHeaderSize;
         1949  +      rc = zonefileFileRead(pFd, aIdx, nIdx, iOff);
         1950  +    }
         1951  +  }
         1952  +
         1953  +  if( rc==SQLITE_OK && pCmp ){
         1954  +    u8 *aUn = 0;
         1955  +    int nUn = 0;
         1956  +    rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn);
         1957  +    if( rc==SQLITE_ERROR ){
         1958  +      *pzErr = sqlite3_mprintf("failed to uncompress index");
         1959  +    }
         1960  +    sqlite3_free(aIdx);
         1961  +    aIdx = aUn;
         1962  +    nIdx = nUn;
         1963  +  }
         1964  +
         1965  +  if( rc!=SQLITE_OK ){
         1966  +    sqlite3_free(aIdx);
         1967  +    aIdx = 0;
         1968  +    nIdx = 0;
         1969  +  }
         1970  +
         1971  +  *paIdx = aIdx;
         1972  +  *pnIdx = nIdx;
         1973  +  return rc;
         1974  +}
         1975  +
         1976  +
         1977  +static int zonefilePopulateIndex(
         1978  +  ZonefileFilesTab *pTab,
         1979  +  const char *zFile,
         1980  +  i64 iFileid
         1981  +){
         1982  +  ZonefileHeader hdr = { 0 };
         1983  +  int rc;
         1984  +  FILE *pFd = zonefileFileOpen(zFile, 0, &pTab->base.zErrMsg);
         1985  +
         1986  +  if( pFd==0 ){
         1987  +    rc = SQLITE_ERROR;
         1988  +  }else{
         1989  +    rc = zonefileReadHeader(pFd, zFile, &hdr, &pTab->base.zErrMsg);
         1990  +  }
         1991  +
         1992  +  if( rc==SQLITE_OK && hdr.numKeys>0 ){
         1993  +    u8 *aKey;                     /* Entire KeyOffsets array */
         1994  +    int nKey;                     /* Size of buffer aKey[] in bytes */
         1995  +    int i;
         1996  +
         1997  +    rc = zonefileLoadIndex(&hdr, pFd, &aKey, &nKey, &pTab->base.zErrMsg);
         1998  +
         1999  +    if( rc==SQLITE_OK && pTab->pInsertIdx==0 ){
         2000  +      rc = zonefilePrepare(pTab->db, &pTab->pInsertIdx, &pTab->base.zErrMsg,
         2001  +          "INSERT INTO %Q.'%q_shadow_idx'(k, fileid, fofst, fsz, ofst, sz)"
         2002  +          "VALUES(?,?,?,?,?,?)",
         2003  +          pTab->zDb, pTab->zBase
         2004  +      );
         2005  +    }
         2006  +
         2007  +    for(i=0; (u32)i<hdr.numKeys && rc==SQLITE_OK; i++){
         2008  +      u8 *aEntry = &aKey[4*hdr.numFrames + ZONEFILE_SZ_KEYOFFSETS_ENTRY * i];
         2009  +      int iFrame = zonefileGet32(&aEntry[8]);
         2010  +      i64 iFrameOff = 0;          /* Offset of frame */
         2011  +      int szFrame;                /* Compressed size of frame */
         2012  +      i64 iOff;                   /* Offset of blob within uncompressed frame */
         2013  +      int sz;                     /* Size of blob within uncompressed frame */
         2014  +
         2015  +      szFrame = zonefileGet32(&aKey[iFrame*4]);
         2016  +      if( iFrame>0 ){
         2017  +        iFrameOff = zonefileGet32(&aKey[(iFrame-1)*4]);
         2018  +        szFrame -= (int)iFrameOff;
         2019  +      }
         2020  +      iFrameOff += hdr.byteOffsetFrames;
         2021  +      iOff = (i64)zonefileGet32(&aEntry[12]);
         2022  +      sz = (int)zonefileGet32(&aEntry[16]);
         2023  +
         2024  +      sqlite3_bind_int64(pTab->pInsertIdx, 1, (i64)zonefileGet64(&aEntry[0]));
         2025  +      sqlite3_bind_int64(pTab->pInsertIdx, 2, iFileid);
         2026  +      if( hdr.encryptionType || hdr.compressionTypeContent ){
         2027  +        sqlite3_bind_int64(pTab->pInsertIdx, 5, iOff);
         2028  +        sqlite3_bind_int(pTab->pInsertIdx, 6, sz);
         2029  +      }else{
         2030  +        iFrameOff += iOff;
         2031  +        szFrame = sz;
         2032  +      }
         2033  +      sqlite3_bind_int64(pTab->pInsertIdx, 3, iFrameOff);
         2034  +      sqlite3_bind_int(pTab->pInsertIdx, 4, szFrame);
         2035  +
         2036  +      sqlite3_step(pTab->pInsertIdx);
         2037  +      rc = sqlite3_reset(pTab->pInsertIdx);
         2038  +      if( rc!=SQLITE_OK ){
         2039  +        pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
         2040  +      }
         2041  +    }
         2042  +    sqlite3_free(aKey);
         2043  +  }
         2044  +
         2045  +  zonefileFileClose(pFd);
         2046  +  return rc;
         2047  +}
         2048  +
         2049  +/*
         2050  +** zonefile_files virtual table module xUpdate method.
         2051  +**
         2052  +** A delete specifies a single argument - the rowid of the row to remove.
         2053  +** 
         2054  +** Update and insert operations pass:
         2055  +**
         2056  +**   1. The "old" rowid, or NULL.
         2057  +**   2. The "new" rowid.
         2058  +**   3. Values for each of the 3 columns: (filename,ekey,header)
         2059  +*/
         2060  +static int zffUpdate(
         2061  +  sqlite3_vtab *pVtab, 
         2062  +  int nVal, 
         2063  +  sqlite3_value **apVal, 
         2064  +  sqlite_int64 *pRowid
         2065  +){
         2066  +  int rc = SQLITE_OK;
         2067  +  int bUpdateKey = 0;
         2068  +  ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab;
         2069  +
         2070  +  if( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ){
         2071  +    if( nVal>1 && sqlite3_value_nochange(apVal[2]) ){
         2072  +      bUpdateKey = 1;
         2073  +    }else{
         2074  +      if( pTab->pDelete==0 ){
         2075  +        rc = zonefilePrepare(pTab->db, &pTab->pDelete, &pVtab->zErrMsg,
         2076  +            "DELETE FROM %Q.'%q_shadow_file' WHERE fileid=?",
         2077  +            pTab->zDb, pTab->zBase
         2078  +            );
         2079  +      }
         2080  +      if( rc==SQLITE_OK && pTab->pDeleteIdx==0 ){
         2081  +        rc = zonefilePrepare(pTab->db, &pTab->pDeleteIdx, &pVtab->zErrMsg,
         2082  +            "DELETE FROM %Q.'%q_shadow_idx' WHERE fileid=?",
         2083  +            pTab->zDb, pTab->zBase
         2084  +            );
         2085  +      }
         2086  +
         2087  +      if( rc==SQLITE_OK ){
         2088  +        sqlite3_bind_value(pTab->pDelete, 1, apVal[0]);
         2089  +        sqlite3_step(pTab->pDelete);
         2090  +        rc = sqlite3_reset(pTab->pDelete);
         2091  +      }
         2092  +      if( rc==SQLITE_OK ){
         2093  +        sqlite3_bind_value(pTab->pDeleteIdx, 1, apVal[0]);
         2094  +        sqlite3_step(pTab->pDeleteIdx);
         2095  +        rc = sqlite3_reset(pTab->pDeleteIdx);
         2096  +      }
         2097  +    }
         2098  +  }
         2099  +  if( nVal>1 ){
         2100  +    i64 iFileid = 0;
         2101  +    if( bUpdateKey ){
         2102  +      iFileid = sqlite3_value_int64(apVal[0]);
         2103  +    }else{
         2104  +      const char *zFile = (const char*)sqlite3_value_text(apVal[2]);
         2105  +
         2106  +      if( pTab->pInsert==0 ){
         2107  +        rc = zonefilePrepare(pTab->db, &pTab->pInsert, &pVtab->zErrMsg,
         2108  +            "INSERT INTO %Q.'%q_shadow_file'(filename) VALUES(?)",
         2109  +            pTab->zDb, pTab->zBase
         2110  +        );
         2111  +      }
         2112  +
         2113  +      /* Add the new entry to the %_shadow_file table. */
         2114  +      if( rc==SQLITE_OK ){
         2115  +        sqlite3_bind_text(pTab->pInsert, 1, zFile, -1, SQLITE_TRANSIENT);
         2116  +        sqlite3_step(pTab->pInsert);
         2117  +        rc = sqlite3_reset(pTab->pInsert);
         2118  +      }
         2119  +
         2120  +      /* Populate the %_shadow_idx table with entries for all keys in
         2121  +      ** the zonefile just added to %_shadow_file.  */
         2122  +      if( rc==SQLITE_OK ){
         2123  +        iFileid = sqlite3_last_insert_rowid(pTab->db);
         2124  +        rc = zonefilePopulateIndex(pTab, zFile, iFileid);
         2125  +      }
         2126  +    }
         2127  +
         2128  +    if( rc==SQLITE_OK ){
         2129  +      int nKey = sqlite3_value_bytes(apVal[3]);
         2130  +      const u8 *aKey = (const u8*)sqlite3_value_blob(apVal[3]);
         2131  +      rc = zonefileKeyStore(
         2132  +          pTab->pGlobal, pTab->zDb, pTab->zBase, iFileid, aKey, nKey
         2133  +      );
         2134  +    }
         2135  +  }
         2136  +
         2137  +  return rc;
         2138  +}
         2139  +
         2140  +/* Each entry in the frame-cache is represented by an instance of the
         2141  +** following structure.  */
         2142  +typedef struct ZonefileFrame ZonefileFrame;
         2143  +struct ZonefileFrame {
         2144  +  i64 iFileid;               /* Fileid for this frame */
         2145  +  i64 iFrameOff;             /* Offset of frame in file */
         2146  +  int nBuf;                  /* Size of aBuf[] in bytes */
         2147  +  u8 *aBuf;                  /* Buffer containing uncompressed frame data */
         2148  +  ZonefileFrame *pLruNext;   /* Next element in LRU list */
         2149  +  ZonefileFrame *pLruPrev;   /* Previous element in LRU list */
         2150  +  ZonefileFrame *pHashNext;  /* Next element in same hash bucket */
         2151  +};
         2152  +
         2153  +typedef struct ZonefileTab ZonefileTab;
         2154  +struct ZonefileTab {
         2155  +  sqlite3_vtab base;         /* Base class - must be first */
         2156  +  sqlite3 *db;
         2157  +  sqlite3_stmt *pIdToName;   /* Translate fileid to filename */
         2158  +  ZonefileGlobal *pGlobal;
         2159  +  char *zName;               /* Name of this table */
         2160  +  char *zDb;                 /* Name of db containing this table */
         2161  +
         2162  +  /* The following variables are used to implement the frame-cache */
         2163  +  int nCacheSize;            /* Configured cache size */
         2164  +  int nCacheEntry;           /* Current number of entries in cache */
         2165  +  int nCacheBucket;          /* Number of buckets in frame cache hash table */
         2166  +  ZonefileFrame **aCache;    /* Array of hash buckets */
         2167  +  ZonefileFrame *pCacheFirst;/* Most recently used frame */
         2168  +  ZonefileFrame *pCacheLast; /* Least recently used frame (first to discard) */
         2169  +};
         2170  +
         2171  +typedef struct ZonefileCsr ZonefileCsr;
         2172  +struct ZonefileCsr {
         2173  +  sqlite3_vtab_cursor base;  /* Base class - must be first */
         2174  +  sqlite3_stmt *pSelect;     /* SELECT on %_shadow_idx table */
         2175  +};
         2176  +
         2177  +/*
         2178  +** Attempt to interpret the contents of string z as an integer. If
         2179  +** successful, set (*piVal) to the integer value and return SQLITE_OK.
         2180  +** Otherwise, return SQLITE_ERROR.
         2181  +*/
         2182  +static int zonefileParseInteger(const char *z, int *piVal){
         2183  +  *piVal = atoi(z);
         2184  +  return SQLITE_OK;
         2185  +}
         2186  +
         2187  +/*
         2188  +** Return true character i is considered to be whitespace.
         2189  +*/
         2190  +static int zonefile_isspace(char i){
         2191  +  return (i==' ');
         2192  +}
         2193  +
         2194  +/*
         2195  +** This function is called as part of constructing zonefile virtual table
         2196  +** pTab. Argument zOption is the full text of a parameter (column name)
         2197  +** specified as part of the CREATE VIRTUAL TABLE statement. This function
         2198  +** attempts to interpret the parameter and update structure pTab 
         2199  +** accordingly. If successful, SQLITE_OK is returned. Otherwise, an
         2200  +** SQLite error code is returned and (*pzErr) is left pointing to
         2201  +** a buffer containing an English language error message. It is the
         2202  +** responsibility of the caller to eventually free this buffer using
         2203  +** sqlite3_free().
         2204  +*/
         2205  +static int zonefileParseOption(
         2206  +  ZonefileTab *pTab,              /* Zonefile vtab under construction */
         2207  +  const char *zOption,            /* Text of option (column name) */
         2208  +  char **pzErr                    /* OUT: Error message */
         2209  +){
         2210  +  const char *z = zOption;
         2211  +  const char *zOpt = z;
         2212  +  int nOpt;
         2213  +  const char *zVal;
         2214  +  int rc = SQLITE_OK;
         2215  +
         2216  +  /* Skip until EOF, whitespace or "=" */
         2217  +  assert( 0==zonefile_isspace(*z) );
         2218  +  while( *z && !zonefile_isspace(*z) && *z!='=' ) z++;
         2219  +  nOpt = z-zOpt;
         2220  +
         2221  +  /* Skip whitespace. Then check there is an "=". */
         2222  +  while( zonefile_isspace(*z) ) z++;
         2223  +  if( *z!='=' ) goto parse_error;
         2224  +  z++;
         2225  +  while( zonefile_isspace(*z) ) z++;
         2226  +  zVal = z;
         2227  +
         2228  +  if( nOpt==9 && sqlite3_strnicmp(zOpt, "cachesize", 9)==0 ){
         2229  +    rc = zonefileParseInteger(zVal, &pTab->nCacheSize);
         2230  +  }else{
         2231  +    goto parse_error;
         2232  +  }
         2233  +
         2234  +  return rc;
         2235  +
         2236  + parse_error:
         2237  +  *pzErr = sqlite3_mprintf("parse error in option: %s", zOption);
         2238  +  return SQLITE_ERROR;
         2239  +}
         2240  +
         2241  +/*
         2242  +** This function does the work of xCreate (if bCreate!=0) or xConnect
         2243  +** (if bCreate==0) for the zonefile module.
         2244  +**
         2245  +**   argv[0]   -> module name  ("zonefile")
         2246  +**   argv[1]   -> database name
         2247  +**   argv[2]   -> table name
         2248  +*/
         2249  +static int zonefileCreateConnect(
         2250  +  int bCreate,
         2251  +  void *pAux,
         2252  +  sqlite3 *db,
         2253  +  int argc, const char *const*argv,
         2254  +  sqlite3_vtab **ppVtab,
         2255  +  char **pzErr
         2256  +){
         2257  +  ZonefileTab *p;
         2258  +  const char *zName = argv[2];
         2259  +  const char *zDb = argv[1];
         2260  +  int nName = strlen(zName);
         2261  +  int nDb = strlen(zDb);
         2262  +  int rc = SQLITE_OK;
         2263  +
         2264  +  p = (ZonefileTab*)sqlite3_malloc(sizeof(ZonefileTab) + nName+1 + nDb+1);
         2265  +  if( !p ){
         2266  +    rc = SQLITE_NOMEM;
         2267  +  }else{
         2268  +    int i;
         2269  +    memset(p, 0, sizeof(ZonefileTab));
         2270  +    p->zName = (char*)&p[1];
         2271  +    memcpy(p->zName, zName, nName+1);
         2272  +    p->zDb = &p->zName[nName+1];
         2273  +    memcpy(p->zDb, zDb, nDb+1);
         2274  +    p->db = db;
         2275  +    p->pGlobal = (ZonefileGlobal*)pAux;
         2276  +  
         2277  +    if( bCreate ){
         2278  +      char *zSql = sqlite3_mprintf(
         2279  +          "CREATE TABLE %Q.'%q_shadow_idx'("
         2280  +          "  k INTEGER PRIMARY KEY,"
         2281  +          "  fileid INTEGER,"
         2282  +          "  fofst INTEGER,"
         2283  +          "  fsz INTEGER,"
         2284  +          "  ofst INTEGER,"
         2285  +          "  sz INTEGER"
         2286  +          ");"
         2287  +          "CREATE TABLE %Q.'%q_shadow_file'(" 
         2288  +          "  filename TEXT,"
         2289  +          "  fileid INTEGER PRIMARY KEY"
         2290  +          ");" 
         2291  +          "CREATE VIRTUAL TABLE %Q.'%q_files' USING zonefile_files;",
         2292  +          zDb, zName, zDb, zName, zDb, zName
         2293  +      );
         2294  +  
         2295  +      if( zSql==0 ){
         2296  +        rc = SQLITE_NOMEM;
         2297  +      }else{
         2298  +        rc = sqlite3_exec(db, zSql, 0, 0, pzErr);
         2299  +        sqlite3_free(zSql);
         2300  +      }
         2301  +    }
         2302  +    
         2303  +    if( rc==SQLITE_OK ){
         2304  +      rc = sqlite3_declare_vtab(db, ZONEFILE_SCHEMA);
         2305  +    }
         2306  +
         2307  +    for(i=3; i<argc && rc==SQLITE_OK; i++){
         2308  +      rc = zonefileParseOption(p, argv[i], pzErr);
         2309  +    }
         2310  +    if( rc==SQLITE_OK && p->nCacheSize<1 ) p->nCacheSize = 1;
         2311  +  }
         2312  +
         2313  +  if( rc!=SQLITE_OK ){
         2314  +    sqlite3_free(p);
         2315  +    p = 0;
         2316  +  }
         2317  +  *ppVtab = (sqlite3_vtab*)p;
         2318  +  return rc;
         2319  +}
         2320  +
         2321  +/* 
         2322  +** zonefile virtual table module xCreate method.
         2323  +*/
         2324  +static int zonefileCreate(
         2325  +  sqlite3 *db,
         2326  +  void *pAux,
         2327  +  int argc, const char *const*argv,
         2328  +  sqlite3_vtab **ppVtab,
         2329  +  char **pzErr
         2330  +){
         2331  +  return zonefileCreateConnect(1, pAux, db, argc, argv, ppVtab, pzErr);
         2332  +}
         2333  +
         2334  +/* 
         2335  +** zonefile virtual table module xConnect method.
         2336  +*/
         2337  +static int zonefileConnect(
         2338  +  sqlite3 *db,
         2339  +  void *pAux,
         2340  +  int argc, const char *const*argv,
         2341  +  sqlite3_vtab **ppVtab,
         2342  +  char **pzErr
         2343  +){
         2344  +  return zonefileCreateConnect(0, pAux, db, argc, argv, ppVtab, pzErr);
         2345  +}
         2346  +
         2347  +/* 
         2348  +** Zonefile virtual table module xBestIndex method.
         2349  +**
         2350  +** Equality and range constraints on either the rowid or column "k" (which
         2351  +** are the same thing) are processed. Bits in the idxNum parameter are
         2352  +** set to indicate the constraints present:
         2353  +**
         2354  +**   0x01:   k == ? 
         2355  +**   0x02:   k <  ? 
         2356  +**   0x04:   k <= ? 
         2357  +**   0x08:   k >  ? 
         2358  +**   0x10:   k >= ? 
         2359  +**
         2360  +** Only some combinations are valid:
         2361  +**
         2362  +**   * If an == constraint is found, no other bits are set.
         2363  +**   * If a < constraint is present, any <= is ignored.
         2364  +**   * If a > constraint is present, any >= is ignored.
         2365  +*/
         2366  +static int zonefileBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pInfo){
         2367  +  int iEq = -1;
         2368  +  int iLt = -1;
         2369  +  int iLe = -1;
         2370  +  int iGt = -1;
         2371  +  int iGe = -1;
         2372  +  int i;
         2373  +  int idxNum = 0;
         2374  +  double cost = 1000000000.0;
         2375  +
         2376  +  for(i=0; i<pInfo->nConstraint; i++){
         2377  +    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
         2378  +    if( p->usable && p->iColumn<=0 ){
         2379  +      switch( p->op ){
         2380  +        case SQLITE_INDEX_CONSTRAINT_EQ: iEq = i; break;
         2381  +        case SQLITE_INDEX_CONSTRAINT_LT: iLt = i; break;
         2382  +        case SQLITE_INDEX_CONSTRAINT_LE: iLe = i; break;
         2383  +        case SQLITE_INDEX_CONSTRAINT_GT: iGt = i; break;
         2384  +        case SQLITE_INDEX_CONSTRAINT_GE: iGe = i; break;
         2385  +      }
         2386  +    }
         2387  +  }
         2388  +
         2389  +  if( iEq>=0 ){
         2390  +    cost = 10.0;
         2391  +    idxNum = 0x01;
         2392  +    pInfo->aConstraintUsage[iEq].argvIndex = 1;
         2393  +  }else{
         2394  +    int iIdx = 1;
         2395  +    if( iLt>=0 ){
         2396  +      pInfo->aConstraintUsage[iLt].argvIndex = iIdx++;
         2397  +      idxNum |= 0x02;
         2398  +    }else if( iLe>=0 ){
         2399  +      pInfo->aConstraintUsage[iLe].argvIndex = iIdx++;
         2400  +      idxNum |= 0x04;
         2401  +    }
         2402  +    if( iGt>=0 ){
         2403  +      pInfo->aConstraintUsage[iGt].argvIndex = iIdx++;
         2404  +      idxNum |= 0x08;
         2405  +    }else if( iGe>=0 ){
         2406  +      pInfo->aConstraintUsage[iGe].argvIndex = iIdx++;
         2407  +      idxNum |= 0x10;
         2408  +    }
         2409  +
         2410  +    if( iIdx==2 ) cost = 10000.0;
         2411  +    if( iIdx==3 ) cost = 100.0;
         2412  +  }
         2413  +
         2414  +  pInfo->idxNum = idxNum;
         2415  +  pInfo->estimatedCost = cost;
         2416  +
         2417  +  return SQLITE_OK;
         2418  +}
         2419  +
         2420  +/* 
         2421  +** zonefile virtual table module xOpen method.
         2422  +*/
         2423  +static int zonefileOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
         2424  +  ZonefileCsr *pCsr;
         2425  +  pCsr = (ZonefileCsr*)sqlite3_malloc(sizeof(ZonefileCsr));
         2426  +  if( pCsr==0 ){
         2427  +    return SQLITE_NOMEM;
         2428  +  }
         2429  +  memset(pCsr, 0, sizeof(ZonefileCsr));
         2430  +  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
         2431  +  return SQLITE_OK;
         2432  +}
         2433  +
         2434  +/* 
         2435  +** Reset a ZonefileCsr object to the state it is in immediately after
         2436  +** it is allocated by zffOpen().
         2437  +*/
         2438  +static void zonefileCursorReset(ZonefileCsr *pCsr){
         2439  +  sqlite3_finalize(pCsr->pSelect);
         2440  +  pCsr->pSelect = 0;
         2441  +}
         2442  +
         2443  +/* 
         2444  +** zonefile virtual table module xClose method.
         2445  +*/
         2446  +static int zonefileClose(sqlite3_vtab_cursor *cur){
         2447  +  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
         2448  +  zonefileCursorReset(pCsr);
         2449  +  sqlite3_free(pCsr);
         2450  +  return SQLITE_OK;
         2451  +}
         2452  +
         2453  +/* 
         2454  +** zonefile virtual table module xNext method.
         2455  +*/
         2456  +static int zonefileNext(sqlite3_vtab_cursor *cur){
         2457  +  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
         2458  +  int rc = SQLITE_OK;
         2459  +  if( SQLITE_ROW!=sqlite3_step(pCsr->pSelect) ){
         2460  +    rc = sqlite3_finalize(pCsr->pSelect);
         2461  +    if( rc!=SQLITE_OK ){
         2462  +      ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
         2463  +      pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
         2464  +    }
         2465  +    pCsr->pSelect = 0;
         2466  +  }
         2467  +  return rc;
         2468  +}
         2469  +
         2470  +/* 
         2471  +** zonefile virtual table module xFilter method.
         2472  +*/
         2473  +static int zonefileFilter(
         2474  +  sqlite3_vtab_cursor *cur, 
         2475  +  int idxNum, const char *idxStr,
         2476  +  int argc, sqlite3_value **argv
         2477  +){
         2478  +  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
         2479  +  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
         2480  +  int rc;
         2481  +
         2482  +  const char *z1 = 0;
         2483  +  const char *z2 = 0;
         2484  +
         2485  +  if( idxNum & 0x01 ){
         2486  +    z1 = "k = ?";
         2487  +  }else{
         2488  +    if( idxNum & 0x02 ) { z1 = "k < ?"; }
         2489  +    if( idxNum & 0x04 ) { z1 = "k <= ?"; }
         2490  +    if( idxNum & 0x08 ) { if( z1 ) z2 = "k > ?"; else z1 = "k > ?"; }
         2491  +    if( idxNum & 0x10 ) { if( z1 ) z2 = "k >= ?"; else z1 = "k >= ?"; }
         2492  +  }
         2493  +
         2494  +  rc = zonefilePrepare(pTab->db, &pCsr->pSelect, &pTab->base.zErrMsg, 
         2495  +      "SELECT k, fileid, fofst, fsz, ofst, sz FROM %Q.'%q_shadow_idx'%s%s%s%s",
         2496  +      pTab->zDb, pTab->zName,
         2497  +      z1 ? " WHERE " : "", z1, 
         2498  +      z2 ? " AND " : "", z2
         2499  +  );
         2500  +
         2501  +  if( z1 ) sqlite3_bind_value(pCsr->pSelect, 1, argv[0]);
         2502  +  if( z2 ) sqlite3_bind_value(pCsr->pSelect, 2, argv[1]);
         2503  +
         2504  +  if( rc==SQLITE_OK ){
         2505  +    rc = zonefileNext(cur);
         2506  +  }
         2507  +
         2508  +  return rc;
         2509  +}
         2510  +
         2511  +/*
         2512  +** zonefile virtual table module xEof method.
         2513  +*/
         2514  +static int zonefileEof(sqlite3_vtab_cursor *cur){
         2515  +  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
         2516  +  return (pCsr->pSelect==0);
         2517  +}
         2518  +
         2519  +static void zonefileFree(void *p){
         2520  +  sqlite3_free(p);
         2521  +}
         2522  +
         2523  +static int zonefileGetFile(
         2524  +  sqlite3_context *pCtx,          /* Leave error message here */
         2525  +  ZonefileCsr *pCsr,              /* Cursor object */
         2526  +  const char **pzFile             /* OUT: Pointer to current file */
         2527  +){
         2528  +  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
         2529  +  int rc = SQLITE_OK;
         2530  +  i64 iFileid;
         2531  +
         2532  +  if( pTab->pIdToName==0 ){
         2533  +    rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, 
         2534  +        "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?",
         2535  +        pTab->zDb, pTab->zName
         2536  +    );
         2537  +    if( rc==SQLITE_ERROR ) zonefileTransferError(pCtx);
         2538  +  }
         2539  +
         2540  +  if( rc==SQLITE_OK ){
         2541  +    iFileid = sqlite3_column_int64(pCsr->pSelect,1);
         2542  +    sqlite3_bind_int64(pTab->pIdToName, 1, iFileid);
         2543  +    if( SQLITE_ROW==sqlite3_step(pTab->pIdToName) ){
         2544  +      *pzFile = (const char*)sqlite3_column_text(pTab->pIdToName, 0);
         2545  +    }else{
         2546  +      rc = sqlite3_reset(pTab->pIdToName);
         2547  +      if( rc==SQLITE_OK ){
         2548  +        rc = SQLITE_CORRUPT_VTAB;
         2549  +      }else{
         2550  +        zonefileTransferError(pCtx);
         2551  +      }
         2552  +    }
         2553  +  }
         2554  +
         2555  +  return rc;
         2556  +}
         2557  +
         2558  +static void zonefileReleaseFile(ZonefileCsr *pCsr){
         2559  +  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
         2560  +  sqlite3_reset(pTab->pIdToName);
         2561  +}
         2562  +
         2563  +static int zonefileValueReadDirect(sqlite3_context *pCtx, ZonefileCsr *pCsr){
         2564  +  i64 iOff = sqlite3_column_int64(pCsr->pSelect, 2);
         2565  +  int sz = sqlite3_column_int(pCsr->pSelect, 3);
         2566  +
         2567  +  FILE *pFd = 0;                  /* File handle open on zonefile */
         2568  +  u8 *aBuf;                       /* Buffer to read blob into */
         2569  +  int rc;                         /* Return code */
         2570  +
         2571  +  aBuf = sqlite3_malloc(sz);
         2572  +  if( aBuf==0 ){
         2573  +    rc = SQLITE_NOMEM;
         2574  +  }else{
         2575  +    const char *zFile = 0;
         2576  +    /* Open the file to read the blob from */
         2577  +    rc = zonefileGetFile(pCtx, pCsr, &zFile);
         2578  +    if( rc==SQLITE_OK ){
         2579  +      char *zErr = 0;
         2580  +      pFd = zonefileFileOpen(zFile, 0, &zErr);
         2581  +      if( pFd==0 ){ 
         2582  +        sqlite3_result_error(pCtx, zErr, -1);
         2583  +        sqlite3_free(zErr);
         2584  +        rc = SQLITE_ERROR;
         2585  +      }
         2586  +      zonefileReleaseFile(pCsr);
         2587  +    }
         2588  +  }
         2589  +
         2590  +  if( rc==SQLITE_OK ){
         2591  +    rc = zonefileFileRead(pFd, aBuf, sz, iOff);
         2592  +    if( rc==SQLITE_OK ){
         2593  +      sqlite3_result_blob(pCtx, aBuf, sz, zonefileFree);
         2594  +      aBuf = 0;
         2595  +    }
         2596  +  }
         2597  +
         2598  +  zonefileFileClose(pFd);
         2599  +  sqlite3_free(aBuf);
         2600  +  return rc;
         2601  +}
         2602  +
         2603  +#ifndef NDEBUG
         2604  +static void zonefileCacheCheck(ZonefileTab *pTab){
         2605  +  ZonefileFrame *p;
         2606  +  int n = 0;
         2607  +  for(p=pTab->pCacheFirst; p; p=p->pLruNext){
         2608  +    assert( p!=pTab->pCacheFirst || p->pLruPrev==0 );
         2609  +    assert( p!=pTab->pCacheLast || p->pLruNext==0 );
         2610  +    assert( p==pTab->pCacheFirst || p->pLruPrev->pLruNext==p );
         2611  +    assert( p==pTab->pCacheLast || p->pLruNext->pLruPrev==p );
         2612  +    n++;
         2613  +  }
         2614  +  assert( n==pTab->nCacheEntry );
         2615  +}
         2616  +#else
         2617  +# define zonefileCacheCheck(x)
         2618  +#endif
         2619  +
         2620  +/*
         2621  +** Search the frame-cache belonging to virtual table pTab for an entry
         2622  +** corresponding to the frame at byte offset iFrameOff of the file with
         2623  +** file id iFile. If such an entry is found, return a pointer to it.
         2624  +** Otherwise, if no such entry exists, return NULL.
         2625  +*/
         2626  +static ZonefileFrame *zonefileCacheFind(
         2627  +  ZonefileTab *pTab, 
         2628  +  i64 iFile, 
         2629  +  i64 iFrameOff
         2630  +){
         2631  +  ZonefileFrame *pFrame;
         2632  +  for(pFrame=pTab->pCacheFirst; pFrame; pFrame=pFrame->pLruNext){
         2633  +    if( pFrame->iFileid==iFile && pFrame->iFrameOff==iFrameOff ){
         2634  +      /* Found a match. Move it to the front of the LRU list and return
         2635  +      ** a pointer to it. */
         2636  +      assert( (pFrame->pLruPrev==0)==(pFrame==pTab->pCacheFirst) );
         2637  +      assert( (pFrame->pLruNext==0)==(pFrame==pTab->pCacheLast) );
         2638  +      if( pFrame->pLruPrev ){
         2639  +        assert( pFrame==pFrame->pLruPrev->pLruNext );
         2640  +        pFrame->pLruPrev->pLruNext = pFrame->pLruNext;
         2641  +
         2642  +        if( pFrame->pLruNext ){
         2643  +          assert( pFrame==pFrame->pLruNext->pLruPrev );
         2644  +          pFrame->pLruNext->pLruPrev = pFrame->pLruPrev;
         2645  +        }else{
         2646  +          assert( pFrame==pTab->pCacheLast );
         2647  +          pTab->pCacheLast = pFrame->pLruPrev;
         2648  +          pTab->pCacheLast->pLruNext = 0;
         2649  +        }
         2650  +
         2651  +        pFrame->pLruPrev = 0;
         2652  +        pFrame->pLruNext = pTab->pCacheFirst;
         2653  +        pTab->pCacheFirst->pLruPrev = pFrame;
         2654  +        pTab->pCacheFirst = pFrame;
         2655  +      }
         2656  +
         2657  +      zonefileCacheCheck(pTab);
         2658  +      return pFrame;
         2659  +    }
         2660  +  }
         2661  +  return 0;
         2662  +}
         2663  +
         2664  +/*
         2665  +** Return the index of the hash bucket that iFileid/iFrameOff belongs to.
         2666  +*/
         2667  +int zonefileCacheHash(ZonefileTab *pTab, i64 iFileid, i64 iFrameOff){
         2668  +  u32 h;
         2669  +  h = (iFileid & 0xFFFFFFFF) ^ (iFrameOff & 0xFFFFFFFF);
         2670  +  return (h % pTab->nCacheBucket);
         2671  +}
         2672  +
         2673  +/*
         2674  +** Store the frame object passed as the second argument in the frame
         2675  +** cache belonging to table pTab.
         2676  +*/
         2677  +static int zonefileCacheStore(ZonefileTab *pTab, ZonefileFrame *pFrame){
         2678  +  int h;
         2679  +
         2680  +  /* Allocate the hash table if it has not already been allocated. */
         2681  +  if( pTab->aCache==0 ){
         2682  +    int nByte = pTab->nCacheSize * 2 * sizeof(ZonefileFrame*);
         2683  +    pTab->aCache = (ZonefileFrame**)sqlite3_malloc(nByte);
         2684  +    if( pTab->aCache==0 ){
         2685  +      sqlite3_free(pFrame);
         2686  +      return SQLITE_NOMEM;
         2687  +    }
         2688  +    memset(pTab->aCache, 0, nByte);
         2689  +    pTab->nCacheBucket = pTab->nCacheSize * 2;
         2690  +  }
         2691  +
         2692  +  /* Add the new entry to the hash table */
         2693  +  h = zonefileCacheHash(pTab, pFrame->iFileid, pFrame->iFrameOff);
         2694  +  assert( h>=0 && h<pTab->nCacheBucket );
         2695  +  pFrame->pHashNext = pTab->aCache[h];
         2696  +  pTab->aCache[h] = pFrame;
         2697  +
         2698  +  /* Add the new entry to the LRU list */
         2699  +  pFrame->pLruPrev = 0;
         2700  +  pFrame->pLruNext = pTab->pCacheFirst;
         2701  +  pTab->pCacheFirst = pFrame;
         2702  +  if( pFrame->pLruNext==0 ){
         2703  +    pTab->pCacheLast = pFrame;
         2704  +  }else{
         2705  +    pFrame->pLruNext->pLruPrev = pFrame;
         2706  +  }
         2707  +  pTab->nCacheEntry++;
         2708  +
         2709  +  if( pTab->nCacheEntry>pTab->nCacheSize ){
         2710  +    ZonefileFrame **pp;
         2711  +
         2712  +    /* Remove the oldest entry from the LRU list. */
         2713  +    ZonefileFrame *pLast = pTab->pCacheLast;
         2714  +    assert( pTab->pCacheLast );
         2715  +    assert( pTab->nCacheEntry>1 );
         2716  +    pTab->pCacheLast = pLast->pLruPrev;
         2717  +    pTab->pCacheLast->pLruNext = 0;
         2718  +    pTab->nCacheEntry--;
         2719  +
         2720  +    /* Remove the same entry from the hash table. */
         2721  +    h = zonefileCacheHash(pTab, pLast->iFileid, pLast->iFrameOff);
         2722  +    assert( h>=0 && h<pTab->nCacheBucket );
         2723  +    for(pp=&pTab->aCache[h]; *pp!=pLast; pp=&((*pp)->pHashNext));
         2724  +    *pp = pLast->pHashNext;
         2725  +    sqlite3_free(pLast);
         2726  +  }
         2727  +  zonefileCacheCheck(pTab);
         2728  +
         2729  +  return SQLITE_OK;
         2730  +}
         2731  +
         2732  +/*
         2733  +** Delete all resources associated with the frame-cache for table pTab.
         2734  +*/
         2735  +static void zonefileCacheDelete(ZonefileTab *pTab){
         2736  +  ZonefileFrame *p;
         2737  +  ZonefileFrame *pNext;
         2738  +  for(p=pTab->pCacheFirst; p; p=pNext){
         2739  +    pNext = p->pLruNext;
         2740  +    sqlite3_free(p);
         2741  +  }
         2742  +  sqlite3_free(pTab->aCache);
         2743  +}
         2744  +
         2745  +static int zonefileValueReadCache(sqlite3_context *pCtx, ZonefileCsr *pCsr){
         2746  +  int rc = SQLITE_OK;
         2747  +  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
         2748  +  ZonefileFrame *pFrame = 0;
         2749  +  i64 iFile = sqlite3_column_int64(pCsr->pSelect, 1);
         2750  +  i64 iFrameOff = sqlite3_column_int64(pCsr->pSelect, 2);
         2751  +  i64 iKeyOff = sqlite3_column_int64(pCsr->pSelect, 4);
         2752  +  int nKeySz = sqlite3_column_int(pCsr->pSelect, 5);
         2753  +
         2754  +  /* Check if this frame is already in the cache. If not, read it from 
         2755  +  ** the file.  */
         2756  +  pFrame = zonefileCacheFind(pTab, iFile, iFrameOff);
         2757  +  if( pFrame==0 ){
         2758  +    const char *zFile = 0;
         2759  +    char *zErr = 0;
         2760  +    FILE *pFd = 0;
         2761  +    ZonefileHeader hdr = { 0 };
         2762  +    ZonefileCompress *pCmpMethod = 0;
         2763  +    ZonefileCodec *pCodec = 0;
         2764  +    void *pCmp = 0;
         2765  +
         2766  +    /* Open the file to read the blob from */
         2767  +    rc = zonefileGetFile(pCtx, pCsr, &zFile);
         2768  +    if( rc==SQLITE_OK ){
         2769  +      pFd = zonefileFileOpen(zFile, 0, &zErr);
         2770  +      if( pFd==0 ) rc = SQLITE_ERROR;
         2771  +    }
         2772  +
         2773  +    /* Read the zonefile header */
         2774  +    if( rc==SQLITE_OK ){
         2775  +      rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
         2776  +    }
         2777  +
         2778  +    /* Find the compression method and open the compressor handle. */
         2779  +    if( rc==SQLITE_OK ){
         2780  +      rc = zonefileFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr);
         2781  +    }
         2782  +    if( pCmpMethod ){
         2783  +      int nDict = 0;
         2784  +      u8 *aDict = 0;
         2785  +      assert( rc==SQLITE_OK );
         2786  +      if( hdr.byteOffsetDictionary ){
         2787  +        nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary;
         2788  +        aDict = sqlite3_malloc(nDict);
         2789  +        if( aDict==0 ){
         2790  +          rc = SQLITE_NOMEM;
         2791  +        }else{
         2792  +          rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary);
         2793  +        }
         2794  +      }
         2795  +      if( rc==SQLITE_OK ){
         2796  +        rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
         2797  +      }
         2798  +      sqlite3_free(aDict);
         2799  +    }
         2800  +
         2801  +    /* Find the encryption method and key. */
         2802  +    if( rc==SQLITE_OK && hdr.encryptionType ){
         2803  +      const u8 *a = 0;
         2804  +      int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFile, &a);
         2805  +      if( n==0 ){
         2806  +        zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
         2807  +        rc = SQLITE_ERROR;
         2808  +      }else{
         2809  +        rc = zonefileCodecCreate(hdr.encryptionType, 0, (u8*)a,n,&pCodec,&zErr);
         2810  +      }
         2811  +    }
         2812  +
         2813  +    /* Read some data into memory. */
         2814  +    if( rc==SQLITE_OK ){
         2815  +      int szFrame = sqlite3_column_int(pCsr->pSelect, 3);
         2816  +
         2817  +      pFrame = (ZonefileFrame*)sqlite3_malloc(szFrame + sizeof(ZonefileFrame));
         2818  +      if( pFrame==0 ){
         2819  +        rc = SQLITE_NOMEM;
         2820  +      }else{
         2821  +        memset(pFrame, 0, sizeof(ZonefileFrame));
         2822  +        pFrame->aBuf = (u8*)&pFrame[1];
         2823  +        pFrame->nBuf = szFrame;
         2824  +        pFrame->iFrameOff = iFrameOff;
         2825  +        pFrame->iFileid = iFile;
         2826  +        rc = zonefileFileRead(pFd, pFrame->aBuf, szFrame, iFrameOff);
         2827  +      }
         2828  +    }
         2829  +
         2830  +    /* Decrypt data if necessary */
         2831  +    if( rc==SQLITE_OK && pCodec ){
         2832  +      zonefileCodecDecode(pCodec, pFrame->aBuf, pFrame->nBuf);
         2833  +      pFrame->nBuf -= zonefileCodecNonceSize(pCodec);
         2834  +    }
         2835  +
         2836  +    /* Uncompress data if required */
         2837  +    if( rc==SQLITE_OK && pCmpMethod ){
         2838  +      ZonefileFrame *p = 0;
         2839  +      int nOut = pCmpMethod->xUncompressSize(pCmp, pFrame->aBuf, pFrame->nBuf);
         2840  +
         2841  +      p = (ZonefileFrame*)sqlite3_malloc(nOut + sizeof(ZonefileFrame));
         2842  +      if( p==0 ){
         2843  +        rc = SQLITE_NOMEM;
         2844  +      }else{
         2845  +        memset(p, 0, sizeof(ZonefileFrame));
         2846  +        p->aBuf = (u8*)&p[1];
         2847  +        p->nBuf = nOut;
         2848  +        p->iFrameOff = iFrameOff;
         2849  +        p->iFileid = iFile;
         2850  +        rc = pCmpMethod->xUncompress(
         2851  +            pCmp, p->aBuf, p->nBuf, pFrame->aBuf, pFrame->nBuf
         2852  +        );
         2853  +        sqlite3_free(pFrame);
         2854  +        pFrame = p;
         2855  +      }
         2856  +    }
         2857  +
         2858  +    if( rc!=SQLITE_OK ){
         2859  +      sqlite3_free(pFrame);
         2860  +      pFrame = 0;
         2861  +    }else{
         2862  +      rc = zonefileCacheStore(pTab, pFrame);
         2863  +      if( rc!=SQLITE_OK ) pFrame = 0;
         2864  +    }
         2865  +    zonefileReleaseFile(pCsr);
         2866  +    zonefileFileClose(pFd);
         2867  +    zonefileCodecDestroy(pCodec);
         2868  +    if( pCmpMethod ) pCmpMethod->xClose(pCmp);
         2869  +
         2870  +    if( zErr ){
         2871  +      assert( rc!=SQLITE_OK );
         2872  +      sqlite3_result_error(pCtx, zErr, -1);
         2873  +      sqlite3_free(zErr);
         2874  +    }
         2875  +  }
         2876  +
         2877  +  if( pFrame ){
         2878  +    assert( rc==SQLITE_OK );
         2879  +    sqlite3_result_blob(pCtx, &pFrame->aBuf[iKeyOff], nKeySz, SQLITE_TRANSIENT);
         2880  +  }
         2881  +
         2882  +  return rc;
         2883  +}
         2884  +
         2885  +/* 
         2886  +** zonefile virtual table module xColumn method.
         2887  +*/
         2888  +static int zonefileColumn(
         2889  +  sqlite3_vtab_cursor *cur, 
         2890  +  sqlite3_context *pCtx, 
         2891  +  int i
         2892  +){
         2893  +  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
         2894  +  int rc = SQLITE_OK;
         2895  +  switch( i ){
         2896  +    case 0: /* k */
         2897  +      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 0));
         2898  +      break;
         2899  +    case 1: /* v */
         2900  +      if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){
         2901  +        rc = zonefileValueReadDirect(pCtx, pCsr);
         2902  +      }else{
         2903  +        rc = zonefileValueReadCache(pCtx, pCsr);
         2904  +      }
         2905  +      break;
         2906  +    case 2: /* fileid */
         2907  +      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 1));
         2908  +      break;
         2909  +    default: { /* sz */
         2910  +      int iCol;
         2911  +      if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){
         2912  +        iCol = 3;
         2913  +      }else{
         2914  +        iCol = 5;
         2915  +      }
         2916  +      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, iCol));
         2917  +      break;
         2918  +    }
         2919  +  }
         2920  +  return rc;
         2921  +}
         2922  +
         2923  +/* 
         2924  +** zonefile virtual table module xDisconnect method.
         2925  +*/
         2926  +static int zonefileDisconnect(sqlite3_vtab *pVtab){
         2927  +  ZonefileTab *pTab = (ZonefileTab*)pVtab;
         2928  +  zonefileCacheDelete(pTab);
         2929  +  sqlite3_finalize(pTab->pIdToName);
         2930  +  sqlite3_free(pTab);
         2931  +  return SQLITE_OK;
         2932  +}
         2933  +
         2934  +/* 
         2935  +** zonefile virtual table module xDestroy method.
         2936  +*/
         2937  +static int zonefileDestroy(sqlite3_vtab *pVtab){
         2938  +  ZonefileTab *pTab = (ZonefileTab*)pVtab;
         2939  +  int rc = SQLITE_OK;
         2940  +  char *zSql = sqlite3_mprintf(
         2941  +      "DROP TABLE IF EXISTS %Q.'%q_shadow_idx';"
         2942  +      "DROP TABLE IF EXISTS %Q.'%q_shadow_file';"
         2943  +      "DROP TABLE IF EXISTS %Q.'%q_shadow_frame';"
         2944  +      "DROP TABLE IF EXISTS %Q.'%q_files';",
         2945  +      pTab->zDb, pTab->zName, pTab->zDb, pTab->zName, 
         2946  +      pTab->zDb, pTab->zName, pTab->zDb, pTab->zName
         2947  +  );
         2948  +  if( zSql==0 ){
         2949  +    rc = SQLITE_NOMEM;
         2950  +  }else{
         2951  +    rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0);
         2952  +    sqlite3_free(zSql);
         2953  +  }
         2954  +
         2955  +  if( rc==SQLITE_OK ){
         2956  +    zonefileDisconnect(pVtab);
         2957  +  }
         2958  +  return rc;
         2959  +}
         2960  +
         2961  +/* 
         2962  +** zonefile virtual table module xRowid method.
         2963  +*/
         2964  +static int zonefileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
         2965  +  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
         2966  +  *pRowid = sqlite3_column_int64(pCsr->pSelect, 0);
         2967  +  return SQLITE_OK;
         2968  +}
         2969  +
         2970  +/*
         2971  +** Register the "zonefile" extensions.
         2972  +*/
         2973  +static int zonefileRegister(sqlite3 *db){
         2974  +  static sqlite3_module filesModule = {
         2975  +    0,                            /* iVersion */
         2976  +    zffCreate,                    /* xCreate - create a table */
         2977  +    zffConnect,                   /* xConnect - connect to an existing table */
         2978  +    zffBestIndex,                 /* xBestIndex - Determine search strategy */
         2979  +    zffDisconnect,                /* xDisconnect - Disconnect from a table */
         2980  +    zffDisconnect,                /* xDestroy - Drop a table */
         2981  +    zffOpen,                      /* xOpen - open a cursor */
         2982  +    zffClose,                     /* xClose - close a cursor */
         2983  +    zffFilter,                    /* xFilter - configure scan constraints */
         2984  +    zffNext,                      /* xNext - advance a cursor */
         2985  +    zffEof,                       /* xEof */
         2986  +    zffColumn,                    /* xColumn - read data */
         2987  +    zffRowid,                     /* xRowid - read data */
         2988  +    zffUpdate,                    /* xUpdate - write data */
         2989  +    0,                            /* xBegin - begin transaction */
         2990  +    0,                            /* xSync - sync transaction */
         2991  +    0,                            /* xCommit - commit transaction */
         2992  +    0,                            /* xRollback - rollback transaction */
         2993  +    0,                            /* xFindFunction - function overloading */
         2994  +    0,                            /* xRename - rename the table */
         2995  +    0,                            /* xSavepoint */
         2996  +    0,                            /* xRelease */
         2997  +    0                             /* xRollbackTo */
         2998  +  };
         2999  +  static sqlite3_module zonefileModule = {
         3000  +    0,                            /* iVersion */
         3001  +    zonefileCreate,               /* xCreate - create a table */
         3002  +    zonefileConnect,              /* xConnect - connect to an existing table */
         3003  +    zonefileBestIndex,            /* xBestIndex - Determine search strategy */
         3004  +    zonefileDisconnect,           /* xDisconnect - Disconnect from a table */
         3005  +    zonefileDestroy,              /* xDestroy - Drop a table */
         3006  +    zonefileOpen,                 /* xOpen - open a cursor */
         3007  +    zonefileClose,                /* xClose - close a cursor */
         3008  +    zonefileFilter,               /* xFilter - configure scan constraints */
         3009  +    zonefileNext,                 /* xNext - advance a cursor */
         3010  +    zonefileEof,                  /* xEof */
         3011  +    zonefileColumn,               /* xColumn - read data */
         3012  +    zonefileRowid,                /* xRowid - read data */
         3013  +    0,                            /* xUpdate - write data */
         3014  +    0,                            /* xBegin - begin transaction */
         3015  +    0,                            /* xSync - sync transaction */
         3016  +    0,                            /* xCommit - commit transaction */
         3017  +    0,                            /* xRollback - rollback transaction */
         3018  +    0,                            /* xFindFunction - function overloading */
         3019  +    0,                            /* xRename - rename the table */
         3020  +    0,                            /* xSavepoint */
         3021  +    0,                            /* xRelease */
         3022  +    0                             /* xRollbackTo */
         3023  +  };
         3024  +
         3025  +  struct Func {
         3026  +    const char *z;
         3027  +    int n;
         3028  +    void (*x)(sqlite3_context*,int,sqlite3_value**);
         3029  +  } aFunc[] = {
         3030  +    { "zonefile_write", 2, zonefileWriteFunc },
         3031  +    { "zonefile_write", 3, zonefileWriteFunc },
         3032  +  };
         3033  +  ZonefileGlobal *pGlobal = 0;
         3034  +
         3035  +  int rc = SQLITE_OK;
         3036  +  int i;
         3037  +
         3038  +  for(i=0; rc==SQLITE_OK && i<sizeof(aFunc)/sizeof(aFunc[0]); i++){
         3039  +    struct Func *p = &aFunc[i];
         3040  +    rc = sqlite3_create_function(db, p->z, p->n, SQLITE_ANY, 0, p->x, 0, 0);
         3041  +  }
         3042  +
         3043  +  if( rc==SQLITE_OK ){
         3044  +    pGlobal = (ZonefileGlobal*)sqlite3_malloc(sizeof(ZonefileGlobal));
         3045  +    if( pGlobal==0 ){
         3046  +      rc = SQLITE_NOMEM;
         3047  +    }else{
         3048  +      memset(pGlobal, 0, sizeof(ZonefileGlobal));
         3049  +    }
         3050  +  }
         3051  +
         3052  +  if( rc==SQLITE_OK ){
         3053  +    rc = sqlite3_create_module(
         3054  +        db, "zonefile_files", &filesModule, (void*)pGlobal
         3055  +    );
         3056  +  }
         3057  +  if( rc==SQLITE_OK ){
         3058  +    rc = sqlite3_create_module_v2(db, "zonefile", &zonefileModule, 
         3059  +        (void*)pGlobal, zonefileKeyDestroy
         3060  +    );
         3061  +    pGlobal = 0;
         3062  +  }
         3063  +
         3064  +  sqlite3_free(pGlobal);
         3065  +  return rc;
         3066  +}
         3067  +
         3068  +#else         /* SQLITE_OMIT_VIRTUALTABLE */
         3069  +# define zonefileRegister(x) SQLITE_OK
         3070  +#endif
         3071  +
         3072  +#ifdef _WIN32
         3073  +__declspec(dllexport)
         3074  +#endif
         3075  +int sqlite3_zonefile_init(
         3076  +  sqlite3 *db, 
         3077  +  char **pzErrMsg, 
         3078  +  const sqlite3_api_routines *pApi
         3079  +){
         3080  +  SQLITE_EXTENSION_INIT2(pApi);
         3081  +  (void)pzErrMsg;  /* Unused parameter */
         3082  +  return zonefileRegister(db);
         3083  +}

Added ext/zonefile/zonefile1.test.

            1  +# 2018 Feb 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# The focus of this file is testing the zonefile extension.
           13  +#
           14  +
           15  +if {![info exists testdir]} {
           16  +  set testdir [file join [file dirname [info script]] .. .. test]
           17  +}
           18  +source [file join $testdir tester.tcl]
           19  +set testprefix zonefile1
           20  +load_static_extension db zonefile
           21  +
           22  +do_execsql_test 1.0 {
           23  +  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
           24  +  INSERT INTO zz VALUES(1, -1, -1, randomblob(100));
           25  +  INSERT INTO zz VALUES(2, -1, -1, randomblob(100));
           26  +  INSERT INTO zz VALUES(3, -1, -1, randomblob(100));
           27  +}
           28  +
           29  +do_execsql_test 1.1 {
           30  +  SELECT zonefile_write('test.zonefile', 'zz');
           31  +} {{}}
           32  +
           33  +do_execsql_test 1.2 {
           34  +  CREATE VIRTUAL TABLE z1 USING zonefile;
           35  +  SELECT name FROM sqlite_master WHERE name LIKE 'z1%' ORDER BY 1;
           36  +} {z1 z1_files z1_shadow_file z1_shadow_idx}
           37  +
           38  +do_execsql_test 1.3 {
           39  +  INSERT INTO z1_files(filename) VALUES('test.zonefile');
           40  +  SELECT filename, 
           41  +         json_extract(header, '$.magicNumber'),
           42  +         json_extract(header, '$.numFrames'),
           43  +         json_extract(header, '$.numKeys')
           44  +         FROM z1_files;
           45  +} {test.zonefile 1179332920 1 3}
           46  +
           47  +do_execsql_test 1.4 { SELECT count(*) FROM z1_shadow_idx } 3
           48  +
           49  +do_execsql_test 1.5.1 { SELECT k FROM z1 } {1 2 3}
           50  +do_execsql_test 1.5.2 { SELECT fileid FROM z1 } {1 1 1}
           51  +do_execsql_test 1.5.4 { SELECT sz FROM z1 } {100 100 100}
           52  +
           53  +do_execsql_test 1.5.5 {
           54  +  SELECT zz.v==z1.v FROM zz, z1 WHERE zz.k=z1.k
           55  +} {1 1 1}
           56  +
           57  +do_execsql_test 1.5 {
           58  +  DELETE FROM z1_files;
           59  +  SELECT * FROM z1_files;
           60  +} {}
           61  +
           62  +do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0
           63  +
           64  +do_execsql_test 1.7 { DROP TABLE z1 }
           65  +
           66  +do_execsql_test 1.8 {
           67  +  SELECT * FROM sqlite_master WHERE name LIKE 'z1%';
           68  +}
           69  +
           70  +#-------------------------------------------------------------------------
           71  +# Figure out which compression algorithms, if any, are supported by
           72  +# this build. Populate the global list $COMPRESSION_METHODS with the
           73  +# result.
           74  +reset_db
           75  +load_static_extension db zonefile
           76  +do_execsql_test 2.0 {
           77  +  CREATE TABLE bb(
           78  +    k INTEGER PRIMARY KEY, 
           79  +    frame INTEGER DEFAULT -1, 
           80  +    idx INTEGER DEFAULT -1, 
           81  +    v BLOB
           82  +  );
           83  +  INSERT INTO bb(k, v) VALUES(1, randomblob(100));
           84  +}
           85  +set COMPRESSION_METHODS [list]
           86  +foreach cmp {
           87  +  none zlib zstd zstd_global_dict lz4 lz4hc brotli nosuchcmp
           88  +} {
           89  +  set res [catchsql {
           90  +    WITH p(n,v) AS (
           91  +        VALUES('compressionTypeContent', $cmp)
           92  +    )
           93  +    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
           94  +  }]
           95  +
           96  +  if {[lindex $res 0]==0} {
           97  +    lappend COMPRESSION_METHODS $cmp
           98  +  }
           99  +}
          100  +
          101  +# Check that it is not possible to use zstd_global_dict to compress
          102  +# the zonefile index.
          103  +#
          104  +if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} {
          105  +  do_catchsql_test 2.1 {
          106  +    WITH p(n,v) AS (
          107  +        VALUES('compressionTypeIndexData', 'zstd_global_dict')
          108  +    )
          109  +    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
          110  +  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
          111  +}
          112  +
          113  +set extra_header 0
          114  +set cachesize 0
          115  +foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
          116  +  if {$cmpidx == "zstd_global_dict"} continue
          117  +  reset_db
          118  +  load_static_extension db zonefile
          119  +
          120  +  set tn "$cmp/$cmpidx"
          121  +  set extra_header [expr {$extra_header ? 0 : 100}]
          122  +  set cachesize [expr {$cachesize ? 0 : 10}]
          123  +
          124  +  do_execsql_test 2.$tn.0.1 {
          125  +    CREATE TABLE zz(
          126  +      k INTEGER PRIMARY KEY, 
          127  +      frame INTEGER DEFAULT -1, 
          128  +      idx INTEGER DEFAULT -1, 
          129  +      v BLOB
          130  +    );
          131  +    CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB);
          132  +  }
          133  +
          134  +  do_execsql_test 2.$tn.0.2 "
          135  +    CREATE VIRTUAL TABLE zone USING zonefile(cachesize = $cachesize)
          136  +  " {}
          137  +  
          138  +  set nMinByte   0
          139  +  set nMaxByte 444
          140  +  foreach {zonefile lKey} {
          141  +    test1.zonefile {195 1238 298 405 297}
          142  +    test2.zonefile {124 1624 82 1929}
          143  +    test3.zonefile {932 683 1751 410 41}
          144  +    test4.zonefile {427 1491}
          145  +    test5.zonefile {1004 473 801 394 1672 816 1577}
          146  +    test6.zonefile {1374 1454 1005}
          147  +    test7.zonefile {450 241 319 133}
          148  +    test8.zonefile {1414 900 1406 1917 127 673}
          149  +    test9.zonefile {1192 226 988 1292 718 1345 1675}
          150  +    test10.zonefile {314}
          151  +    test11.zonefile {1177 1597 60 532 291 1164 812}
          152  +    test12.zonefile {1168 1290 1585 939 1916}
          153  +    test13.zonefile {644 1784 1476 1283 433 506}
          154  +    test14.zonefile {1141 1547 1506 364}
          155  +    test15.zonefile {1756 1885 844 1880 1896 354}
          156  +    test16.zonefile {1383 1928 1371}
          157  +    test17.zonefile {93}
          158  +    test18.zonefile {1067}
          159  +    test19.zonefile {642}
          160  +    test20.zonefile {1380 1857}
          161  +    test21.zonefile {288 293 1968 1207 1739 231 300}
          162  +    test22.zonefile {651 1007 607 830 299 1431}
          163  +    test23.zonefile {81 1651 543 1949 256 119 1088}
          164  +    test24.zonefile {1278 2024 682 1115 194 636 1804}
          165  +    test25.zonefile {514 1155 171 2015 791}
          166  +    test26.zonefile {1615 1228 147 1464}
          167  +    test27.zonefile {55 1130 781 678 78}
          168  +    test28.zonefile {1981 1401 1178}
          169  +    test29.zonefile {1754 864 183 1953 1901}
          170  +    test30.zonefile {1461 817}
          171  +    test31.zonefile {1720 1722 686 1833}
          172  +  } {
          173  +    forcedelete $zonefile
          174  +    execsql { DELETE FROM zz; }
          175  +    foreach k $lKey {
          176  +      execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
          177  +    }
          178  +    execsql { INSERT INTO rt SELECT k, v FROM zz }
          179  +    execsql { 
          180  +      WITH p(n,v) AS (
          181  +        VALUES('maxAutoFrameSize', 2000) UNION ALL
          182  +        VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
          183  +        VALUES('compressionTypeContent', $cmp) UNION ALL
          184  +        VALUES('debugExtendedHeaderSize', $extra_header)
          185  +      )
          186  +      SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
          187  +      INSERT INTO zone_files(filename) VALUES($zonefile);
          188  +    }
          189  +  }
          190  +  
          191  +  do_execsql_test 2.$tn.1 {
          192  +    SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v
          193  +  }
          194  +  do_execsql_test 2.$tn.2 {
          195  +    SELECT count(*) FROM zone JOIN rt USING (k);
          196  +  } {135}
          197  +  do_execsql_test 2.$tn.3 {
          198  +    SELECT filename, 
          199  +           json_extract(header, '$.numKeys'),
          200  +           json_extract(header, '$.numFrames')
          201  +    FROM zone_files 
          202  +    WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
          203  +    ORDER BY 1
          204  +  } {
          205  +    test19.zonefile 1 1
          206  +    test20.zonefile 2 2
          207  +    test21.zonefile 7 4
          208  +  }
          209  +}}
          210  +
          211  +#--------------------------------------------------------------------------
          212  +#
          213  +reset_db
          214  +load_static_extension db zonefile
          215  +do_execsql_test 3.0 {
          216  +  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
          217  +  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
          218  +  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
          219  +  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
          220  +  WITH p(n,v) AS (
          221  +      VALUES('maxAutoFrameSize', 2000) UNION ALL
          222  +      VALUES('encryptionType', 'xor') UNION ALL
          223  +      VALUES('debugEncryptionKeyText', 1) UNION ALL
          224  +      VALUES('encryptionKey', '0123456789')
          225  +  )
          226  +  SELECT zonefile_write('test.zonefile', 'dd', json_group_object(n, v)) FROM p;
          227  +} {{}}
          228  +
          229  +do_execsql_test 3.1 {
          230  +  CREATE VIRTUAL TABLE cc USING zonefile;
          231  +  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','0123456789');
          232  +  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
          233  +} {1 1 1}
          234  +
          235  +do_execsql_test 3.2.1 {
          236  +  DELETE FROM cc_files;
          237  +  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','abcdefghij');
          238  +  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
          239  +} {0 0 0}
          240  +
          241  +do_execsql_test 3.2.2 {
          242  +  SELECT rowid,sz FROM cc;
          243  +} {1000 44 1001 55 1002 66}
          244  +
          245  +do_execsql_test 3.3 {
          246  +  UPDATE cc_files SET ekey = '0123456789';
          247  +  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
          248  +} {1 1 1}
          249  +
          250  +close [open test.zonefile w+]
          251  +do_catchsql_test 3.4 {
          252  +  SELECT header FROM cc_files
          253  +} {1 {failed to read zonefile header from file "test.zonefile"}}
          254  +
          255  +forcedelete test.zonefile
          256  +do_catchsql_test 3.5 {
          257  +  SELECT header FROM cc_files
          258  +} {1 {failed to open file "test.zonefile" for reading}}
          259  +
          260  +do_execsql_test 3.6 {
          261  +  SELECT ekey FROM cc_files
          262  +} {{}}
          263  +
          264  +forcedelete test.zonefile
          265  +do_catchsql_test 3.7 {
          266  +  SELECT * FROM cc;
          267  +} {1 {failed to open file "test.zonefile" for reading}}
          268  +
          269  +#-------------------------------------------------------------------------
          270  +# Check that a file that uses an unknown compression method is handled
          271  +# correctly (an error is returned).
          272  +#
          273  +reset_db
          274  +load_static_extension db zonefile
          275  +do_execsql_test 4.0 {
          276  +  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
          277  +  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
          278  +  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
          279  +  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
          280  +  SELECT zonefile_write('test.zonefile', 'dd');
          281  +  CREATE VIRTUAL TABLE x1 USING zonefile;
          282  +} {{}}
          283  +
          284  +do_test 4.1 {
          285  +  hexio_write test.zonefile 5 77
          286  +} {1}
          287  +do_execsql_test 4.2 {
          288  +  INSERT INTO x1_files(filename) VALUES('test.zonefile');
          289  +} {}
          290  +do_catchsql_test 4.3 {
          291  +  SELECT * FROM x1
          292  +} {1 {unsupported compression method: 119}}
          293  +do_test 4.4 {
          294  +  hexio_write test.zonefile 4 77
          295  +} {1}
          296  +do_catchsql_test 4.5 {
          297  +  DELETE FROM x1_files;
          298  +  INSERT INTO x1_files(filename) VALUES('test.zonefile');
          299  +} {1 {unsupported compression method: 119}}
          300  +
          301  +do_test 4.6 {
          302  +  hexio_write test.zonefile 0 00
          303  +} {1}
          304  +do_catchsql_test 4.7 {
          305  +  INSERT INTO x1_files(filename) VALUES('test.zonefile');
          306  +} {1 {failed to read zonefile header from file "test.zonefile"}}
          307  +
          308  +#-------------------------------------------------------------------------
          309  +# Test using various types in the "frame" field of an input table.
          310  +#
          311  +reset_db
          312  +load_static_extension db zonefile
          313  +if {[lsearch $COMPRESSION_METHODS zlib]>=0} {
          314  +  do_execsql_test 5.0 {
          315  +    CREATE TABLE "a b"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
          316  +    INSERT INTO "a b" VALUES(1, 0.5, -1, randomblob(44));
          317  +    INSERT INTO "a b" VALUES(2, 0.5, -1, randomblob(55));
          318  +    INSERT INTO "a b" VALUES(3, 1.5, -1, randomblob(55));
          319  +    INSERT INTO "a b" VALUES(4, 1.5, -1, randomblob(55));
          320  +    INSERT INTO "a b" VALUES(5,   2, -1, randomblob(55));
          321  +    INSERT INTO "a b" VALUES(6,   2, -1, randomblob(55));
          322  +    INSERT INTO "a b" VALUES(7, 200, -1, randomblob(55));
          323  +    INSERT INTO "a b" VALUES(8, 200, -1, randomblob(55));
          324  +    INSERT INTO "a b" VALUES(9, 300, -1, randomblob(55));
          325  +    INSERT INTO "a b" VALUES(10, 300, -1, randomblob(55));
          326  +    INSERT INTO "a b" VALUES(11, NULL, -1, randomblob(55));
          327  +    INSERT INTO "a b" VALUES(12, NULL, -1, randomblob(55));
          328  +    INSERT INTO "a b" VALUES(13, 'f1', -1, randomblob(55));
          329  +    INSERT INTO "a b" VALUES(14, 'f1', -1, randomblob(55));
          330  +    INSERT INTO "a b" VALUES(15, 'frame2', -1, randomblob(55));
          331  +    INSERT INTO "a b" VALUES(16, 'frame2', -1, randomblob(55));
          332  +    INSERT INTO "a b" VALUES(17, x'1234', -1, randomblob(55));
          333  +    INSERT INTO "a b" VALUES(18, x'1234', -1, randomblob(55));
          334  +    INSERT INTO "a b" VALUES(19, x'abcd', -1, randomblob(55));
          335  +    INSERT INTO "a b" VALUES(20, x'abcd', -1, randomblob(55));
          336  +  
          337  +    SELECT zonefile_write('test.zonefile', 'a b',
          338  +      '{"compressionTypeContent":"zlib"}'
          339  +    );
          340  +  } {{}}
          341  +
          342  +  do_execsql_test 5.1 {
          343  +    CREATE VIRTUAL TABLE abc USING zonefile; 
          344  +    INSERT INTO abc_files(filename) VALUES('test.zonefile');
          345  +    SELECT group_concat(k) FROM abc_shadow_idx GROUP BY fofst
          346  +  } {
          347  +    11,12   1,2   3,4   5,6   7,8  
          348  +     9,10  13,14 15,16 17,18 19,20
          349  +  }
          350  +}
          351  +
          352  +do_execsql_test 6.0 {
          353  +  CREATE TABLE "ab"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
          354  +  INSERT INTO "ab" VALUES(1, 0.5, -1, randomblob(44));
          355  +  INSERT INTO "ab" VALUES(2, 0.5, -1, randomblob(55));
          356  +  INSERT INTO "ab" VALUES(3, 1.5, -1, randomblob(55));
          357  +  INSERT INTO "ab" VALUES(4, 1.5, -1, randomblob(55));
          358  +}
          359  +do_catchsql_test 6.1.1 {
          360  +  SELECT zonefile_write('test.zonefile', 'ab',
          361  +      '{"debugExtendedHeaderSize":-1}'
          362  +  );
          363  +} {1 {debugExtendedHeaderSize value out of range: -1}}
          364  +do_catchsql_test 6.1.2 {
          365  +  SELECT zonefile_write('test.zonefile', 'ab',
          366  +      '{"debugExtendedHeaderSize":256}'
          367  +  );
          368  +} {1 {debugExtendedHeaderSize value out of range: 256}}
          369  +
          370  +do_catchsql_test 6.2 {
          371  +  SELECT zonefile_write('test.zonefile', 'ab',
          372  +      '{"unknown":256}'
          373  +  );
          374  +} {1 {unknown parameter name: "unknown"}}
          375  +
          376  +forcedelete test.dir
          377  +file mkdir test.dir
          378  +do_catchsql_test 6.3 {
          379  +  SELECT zonefile_write('test.dir', 'ab');
          380  +} {1 {failed to open file "test.dir" for writing}}
          381  +
          382  +do_catchsql_test 6.4 {
          383  +  CREATE VIRTUAL TABLE zzz USING zonefile;
          384  +  INSERT INTO zzz_files(filename) VALUES('nosuchfile.zonefile');
          385  +} {1 {failed to open file "nosuchfile.zonefile" for reading}}
          386  +
          387  +if {$tcl_platform(platform)=="windows"} {
          388  +  do_catchsql_test 6.5 {
          389  +    INSERT INTO zzz_files(filename) VALUES('test.dir');
          390  +  } {1 {failed to open file "test.dir" for reading}}
          391  +} else {
          392  +  do_catchsql_test 6.5 {
          393  +    INSERT INTO zzz_files(filename) VALUES('test.dir');
          394  +  } {1 {failed to read zonefile header from file "test.dir"}}
          395  +}
          396  +
          397  +#-------------------------------------------------------------------------
          398  +# Check that errors generated when building a dictionary are handled.
          399  +# The zstd library routines for building a dictionary throw an error
          400  +# if they are provided with too little data.
          401  +#
          402  +# Also test that zstd_global_dict cannot be used to compress the zonefile
          403  +# index (as there is nowhere in the file format to store the dictionary
          404  +# for this compression).
          405  +#
          406  +reset_db
          407  +load_static_extension db zonefile
          408  +if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} {
          409  +  do_execsql_test 7.0 {
          410  +    CREATE TABLE "ab"(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
          411  +    INSERT INTO "ab" VALUES(1, -1, -1, 'abc');
          412  +  }
          413  +
          414  +  do_catchsql_test 7.1 {
          415  +    SELECT zonefile_write('test.zonefile', 'ab',
          416  +      '{"compressionTypeContent":"zstd_global_dict"}'
          417  +    );
          418  +  } {1 {error generating dictionary}}
          419  +
          420  +  do_catchsql_test 7.2 {
          421  +    SELECT zonefile_write('test.zonefile', 'ab',
          422  +      '{"compressionTypeIndexData":"zstd_global_dict"}'
          423  +    );
          424  +  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
          425  +}
          426  +
          427  +#-------------------------------------------------------------------------
          428  +#
          429  +reset_db
          430  +load_static_extension db zonefile
          431  +do_catchsql_test 8.1 {
          432  +  CREATE VIRTUAL TABLE one USING zonefile_files;
          433  +} {1 {do not create zonefile_files tables directly!}}
          434  +do_catchsql_test 8.2 {
          435  +  CREATE VIRTUAL TABLE onetwothree USING zonefile_files;
          436  +} {1 {do not create zonefile_files tables directly!}}
          437  +
          438  +#-------------------------------------------------------------------------
          439  +# A zone file containing zero keys.
          440  +#
          441  +reset_db
          442  +load_static_extension db zonefile
          443  +do_execsql_test 9.0 {
          444  +  CREATE TABLE s(k INTEGER PRIMARY KEY,frame INTEGER,idx INTEGER,v BLOB);
          445  +  SELECT zonefile_write('test.zonefile', 's');
          446  +} {{}}
          447  +do_execsql_test 9.1 {
          448  +  CREATE VIRTUAL TABLE ss USING zonefile;
          449  +  INSERT INTO ss_files(filename) VALUES('test.zonefile');
          450  +  SELECT * FROM ss;
          451  +} {}
          452  +
          453  +#-------------------------------------------------------------------------
          454  +# Test UPDATE and DELETE statements on the %_files virtual table.
          455  +#
          456  +reset_db
          457  +load_static_extension db zonefile
          458  +do_execsql_test 10.0 {
          459  +  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
          460  +  INSERT INTO data(k, v) VALUES(1, randomblob(100));
          461  +  INSERT INTO data(k, v) VALUES(2, randomblob(100));
          462  +  INSERT INTO data(k, v) VALUES(3, randomblob(100));
          463  +  INSERT INTO data(k, v) VALUES(4, randomblob(100));
          464  +  INSERT INTO data(k, v) VALUES(5, randomblob(100));
          465  +  INSERT INTO data(k, v) VALUES(6, randomblob(100));
          466  +  INSERT INTO data(k, v) VALUES(7, randomblob(100));
          467  +  INSERT INTO data(k, v) VALUES(8, randomblob(100));
          468  +  INSERT INTO data(k, v) VALUES(9, randomblob(100));
          469  +
          470  +  CREATE VIEW v1 AS 
          471  +    SELECT k, -1 AS frame, -1 AS idx, v
          472  +    FROM data WHERE k IN (1,2,3);
          473  +  SELECT zonefile_write('test1.zonefile', 'v1');
          474  +
          475  +  CREATE VIEW v2 AS 
          476  +    SELECT k, -1 AS frame, -1 AS idx, v
          477  +    FROM data WHERE k IN (4,5,6);
          478  +  SELECT zonefile_write('test2.zonefile', 'v2');
          479  +
          480  +  CREATE VIEW v3 AS 
          481  +    SELECT k, -1 AS frame, -1 AS idx, v
          482  +    FROM data WHERE k IN (7,8,9);
          483  +  SELECT zonefile_write('test3.zonefile', 'v3');
          484  +} {{} {} {}}
          485  +
          486  +do_execsql_test 10.1 {
          487  +  CREATE VIRTUAL TABLE "a b" USING zonefile;
          488  +  INSERT INTO "a b_files"(filename) VALUES('test1.zonefile');
          489  +  SELECT k FROM "a b"
          490  +} {1 2 3}
          491  +
          492  +do_execsql_test 10.2 {
          493  +  UPDATE "a b_files" SET filename = 'test2.zonefile';
          494  +  SELECT k FROM "a b"
          495  +} {4 5 6}
          496  +
          497  +do_execsql_test 10.3 {
          498  +  INSERT INTO "a b_files"(filename) VALUES('test1.zonefile');
          499  +  INSERT INTO "a b_files"(filename) VALUES('test3.zonefile');
          500  +  SELECT k FROM "a b"
          501  +} {1 2 3 4 5 6 7 8 9}
          502  +
          503  +do_execsql_test 10.4 {
          504  +  DELETE FROM "a b_files" WHERE filename = 'test2.zonefile';
          505  +  SELECT k FROM "a b"
          506  +} {1 2 3 7 8 9}
          507  +
          508  +do_execsql_test 10.5 {
          509  +  DELETE FROM "a b_files" WHERE filename = 'test1.zonefile';
          510  +  SELECT k FROM "a b"
          511  +} {7 8 9}
          512  +
          513  +#-------------------------------------------------------------------------
          514  +#
          515  +reset_db
          516  +load_static_extension db zonefile
          517  +do_execsql_test 11.0 {
          518  +  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
          519  +  INSERT INTO data(k, v) VALUES(-1, randomblob(100));
          520  +  INSERT INTO data(k, v) VALUES(1000, randomblob(100));
          521  +  INSERT INTO data(k, v) VALUES(-1000, randomblob(100));
          522  +  INSERT INTO data(k, v) VALUES(9223372036854775807, randomblob(100));
          523  +  INSERT INTO data(k, v) VALUES(-9223372036854775807, randomblob(100));
          524  +  INSERT INTO data(k, v) VALUES(-9223372036854775808, randomblob(100));
          525  +
          526  +  CREATE VIEW v1 AS SELECT k, -1 AS frame, -1 AS idx, v FROM data;
          527  +  SELECT zonefile_write('test1.zonefile', 'v1');
          528  +} {{}}
          529  +
          530  +do_execsql_test 11.1 {
          531  +  CREATE VIRTUAL TABLE one USING zonefile;
          532  +  INSERT INTO "one_files"(filename) VALUES('test1.zonefile');
          533  +  SELECT k FROM one
          534  +} {-9223372036854775808 -9223372036854775807 -1000 -1 1000 9223372036854775807}
          535  +
          536  +do_execsql_test 11.2 {
          537  +  SELECT count(*) FROM data JOIN one USING (k) WHERE one.v==data.v
          538  +} 6
          539  +
          540  +#-------------------------------------------------------------------------
          541  +#
          542  +do_catchsql_test 12.1 {
          543  +  CREATE VIRTUAL TABLE nm USING zonefile(abcd)
          544  +} {1 {parse error in option: abcd}}
          545  +do_catchsql_test 12.2 {
          546  +  CREATE VIRTUAL TABLE nm USING zonefile(a=b)
          547  +} {1 {parse error in option: a=b}}
          548  +do_catchsql_test 12.3 {
          549  +  CREATE VIRTUAL TABLE nm USING zonefile(cachesiza=b)
          550  +} {1 {parse error in option: cachesiza=b}}
          551  +
          552  +#-------------------------------------------------------------------------
          553  +#
          554  +reset_db
          555  +load_static_extension db zonefile
          556  +do_execsql_test 13.0 {
          557  +  CREATE TABLE data(k INTEGER PRIMARY KEY, v BLOB);
          558  +  WITH s(i) AS (
          559  +    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<250
          560  +  )
          561  +  INSERT INTO data SELECT i, randomblob(100) FROM s;
          562  +  CREATE VIEW v1 AS SELECT k, -1 AS frame, -1 AS idx, v FROM data;
          563  +  SELECT zonefile_write('test1.zonefile', 'v1');
          564  +
          565  +  CREATE VIRTUAL TABLE nm USING zonefile;
          566  +  INSERT INTO nm_files(filename) VALUES('test1.zonefile');
          567  +} {{}}
          568  +
          569  +foreach {tn cond} {
          570  +  1   "k > 30"
          571  +  2   "k >= 100"
          572  +  3   "k <= 100"
          573  +  4   "k < 55"
          574  +  5   "k LIKE '1%'"
          575  +  6   "k BETWEEN 10 AND 20"
          576  +  7   "k > 100 AND k < 200"
          577  +} {
          578  +  do_execsql_test 13.1.$tn.1 [subst {
          579  +    SELECT count(*) FROM nm WHERE $cond
          580  +  }] [db one "SELECT count(*) FROM data WHERE $cond"]
          581  +
          582  +  do_execsql_test 13.1.$tn.2 [subst {
          583  +    SELECT count(*) FROM nm WHERE $cond AND
          584  +      v!=(SELECT v FROM data WHERE k=nm.k);
          585  +  }] 0
          586  +}
          587  +
          588  +close [open test1.zonefile w+]
          589  +do_catchsql_test 13.2.1 {
          590  +  SELECT * FROM nm WHERE k=24;
          591  +} {1 {SQL logic error}}
          592  +forcedelete test1.zonefile
          593  +do_catchsql_test 13.2.2 {
          594  +  SELECT * FROM nm WHERE k=24;
          595  +} {1 {failed to open file "test1.zonefile" for reading}}
          596  +
          597  +do_catchsql_test 13.3.1 {
          598  +  DELETE FROM nm_shadow_file;
          599  +  SELECT * FROM nm WHERE k=24;
          600  +} {1 {database disk image is malformed}}
          601  +do_catchsql_test 13.3.2 {
          602  +  DROP TABLE nm_shadow_file;
          603  +  SELECT * FROM nm WHERE k=24;
          604  +} {1 {no such table: main.nm_shadow_file}}
          605  +db close
          606  +sqlite3 db test.db
          607  +load_static_extension db zonefile
          608  +do_catchsql_test 13.3.3 {
          609  +  SELECT * FROM nm WHERE k=24;
          610  +} {1 {no such table: main.nm_shadow_file}}
          611  +
          612  +#-------------------------------------------------------------------------
          613  +#
          614  +reset_db
          615  +load_static_extension db zonefile
          616  +do_execsql_test 14.0 {
          617  +  CREATE TABLE data(k INTEGER PRIMARY KEY, frame, idx, v BLOB);
          618  +  INSERT INTO data VALUES(1, 1, -1, randomblob(200));
          619  +  INSERT INTO data VALUES(2, 2, -1, randomblob(200));
          620  +  INSERT INTO data VALUES(3, 3, -1, randomblob(200));
          621  +  SELECT zonefile_write('test.zonefile', 'data',
          622  +  '{"encryptionType":"xor","encryptionKey":"pass","debugEncryptionKeyText":1}'
          623  +  );
          624  +
          625  +  CREATE VIRTUAL TABLE nm USING zonefile(cachesize=2);
          626  +  INSERT INTO nm_files(filename,ekey) VALUES('test.zonefile','pass');
          627  +} {{}}
          628  +
          629  +set i 0
          630  +foreach id {1 2 3 2 3 1} {
          631  +  do_execsql_test 14.1.$i {
          632  +    SELECT data.v=nm.v FROM data,nm WHERE data.k=$id AND nm.k=$id
          633  +  } 1
          634  +  incr i
          635  +}
          636  +
          637  +if {[file exists /dev/null]} {
          638  +  do_catchsql_test 14.2 {
          639  +    INSERT INTO nm_files(filename) VALUES('/dev/null');
          640  +  } {1 {failed to read zonefile header from file "/dev/null"}}
          641  +}
          642  +
          643  +finish_test
          644  +

Added ext/zonefile/zonefileenc.test.

            1  +# 2018 Feb 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# The focus of this file is testing the zonefile extension.
           13  +#
           14  +
           15  +package require Tcl 8.6
           16  +
           17  +if {![info exists testdir]} {
           18  +  set testdir [file join [file dirname [info script]] .. .. test]
           19  +}
           20  +source [file join $testdir tester.tcl]
           21  +set testprefix zonefileenc
           22  +
           23  +
           24  +foreach {tn code} {
           25  +  1 { 
           26  +    set K {
           27  +      braking bramble brambles brambly
           28  +      bran branch branched branches
           29  +      branching branchings brand branded
           30  +    }
           31  +    set textkey 1
           32  +  }
           33  +  2 {
           34  +    set K {
           35  +      5e008542742ce0442e37cbf2512e9492    c91c26e0573ca3464e037568c51126da
           36  +      e90e17489c1aef80ac620c9059271a5a    163338707cbe4c72b18d1058a42c5c78
           37  +      5c6b1e7c7c9e8e4a8d8fdc30dfc11bea    ff1012687828ecaac6c9ca86ea0f895e
           38  +      a203f25eb11d4c6afa841dfcf7cd0be0    b6c71e38ca914c460926ef90db39dba0
           39  +      b38255d031d026c258a0a41a9a75d46a    adccca5e5ffa3a7625144a345713aef0
           40  +      cd423b38b73e42ce5894405e6d0e08c0    b460ad2e370a0386726d6ea46e7b0bac
           41  +      503b81de72cb3ef87d9346a850040000    369c290a464a6b88bfd9d1c4755afd42
           42  +      a8a9343efca528f2bf23a972be49dd66    e366b5226bfe3fd0010fa814aae3b996
           43  +      4cad7e80124c2cd447131bae377e60f6    4a0fd2f054e1b08cad0de2dc6aa93246
           44  +      8a23c85e3337da2c97d498f806870fa8    8d14e1f055fd9bec7d07cf0e8baae042
           45  +      7f6954b0dc373028ab3b030aaf44dd58    d220164c3898435a946de6bcbb478cc4
           46  +      566af7ea88ba4ff87fd868e858cf98ea    a5405832235e8f601516f9c49767bdac
           47  +      1bd5b4dc6b54e5ca92ba67d20bf65740    59da30e203bf73840e38e108b83ddb82
           48  +      e516924c2cdf3114f10f2f0e1bdabbc6    b55dd27222a39764222838007e749984
           49  +      190ae9f81b86a5a024e3b97ee2a7121c    469660843a9a9e507d0fb43e92029296
           50  +      e6e600bc063aad12f6387beef650c48a    3097be5c3a52a2f00747587add01b550
           51  +    }
           52  +    set textkey 0
           53  +  }
           54  +} {
           55  +  reset_db
           56  +  load_static_extension db zonefile
           57  +  set nFile 100
           58  +  eval $code
           59  +
           60  +  do_execsql_test 1.$tn.0 {
           61  +    CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
           62  +    CREATE TABLE rr(k INTEGER PRIMARY KEY, v);
           63  +  }
           64  +  do_test 1.$tn.1 {
           65  +    for {set i 0} {$i < $nFile} {incr i} {
           66  +      set k [lindex $K [expr $i % [llength $K]]]
           67  +      execsql {
           68  +        DELETE FROM zz;
           69  +        INSERT INTO zz VALUES($i*10+1, 1, -1, randomblob(100));
           70  +        INSERT INTO zz VALUES($i*10+2, 2, -1, randomblob(100));
           71  +        INSERT INTO zz VALUES($i*10+3, 1, -1, randomblob(100));
           72  +        INSERT INTO rr SELECT k,v FROM zz;
           73  +  
           74  +        WITH p(n,v) AS (
           75  +            VALUES('encryptionType', 'xor') UNION ALL
           76  +            VALUES('debugEncryptionKeyText', $textkey) UNION ALL
           77  +            VALUES('encryptionKey', $k)
           78  +        )
           79  +        SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
           80  +            json_group_object(n, v)
           81  +        ) FROM p;
           82  +      }
           83  +    }
           84  +  } {}
           85  +  
           86  +  proc k {i} { 
           87  +    set val [lindex $::K [expr $i % [llength $::K]]]
           88  +    if {$::textkey==0} {
           89  +      return [binary decode hex $val]
           90  +    }
           91  +    return $val
           92  +  }
           93  +  db func k k
           94  +  
           95  +  do_execsql_test 1.$tn.2 {
           96  +    CREATE VIRTUAL TABLE gg USING zonefile;
           97  +  }
           98  +  for {set i 0} {$i < $nFile} {incr i} {
           99  +    do_execsql_test 1.$tn.2.$i { 
          100  +      INSERT INTO gg_files(filename, ekey) 
          101  +        VALUES('test' || $i || '.zonefile', k($i));
          102  +      SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
          103  +    } 0
          104  +  }
          105  +  
          106  +  db close
          107  +  sqlite3 db test.db
          108  +  load_static_extension db zonefile
          109  +  db func k k
          110  +  
          111  +  do_catchsql_test 1.$tn.3 {
          112  +    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
          113  +  } {1 {missing encryption key for file "test0.zonefile"}}
          114  +  do_execsql_test 1.$tn.4 {
          115  +    UPDATE gg_files SET ekey = k(0) WHERE filename='test0.zonefile';
          116  +  }
          117  +  do_execsql_test 1.$tn.4.2 {
          118  +    SELECT count(*) FROM rr JOIN gg USING(k) 
          119  +      WHERE rr.v==gg.v AND k IN (1,2,3);
          120  +  } {3}
          121  +  do_catchsql_test 1.5 {
          122  +    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
          123  +  } {1 {missing encryption key for file "test1.zonefile"}}
          124  +  
          125  +  do_execsql_test 1.$tn.6 {
          126  +    UPDATE gg_files SET ekey = k(rowid-1);
          127  +  }
          128  +  do_execsql_test 1.$tn.7 {
          129  +    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
          130  +  } {0}
          131  +  do_execsql_test 1.$tn.8 {
          132  +    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
          133  +  } {300}
          134  +  
          135  +  forcedelete test.db2
          136  +  do_execsql_test 1.$tn.9.1 {
          137  +    ATTACH 'test.db2' AS maing;
          138  +    CREATE VIRTUAL TABLE maing.g USING zonefile;
          139  +    INSERT INTO g_files(filename) SELECT filename FROM gg_files;
          140  +  }
          141  +  do_catchsql_test 1.$tn.9.2 {
          142  +    SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v!=g.v;
          143  +  } {1 {missing encryption key for file "test0.zonefile"}}
          144  +  do_execsql_test 1.$tn.9.3 {
          145  +    UPDATE g_files SET ekey = k(rowid-1);
          146  +    SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v==g.v;
          147  +  } {300}
          148  +  
          149  +  do_execsql_test 1.$tn.10 {
          150  +    SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
          151  +  } {300}
          152  +}
          153  +#-------------------------------------------------------------------------
          154  +
          155  +reset_db
          156  +load_static_extension db zonefile
          157  +
          158  +do_execsql_test 2.0 {
          159  +  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
          160  +}
          161  +foreach {tn alg id} {
          162  +  1 aes_128_ctr 1
          163  +  2 aes_128_cbc 2
          164  +  3 AES_256_CTR 3
          165  +  4 Aes_256_CBC 4
          166  +} {
          167  +  do_catchsql_test 2.1.$tn {
          168  +    WITH p(n,v) AS (
          169  +        VALUES('encryptionType', $alg) UNION ALL
          170  +        VALUES('debugEncryptionKeyText', 1) UNION ALL
          171  +        VALUES('encryptionKey', 'secret')
          172  +    )
          173  +    SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
          174  +        json_group_object(n, v)
          175  +    ) FROM p;
          176  +  } "1 {unsupported encryption method: $id}"
          177  +}
          178  +
          179  +foreach {tn alg} {
          180  +  1 nosuchmethod! 
          181  +} {
          182  +  do_catchsql_test 2.2.$tn {
          183  +    WITH p(n,v) AS (
          184  +        VALUES('encryptionType', $alg) UNION ALL
          185  +        VALUES('debugEncryptionKeyText', 1) UNION ALL
          186  +        VALUES('encryptionKey', 'secret')
          187  +    )
          188  +    SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
          189  +        json_group_object(n, v)
          190  +    ) FROM p;
          191  +  } "1 {unknown encryption method: $alg}"
          192  +}
          193  +
          194  +#-------------------------------------------------------------------------
          195  +# Test some hash collisions in the encryption key table.
          196  +#
          197  +
          198  +# This is the same hash function used internally to store keys.
          199  +#
          200  +proc hash {zDb zTab iFile} {
          201  +  binary scan $zDb c*  A
          202  +    binary scan $zTab c* B
          203  +    set h 0
          204  +    foreach i $A { set h [expr ($h + ($h << 3) + $i) & 0xFFFFFFFF] }
          205  +  foreach i $B { set h [expr ($h + ($h << 3) + $i) & 0xFFFFFFFF] }
          206  +  return [expr $h ^ $iFile]
          207  +}
          208  +
          209  +do_test 3.0 {
          210  +  set h1 [expr [hash main zone 1] % 512]
          211  +    for {set i 0} {1} {incr i} {
          212  +      set h2 [expr [hash "aux$i" zone 1] % 512]
          213  +        if {$h1==$h2} break
          214  +    }
          215  +  set i
          216  +} 52
          217  +
          218  +reset_db
          219  +load_static_extension db zonefile
          220  +forcedelete test.db2
          221  +
          222  +do_execsql_test 3.1 {
          223  +  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
          224  +  INSERT INTO zz VALUES(222, -1, -1, randomblob(60));
          225  +  WITH p(n,v) AS (
          226  +      VALUES('encryptionType', 'xor') UNION ALL
          227  +      VALUES('debugEncryptionKeyText', 1) UNION ALL
          228  +      VALUES('encryptionKey', 'pass')
          229  +  )
          230  +  SELECT zonefile_write('test1.zonefile', 'zz', 
          231  +      json_group_object(n, v)
          232  +  ) FROM p;
          233  +
          234  +  DELETE FROM zz;
          235  +  INSERT INTO zz VALUES(333, -1, -1, randomblob(80));
          236  +  WITH p(n,v) AS (
          237  +      VALUES('encryptionType', 'xor') UNION ALL
          238  +      VALUES('debugEncryptionKeyText', 1) UNION ALL
          239  +      VALUES('encryptionKey', 'pass')
          240  +  )
          241  +  SELECT zonefile_write('test2.zonefile', 'zz', 
          242  +      json_group_object(n, v)
          243  +  ) FROM p;
          244  +} {{} {}}
          245  +
          246  +do_execsql_test 3.2 {
          247  +  ATTACH 'test.db2' AS aux52;
          248  +  CREATE VIRTUAL TABLE main.zone USING zonefile;
          249  +  CREATE VIRTUAL TABLE aux52.zone USING zonefile;
          250  +  INSERT INTO main.zone_files(filename, ekey) VALUES('test1.zonefile', 'pass');
          251  +  INSERT INTO aux52.zone_files(filename, ekey) VALUES('test2.zonefile', 'pass');
          252  +}
          253  +
          254  +do_execsql_test 3.3 {
          255  +  SELECT v IS NULL FROM main.zone;
          256  +  SELECT v IS NULL FROM aux52.zone;
          257  +} {0 0}
          258  +
          259  +do_test 3.4 {
          260  +  set h1 [expr [hash main zone 1] % 512]
          261  +  for {set i 0} {1} {incr i} {
          262  +    set h2 [expr [hash "aux$i" zone 2] % 512]
          263  +    if {$h1==$h2} break
          264  +  }
          265  +  set i
          266  +} 682
          267  +
          268  +forcedelete test.db3
          269  +do_execsql_test 3.5 {
          270  +  ATTACH 'test.db3' AS aux682;
          271  +  CREATE VIRTUAL TABLE aux682.zone USING zonefile;
          272  +  INSERT INTO aux682.zone_files(filename, ekey) VALUES('test1.zonefile','pass');
          273  +  INSERT INTO aux682.zone_files(filename, ekey) VALUES('test2.zonefile','pass');
          274  +  INSERT INTO main.zone_files(filename, ekey) VALUES('test2.zonefile','pass');
          275  +}
          276  +
          277  +do_execsql_test 3.6 {
          278  +  SELECT v IS NULL FROM main.zone;
          279  +  SELECT v IS NULL FROM aux682.zone;
          280  +  SELECT v IS NULL FROM main.zone;
          281  +} {0 0 0 0 0 0}
          282  +
          283  +
          284  +finish_test
          285  +

Added ext/zonefile/zonefilefault.test.

            1  +# 2018 Feb 11
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# The focus of this file is testing the zonefile extension.
           13  +#
           14  +
           15  +if {![info exists testdir]} {
           16  +  set testdir [file join [file dirname [info script]] .. .. test]
           17  +}
           18  +source [file join $testdir tester.tcl]
           19  +source [file join $testdir malloc_common.tcl]
           20  +set testprefix zonefilefault
           21  +load_static_extension db zonefile
           22  +
           23  +do_execsql_test 1.0 {
           24  +  CREATE TABLE tt(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
           25  +  INSERT INTO tt VALUES(1, -1, -1, randomblob(100));
           26  +  INSERT INTO tt VALUES(2, -1, -1, randomblob(200));
           27  +  INSERT INTO tt VALUES(3,  1, -1, randomblob(300));
           28  +  INSERT INTO tt VALUES(4,  2, -1, randomblob(400));
           29  +}
           30  +
           31  +do_faultsim_test 1.1 -faults oom* -prep {
           32  +  sqlite3 db test.db
           33  +  load_static_extension db zonefile
           34  +} -body {
           35  +  execsql {
           36  +    SELECT zonefile_write('test.zonefile', 'tt',
           37  +        '{"encryptionType":"xor", "encryptionKey":"secret",
           38  +          "debugEncryptionKeyText":1
           39  +        }'
           40  +    );
           41  +  }
           42  +} -test {
           43  +  faultsim_test_result {0 {{}}}
           44  +}
           45  +
           46  +set sql {
           47  +    SELECT zonefile_write('test.zonefile', 'tt',
           48  +        '{"compressionTypeContent":"zstd_global_dict"}'
           49  +    );
           50  +}
           51  +set HAVE_ZSTD 0
           52  +if {[catch { db eval $sql }]==0} {
           53  +  set HAVE_ZSTD 1
           54  +}
           55  +
           56  +if {$HAVE_ZSTD} {
           57  +  do_faultsim_test 1.2 -faults oom* -prep {
           58  +    sqlite3 db test.db
           59  +    load_static_extension db zonefile
           60  +  } -body {
           61  +    execsql {
           62  +      SELECT zonefile_write('test.zonefile', 'tt',
           63  +          '{"compressionTypeContent":"zstd_global_dict"}'
           64  +      );
           65  +    }
           66  +  } -test {
           67  +    faultsim_test_result {0 {{}}}
           68  +  }
           69  +}
           70  +
           71  +do_execsql_test 1.3.0 { UPDATE tt SET frame = NULL; }
           72  +do_faultsim_test 1.3 -faults oom* -prep {
           73  +  sqlite3 db test.db
           74  +  load_static_extension db zonefile
           75  +} -body {
           76  +  execsql {
           77  +    SELECT zonefile_write('test.zonefile', 'tt');
           78  +  }
           79  +} -test {
           80  +  faultsim_test_result {0 {{}}}
           81  +}
           82  +
           83  +do_execsql_test 1.4.0 {
           84  +  INSERT INTO tt VALUES(5, -1, -1, randomblob(100));
           85  +  INSERT INTO tt VALUES(6, -1, -1, randomblob(100));
           86  +  INSERT INTO tt VALUES(7, -1, -1, randomblob(100));
           87  +  INSERT INTO tt VALUES(8, -1, -1, randomblob(100));
           88  +  INSERT INTO tt VALUES(9, -1, -1, randomblob(100));
           89  +  CREATE VIRTUAL TABLE ttz USING zonefile;
           90  +}
           91  +if {$HAVE_ZSTD} {
           92  +  faultsim_save_and_close
           93  +  do_faultsim_test 1.4 -faults oom* -prep {
           94  +    faultsim_restore_and_reopen
           95  +    load_static_extension db zonefile
           96  +  } -body {
           97  +    execsql {
           98  +      SELECT zonefile_write('test.zonefile', 'tt',
           99  +          '{"compressionTypeIndexData":"zstd"}'
          100  +      );
          101  +    }
          102  +  } -test {
          103  +    faultsim_test_result {0 {{}}}
          104  +  }
          105  +
          106  +  faultsim_save_and_close
          107  +  do_faultsim_test 1.5 -faults oom* -prep {
          108  +    faultsim_restore_and_reopen
          109  +    load_static_extension db zonefile
          110  +  } -body {
          111  +    execsql {
          112  +      INSERT INTO ttz_files(filename) VALUES('test.zonefile');
          113  +    }
          114  +  } -test {
          115  +    faultsim_test_result {0 {}}
          116  +  }
          117  +}
          118  +
          119  +#-------------------------------------------------------------------------
          120  +#
          121  +do_execsql_test 2.0 {
          122  +  SELECT zonefile_write('test.zonefile', 'tt',
          123  +      '{"encryptionType":"xor", "encryptionKey":"secret",
          124  +          "debugEncryptionKeyText":1
          125  +      }'
          126  +  );
          127  +  CREATE VIRTUAL TABLE zz USING zonefile;
          128  +} {{}}
          129  +
          130  +faultsim_save_and_close
          131  +do_faultsim_test 2.1 -faults oom* -prep {
          132  +  faultsim_restore_and_reopen
          133  +  load_static_extension db zonefile
          134  +} -body {
          135  +  execsql {
          136  +    INSERT INTO zz_files(filename, ekey) VALUES('test.zonefile', 'secret')
          137  +  }
          138  +} -test {
          139  +  faultsim_test_result {0 {}}
          140  +}
          141  +
          142  +faultsim_save_and_close
          143  +do_faultsim_test 2.2 -faults oom* -prep {
          144  +  faultsim_restore_and_reopen
          145  +  load_static_extension db zonefile
          146  +} -body {
          147  +  execsql { SELECT json_extract(header, '$.magicNumber') FROM zz_files }
          148  +} -test {
          149  +  faultsim_test_result {0 1179332920}
          150  +}
          151  +
          152  +do_faultsim_test 2.3 -faults oom* -prep {
          153  +  faultsim_restore_and_reopen
          154  +  load_static_extension db zonefile
          155  +} -body {
          156  +  execsql { DELETE FROM zz_files }
          157  +} -test {
          158  +  faultsim_test_result {0 {}}
          159  +}
          160  +
          161  +#-------------------------------------------------------------------------
          162  +#
          163  +reset_db
          164  +faultsim_save_and_close
          165  +do_faultsim_test 3.1 -faults oom* -prep {
          166  +  faultsim_restore_and_reopen
          167  +  load_static_extension db zonefile
          168  +} -body {
          169  +  execsql { CREATE VIRTUAL TABLE t1 USING zonefile(cachesize=5) }
          170  +} -test {
          171  +  faultsim_test_result {0 {}}
          172  +}
          173  +
          174  +faultsim_save_and_close
          175  +do_faultsim_test 3.2 -faults oom* -prep {
          176  +  faultsim_restore_and_reopen
          177  +  load_static_extension db zonefile
          178  +} -body {
          179  +  execsql { DROP TABLE t1 }
          180  +} -test {
          181  +  faultsim_test_result {0 {}}
          182  +}
          183  +
          184  +#-------------------------------------------------------------------------
          185  +#
          186  +reset_db
          187  +load_static_extension db zonefile
          188  +do_execsql_test 4.0 {
          189  +  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
          190  +  INSERT INTO zz VALUES(1, -1, -1, randomblob(100));
          191  +  INSERT INTO zz VALUES(2, -1, -1, randomblob(100));
          192  +  INSERT INTO zz VALUES(3, -1, -1, randomblob(100));
          193  +  SELECT zonefile_write('test.zonefile', 'zz');
          194  +  CREATE VIRTUAL TABLE zone USING zonefile;
          195  +  INSERT INTO zone_files(filename) VALUES('test.zonefile');
          196  +} {{}}
          197  +
          198  +faultsim_save_and_close
          199  +do_faultsim_test 4.1 -faults oom* -prep {
          200  +  faultsim_restore_and_reopen
          201  +  load_static_extension db zonefile
          202  +} -body {
          203  +  execsql { SELECT v IS NULL FROM zone WHERE k = 2 }
          204  +} -test {
          205  +  faultsim_test_result {0 0}
          206  +}
          207  +
          208  +
          209  +if {$HAVE_ZSTD} {
          210  +  set params {
          211  +      {"encryptionType":"xor","encryptionKey":"pass",
          212  +       "compressionTypeContent":"zstd_global_dict",
          213  +          "debugEncryptionKeyText":1
          214  +      }
          215  +  }
          216  +} else {
          217  +  set params { 
          218  +      {"encryptionType":"xor","encryptionKey":"pass",
          219  +          "debugEncryptionKeyText":1
          220  +      }
          221  +  }
          222  +}
          223  +do_execsql_test 4.2 {
          224  +  SELECT zonefile_write('test.zonefile', 'zz', $params);
          225  +  CREATE VIRTUAL TABLE zone2 USING zonefile;
          226  +  INSERT INTO zone2_files(filename,ekey) VALUES('test.zonefile','pass');
          227  +} {{}}
          228  +
          229  +faultsim_save_and_close
          230  +do_faultsim_test 4.3 -faults oom* -prep {
          231  +  faultsim_restore_and_reopen
          232  +  load_static_extension db zonefile
          233  +  execsql { UPDATE zone2_files SET ekey='pass' }
          234  +} -body {
          235  +  execsql { SELECT v IS NULL FROM zone2 WHERE k = 2 }
          236  +} -test {
          237  +  faultsim_test_result {0 0}
          238  +}
          239  +
          240  +#-------------------------------------------------------------------------
          241  +reset_db
          242  +do_faultsim_test 4.4 -faults oom* -prep {
          243  +  faultsim_restore_and_reopen
          244  +} -body {
          245  +  load_static_extension db zonefile
          246  +} -test {
          247  +  faultsim_test_result {0 {}} {1 {initialization of zonefile failed: }}
          248  +}
          249  +
          250  +finish_test
          251  +

Changes to main.mk.

   368    368     $(TOP)/ext/misc/series.c \
   369    369     $(TOP)/ext/misc/spellfix.c \
   370    370     $(TOP)/ext/misc/totype.c \
   371    371     $(TOP)/ext/misc/unionvtab.c \
   372    372     $(TOP)/ext/misc/wholenumber.c \
   373    373     $(TOP)/ext/misc/vfslog.c \
   374    374     $(TOP)/ext/misc/zipfile.c \
          375  +  $(TOP)/ext/zonefile/zonefile.c \
   375    376     $(TOP)/ext/fts5/fts5_tcl.c \
   376    377     $(TOP)/ext/fts5/fts5_test_mi.c \
   377    378     $(TOP)/ext/fts5/fts5_test_tok.c
   378    379   
   379    380   
   380    381   #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
   381    382   #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c

Changes to src/test1.c.

  6988   6988     extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*);
  6989   6989     extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*);
  6990   6990     extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*);
  6991   6991     extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*);
  6992   6992   #ifdef SQLITE_HAVE_ZLIB
  6993   6993     extern int sqlite3_zipfile_init(sqlite3*,char**,const sqlite3_api_routines*);
  6994   6994   #endif
         6995  +  extern int sqlite3_zonefile_init(sqlite3*,char**,const sqlite3_api_routines*);
  6995   6996     static const struct {
  6996   6997       const char *zExtName;
  6997   6998       int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
  6998   6999     } aExtension[] = {
  6999   7000       { "amatch",                sqlite3_amatch_init               },
  7000   7001       { "carray",                sqlite3_carray_init               },
  7001   7002       { "closure",               sqlite3_closure_init              },
................................................................................
  7012   7013       { "spellfix",              sqlite3_spellfix_init             },
  7013   7014       { "totype",                sqlite3_totype_init               },
  7014   7015       { "unionvtab",             sqlite3_unionvtab_init            },
  7015   7016       { "wholenumber",           sqlite3_wholenumber_init          },
  7016   7017   #ifdef SQLITE_HAVE_ZLIB
  7017   7018       { "zipfile",               sqlite3_zipfile_init              },
  7018   7019   #endif
         7020  +    { "zonefile",              sqlite3_zonefile_init             },
  7019   7021     };
  7020   7022     sqlite3 *db;
  7021   7023     const char *zName;
  7022   7024     int i, j, rc;
  7023   7025     char *zErrMsg = 0;
  7024   7026     if( objc<3 ){
  7025   7027       Tcl_WrongNumArgs(interp, 1, objv, "DB NAME ...");