/ Check-in [d9d5cc62]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Modifications to the zonefile module to make it easier to add a cache of uncompressed frame content.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256: d9d5cc62f11058f9ba560381367ff4765dbbde08184e55abdb50ae1b6bf4a016
User & Date: dan 2018-02-21 21:15:45
Context
2018-02-22
16:46
Add an LRU cache of uncompressed frame content to the zonefile virtual table implementation. check-in: 883e7e75 user: dan tags: zonefile
2018-02-21
21:15
Modifications to the zonefile module to make it easier to add a cache of uncompressed frame content. check-in: d9d5cc62 user: dan tags: zonefile
16:36
Have zonefile store encryption keys in a hash-table instead of a linked list. Add extra tests for key management. check-in: 3a63ea65 user: dan tags: zonefile
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/zonefile/zonefile.c.

    12     12   */
    13     13   
    14     14   #include "sqlite3ext.h"
    15     15   SQLITE_EXTENSION_INIT1
    16     16   #ifndef SQLITE_OMIT_VIRTUALTABLE
    17     17   
    18     18   #include <stdio.h>
           19  +#include <stdlib.h>
    19     20   #include <string.h>
    20     21   #include <assert.h>
           22  +
           23  +/*
           24  +** Default values for various zonefile_write() parameters.
           25  +*/
           26  +#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
           27  +#define ZONEFILE_DEFAULT_ENCRYPTION       1
           28  +#define ZONEFILE_DEFAULT_COMPRESSION      0
           29  +#define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)
           30  +
           31  +/* 
           32  +** Value to use for the first 4 bytes of a zonefile file header.
           33  +*/
           34  +#define ZONEFILE_MAGIC_NUMBER 0x464B3138
           35  +
           36  +/*
           37  +** Size of a zonefile header. And of each entry in the 
           38  +** ZonefileIndex.keyOffsets array.
           39  +*/
           40  +#define ZONEFILE_SZ_HEADER           32
           41  +#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20
           42  +
           43  +/*
           44  +** Constants for supported compression types. These are copied from the
           45  +** published file format spec.
           46  +*/
           47  +#define ZONEFILE_COMPRESSION_NONE             0
           48  +#define ZONEFILE_COMPRESSION_ZSTD             1
           49  +#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
           50  +#define ZONEFILE_COMPRESSION_ZLIB             3
           51  +#define ZONEFILE_COMPRESSION_BROTLI           4
           52  +#define ZONEFILE_COMPRESSION_LZ4              5
           53  +#define ZONEFILE_COMPRESSION_LZ4HC            6
           54  +
           55  +/*
           56  +** Schema for "zonefile" virtual table.
           57  +*/
           58  +#define ZONEFILE_SCHEMA          \
           59  +  "CREATE TABLE z1("             \
           60  +  "  k INTEGER PRIMARY KEY,"     \
           61  +  "  v BLOB,"                    \
           62  +  "  fileid INTEGER,"            \
           63  +  "  sz INTEGER"                 \
           64  +  ")"
           65  +
           66  +/*
           67  +** Schema for "zonefile_files" virtual table.
           68  +*/
           69  +#define ZONEFILE_FILES_SCHEMA    \
           70  +  "CREATE TABLE z2("             \
           71  +  "  filename TEXT,"             \
           72  +  "  ekey BLOB,"                 \
           73  +  "  header JSON HIDDEN"         \
           74  +  ")"
           75  +
    21     76   
    22     77   #ifndef SQLITE_AMALGAMATION
    23     78   typedef sqlite3_int64 i64;
    24     79   typedef sqlite3_uint64 u64;
    25     80   typedef unsigned char u8;
    26     81   typedef unsigned short u16;
    27     82   typedef unsigned long u32;
................................................................................
    35     90   # define NEVER(X)       ((X)?(assert(0),1):0)
    36     91   #else
    37     92   # define ALWAYS(X)      (X)
    38     93   # define NEVER(X)       (X)
    39     94   #endif
    40     95   #endif   /* SQLITE_AMALGAMATION */
    41     96   
           97  +/*
           98  +** Forward declarations for encryption/decryption functions.
           99  +**
          100  +** If this module is not compiled with SQLITE_HAVE_ZONEFILE_CODEC, then
          101  +** implementations of the following type and functions that support the
          102  +** mock encryption method "xor" only are provided. Alternatively, the
          103  +** application may append a more functional implementation of the following 
          104  +** type and functions to this file before compiling it with
          105  +** SQLITE_HAVE_ZONEFILE_CODEC defined.
          106  +*/
          107  +typedef struct ZonefileCodec ZonefileCodec;
          108  +static int zonefileCodecCreate(int,unsigned char*,int,ZonefileCodec**,char**);
          109  +static int zonefileCodecNonceSize(ZonefileCodec*);
          110  +static void zonefileCodecEncode(ZonefileCodec*, unsigned char*, int);
          111  +static void zonefileCodecDecode(ZonefileCodec*, unsigned char*, int);
          112  +static void zonefileCodecDestroy(ZonefileCodec*);
          113  +
    42    114   #ifndef SQLITE_HAVE_ZONEFILE_CODEC
    43    115   typedef struct ZonefileCodec ZonefileCodec;
    44    116   
    45    117   struct ZonefileCodec {
    46    118     u8 aKey[16];
    47    119   };
    48    120   
................................................................................
   153    225     i64 iFileid;                    /* File id */
   154    226     const char *zKey;               /* Key buffer */
   155    227     int nKey;                       /* Size of zKey in bytes */
   156    228     u32 iHash;                      /* zonefileKeyHash() value */
   157    229     ZonefileKey *pHashNext;         /* Next colliding key in hash table */
   158    230   };
   159    231   
          232  +/*
          233  +** Return a 32-bit hash value for the three arguments.
          234  +*/
   160    235   static u32 zonefileKeyHash(
   161    236     const char *zDb,
   162    237     const char *zTab,
   163    238     i64 iFileid
   164    239   ){
   165    240     u32 iHash = 0;
   166    241     int i;
................................................................................
   298    373         sqlite3_free(pKey);
   299    374       }
   300    375     }
   301    376     sqlite3_free(pGlobal->aHash);
   302    377     sqlite3_free(pGlobal);
   303    378   }
   304    379   
   305         -
   306         -#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
   307         -#define ZONEFILE_DEFAULT_ENCRYPTION       1
   308         -#define ZONEFILE_DEFAULT_COMPRESSION      0
   309         -#define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)
   310         -
   311         -#define ZONEFILE_MAGIC_NUMBER 0x464B3138
   312         -
   313         -#define ZONEFILE_SZ_HEADER           32
   314         -#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20
   315         -
   316         -
   317         -#define ZONEFILE_COMPRESSION_NONE             0
   318         -#define ZONEFILE_COMPRESSION_ZSTD             1
   319         -#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
   320         -#define ZONEFILE_COMPRESSION_ZLIB             3
   321         -#define ZONEFILE_COMPRESSION_BROTLI           4
   322         -#define ZONEFILE_COMPRESSION_LZ4              5
   323         -#define ZONEFILE_COMPRESSION_LZ4HC            6
   324         -
   325         -
          380  +/*
          381  +** Write value v to buffer aBuf as an unsigned 32-bit big-endian integer.
          382  +*/
   326    383   static void zonefilePut32(u8 *aBuf, u32 v){
   327    384     aBuf[0] = (v >> 24) & 0xFF;
   328    385     aBuf[1] = (v >> 16) & 0xFF;
   329    386     aBuf[2] = (v >>  8) & 0xFF;
   330    387     aBuf[3] = v & 0xFF;
   331    388   }
          389  +
          390  +/*
          391  +** Read and return an unsigned 32-bit big-endian integer from buffer aBuf.
          392  +*/
   332    393   static u32 zonefileGet32(const u8 *aBuf){
   333    394     return (((u32)aBuf[0]) << 24)
   334    395          + (((u32)aBuf[1]) << 16)
   335    396          + (((u32)aBuf[2]) <<  8)
   336    397          + (((u32)aBuf[3]) <<  0);
   337    398   }
   338    399   
          400  +/*
          401  +** Generic xOpen, xClose and xUncompressSize methods used by a few
          402  +** different compression method bindings.
          403  +*/
   339    404   static int zfGenericOpen(void **pp, u8 *aDict, int nDict){
   340    405     *pp = 0;
   341    406     return SQLITE_OK;
   342    407   }
   343    408   static void zfGenericClose(void *p){
   344    409   }
   345    410   static int zfGenericUncompressSize(
................................................................................
   648    713     return 0;
   649    714   }
   650    715   
   651    716   /* End of code for compression routines
   652    717   **************************************************************************/
   653    718   
   654    719   
   655         -
   656         -#define ZONEFILE_SCHEMA          \
   657         -  "CREATE TABLE z1("             \
   658         -  "  k INTEGER PRIMARY KEY,"     \
   659         -  "  v BLOB,"                    \
   660         -  "  fileid INTEGER,"            \
   661         -  "  sz INTEGER"                 \
   662         -  ")"
   663         -
   664         -#define ZONEFILE_FILES_SCHEMA    \
   665         -  "CREATE TABLE z2("             \
   666         -  "  filename TEXT,"             \
   667         -  "  ekey BLOB,"                 \
   668         -  "  header JSON HIDDEN"         \
   669         -  ")"
   670         -
   671         -
   672         -
   673    720   /*
   674    721   ** A structure to store the parameters for a single zonefile_write()
   675         -** invocation. 
          722  +** invocation. An instance of this structure is populated based on
          723  +** the json parameters passed to zonefile_write() by function
          724  +** zonefileGetParams();
   676    725   */
   677    726   typedef struct ZonefileParam ZonefileParam;
   678    727   struct ZonefileParam {
   679    728     ZonefileCompress *pCmpIdx;      /* For compressing the index */
   680    729     ZonefileCompress *pCmpData;     /* For compressing each frame */
   681    730     int encryptionType;
   682    731     int maxAutoFrameSize;
................................................................................
   699    748     u8 encryptionType;
   700    749     u8 encryptionKeyIdx;
   701    750     u8 extendedHeaderVersion;
   702    751     u8 extendedHeaderSize;
   703    752   };
   704    753   
   705    754   /*
   706         -** Buffer structure used by the zonefile_write() implementation.
          755  +** Resizable buffer structure used by the zonefile_write() implementation.
   707    756   */
   708    757   typedef struct ZonefileBuffer ZonefileBuffer;
   709    758   struct ZonefileBuffer {
   710    759     u8 *a;
   711    760     int n;
   712    761     int nAlloc;
   713    762   };
................................................................................
   753    802       sqlite3_free(zSql);
   754    803     }else{
   755    804       rc = SQLITE_NOMEM;
   756    805     }
   757    806     return rc;
   758    807   }
   759    808   
   760         -
   761         -static sqlite3_stmt *zonefileCtxPrepare(
   762         -  sqlite3_context *pCtx,
   763         -  const char *zFmt,
   764         -  ...
   765         -){
   766         -  sqlite3_stmt *pRet = 0;
   767         -  va_list ap;
   768         -  char *zSql;
   769         -  va_start(ap, zFmt);
   770         -  zSql = sqlite3_vmprintf(zFmt, ap);
   771         -  if( zSql ){
   772         -    sqlite3 *db = sqlite3_context_db_handle(pCtx);
   773         -    int rc = sqlite3_prepare(db, zSql, -1, &pRet, 0);
   774         -    if( rc!=SQLITE_OK ){
   775         -      zonefileTransferError(pCtx);
   776         -    }
   777         -    sqlite3_free(zSql);
   778         -  }else{
   779         -    sqlite3_result_error_nomem(pCtx);
   780         -  }
   781         -  return pRet;
   782         -}
   783         -
   784    809   /*
   785    810   ** Return zero if the two SQL values passed as arguments are equal, or
   786    811   ** non-zero otherwise. Values with different types are considered unequal,
   787    812   ** even if they both contain the same numeric value (e.g. 2 and 2.0).
   788    813   */
   789    814   static int zonefileCompareValue(sqlite3_value *p1, sqlite3_value *p2){
   790    815     int eType;
................................................................................
  1130   1155         zonefileCtxError(pCtx, "error in compressor construction");
  1131   1156         goto zone_write_out;
  1132   1157       }
  1133   1158     }
  1134   1159   
  1135   1160     /* Prepare the SQL statement used to read data from the source table. This
  1136   1161     ** also serves to verify the suitability of the source table schema. */
  1137         -  pStmt = zonefileCtxPrepare(pCtx, 
         1162  +  rc = zonefilePrepare(sqlite3_context_db_handle(pCtx), &pStmt, &zErr,
  1138   1163         "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl
  1139   1164     );
  1140         -  if( pStmt==0 ) goto zone_write_out;
  1141   1165   
  1142   1166     /* Open the file-handle used to write out the zonefile */ 
  1143         -  pFd = zonefileFileOpen(zFile, 1, &zErr);
         1167  +  if( rc==SQLITE_OK ){
         1168  +    pFd = zonefileFileOpen(zFile, 1, &zErr);
         1169  +  }
  1144   1170     if( pFd==0 ){
  1145   1171       sqlite3_result_error(pCtx, zErr, -1);
  1146   1172       sqlite3_free(zErr);
  1147   1173       goto zone_write_out;
  1148   1174     }
  1149   1175   
  1150   1176     /* If the data compressor uses a global dictionary, create the dictionary
................................................................................
  1889   1915   struct ZonefileTab {
  1890   1916     sqlite3_vtab base;         /* Base class - must be first */
  1891   1917     sqlite3 *db;
  1892   1918     sqlite3_stmt *pIdToName;   /* Translate fileid to filename */
  1893   1919     ZonefileGlobal *pGlobal;
  1894   1920     char *zName;               /* Name of this table */
  1895   1921     char *zDb;                 /* Name of db containing this table */
         1922  +  int nCacheSize;
  1896   1923   };
  1897   1924   
  1898   1925   typedef struct ZonefileCsr ZonefileCsr;
  1899   1926   struct ZonefileCsr {
  1900   1927     sqlite3_vtab_cursor base;  /* Base class - must be first */
  1901   1928     sqlite3_stmt *pSelect;     /* SELECT on %_shadow_idx table */
  1902   1929   };
         1930  +
         1931  +typedef struct ZonefileFrame ZonefileFrame;
         1932  +struct ZonefileFrame {
         1933  +  i64 iFileid;               /* Fileid for this frame */
         1934  +  i64 iFrameOff;             /* Offset of frame in file */
         1935  +  int nBuf;                  /* Size of aBuf[] in bytes */
         1936  +  u8 *aBuf;                  /* Buffer containing uncompressed frame data */
         1937  +};
         1938  +
         1939  +/*
         1940  +** Attempt to interpret the contents of string z as an integer. If
         1941  +** successful, set (*piVal) to the integer value and return SQLITE_OK.
         1942  +** Otherwise, return SQLITE_ERROR.
         1943  +*/
         1944  +static int zonefileParseInteger(const char *z, int *piVal){
         1945  +  *piVal = atoi(z);
         1946  +  return SQLITE_OK;
         1947  +}
         1948  +
         1949  +/*
         1950  +** Return true character i is considered to be whitespace.
         1951  +*/
         1952  +static int zonefile_isspace(char i){
         1953  +  return (i==' ');
         1954  +}
         1955  +
         1956  +/*
         1957  +** This function is called as part of constructing zonefile virtual table
         1958  +** pTab. Argument zOption is the full text of a parameter (column name)
         1959  +** specified as part of the CREATE VIRTUAL TABLE statement. This function
         1960  +** attempts to interpret the parameter and update structure pTab 
         1961  +** accordingly. If successful, SQLITE_OK is returned. Otherwise, an
         1962  +** SQLite error code is returned and (*pzErr) is left pointing to
         1963  +** a buffer containing an English language error message. It is the
         1964  +** responsibility of the caller to eventually free this buffer using
         1965  +** sqlite3_free().
         1966  +*/
         1967  +static int zonefileParseOption(
         1968  +  ZonefileTab *pTab,              /* Zonefile vtab under construction */
         1969  +  const char *zOption,            /* Text of option (column name) */
         1970  +  char **pzErr                    /* OUT: Error message */
         1971  +){
         1972  +  const char *z = zOption;
         1973  +  const char *zOpt;
         1974  +  int nOpt;
         1975  +  const char *zVal;
         1976  +  int rc = SQLITE_OK;
         1977  +
         1978  +  /* Skip leading whitespace */
         1979  +  while( zonefile_isspace(*z) ) z++;
         1980  +  zOpt = z;
         1981  +
         1982  +  /* Skip until EOF, whitespace or "=" */
         1983  +  while( *z && !zonefile_isspace(*z) && *z!='=' ) z++;
         1984  +  nOpt = z-zOpt;
         1985  +
         1986  +  /* Skip whitespace. Then check there is an "=". */
         1987  +  while( zonefile_isspace(*z) ) z++;
         1988  +  if( *z!='=' ) goto parse_error;
         1989  +  z++;
         1990  +  while( zonefile_isspace(*z) ) z++;
         1991  +  zVal = z;
         1992  +
         1993  +  if( nOpt==9 && sqlite3_strnicmp(zOpt, "cachesize", 9)==0 ){
         1994  +    rc = zonefileParseInteger(zVal, &pTab->nCacheSize);
         1995  +  }else{
         1996  +    goto parse_error;
         1997  +  }
         1998  +
         1999  +  return rc;
         2000  +
         2001  + parse_error:
         2002  +  *pzErr = sqlite3_mprintf("parse error in option: %s", zOption);
         2003  +  return SQLITE_ERROR;
         2004  +}
  1903   2005   
  1904   2006   /*
  1905   2007   ** This function does the work of xCreate (if bCreate!=0) or xConnect
  1906   2008   ** (if bCreate==0) for the zonefile module.
  1907   2009   **
  1908   2010   **   argv[0]   -> module name  ("zonefile")
  1909   2011   **   argv[1]   -> database name
................................................................................
  1924   2026     int nDb = strlen(zDb);
  1925   2027     int rc = SQLITE_OK;
  1926   2028   
  1927   2029     p = (ZonefileTab*)sqlite3_malloc(sizeof(ZonefileTab) + nName+1 + nDb+1);
  1928   2030     if( !p ){
  1929   2031       rc = SQLITE_NOMEM;
  1930   2032     }else{
         2033  +    int i;
  1931   2034       memset(p, 0, sizeof(ZonefileTab));
  1932   2035       p->zName = (char*)&p[1];
  1933   2036       memcpy(p->zName, zName, nName+1);
  1934   2037       p->zDb = &p->zName[nName+1];
  1935   2038       memcpy(p->zDb, zDb, nDb+1);
  1936   2039       p->db = db;
  1937   2040       p->pGlobal = (ZonefileGlobal*)pAux;
................................................................................
  1961   2064           sqlite3_free(zSql);
  1962   2065         }
  1963   2066       }
  1964   2067       
  1965   2068       if( rc==SQLITE_OK ){
  1966   2069         rc = sqlite3_declare_vtab(db, ZONEFILE_SCHEMA);
  1967   2070       }
         2071  +
         2072  +    for(i=3; i<argc && rc==SQLITE_OK; i++){
         2073  +      zonefileParseOption(p, argv[i], pzErr);
         2074  +    }
  1968   2075     }
  1969   2076   
  1970   2077     if( rc!=SQLITE_OK ){
  1971   2078       sqlite3_free(p);
  1972   2079       p = 0;
  1973   2080     }
  1974   2081     *ppVtab = (sqlite3_vtab*)p;
................................................................................
  2272   2379   }
  2273   2380   
  2274   2381   static void zonefileReleaseFile(ZonefileCsr *pCsr){
  2275   2382     ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  2276   2383     sqlite3_reset(pTab->pIdToName);
  2277   2384   }
  2278   2385   
  2279         -static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){
  2280         -  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  2281         -  const char *zFile = 0;
  2282         -  char *zErr = 0;
  2283         -  FILE *pFd = 0;
  2284         -  int rc = SQLITE_OK;
  2285         -  ZonefileHeader hdr;
  2286         -  ZonefileCompress *pCmpMethod = 0;
  2287         -  ZonefileCodec *pCodec = 0;
  2288         -  void *pCmp = 0;
  2289         -
  2290         -  /* Open the file to read the blob from */
  2291         -  rc = zonefileGetFile(pCtx, pCsr, &zFile);
  2292         -  if( rc==SQLITE_OK ){
  2293         -    pFd = zonefileFileOpen(zFile, 0, &zErr);
  2294         -    if( pFd==0 ) rc = SQLITE_ERROR;
  2295         -  }
  2296         -
  2297         -  /* Read the zonefile header */
  2298         -  if( rc==SQLITE_OK ){
  2299         -    rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
         2386  +static int zonefileValueReadDirect(sqlite3_context *pCtx, ZonefileCsr *pCsr){
         2387  +  i64 iOff = sqlite3_column_int64(pCsr->pSelect, 2);
         2388  +  int sz = sqlite3_column_int(pCsr->pSelect, 3);
         2389  +
         2390  +  FILE *pFd = 0;                  /* File handle open on zonefile */
         2391  +  u8 *aBuf;                       /* Buffer to read blob into */
         2392  +  int rc;                         /* Return code */
         2393  +
         2394  +  aBuf = sqlite3_malloc(sz);
         2395  +  if( aBuf==0 ){
         2396  +    rc = SQLITE_NOMEM;
         2397  +  }else{
         2398  +    const char *zFile = 0;
         2399  +    /* Open the file to read the blob from */
         2400  +    rc = zonefileGetFile(pCtx, pCsr, &zFile);
         2401  +    if( rc==SQLITE_OK ){
         2402  +      char *zErr = 0;
         2403  +      pFd = zonefileFileOpen(zFile, 0, &zErr);
         2404  +      if( pFd==0 ){ 
         2405  +        sqlite3_result_error(pCtx, zErr, -1);
         2406  +        sqlite3_free(zErr);
         2407  +        rc = SQLITE_ERROR;
         2408  +      }
         2409  +      zonefileReleaseFile(pCsr);
         2410  +    }
  2300   2411     }
  2301   2412   
  2302         -  /* Find the compression method and open the compressor handle. */
  2303   2413     if( rc==SQLITE_OK ){
  2304         -    rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr);
  2305         -  }
  2306         -  if( pCmpMethod ){
  2307         -    int nDict = 0;
  2308         -    u8 *aDict = 0;
  2309         -    assert( rc==SQLITE_OK );
  2310         -    if( hdr.byteOffsetDictionary ){
  2311         -      nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary;
  2312         -      aDict = sqlite3_malloc(nDict);
  2313         -      if( aDict==0 ){
  2314         -        rc = SQLITE_NOMEM;
  2315         -      }else{
  2316         -        rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary);
  2317         -      }
  2318         -    }
         2414  +    rc = zonefileFileRead(pFd, aBuf, sz, iOff);
  2319   2415       if( rc==SQLITE_OK ){
  2320         -      rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
  2321         -    }
  2322         -    sqlite3_free(aDict);
  2323         -  }
  2324         -
  2325         -  /* Find the encryption method and key. */
  2326         -  if( hdr.encryptionType ){
  2327         -    i64 iFileid = sqlite3_column_int64(pCsr->pSelect, 1);
  2328         -    const char *z = 0;
  2329         -    int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFileid, &z);
  2330         -    if( n==0 ){
  2331         -      zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
  2332         -      rc = SQLITE_ERROR;
  2333         -    }else{
  2334         -      rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr);
         2416  +      sqlite3_result_blob(pCtx, aBuf, sz, zonefileFree);
         2417  +      aBuf = 0;
  2335   2418       }
  2336   2419     }
  2337   2420   
  2338         -  /* Read some data into memory. If the data is uncompressed, then just
  2339         -  ** the required record is read. Otherwise, the entire frame is read
  2340         -  ** into memory.  */
  2341         -  if( rc==SQLITE_OK ){
  2342         -    i64 iFrameOff = sqlite3_column_int64(pCsr->pSelect, 2);
  2343         -    int szFrame = sqlite3_column_int(pCsr->pSelect, 3);
  2344         -    u8 *aBuf = sqlite3_malloc(szFrame);
  2345         -
  2346         -    if( aBuf==0 ){
  2347         -      rc = SQLITE_NOMEM;
  2348         -    }else{
  2349         -      rc = zonefileFileRead(pFd, aBuf, szFrame, iFrameOff);
  2350         -      if( rc!=SQLITE_OK ){
  2351         -        zErr = sqlite3_mprintf(
  2352         -            "failed to read %d bytes at offset %d from file \"%s\"", 
  2353         -            szFrame, iFrameOff, zFile
         2421  +  zonefileFileClose(pFd);
         2422  +  sqlite3_free(aBuf);
         2423  +  return rc;
         2424  +}
         2425  +
         2426  +static ZonefileFrame *zonefileCacheFind(
         2427  +  ZonefileTab *pTab, 
         2428  +  i64 iFile, 
         2429  +  i64 iFrameOff
         2430  +){
         2431  +  return 0;
         2432  +}
         2433  +
         2434  +static void zonefileCacheStore(
         2435  +  ZonefileTab *pTab,
         2436  +  ZonefileFrame *pFrame
         2437  +){
         2438  +}
         2439  +
         2440  +static int zonefileValueReadCache(sqlite3_context *pCtx, ZonefileCsr *pCsr){
         2441  +  int rc = SQLITE_OK;
         2442  +  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
         2443  +  ZonefileFrame *pFrame = 0;
         2444  +  i64 iFile = sqlite3_column_int64(pCsr->pSelect, 1);
         2445  +  i64 iFrameOff = sqlite3_column_int64(pCsr->pSelect, 2);
         2446  +  i64 iKeyOff = sqlite3_column_int64(pCsr->pSelect, 4);
         2447  +  int nKeySz = sqlite3_column_int64(pCsr->pSelect, 5);
         2448  +
         2449  +  /* Check if this frame is already in the cache. If not, read it from 
         2450  +  ** the file.  */
         2451  +  pFrame = zonefileCacheFind(pTab, iFile, iFrameOff);
         2452  +  if( pFrame==0 ){
         2453  +    const char *zFile = 0;
         2454  +    char *zErr = 0;
         2455  +    FILE *pFd = 0;
         2456  +    ZonefileHeader hdr;
         2457  +    ZonefileCompress *pCmpMethod = 0;
         2458  +    ZonefileCodec *pCodec = 0;
         2459  +    void *pCmp = 0;
         2460  +
         2461  +    /* Open the file to read the blob from */
         2462  +    rc = zonefileGetFile(pCtx, pCsr, &zFile);
         2463  +    if( rc==SQLITE_OK ){
         2464  +      pFd = zonefileFileOpen(zFile, 0, &zErr);
         2465  +      if( pFd==0 ) rc = SQLITE_ERROR;
         2466  +    }
         2467  +
         2468  +    /* Read the zonefile header */
         2469  +    if( rc==SQLITE_OK ){
         2470  +      rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
         2471  +    }
         2472  +
         2473  +    /* Find the compression method and open the compressor handle. */
         2474  +    if( rc==SQLITE_OK ){
         2475  +      rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr);
         2476  +    }
         2477  +    if( pCmpMethod ){
         2478  +      int nDict = 0;
         2479  +      u8 *aDict = 0;
         2480  +      assert( rc==SQLITE_OK );
         2481  +      if( hdr.byteOffsetDictionary ){
         2482  +        nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary;
         2483  +        aDict = sqlite3_malloc(nDict);
         2484  +        if( aDict==0 ){
         2485  +          rc = SQLITE_NOMEM;
         2486  +        }else{
         2487  +          rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary);
         2488  +        }
         2489  +      }
         2490  +      if( rc==SQLITE_OK ){
         2491  +        rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
         2492  +      }
         2493  +      sqlite3_free(aDict);
         2494  +    }
         2495  +
         2496  +    /* Find the encryption method and key. */
         2497  +    if( hdr.encryptionType ){
         2498  +      const char *z = 0;
         2499  +      int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFile, &z);
         2500  +      if( n==0 ){
         2501  +        zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
         2502  +        rc = SQLITE_ERROR;
         2503  +      }else{
         2504  +        rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr);
         2505  +      }
         2506  +    }
         2507  +
         2508  +    /* Read some data into memory. */
         2509  +    if( rc==SQLITE_OK ){
         2510  +      int szFrame = sqlite3_column_int(pCsr->pSelect, 3);
         2511  +
         2512  +      pFrame = (ZonefileFrame*)sqlite3_malloc(szFrame + sizeof(ZonefileFrame));
         2513  +      if( pFrame==0 ){
         2514  +        rc = SQLITE_NOMEM;
         2515  +      }else{
         2516  +        memset(pFrame, 0, sizeof(ZonefileFrame));
         2517  +        pFrame->aBuf = (u8*)&pFrame[1];
         2518  +        pFrame->nBuf = szFrame;
         2519  +        pFrame->iFrameOff = iFrameOff;
         2520  +        pFrame->iFileid = iFile;
         2521  +        rc = zonefileFileRead(pFd, pFrame->aBuf, szFrame, iFrameOff);
         2522  +      }
         2523  +    }
         2524  +
         2525  +    /* Decrypt data if necessary */
         2526  +    if( rc==SQLITE_OK && pCodec ){
         2527  +      zonefileCodecDecode(pCodec, pFrame->aBuf, pFrame->nBuf);
         2528  +      pFrame->nBuf -= zonefileCodecNonceSize(pCodec);
         2529  +    }
         2530  +
         2531  +    /* Uncompress data if required */
         2532  +    if( rc==SQLITE_OK && pCmpMethod ){
         2533  +      ZonefileFrame *p = 0;
         2534  +      int nOut = pCmpMethod->xUncompressSize(pCmp, pFrame->aBuf, pFrame->nBuf);
         2535  +
         2536  +      p = (ZonefileFrame*)sqlite3_malloc(nOut + sizeof(ZonefileFrame));
         2537  +      if( p==0 ){
         2538  +        rc = SQLITE_NOMEM;
         2539  +      }else{
         2540  +        p->aBuf = (u8*)&p[1];
         2541  +        p->nBuf = nOut;
         2542  +        p->iFrameOff = iFrameOff;
         2543  +        p->iFileid = iFile;
         2544  +        rc = pCmpMethod->xUncompress(
         2545  +            pCmp, p->aBuf, p->nBuf, pFrame->aBuf, pFrame->nBuf
  2354   2546           );
         2547  +        sqlite3_free(pFrame);
         2548  +        pFrame = p;
  2355   2549         }
  2356   2550       }
  2357   2551   
  2358         -    if( rc==SQLITE_OK ){
  2359         -      if( pCodec==0 && pCmpMethod==0 ){
  2360         -        sqlite3_result_blob(pCtx, aBuf, szFrame, zonefileFree);
  2361         -        aBuf = 0;
  2362         -      }else{
  2363         -        i64 iOff = sqlite3_column_int64(pCsr->pSelect, 4);
  2364         -        int sz = sqlite3_column_int(pCsr->pSelect, 5);
         2552  +    if( rc!=SQLITE_OK ){
         2553  +      sqlite3_free(pFrame);
         2554  +      pFrame = 0;
         2555  +    }
         2556  +    zonefileReleaseFile(pCsr);
         2557  +    zonefileFileClose(pFd);
         2558  +    zonefileCodecDestroy(pCodec);
         2559  +    if( pCmpMethod ) pCmpMethod->xClose(pCmp);
  2365   2560   
  2366         -        /* Decrypt the data if necessary */
  2367         -        if( pCodec ){
  2368         -          zonefileCodecDecode(pCodec, aBuf, szFrame);
  2369         -          szFrame -= zonefileCodecNonceSize(pCodec);
  2370         -        }
  2371         -        if( pCmpMethod ){
  2372         -          rc = zonefileCtxUncompress(
  2373         -              pCtx, pCmpMethod, pCmp, aBuf, szFrame, iOff, sz
  2374         -          );
  2375         -        }else{
  2376         -          sqlite3_result_blob(pCtx, &aBuf[iOff], sz, SQLITE_TRANSIENT);
  2377         -        }
  2378         -      }
         2561  +    if( zErr ){
         2562  +      assert( rc!=SQLITE_OK );
         2563  +      sqlite3_result_error(pCtx, zErr, -1);
         2564  +      sqlite3_free(zErr);
  2379   2565       }
  2380         -    sqlite3_free(aBuf);
         2566  +  }
         2567  +
         2568  +  if( pFrame ){
         2569  +    assert( rc==SQLITE_OK );
         2570  +    sqlite3_result_blob(pCtx, &pFrame->aBuf[iKeyOff], nKeySz, SQLITE_TRANSIENT);
         2571  +    sqlite3_free(pFrame);
  2381   2572     }
  2382   2573   
  2383         -  zonefileReleaseFile(pCsr);
  2384         -  if( zErr ){
  2385         -    assert( rc!=SQLITE_OK );
  2386         -    sqlite3_result_error(pCtx, zErr, -1);
  2387         -    sqlite3_free(zErr);
  2388         -  }
  2389         -  zonefileFileClose(pFd);
  2390         -  zonefileCodecDestroy(pCodec);
  2391         -  if( pCmpMethod ) pCmpMethod->xClose(pCmp);
  2392   2574     return rc;
  2393   2575   }
  2394   2576   
  2395   2577   /* 
  2396   2578   ** zonefile virtual table module xColumn method.
  2397   2579   */
  2398   2580   static int zonefileColumn(
................................................................................
  2403   2585     ZonefileCsr *pCsr = (ZonefileCsr*)cur;
  2404   2586     int rc = SQLITE_OK;
  2405   2587     switch( i ){
  2406   2588       case 0: /* k */
  2407   2589         sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 0));
  2408   2590         break;
  2409   2591       case 1: /* v */
  2410         -      rc = zonefileGetValue(pCtx, pCsr);
         2592  +      if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){
         2593  +        rc = zonefileValueReadDirect(pCtx, pCsr);
         2594  +      }else{
         2595  +        rc = zonefileValueReadCache(pCtx, pCsr);
         2596  +      }
  2411   2597         break;
  2412   2598       case 2: /* fileid */
  2413   2599         sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pSelect, 1));
  2414   2600         break;
  2415   2601       default: { /* sz */
  2416   2602         int iCol;
  2417   2603         if( sqlite3_column_type(pCsr->pSelect, 5)==SQLITE_NULL ){