SQLite

# define ALWAYS(X)      ((X)?1:(assert(0),0))
# define NEVER(X)       ((X)?(assert(0),1):0)
#else
# define ALWAYS(X)      (X)
# define NEVER(X)       (X)
#endif
#endif   /* SQLITE_AMALGAMATION */

#ifndef SQLITE_HAVE_ZONEFILE_CODEC
typedef struct ZonefileCodec ZonefileCodec;

struct ZonefileCodec {
  u8 aKey[16];
};

/* Create a new encryption module instance using algorithm iAlg.
**
**   iAlg==1   AES128 CTR
**   iAlg==2   AES128 CBC
**   iAlg==3   AES256 CTR
**   iAlg==4   AES256 CBC
**   iAlg==5   XOR          Testing use only
**
** If the requested algorithm is not available, the routine returns
** a NULL pointer.  NULL is also returned on a OOM error.
**
** Use zonefileCodecDestroy() to reclaim memory.
*/
static int zonefileCodecCreate(
  int iAlg, 
  unsigned char *pKey, int nKey, 
  ZonefileCodec **pp, 
  char **pzErr
){
  ZonefileCodec *pRet;
  int rc = SQLITE_OK;
  
  if( iAlg!=5 ){
    *pzErr = sqlite3_mprintf("unsupported encryption method: %d", iAlg);
    rc = SQLITE_ERROR;
  }else{
    *pp = pRet = (ZonefileCodec*)sqlite3_malloc(sizeof(ZonefileCodec));
    if( pRet==0 ){
      rc = SQLITE_NOMEM;
    }else{
      int i;
      for(i=0; i<sizeof(pRet->aKey); i++){
        pRet->aKey[i] = pKey[i % nKey];
      }
    }
  }

  return rc;
}

/* Return the size of the nonce used for the given encryption module */
static int zonefileCodecNonceSize(ZonefileCodec *pCodec){
  return 16;
}

/* Encrypt in-place.
**
** The size of the content will grow by the nonce size.  Hence, the
** buffer must have at least nonce bytes of extra space available at
** the end to accommodate that growth.  When persisting results, be
** sure to include the extra bytes.
*/
static void zonefileCodecEncode(
  ZonefileCodec *pCodec, 
  unsigned char *pIn, int nIn
){
  int i;
  u8 *aNonce = &pIn[nIn];
  sqlite3_randomness(16, aNonce);
  for(i=0; i<nIn; i++){
    pIn[i] = pIn[i] ^ aNonce[i%16] ^ pCodec->aKey[i%16];
  }
}

/* Decrypt in-place.
**
** The size of the decrypted text will be less than the input buffer
** by nonce-size bytes.
*/
static void zonefileCodecDecode(
  ZonefileCodec *pCodec, 
  unsigned char *pIn, int nIn
){
  int i;
  u8 *aNonce = &pIn[nIn-16];
  for(i=0; i<nIn-16; i++){
    pIn[i] = pIn[i] ^ aNonce[i%16] ^ pCodec->aKey[i%16];
  }
}

/* Destroy an encryption module.
** It is harmless to pass in a NULL pointer.
*/
static void zonefileCodecDestroy(ZonefileCodec *pCodec){
  sqlite3_free(pCodec);
}
#endif                           /* SQLITE_HAVE_ZONEFILE_CODEC */

typedef struct ZonefileGlobal ZonefileGlobal;
typedef struct ZonefileKey ZonefileKey;
struct ZonefileGlobal {
  ZonefileKey *pList;
};
struct ZonefileKey {
  const char *zName;              /* Table name */
  const char *zDb;                /* Database name */
  i64 iFileid;                    /* File id */
  const char *zKey;               /* Key buffer */
  int nKey;                       /* Size of zKey in bytes */
  ZonefileKey *pNext;             /* Next key on same db connection */
};


#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
#define ZONEFILE_DEFAULT_ENCRYPTION       1
#define ZONEFILE_DEFAULT_COMPRESSION      0
#define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)

#define ZONEFILE_MAGIC_NUMBER 0x464B3138

#define ZONEFILE_SZ_HEADER           32
#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20

    0,
    zfLz4CompressBound, zfLz4hcCompress, 
    zfGenericUncompressSize, zfLz4Uncompress
  },
#endif /* SQLITE_HAVE_LZ4 */
};

static int zonefileCompress(
  const char *zName,              /* Name of requested compression method */
  ZonefileCompress **pp,          /* OUT: Pointer to compression object */
  char **pzErr                    /* OUT: Error message */
){
  int i;
  assert( *pzErr==0 );
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
    if( sqlite3_stricmp(aZonefileCompress[i].zName, zName)==0 ){
      *pp = &aZonefileCompress[i];
      return SQLITE_OK;
    }
  }
  *pzErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName);
  return SQLITE_ERROR;
}

static ZonefileCompress *zonefileCompressByValue(int eType){
  int i;
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
    if( aZonefileCompress[i].eType==eType ){
      return &aZonefileCompress[i];

/*
** A structure to store the parameters for a single zonefile_write()
** invocation. 
*/
typedef struct ZonefileParam ZonefileParam;
struct ZonefileParam {
  ZonefileCompress *pCmpIdx;      /* For compressing the index */
  ZonefileCompress *pCmpData;     /* For compressing each frame */
  int encryptionType;
  int maxAutoFrameSize;
  int debugExtendedHeaderSize;    /* Size of extended header */
  char *encryptionKey;            /* Encryption key */
};

/*
** A structure to store a deserialized zonefile header in.
*/
typedef struct ZonefileHeader ZonefileHeader;
struct ZonefileHeader {

int zonefileIsAutoFrame(sqlite3_value *pFrame){
  return (
      sqlite3_value_type(pFrame)==SQLITE_INTEGER 
   && sqlite3_value_int64(pFrame)==-1
  );
}

static int zonefileEncryption(const char *zName, int *peType, char **pzErr){
  struct Encryption {
    const char *zName;
    int eType;
  } a[] = {
    {"NONE", 0}, 
    {"AES_128_CTR", 1}, 
    {"AES_128_CBC", 2}, 
    {"AES_256_CTR", 3}, 
    {"AES_256_CBC", 4}, 
    {"XOR",         5}, 
  };
  int i;

  for(i=0; i<sizeof(a)/sizeof(a[0]); i++){
    if( 0==sqlite3_stricmp(zName, a[i].zName) ){
      *peType = a[i].eType;
      return SQLITE_OK;
    }
  }

  *pzErr = sqlite3_mprintf("unknown encryption type: %s", zName);
  return SQLITE_ERROR;
}

static int zonefileGetParams(
  sqlite3_context *pCtx,          /* Leave any error message here */
  const char *zJson,              /* JSON configuration parameter (or NULL) */
  ZonefileParam *p                /* Populate this object before returning */
){
  sqlite3 *db = sqlite3_context_db_handle(pCtx);
  sqlite3_stmt *pStmt = 0;
  char *zErr = 0;
  int rc = SQLITE_OK;
  int rc2;

  memset(p, 0, sizeof(ZonefileParam));
  p->maxAutoFrameSize = ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE;
  p->encryptionType = ZONEFILE_DEFAULT_ENCRYPTION;
  p->pCmpData = p->pCmpIdx = zonefileCompressByValue(0);

  rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
  }
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){

      p->debugExtendedHeaderSize = iVal;
    }else
    if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
      p->maxAutoFrameSize = iVal;
    }else
    if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
      rc = zonefileCompress(zName, &p->pCmpIdx, &zErr);




    }else
    if( sqlite3_stricmp("compressionTypeContent", zKey)==0 ){
      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
      rc = zonefileCompress(zName, &p->pCmpData, &zErr);
    }else
    if( sqlite3_stricmp("encryptionKey", zKey)==0 ){
      const char *zKey = (const char*)sqlite3_column_text(pStmt, 1);
      p->encryptionKey = sqlite3_mprintf("%s", zKey);
      if( p->encryptionKey==0 ) rc = SQLITE_NOMEM;


    }else
    if( sqlite3_stricmp("encryptionType", zKey)==0 ){
      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
      rc = zonefileEncryption(zName, &p->encryptionType, &zErr);
    }else{
      rc = SQLITE_ERROR;
      zErr = sqlite3_mprintf("unknown parameter name: \"%s\"", zKey);
    }
  }
  rc2 = sqlite3_finalize(pStmt);
  if( rc==SQLITE_OK ) rc = rc2;

  if( zErr ){
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
  }else{
    if( p->encryptionKey==0 ){
      p->encryptionType = 0;
    }
  }
  return rc;
}

/*
** Check that there is room in buffer pBuf for at least nByte bytes more 
** data. If not, attempt to allocate more space. If the allocation attempt

  return pFd;
}

static void zonefileFileClose(FILE *pFd){
  if( pFd ) fclose(pFd);
}

static int zonefileAppendData(
  sqlite3_context *pCtx,          /* Leave any error message here */
  ZonefileCompress *pMethod,      /* Compression method object */
  void *pCmp,                     /* Compression handle */
  ZonefileCodec *pCodec,          /* Compression method, if any */
  ZonefileBuffer *pTo,            /* Append new data here */
  ZonefileBuffer *pFrom           /* Input buffer */
){
  int rc = SQLITE_OK;
  if( pFrom->n ){
    int nNonce = pCodec ? zonefileCodecNonceSize(pCodec) : 0;
    int nOrig = pTo->n;
    if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
      if( zonefileBufferGrow(pCtx, pTo, pFrom->n + nNonce) ){
        rc = SQLITE_ERROR;
      }else{
        zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
      }
    }else{
      int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
      if( zonefileBufferGrow(pCtx, pTo, nReq + nNonce) ) return SQLITE_ERROR;
      rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
      pTo->n += nReq;
      if( rc!=SQLITE_OK ){
        return rc;
      }
    }

    /* Encrypt the data just appended to buffer pTo. */
    if( pCodec ){
      zonefileCodecEncode(pCodec, &pTo->a[nOrig], pTo->n - nOrig);
      pTo->n += nNonce;
    }
  }
  return SQLITE_OK;
}

/*
** Append nByte bytes of garbage data to the file opened with pFd. Return
** SQLITE_OK if successful, or SQLITE_ERROR if an error occurs.

  sqlite3_context *pCtx,       /* Context object */
  int objc,                       /* Number of SQL arguments */
  sqlite3_value **objv            /* Array of SQL arguments */
){
  const char *zFile = 0;          /* File to write to */
  const char *zTbl = 0;           /* Database object to read from */
  const char *zJson = 0;          /* JSON configuration parameters */
  ZonefileParam sParam;           /* Decoded JSON parameters */
  int nKey = 0;                   /* Number of keys in new zonefile */
  int nFrame = 0;                 /* Number of frames in new zonefile */
  sqlite3_stmt *pStmt = 0;        /* SQL used to read data from source table */
  FILE *pFd = 0;
  int rc;
  sqlite3_value *pPrev = 0;
  char *zErr = 0;
  void *pCmp = 0;                 /* Data compressor handle */
  u32 iOff;
  ZonefileCodec *pCodec = 0;

  ZonefileBuffer sFrameIdx = {0, 0, 0};     /* Array of frame offsets */
  ZonefileBuffer sKeyIdx = {0, 0, 0};       /* Array of key locations */
  ZonefileBuffer sData = {0, 0, 0};         /* All completed frames so far */
  ZonefileBuffer sFrame = {0, 0, 0};        /* Current frame (uncompressed) */
  ZonefileBuffer sDict = {0, 0, 0};         /* Compression model (if any) */
  ZonefileBuffer sSample = {0,0,0};         /* Sample data for compressor */
  size_t *aSample = 0;                      /* Array of sample sizes */

  u8 aHdr[ZONEFILE_SZ_HEADER];    /* Space to assemble zonefile header */

  assert( objc==2 || objc==3 );
  zFile = (const char*)sqlite3_value_text(objv[0]);
  zTbl = (const char*)sqlite3_value_text(objv[1]);
  if( objc==3 ){
    zJson = (const char*)sqlite3_value_text(objv[2]);
  }
  if( zonefileGetParams(pCtx, zJson, &sParam) ) return;

  if( sParam.encryptionType!=0 ){
    int n = strlen(sParam.encryptionKey);
    rc = zonefileCodecCreate(
        sParam.encryptionType, (u8*)sParam.encryptionKey, n, &pCodec, &zErr
    );
    if( rc!=SQLITE_OK ){
      sqlite3_result_error(pCtx, zErr, -1);
      sqlite3_free(zErr);
      goto zone_write_out;
    }
  }

  /* Check that the index-data compressor is not one that uses an external
  ** dictionary. This is not permitted as there is no sense in using a
  ** dictionary to compress a single blob of data, and the index-data
  ** compressor only ever compresses a single frame. Also, there is nowhere
  ** in the file-format to store such a dictionary for the index-data.  */
  if( sParam.pCmpIdx->xTrain ){
    zonefileCtxError(pCtx, 
        "compressor \"%s\" may not be used to compress the zonefile index", 
        sParam.pCmpIdx->zName
    );
    goto zone_write_out;
  }

  if( sParam.pCmpData->xOpen ){
    rc = sParam.pCmpData->xOpen(&pCmp, 0, 0);
    if( rc!=SQLITE_OK ){
      zonefileCtxError(pCtx, "error in compressor construction");
      goto zone_write_out;
    }
  }

  /* Prepare the SQL statement used to read data from the source table. This

    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
    goto zone_write_out;
  }

  /* If the data compressor uses a global dictionary, create the dictionary
  ** and store it in buffer sDict.  */
  if( sParam.pCmpData->xTrain ){
    int nSample = 0;

    while( SQLITE_ROW==sqlite3_step(pStmt) ){
      int nByte = sqlite3_column_bytes(pStmt, 2);
      const u8 *aByte = (const u8*)sqlite3_column_blob(pStmt, 2);
      if( zonefileBufferGrow(pCtx, &sSample, nByte) ){
        goto zone_write_out;

    }

    if( zonefileBufferGrow(pCtx, &sDict, ZONEFILE_DEFAULT_DICTSIZE) ){
      goto zone_write_out;
    }
    sDict.n = sDict.nAlloc;

    rc = sParam.pCmpData->xTrain(
        pCmp, sDict.a, &sDict.n, sSample.a, aSample, nSample
    );
    if( rc!=SQLITE_OK ){
      zonefileCtxError(pCtx, "error generating dictionary");
      goto zone_write_out;
    }
  }

  while( SQLITE_ROW==sqlite3_step(pStmt) ){
    sqlite3_int64 k = sqlite3_column_int64(pStmt, 0);
    sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
    int nBlob = sqlite3_column_bytes(pStmt, 2);
    const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);

    int bAuto = zonefileIsAutoFrame(pFrame);
    if( sFrame.n>0 ){
      if( zonefileCompareValue(pFrame, pPrev) 
       || (bAuto && (sFrame.n+nBlob)>sParam.maxAutoFrameSize)
      ){
        /* Add new entry to sFrame */
        if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
         || zonefileAppendData(pCtx,sParam.pCmpData,pCmp,pCodec,&sData,&sFrame)
        ){
          goto zone_write_out;
        }
        sFrame.n = 0;
        zonefileAppend32(&sFrameIdx, sData.n);
        sqlite3_value_free(pPrev);
        pPrev = 0;

    if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out;
    zonefileAppendBlob(&sFrame, pBlob, nBlob);
    nKey++;
  }

  if( sFrame.n>0 ){
    if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
     || zonefileAppendData(pCtx, sParam.pCmpData, pCmp, pCodec, &sData, &sFrame)
    ){
      goto zone_write_out;
    }
    zonefileAppend32(&sFrameIdx, sData.n);
    nFrame++;
  }

  /* If a compression method was specified, compress the key-index here */
  if( sParam.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out;
    zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n);
    zonefileBufferFree(&sKeyIdx);
    rc = zonefileAppendData(pCtx, sParam.pCmpIdx, 0, 0, &sKeyIdx, &sFrameIdx);
    sFrameIdx.n = 0;
    if( rc ) goto zone_write_out;
  }

  /* Create the zonefile header in the in-memory buffer */
  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
  aHdr[4] = sParam.pCmpIdx->eType;
  aHdr[5] = sParam.pCmpData->eType;
  iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
  zonefilePut32(&aHdr[6], sDict.n ? iOff+sParam.debugExtendedHeaderSize : 0);
  zonefilePut32(&aHdr[10], iOff + sParam.debugExtendedHeaderSize + sDict.n);
  zonefilePut32(&aHdr[14], nFrame);
  zonefilePut32(&aHdr[18], nKey);
  aHdr[22] = sParam.encryptionType;
  aHdr[23] = 0;                   /* Encryption key index */
  aHdr[24] = 0;                   /* extended header version */
  aHdr[25] = sParam.debugExtendedHeaderSize;
  assert( ZONEFILE_SZ_HEADER>=26 );

  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
  if( rc==SQLITE_OK ) rc = zonefilePad(pFd, sParam.debugExtendedHeaderSize);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
  if( rc ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
    goto zone_write_out;
  }

  if( fclose(pFd) ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
  }
  pFd = 0;

 zone_write_out:
  if( pCmp ) sParam.pCmpData->xClose(pCmp);
  if( pFd ) fclose(pFd);
  sqlite3_value_free(pPrev);
  sqlite3_finalize(pStmt);
  zonefileCodecDestroy(pCodec);
  zonefileBufferFree(&sFrameIdx);
  zonefileBufferFree(&sKeyIdx);
  zonefileBufferFree(&sFrame);
  zonefileBufferFree(&sDict);
  zonefileBufferFree(&sData);
  zonefileBufferFree(&sSample);
  sqlite3_free(aSample);
  sqlite3_free(sParam.encryptionKey);
}

typedef struct ZonefileFilesTab ZonefileFilesTab;
struct ZonefileFilesTab {
  sqlite3_vtab base;              /* Base class - must be first */
  sqlite3 *db;
  char *zBase;                    /* Name of this table */
  char *zDb;                      /* Database containing this table */
  ZonefileGlobal *pGlobal;        /* Global object */
  sqlite3_stmt *pInsert;          /* Insert into the %_shadow_file table */
  sqlite3_stmt *pInsertIdx;       /* Insert into the %_shadow_idx table */
  sqlite3_stmt *pDeleteIdx;       /* Delete by fileid from %_shadow_idx table */
  sqlite3_stmt *pDelete;          /* Delete by rowid from %_shadow_file table */
};

typedef struct ZonefileFilesCsr ZonefileFilesCsr;
struct ZonefileFilesCsr {
  sqlite3_vtab_cursor base;       /* Base class - must be first */
  sqlite3_stmt *pSelect;
};

/*
** This function does the work of xCreate (if bCreate!=0) or xConnect
** (if bCreate==0) for the zonefile_files module.
*/
static int zffCreateConnect(
  int bCreate,
  void *pAux,
  sqlite3 *db,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  ZonefileFilesTab *p;
  const char *zName = argv[2];

    memset(p, 0, sizeof(ZonefileFilesTab));
    p->zBase = (char*)&p[1];
    memcpy(p->zBase, zName, nName-6);
    p->zBase[nName-6] = '\0';
    p->zDb = &p->zBase[nName+1];
    memcpy(p->zDb, zDb, nDb+1);
    p->db = db;
    p->pGlobal = (ZonefileGlobal*)pAux;
    rc = sqlite3_declare_vtab(db, ZONEFILE_FILES_SCHEMA);
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(p);
    p = 0;
  }

static int zffCreate(
  sqlite3 *db,
  void *pAux,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  return zffCreateConnect(1, pAux, db, argc, argv, ppVtab, pzErr);
}

/* 
** zonefile_files virtual table module xConnect method.
*/
static int zffConnect(
  sqlite3 *db,
  void *pAux,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  return zffCreateConnect(0, pAux, db, argc, argv, ppVtab, pzErr);
}

/* 
** zonefile_files virtual table module xDisconnect method.
*/
static int zffDisconnect(sqlite3_vtab *pVtab){
  ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab;

}

/* 
** zonefile_files virtual table module xColumn method.
*/
static int zffColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
  ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
  if( sqlite3_vtab_nochange(ctx) ){
    return SQLITE_OK;
  }
  switch( i ){
    case 0: /* filename */
      sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pSelect, 0));
      break;
    case 1: /* ekey */
      break;
    case 2: { /* header */

  }

  if( rc==SQLITE_OK && hdr.numKeys>0 ){
    u8 *aKey;                     /* Entire KeyOffsets array */
    int nKey;                     /* Size of buffer aKey[] in bytes */
    int i;


    rc = zonefileLoadIndex(&hdr, pFd, &aKey, &nKey, &pTab->base.zErrMsg);

    if( rc==SQLITE_OK && pTab->pInsertIdx==0 ){
      rc = zonefilePrepare(pTab->db, &pTab->pInsertIdx, &pTab->base.zErrMsg,
          "INSERT INTO %Q.'%q_shadow_idx'(k, fileid, frame, ofst, sz)"
          "VALUES(?,?,?,?,?)",
          pTab->zDb, pTab->zBase

    }
    sqlite3_free(aKey);
  }

  zonefileFileClose(pFd);
  return rc;
}

static int zonefileStoreKey(
  ZonefileFilesTab *pTab, 
  i64 iFileid, 
  const char *zKey
){
  ZonefileKey **pp;

  for(pp=&pTab->pGlobal->pList; *pp; pp=&((*pp)->pNext)){
    ZonefileKey *pThis = *pp;
    if( pThis->iFileid==iFileid 
     && 0==sqlite3_stricmp(pTab->zBase, pThis->zName)
     && 0==sqlite3_stricmp(pTab->zDb, pThis->zDb)
    ){
      *pp = pThis->pNext;
      sqlite3_free(pThis);
      break;
    }
  }

  if( zKey ){
    int nKey = strlen(zKey);
    int nDb = strlen(pTab->zDb);
    int nName = strlen(pTab->zBase);
    ZonefileKey *pNew;
    pNew = (ZonefileKey*)sqlite3_malloc(
        sizeof(ZonefileKey) + nKey+1 + nDb+1 + nName+1
        );
    if( pNew==0 ) return SQLITE_NOMEM;
    memset(pNew, 0, sizeof(ZonefileKey));
    pNew->iFileid = iFileid;
    pNew->zKey = (const char*)&pNew[1];
    pNew->nKey = nKey;
    pNew->zDb = &pNew->zKey[nKey+1];
    pNew->zName = &pNew->zDb[nDb+1];
    memcpy((char*)pNew->zKey, zKey, nKey+1);
    memcpy((char*)pNew->zDb, pTab->zDb, nDb+1);
    memcpy((char*)pNew->zName, pTab->zBase, nName+1);

    pNew->pNext = pTab->pGlobal->pList;
    pTab->pGlobal->pList = pNew;
  }
  return SQLITE_OK;
}

/*
** zonefile_files virtual table module xUpdate method.
**
** A delete specifies a single argument - the rowid of the row to remove.
** 
** Update and insert operations pass:

  sqlite3_value **apVal, 
  sqlite_int64 *pRowid
){
  int rc = SQLITE_OK;
  ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab;

  if( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ){
    if( nVal>1 && sqlite3_value_nochange(apVal[2]) ){
      const char *zKey = (const char*)sqlite3_value_text(apVal[3]);
      i64 iFileid = sqlite3_value_int64(apVal[0]);
      return zonefileStoreKey(pTab, iFileid, zKey);
    }else{
      if( pTab->pDelete==0 ){
        rc = zonefilePrepare(pTab->db, &pTab->pDelete, &pVtab->zErrMsg,
            "DELETE FROM %Q.'%q_shadow_file' WHERE fileid=?",
            pTab->zDb, pTab->zBase
            );
      }
      if( rc==SQLITE_OK && pTab->pDeleteIdx==0 ){
        rc = zonefilePrepare(pTab->db, &pTab->pDeleteIdx, &pVtab->zErrMsg,
            "DELETE FROM %Q.'%q_shadow_idx' WHERE fileid=?",
            pTab->zDb, pTab->zBase
            );
      }

      if( rc==SQLITE_OK ){
        sqlite3_bind_value(pTab->pDelete, 1, apVal[0]);
        sqlite3_step(pTab->pDelete);
        rc = sqlite3_reset(pTab->pDelete);
      }
      if( rc==SQLITE_OK ){
        sqlite3_bind_value(pTab->pDeleteIdx, 1, apVal[0]);
        sqlite3_step(pTab->pDeleteIdx);
        rc = sqlite3_reset(pTab->pDeleteIdx);
      }
    }
  }
  if( nVal>1 ){
    i64 iFileid = 0;
    const char *zFile = (const char*)sqlite3_value_text(apVal[2]);

    if( pTab->pInsert==0 ){
      rc = zonefilePrepare(pTab->db, &pTab->pInsert, &pVtab->zErrMsg,
          "INSERT INTO %Q.'%q_shadow_file'(filename) VALUES(?)",
          pTab->zDb, pTab->zBase
      );
    }

    /* Add the new entry to the %_shadow_file table. */
    if( rc==SQLITE_OK ){
      sqlite3_bind_text(pTab->pInsert, 1, zFile, -1, SQLITE_TRANSIENT);
      sqlite3_step(pTab->pInsert);
      rc = sqlite3_reset(pTab->pInsert);
    }

    /* Populate the %_shadow_idx table with entries for all keys in
    ** the zonefile just added to %_shadow_file.  */
    if( rc==SQLITE_OK ){
      iFileid = sqlite3_last_insert_rowid(pTab->db);
      rc = zonefilePopulateIndex(pTab, zFile, iFileid);
    }

    if( rc==SQLITE_OK ){
      const char *zKey = (const char*)sqlite3_value_text(apVal[3]);
      rc = zonefileStoreKey(pTab, iFileid, zKey);
    }
  }

  return rc;
}

typedef struct ZonefileTab ZonefileTab;
struct ZonefileTab {
  sqlite3_vtab base;         /* Base class - must be first */
  sqlite3 *db;
  sqlite3_stmt *pIdToName;   /* Translate fileid to filename */
  ZonefileGlobal *pGlobal;
  char *zName;               /* Name of this table */
  char *zDb;                 /* Name of db containing this table */
};

typedef struct ZonefileCsr ZonefileCsr;
struct ZonefileCsr {
  sqlite3_vtab_cursor base;  /* Base class - must be first */
  sqlite3_stmt *pSelect;     /* SELECT on %_shadow_idx table */
};

/*
** This function does the work of xCreate (if bCreate!=0) or xConnect
** (if bCreate==0) for the zonefile module.
**
**   argv[0]   -> module name  ("zonefile")
**   argv[1]   -> database name
**   argv[2]   -> table name
*/
static int zonefileCreateConnect(
  int bCreate,
  void *pAux,
  sqlite3 *db,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  ZonefileTab *p;
  const char *zName = argv[2];

  }else{
    memset(p, 0, sizeof(ZonefileTab));
    p->zName = (char*)&p[1];
    memcpy(p->zName, zName, nName+1);
    p->zDb = &p->zName[nName+1];
    memcpy(p->zDb, zDb, nDb+1);
    p->db = db;
    p->pGlobal = (ZonefileGlobal*)pAux;
  
    if( bCreate ){
      char *zSql = sqlite3_mprintf(
          "CREATE TABLE %Q.'%q_shadow_idx'(" 
          "  k INTEGER PRIMARY KEY,"
          "  fileid INTEGER,"
          "  frame INTEGER,"

static int zonefileCreate(
  sqlite3 *db,
  void *pAux,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  return zonefileCreateConnect(1, pAux, db, argc, argv, ppVtab, pzErr);
}

/* 
** zonefile virtual table module xConnect method.
*/
static int zonefileConnect(
  sqlite3 *db,
  void *pAux,
  int argc, const char *const*argv,
  sqlite3_vtab **ppVtab,
  char **pzErr
){
  return zonefileCreateConnect(0, pAux, db, argc, argv, ppVtab, pzErr);
}

/* 
** zonefile virtual table module xDisconnect method.
*/
static int zonefileDisconnect(sqlite3_vtab *pVtab){
  ZonefileTab *pTab = (ZonefileTab*)pVtab;

  }else if( rc==SQLITE_ERROR ){
    zonefileCtxError(pCtx, "failed to uncompress frame");
  }

  sqlite3_free(aUn);
  return rc;
}

static int zonefileFindKey(ZonefileTab *pTab, i64 iFileid, const char **pzKey){
  ZonefileKey *pKey;

  for(pKey=pTab->pGlobal->pList; pKey; pKey=pKey->pNext){
    if( pKey->iFileid==iFileid 
     && 0==sqlite3_stricmp(pTab->zName, pKey->zName)
     && 0==sqlite3_stricmp(pTab->zDb, pKey->zDb)
    ){
      *pzKey = pKey->zKey;
      return pKey->nKey;
    }
  }

  return 0;
}

static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){
  ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  const char *zFile = 0;
  char *zErr = 0;
  FILE *pFd = 0;
  int rc = SQLITE_OK;
  u32 iOff = 0;                   /* Offset of frame in file */
  u32 szFrame = 0;                /* Size of frame in bytes */
  int iKeyOff = 0;                /* Offset of record within frame */
  int szKey = 0;                  /* Uncompressed size of record in bytes */
  i64 iFileid = 0;
  ZonefileHeader hdr;
  ZonefileCompress *pCmpMethod = 0;
  ZonefileCodec *pCodec = 0;
  void *pCmp = 0;

  if( pTab->pIdToName==0 ){
    rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, 
        "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?",
        pTab->zDb, pTab->zName
    );
    if( rc!=SQLITE_OK ){
      zonefileTransferError(pCtx);
      return SQLITE_ERROR;
    }
  }

  iKeyOff = sqlite3_column_int(pCsr->pSelect, 3);
  szKey = sqlite3_column_int(pCsr->pSelect, 4);

  /* Open the file to read the blob from */
  iFileid = sqlite3_column_int64(pCsr->pSelect,1);
  sqlite3_bind_int64(pTab->pIdToName, 1, iFileid);
  if( SQLITE_ROW==sqlite3_step(pTab->pIdToName) ){
    zFile = (const char*)sqlite3_column_text(pTab->pIdToName, 0);
    pFd = zonefileFileOpen(zFile, 0, &zErr);
  }
  if( zFile==0 ){
    rc = sqlite3_reset(pTab->pIdToName);
    if( rc!=SQLITE_OK ){

      }
    }
    if( rc==SQLITE_OK ){
      rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
    }
    sqlite3_free(aDict);
  }

  if( hdr.encryptionType ){
    const char *zKey = 0;
    int nKey = zonefileFindKey(pTab, iFileid, &zKey);
    if( nKey==0 ){
      zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
      rc = SQLITE_ERROR;
    }else{
      rc = zonefileCodecCreate(hdr.encryptionType,(u8*)zKey,nKey,&pCodec,&zErr);
    }
  }

  /* Find the offset (iOff) and size (szFrame) of the frame that the
  ** record is stored in.  */
  if( rc==SQLITE_OK ){
    int iFrame = sqlite3_column_int(pCsr->pSelect, 2);
    u8 aSpace[8] = {0,0,0,0,0,0,0,0};
    u8 *aOff = aSpace;

    sqlite3_free(aFree);
  }

  /* Read some data into memory. If the data is uncompressed, then just
  ** the required record is read. Otherwise, the entire frame is read
  ** into memory.  */
  if( rc==SQLITE_OK ){
    int nRead;                       /* Bytes of data to read */
    i64 iRead;                       /* Offset to read at */

    if( pCmpMethod || pCodec ){
      nRead = szFrame;
      iRead = iOff;
    }else{
      nRead = szKey;
      iRead = iOff + iKeyOff;
    }
    
    u8 *aBuf = sqlite3_malloc(nRead);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zonefileFileRead(pFd, aBuf, nRead, hdr.byteOffsetFrames + iRead);
      if( rc==SQLITE_OK ){
        if( pCodec ){
          zonefileCodecDecode(pCodec, aBuf, nRead);
        }
        if( pCmpMethod==0 ){
          if( pCodec==0 ){
            sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree);
            aBuf = 0;
          }else{
            sqlite3_result_blob(pCtx, &aBuf[iKeyOff], szKey, SQLITE_TRANSIENT);
          }
        }else{
          rc = zonefileCtxUncompress(
              pCtx, pCmpMethod, pCmp, aBuf, szFrame, iKeyOff, szKey
          );
        }
      }else{
        zErr = sqlite3_mprintf(
            "failed to read %d bytes at offset %d from file \"%s\"", 
            nRead, (int)(hdr.byteOffsetFrames+iRead), zFile
        );
      }
      sqlite3_free(aBuf);
    }
  }

  sqlite3_reset(pTab->pIdToName);
  if( zErr ){
    assert( rc!=SQLITE_OK );
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
  }
  zonefileFileClose(pFd);
  zonefileCodecDestroy(pCodec);
  if( pCmpMethod ) pCmpMethod->xClose(pCmp);
  return rc;
}

/* 
** zonefile virtual table module xColumn method.
*/

** zonefile virtual table module xRowid method.
*/
static int zonefileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  ZonefileCsr *pCsr = (ZonefileCsr*)cur;
  *pRowid = sqlite3_column_int64(pCsr->pSelect, 0);
  return SQLITE_OK;
}

static void zonefileGlobalFree(void *p){
  ZonefileGlobal *pGlobal = (ZonefileGlobal*)p;
  ZonefileKey *pKey;
  ZonefileKey *pNext;
  for(pKey=pGlobal->pList; pKey; pKey=pNext){
    pNext = pKey->pNext;
    sqlite3_free(pKey);
  }
  sqlite3_free(pGlobal);
}

/*
** Register the "zonefile" extensions.
*/
static int zonefileRegister(sqlite3 *db){
  static sqlite3_module filesModule = {
    0,                            /* iVersion */

    const char *z;
    int n;
    void (*x)(sqlite3_context*,int,sqlite3_value**);
  } aFunc[] = {
    { "zonefile_write", 2, zonefileWriteFunc },
    { "zonefile_write", 3, zonefileWriteFunc },
  };
  ZonefileGlobal *pGlobal = 0;

  int rc = SQLITE_OK;
  int i;

  for(i=0; rc==SQLITE_OK && i<sizeof(aFunc)/sizeof(aFunc[0]); i++){
    struct Func *p = &aFunc[i];
    rc = sqlite3_create_function(db, p->z, p->n, SQLITE_ANY, 0, p->x, 0, 0);
  }

  if( rc==SQLITE_OK ){
    pGlobal = (ZonefileGlobal*)sqlite3_malloc(sizeof(ZonefileGlobal));
    if( pGlobal==0 ){
      rc = SQLITE_NOMEM;
    }else{
      memset(pGlobal, 0, sizeof(ZonefileGlobal));
    }
  }

  if( rc==SQLITE_OK ){
    rc = sqlite3_create_module(
        db, "zonefile_files", &filesModule, (void*)pGlobal
    );
  }
  if( rc==SQLITE_OK ){
    rc = sqlite3_create_module_v2(db, "zonefile", &zonefileModule, 
        (void*)pGlobal, zonefileGlobalFree
    );
    pGlobal = 0;
  }

  sqlite3_free(pGlobal);
  return rc;
}

#else         /* SQLITE_OMIT_VIRTUALTABLE */
# define zonefileRegister(x) SQLITE_OK
#endif

    ORDER BY 1
  } {
    test19.zonefile 1 1
    test20.zonefile 2 2
    test21.zonefile 7 4
  }
}}

#--------------------------------------------------------------------------
#
reset_db
load_static_extension db zonefile
do_execsql_test 3.0 {
  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
  WITH p(n,v) AS (
      VALUES('maxAutoFrameSize', 2000) UNION ALL
      VALUES('encryptionType', 'xor') UNION ALL
      VALUES('encryptionKey', '0123456789')
  )
  SELECT zonefile_write('test.zonefile', 'dd', json_group_object(n, v)) FROM p;
} {{}}

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE cc USING zonefile;
  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','0123456789');
  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
} {1 1 1}

do_execsql_test 3.2 {
  DELETE FROM cc_files;
  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','abcdefghij');
  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
} {0 0 0}

do_execsql_test 3.3 {
  UPDATE cc_files SET ekey = '0123456789';
  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
} {1 1 1}



finish_test