# define NEVER(X)       ((X)?(assert(0),1):0)
#else
# define ALWAYS(X)      (X)
# define NEVER(X)       (X)
#endif
#endif   /* SQLITE_AMALGAMATION */






#define ZONEFILE_MAGIC_NUMBER 0x464B3138

#define ZONEFILE_SZ_HEADER           32
#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20

#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
#define ZONEFILE_DEFAULT_ENCRYPTION       0
#define ZONEFILE_DEFAULT_COMPRESSION      0

#define ZONEFILE_COMPRESSION_NONE             0
#define ZONEFILE_COMPRESSION_ZSTD             1
#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
#define ZONEFILE_COMPRESSION_ZLIB             3
#define ZONEFILE_COMPRESSION_BROTLI           4
#define ZONEFILE_COMPRESSION_LZ4              5
................................................................................
static u32 zonefileGet32(const u8 *aBuf){
  return (((u32)aBuf[0]) << 24)
       + (((u32)aBuf[1]) << 16)
       + (((u32)aBuf[2]) <<  8)
       + (((u32)aBuf[3]) <<  0);
}

#ifdef SQLITE_HAVE_ZLIB 




#include <zlib.h>

static int zfZlibOpen(void **pp){
  *pp = 0;
  return SQLITE_OK;
}
static void zfZlibClose(void *p){
}
static int zfZlibCompressBound(void *p, int nSrc){
  return (int)compressBound((uLong)nSrc) + 4;
................................................................................
  void *p, 
  const u8 *aSrc, int nSrc
){
  return (int)zonefileGet32(aSrc);
}
static int zfZlibUncompress(
  void *p, 
  u8 *aDest, int *pnDest, 
  const u8 *aSrc, int nSrc
){
  uLongf destLen = (uLongf)(*pnDest);
  int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4);






















  *pnDest = (int)destLen;

















































  return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
























































}
#endif

typedef struct ZonefileCompress ZonefileCompress;
static struct ZonefileCompress {
  int eType;
  const char *zName;
  int (*xOpen)(void**);
  void (*xClose)(void*);

  int (*xCompressBound)(void*, int nSrc);
  int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc);
  int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc);
  int (*xUncompress)(void*, u8 *aDest, int *pnDest,const u8 *aSrc,int nSrc);
} aZonefileCompress[] = {
  { ZONEFILE_COMPRESSION_NONE,             "none",
    0, 0, 0, 0, 0, 0
  },
#ifdef SQLITE_HAVE_ZSTD
  { ZONEFILE_COMPRESSION_ZSTD,             "zstd",
    0, 0, 0, 0, 0, 0



  },
  { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict",
    0, 0, 0, 0, 0, 0




  },
#endif /* SQLITE_HAVE_ZSTD */
#ifdef SQLITE_HAVE_ZLIB
  { ZONEFILE_COMPRESSION_ZLIB,             "zlib",
    zfZlibOpen, zfZlibClose, 

    zfZlibCompressBound, zfZlibCompress, 
    zfZlibUncompressSize, zfZlibUncompress
  },
#endif /* SQLITE_HAVE_ZLIB */
#ifdef SQLITE_HAVE_BROTLI
  { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
    0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_BROTLI */
#ifdef SQLITE_HAVE_LZ4
  { ZONEFILE_COMPRESSION_LZ4,              "lz4",
    0, 0, 0, 0, 0, 0
  },
  { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",
    0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_LZ4 */
};

static ZonefileCompress *zonefileCompress(const char *zName){
  int i;
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
................................................................................
  "CREATE TABLE z2("             \
  "  filename TEXT,"             \
  "  ekey BLOB,"                 \
  "  header JSON HIDDEN"         \
  ")"


#include <stdio.h>
#include <string.h>
#include <assert.h>

/*
** A structure to store the parameters for a single zonefile_write()
** invocation. 
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
................................................................................

static void zonefileFileClose(FILE *pFd){
  if( pFd ) fclose(pFd);
}

static int zonefileAppendCompressed(
  sqlite3_context *pCtx,          /* Leave any error message here */
  ZonefileCompress *pCmp,

  ZonefileBuffer *pTo,            /* Append new data here */
  ZonefileBuffer *pFrom           /* Input buffer */
){
  int rc = SQLITE_OK;
  if( pCmp->eType==ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
      rc = SQLITE_ERROR;
    }else{
      zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
    }
  }else{
    int nReq = pCmp->xCompressBound(0, pFrom->n);
    if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
    rc = pCmp->xCompress(0, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
    pTo->n += nReq;
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }
  return SQLITE_OK;
}
................................................................................
  int nKey = 0;                   /* Number of keys in new zonefile */
  int nFrame = 0;                 /* Number of frames in new zonefile */
  sqlite3_stmt *pStmt = 0;        /* SQL used to read data from source table */
  FILE *pFd = 0;
  int rc;
  sqlite3_value *pPrev = 0;
  char *zErr = 0;



  ZonefileBuffer sFrameIdx = {0, 0, 0};
  ZonefileBuffer sKeyIdx = {0, 0, 0};
  ZonefileBuffer sData = {0, 0, 0};         /* All completed frames so far */
  ZonefileBuffer sFrame = {0, 0, 0};        /* Current frame (uncompressed) */




  u8 aHdr[ZONEFILE_SZ_HEADER];    /* Space to assemble zonefile header */

  assert( objc==2 || objc==3 );
  zFile = (const char*)sqlite3_value_text(objv[0]);
  zTbl = (const char*)sqlite3_value_text(objv[1]);
  if( objc==3 ){
    zJson = (const char*)sqlite3_value_text(objv[2]);
  }
  if( zonefileGetParams(pCtx, zJson, &sWrite) ) return;






















  /* Prepare the SQL statement used to read data from the source table. This
  ** also serves to verify the suitability of the source table schema. */
  pStmt = zonefileCtxPrepare(pCtx, 
      "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl
  );
  if( pStmt==0 ) return;

  /* Open a file-handle used to write out the zonefile */ 
  pFd = zonefileFileOpen(zFile, 1, &zErr);
  if( pFd==0 ){
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_finalize(pStmt);
    sqlite3_free(zErr);
    return;













































  }

  while( SQLITE_ROW==sqlite3_step(pStmt) ){
    sqlite3_int64 k = sqlite3_column_int64(pStmt, 0);
    sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
    int nBlob = sqlite3_column_bytes(pStmt, 2);
    const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);
................................................................................

    int bAuto = zonefileIsAutoFrame(pFrame);
    if( zonefileCompareValue(pFrame, pPrev) 
     || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
    ){
      /* Add new entry to sFrame */
      if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
       || zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) 
      ){
        goto zone_write_out;
      }
      sFrame.n = 0;
      zonefileAppend32(&sFrameIdx, sData.n);
      sqlite3_value_free(pPrev);
      pPrev = sqlite3_value_dup(pFrame);
................................................................................
    zonefileAppend32(&sKeyIdx, nBlob);

    /* Add uncompressed data for new entry to sFrame */
    if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out;
    zonefileAppendBlob(&sFrame, pBlob, nBlob);
    nKey++;
  }
  if( sFrame.n>0 ){
    if( zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) ){

      goto zone_write_out;
    }
  }
  sqlite3_value_free(pPrev);
  pPrev = 0;

  /* If a compression method was specified, compress the key-index here */
  if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out;
    zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n);
    zonefileBufferFree(&sKeyIdx);
    rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, &sKeyIdx, &sFrameIdx);
    sFrameIdx.n = 0;
    if( rc ) goto zone_write_out;
  }

  /* Create the zonefile header in the in-memory buffer */
  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
  aHdr[4] = sWrite.pCmpIdx->eType;
  aHdr[5] = sWrite.pCmpData->eType;

  zonefilePut32(&aHdr[6], 0);     /* Compression dictionary byte offset */
  zonefilePut32(&aHdr[10], ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n); 
  zonefilePut32(&aHdr[14], nFrame);
  zonefilePut32(&aHdr[18], nKey);
  aHdr[22] = sWrite.encryptionType;
  aHdr[23] = 0;                   /* Encryption key index */
  aHdr[24] = 0;                   /* extended header version */
  aHdr[25] = 0;                   /* extended header size */
  assert( ZONEFILE_SZ_HEADER>=26 );

  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);

  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
  if( rc ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
    goto zone_write_out;
  }

  if( fclose(pFd) ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
  }
  pFd = 0;

 zone_write_out:

  if( pFd ) fclose(pFd);
  sqlite3_finalize(pStmt);
  zonefileBufferFree(&sFrameIdx);
  zonefileBufferFree(&sKeyIdx);
  zonefileBufferFree(&sFrame);

  zonefileBufferFree(&sData);


}

typedef struct ZonefileFilesTab ZonefileFilesTab;
struct ZonefileFilesTab {
  sqlite3_vtab base;              /* Base class - must be first */
  sqlite3 *db;
  char *zBase;                    /* Name of this table */
................................................................................
    );
  }

  return rc;
}

static int zonefileUncompress(
  ZonefileCompress *pCmp,

  u8 *aIn, int nIn,
  u8 **paOut, int *pnOut
){
  int rc;
  int nOut = pCmp->xUncompressSize(0, aIn, nIn);
  u8 *aOut = sqlite3_malloc(nOut);

  assert( pCmp->eType!=ZONEFILE_COMPRESSION_NONE );
  if( aOut==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = pCmp->xUncompress(0, aOut, &nOut, aIn, nIn);
    if( rc ){
      sqlite3_free(aOut);
      aOut = 0;
    }
  }

  *paOut = aOut;
................................................................................
      rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER);
    }
  }

  if( rc==SQLITE_OK && pCmp ){
    u8 *aUn = 0;
    int nUn = 0;
    rc = zonefileUncompress(pCmp, aIdx, nIdx, &aUn, &nUn);
    if( rc==SQLITE_ERROR ){
      *pzErr = sqlite3_mprintf("failed to uncompress index");
    }
    sqlite3_free(aIdx);
    aIdx = aUn;
    nIdx = nUn;
  }
................................................................................

static void zonefileFree(void *p){
  sqlite3_free(p);
}

static int zonefileCtxUncompress(
  sqlite3_context *pCtx, 
  ZonefileCompress *pCmp, 

  u8 *aBuf, int nBuf,
  int iKeyOff, int nKey
){
  int rc;
  u8 *aUn = 0;
  int nUn = 0;
    
  rc = zonefileUncompress(pCmp, aBuf, nBuf, &aUn, &nUn);
  if( rc==SQLITE_OK ){
    sqlite3_result_blob(pCtx, &aUn[iKeyOff], nKey, SQLITE_TRANSIENT);
  }else if( rc==SQLITE_ERROR ){
    zonefileCtxError(pCtx, "failed to uncompress frame");
  }

  sqlite3_free(aUn);
................................................................................
  FILE *pFd = 0;
  int rc = SQLITE_OK;
  u32 iOff = 0;                   /* Offset of frame in file */
  u32 szFrame = 0;                /* Size of frame in bytes */
  int iKeyOff = 0;                /* Offset of record within frame */
  int szKey = 0;                  /* Uncompressed size of record in bytes */
  ZonefileHeader hdr;
  ZonefileCompress *pCmp = 0;


  if( pTab->pIdToName==0 ){
    rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, 
        "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?",
        pTab->zDb, pTab->zName
    );
    if( rc!=SQLITE_OK ){
................................................................................
  }

  /* Read the zonefile header */
  if( rc==SQLITE_OK ){
    rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
  }

  /* Find the compression method */
  if( rc==SQLITE_OK ){
    rc = zfFindCompress(hdr.compressionTypeContent, &pCmp, &zErr);


















  }

  /* Find the offset (iOff) and size (szFrame) of the frame that the
  ** record is stored in.  */
  if( rc==SQLITE_OK ){
    int iFrame = sqlite3_column_int(pCsr->pSelect, 2);
    u8 aSpace[8] = {0,0,0,0,0,0,0,0};
................................................................................
    sqlite3_free(aFree);
  }

  /* Read some data into memory. If the data is uncompressed, then just
  ** the required record is read. Otherwise, the entire frame is read
  ** into memory.  */
  if( rc==SQLITE_OK ){
    int sz = (pCmp ? (int)szFrame : szKey);
    i64 ofst = iOff + (pCmp ? 0 : iKeyOff);
    u8 *aBuf = sqlite3_malloc(sz);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + ofst);
      if( rc==SQLITE_OK ){
        if( pCmp==0 ){
          sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree);
          aBuf = 0;
        }else{
          rc = zonefileCtxUncompress(pCtx, pCmp, aBuf, szFrame, iKeyOff, szKey);


        }
      }else{
        zErr = sqlite3_mprintf(
            "failed to read %d bytes at offset %d from file \"%s\"", 
            sz, (int)(hdr.byteOffsetFrames+ofst), zFile
        );
      }
................................................................................
  sqlite3_reset(pTab->pIdToName);
  if( zErr ){
    assert( rc!=SQLITE_OK );
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
  }
  zonefileFileClose(pFd);

  return rc;
}

/* 
** zonefile virtual table module xColumn method.
*/
static int zonefileColumn(

# define NEVER(X)       ((X)?(assert(0),1):0)
#else
# define ALWAYS(X)      (X)
# define NEVER(X)       (X)
#endif
#endif   /* SQLITE_AMALGAMATION */

#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
#define ZONEFILE_DEFAULT_ENCRYPTION       0
#define ZONEFILE_DEFAULT_COMPRESSION      0
#define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)

#define ZONEFILE_MAGIC_NUMBER 0x464B3138

#define ZONEFILE_SZ_HEADER           32
#define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20





#define ZONEFILE_COMPRESSION_NONE             0
#define ZONEFILE_COMPRESSION_ZSTD             1
#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2
#define ZONEFILE_COMPRESSION_ZLIB             3
#define ZONEFILE_COMPRESSION_BROTLI           4
#define ZONEFILE_COMPRESSION_LZ4              5
................................................................................
static u32 zonefileGet32(const u8 *aBuf){
  return (((u32)aBuf[0]) << 24)
       + (((u32)aBuf[1]) << 16)
       + (((u32)aBuf[2]) <<  8)
       + (((u32)aBuf[3]) <<  0);
}

#include <stdio.h>
#include <string.h>
#include <assert.h>

#ifdef SQLITE_HAVE_ZLIB 
#include <zlib.h>

static int zfZlibOpen(void **pp, u8 *aDict, int nDict){
  *pp = 0;
  return SQLITE_OK;
}
static void zfZlibClose(void *p){
}
static int zfZlibCompressBound(void *p, int nSrc){
  return (int)compressBound((uLong)nSrc) + 4;
................................................................................
  void *p, 
  const u8 *aSrc, int nSrc
){
  return (int)zonefileGet32(aSrc);
}
static int zfZlibUncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){
  uLongf destLen = (uLongf)nDest;
  int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4);
  return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
}
#endif
#ifdef SQLITE_HAVE_ZSTD 
#include <zstd.h>
static int zfZstdOpen(void **pp, u8 *aDict, int nDict){
  *pp = 0;
  return SQLITE_OK;
}
static void zfZstdClose(void *p){
}
static int zfZstdCompressBound(void *p, int nSrc){
  return (int)ZSTD_compressBound((size_t)nSrc);
}
static int zfZstdCompress(
  void *p, 
  u8 *aDest, int *pnDest, 
  const u8 *aSrc, int nSrc
){
  size_t szDest = (size_t)(*pnDest);
  size_t rc = ZSTD_compress(aDest, szDest, aSrc, (size_t)nSrc, 1);
  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
  *pnDest = (int)rc;
  return SQLITE_OK;
}
static int zfZstdUncompressSize(
  void *p, 
  const u8 *aSrc, int nSrc
){
  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
}
static int zfZstdUncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){
  size_t rc = ZSTD_decompress(aDest, (size_t)nDest, aSrc, (size_t)nSrc);
  if( rc!=(size_t)nDest ) return SQLITE_ERROR;
  return SQLITE_OK;
}

#include <zdict.h>
typedef struct ZfZstddict ZfZstddict;
struct ZfZstddict {
  ZSTD_CDict *pCDict;
  ZSTD_CCtx *pCCtx;
  ZSTD_DDict *pDDict;
  ZSTD_DCtx *pDCtx;
};

static void zfZstddictClose(void *p){
  if( p ){
    ZfZstddict *pCmp = (ZfZstddict*)p;
    if( pCmp->pCDict ) ZSTD_freeCDict(pCmp->pCDict);
    if( pCmp->pCCtx ) ZSTD_freeCCtx(pCmp->pCCtx);
    if( pCmp->pDCtx ) ZSTD_freeDCtx(pCmp->pDCtx);
    sqlite3_free(pCmp);
  }
}
static int zfZstddictOpen(void **pp, u8 *aDict, int nDict){
  int rc = SQLITE_OK;
  ZfZstddict *pDict = (ZfZstddict*)sqlite3_malloc(sizeof(ZfZstddict));
  if( pDict==0 ){
    rc = SQLITE_NOMEM;
  }else{
    memset(pDict, 0, sizeof(ZfZstddict));
    if( aDict ){
      pDict->pDDict = ZSTD_createDDict(aDict, nDict);
      pDict->pDCtx = ZSTD_createDCtx();
      if( pDict->pDDict==0 || pDict->pDCtx==0 ){
        zfZstddictClose((void*)pDict);
        pDict = 0;
        rc = SQLITE_ERROR;
      }
    }
  }
  *pp = (void*)pDict;
  return rc;
}
static int zfZstddictTrain(
  void *p,                        /* Compressor handle */
  u8 *aDict, int *pnDict,         /* OUT: Dictionary buffer */
  u8 *aSamp, size_t *aSz, int nSamp  /* IN: Training samples */
){
  ZfZstddict *pCmp = (ZfZstddict*)p;
  size_t sz = ZDICT_trainFromBuffer(aDict, (size_t)*pnDict, aSamp, aSz, nSamp);
  if( ZDICT_isError(sz) ) return SQLITE_ERROR;
  pCmp->pCDict = ZSTD_createCDict(aDict, sz, 1);
  pCmp->pCCtx = ZSTD_createCCtx();
  if( pCmp->pCDict==0 || pCmp->pCCtx==0 ) return SQLITE_ERROR;
  *pnDict = (int)sz;
  return SQLITE_OK;
}
static int zfZstddictCompressBound(void *p, int nSrc){
  return (int)ZSTD_compressBound((size_t)nSrc);
}
static int zfZstddictCompress(
  void *p, 
  u8 *aDest, int *pnDest, 
  const u8 *aSrc, int nSrc
){
  ZfZstddict *pCmp = (ZfZstddict*)p;
  size_t szDest = (size_t)(*pnDest);
  size_t rc;
  assert( pCmp && pCmp->pCDict && pCmp->pCCtx );
  rc = ZSTD_compress_usingCDict(
      pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict
  );
  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
  *pnDest = (int)rc;
  return SQLITE_OK;
}
static int zfZstddictUncompressSize(
  void *p, 
  const u8 *aSrc, int nSrc
){
  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
}
static int zfZstddictUncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){
  ZfZstddict *pCmp = (ZfZstddict*)p;
  size_t rc = ZSTD_decompress_usingDDict(
      pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict
  );
  if( rc!=(size_t)nDest ) return SQLITE_ERROR;
  return SQLITE_OK;
}
#endif

typedef struct ZonefileCompress ZonefileCompress;
static struct ZonefileCompress {
  int eType;
  const char *zName;
  int (*xOpen)(void**, u8 *aDict, int nDict);
  void (*xClose)(void*);
  int (*xTrain)(void*, u8 *aDict, int *pnDict, u8 *a, size_t *aSz, int n);
  int (*xCompressBound)(void*, int nSrc);
  int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc);
  int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc);
  int (*xUncompress)(void*, u8 *aDest, int nDest, const u8 *aSrc, int nSrc);
} aZonefileCompress[] = {
  { ZONEFILE_COMPRESSION_NONE,             "none",
    0, 0, 0, 0, 0, 0, 0
  },
#ifdef SQLITE_HAVE_ZSTD
  { ZONEFILE_COMPRESSION_ZSTD,             "zstd",
    zfZstdOpen, zfZstdClose, 
    0,
    zfZstdCompressBound, zfZstdCompress, 
    zfZstdUncompressSize, zfZstdUncompress
  },
  { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict",

    zfZstddictOpen, zfZstddictClose, 
    zfZstddictTrain,
    zfZstddictCompressBound, zfZstddictCompress, 
    zfZstddictUncompressSize, zfZstddictUncompress
  },
#endif /* SQLITE_HAVE_ZSTD */
#ifdef SQLITE_HAVE_ZLIB
  { ZONEFILE_COMPRESSION_ZLIB,             "zlib",
    zfZlibOpen, zfZlibClose, 
    0,
    zfZlibCompressBound, zfZlibCompress, 
    zfZlibUncompressSize, zfZlibUncompress
  },
#endif /* SQLITE_HAVE_ZLIB */
#ifdef SQLITE_HAVE_BROTLI
  { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
    0, 0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_BROTLI */
#ifdef SQLITE_HAVE_LZ4
  { ZONEFILE_COMPRESSION_LZ4,              "lz4",
    0, 0, 0, 0, 0, 0, 0
  },
  { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",
    0, 0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_LZ4 */
};

static ZonefileCompress *zonefileCompress(const char *zName){
  int i;
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
................................................................................
  "CREATE TABLE z2("             \
  "  filename TEXT,"             \
  "  ekey BLOB,"                 \
  "  header JSON HIDDEN"         \
  ")"






/*
** A structure to store the parameters for a single zonefile_write()
** invocation. 
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
................................................................................

static void zonefileFileClose(FILE *pFd){
  if( pFd ) fclose(pFd);
}

static int zonefileAppendCompressed(
  sqlite3_context *pCtx,          /* Leave any error message here */
  ZonefileCompress *pMethod,      /* Compression method object */
  void *pCmp,                     /* Compression handle */
  ZonefileBuffer *pTo,            /* Append new data here */
  ZonefileBuffer *pFrom           /* Input buffer */
){
  int rc = SQLITE_OK;
  if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
      rc = SQLITE_ERROR;
    }else{
      zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
    }
  }else{
    int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
    if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
    rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
    pTo->n += nReq;
    if( rc!=SQLITE_OK ){
      return rc;
    }
  }
  return SQLITE_OK;
}
................................................................................
  int nKey = 0;                   /* Number of keys in new zonefile */
  int nFrame = 0;                 /* Number of frames in new zonefile */
  sqlite3_stmt *pStmt = 0;        /* SQL used to read data from source table */
  FILE *pFd = 0;
  int rc;
  sqlite3_value *pPrev = 0;
  char *zErr = 0;
  void *pCmp = 0;                 /* Data compressor handle */
  u32 iOff;

  ZonefileBuffer sFrameIdx = {0, 0, 0};     /* Array of frame offsets */
  ZonefileBuffer sKeyIdx = {0, 0, 0};       /* Array of key locations */
  ZonefileBuffer sData = {0, 0, 0};         /* All completed frames so far */
  ZonefileBuffer sFrame = {0, 0, 0};        /* Current frame (uncompressed) */
  ZonefileBuffer sDict = {0, 0, 0};         /* Compression model (if any) */
  ZonefileBuffer sSample = {0,0,0};         /* Sample data for compressor */
  size_t *aSample = 0;                      /* Array of sample sizes */

  u8 aHdr[ZONEFILE_SZ_HEADER];    /* Space to assemble zonefile header */

  assert( objc==2 || objc==3 );
  zFile = (const char*)sqlite3_value_text(objv[0]);
  zTbl = (const char*)sqlite3_value_text(objv[1]);
  if( objc==3 ){
    zJson = (const char*)sqlite3_value_text(objv[2]);
  }
  if( zonefileGetParams(pCtx, zJson, &sWrite) ) return;

  /* Check that the index-data compressor is not one that uses an external
  ** dictionary. This is not permitted as there is no sense in using a
  ** dictionary to compress a single blob of data, and the index-data
  ** compressor only ever compresses a single frame. Also, there is nowhere
  ** in the file-format to store such a dictionary for the index-data.  */
  if( sWrite.pCmpIdx->xTrain ){
    zonefileCtxError(pCtx, 
        "compressor \"%s\" may not be used to compress the zonefile index", 
        sWrite.pCmpIdx->zName
    );
    goto zone_write_out;
  }

  if( sWrite.pCmpData->xOpen ){
    rc = sWrite.pCmpData->xOpen(&pCmp, 0, 0);
    if( rc!=SQLITE_OK ){
      zonefileCtxError(pCtx, "error in compressor construction");
      goto zone_write_out;
    }
  }

  /* Prepare the SQL statement used to read data from the source table. This
  ** also serves to verify the suitability of the source table schema. */
  pStmt = zonefileCtxPrepare(pCtx, 
      "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl
  );
  if( pStmt==0 ) goto zone_write_out;

  /* Open a file-handle used to write out the zonefile */ 
  pFd = zonefileFileOpen(zFile, 1, &zErr);
  if( pFd==0 ){
    sqlite3_result_error(pCtx, zErr, -1);

    sqlite3_free(zErr);

    goto zone_write_out;
  }

  /* If the data compressor uses a global dictionary, create the dictionary
  ** now.  */
  if( sWrite.pCmpData->xTrain ){
    int nSample = 0;

    while( SQLITE_ROW==sqlite3_step(pStmt) ){
      int nByte = sqlite3_column_bytes(pStmt, 2);
      const u8 *aByte = (const u8*)sqlite3_column_blob(pStmt, 2);
      if( zonefileBufferGrow(pCtx, &sSample, nByte) ){
        goto zone_write_out;
      }
      if( (nSample & (nSample-1))==0 ){
        int nNew = nSample ? nSample*2 : 8;
        size_t *aNew = (size_t*)sqlite3_realloc(aSample, sizeof(size_t) * nNew);
        if( aNew==0 ){
          sqlite3_result_error_nomem(pCtx);
          goto zone_write_out;
        }
        aSample = aNew;
      }
      aSample[nSample] = nByte;
      zonefileAppendBlob(&sSample, aByte, nByte);
      nSample++;
    }
    rc = sqlite3_reset(pStmt);
    if( rc!=SQLITE_OK ){
      zonefileTransferError(pCtx);
      goto zone_write_out;
    }

    if( zonefileBufferGrow(pCtx, &sDict, ZONEFILE_DEFAULT_DICTSIZE) ){
      goto zone_write_out;
    }
    sDict.n = sDict.nAlloc;

    rc = sWrite.pCmpData->xTrain(
        pCmp, sDict.a, &sDict.n, sSample.a, aSample, nSample
    );
    if( rc!=SQLITE_OK ){
      zonefileCtxError(pCtx, "error generating dictionary");
      goto zone_write_out;
    }
  }

  while( SQLITE_ROW==sqlite3_step(pStmt) ){
    sqlite3_int64 k = sqlite3_column_int64(pStmt, 0);
    sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
    int nBlob = sqlite3_column_bytes(pStmt, 2);
    const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);
................................................................................

    int bAuto = zonefileIsAutoFrame(pFrame);
    if( zonefileCompareValue(pFrame, pPrev) 
     || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
    ){
      /* Add new entry to sFrame */
      if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
       || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame)
      ){
        goto zone_write_out;
      }
      sFrame.n = 0;
      zonefileAppend32(&sFrameIdx, sData.n);
      sqlite3_value_free(pPrev);
      pPrev = sqlite3_value_dup(pFrame);
................................................................................
    zonefileAppend32(&sKeyIdx, nBlob);

    /* Add uncompressed data for new entry to sFrame */
    if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out;
    zonefileAppendBlob(&sFrame, pBlob, nBlob);
    nKey++;
  }
  if( sFrame.n>0
   && zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) 
  ){
    goto zone_write_out;

  }
  sqlite3_value_free(pPrev);
  pPrev = 0;

  /* If a compression method was specified, compress the key-index here */
  if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out;
    zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n);
    zonefileBufferFree(&sKeyIdx);
    rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, 0, &sKeyIdx,&sFrameIdx);
    sFrameIdx.n = 0;
    if( rc ) goto zone_write_out;
  }

  /* Create the zonefile header in the in-memory buffer */
  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
  aHdr[4] = sWrite.pCmpIdx->eType;
  aHdr[5] = sWrite.pCmpData->eType;
  iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
  zonefilePut32(&aHdr[6], sDict.n ? iOff : 0);
  zonefilePut32(&aHdr[10], iOff + sDict.n);
  zonefilePut32(&aHdr[14], nFrame);
  zonefilePut32(&aHdr[18], nKey);
  aHdr[22] = sWrite.encryptionType;
  aHdr[23] = 0;                   /* Encryption key index */
  aHdr[24] = 0;                   /* extended header version */
  aHdr[25] = 0;                   /* extended header size */
  assert( ZONEFILE_SZ_HEADER>=26 );

  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
  if( rc ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
    goto zone_write_out;
  }

  if( fclose(pFd) ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
  }
  pFd = 0;

 zone_write_out:
  if( pCmp ) sWrite.pCmpData->xClose(pCmp);
  if( pFd ) fclose(pFd);
  sqlite3_finalize(pStmt);
  zonefileBufferFree(&sFrameIdx);
  zonefileBufferFree(&sKeyIdx);
  zonefileBufferFree(&sFrame);
  zonefileBufferFree(&sDict);
  zonefileBufferFree(&sData);
  zonefileBufferFree(&sSample);
  sqlite3_free(aSample);
}

typedef struct ZonefileFilesTab ZonefileFilesTab;
struct ZonefileFilesTab {
  sqlite3_vtab base;              /* Base class - must be first */
  sqlite3 *db;
  char *zBase;                    /* Name of this table */
................................................................................
    );
  }

  return rc;
}

static int zonefileUncompress(
  ZonefileCompress *pMethod,
  void *pCmp,
  u8 *aIn, int nIn,
  u8 **paOut, int *pnOut
){
  int rc;
  int nOut = pMethod->xUncompressSize(pCmp, aIn, nIn);
  u8 *aOut = sqlite3_malloc(nOut);

  assert( pMethod->eType!=ZONEFILE_COMPRESSION_NONE );
  if( aOut==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = pMethod->xUncompress(pCmp, aOut, nOut, aIn, nIn);
    if( rc ){
      sqlite3_free(aOut);
      aOut = 0;
    }
  }

  *paOut = aOut;
................................................................................
      rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER);
    }
  }

  if( rc==SQLITE_OK && pCmp ){
    u8 *aUn = 0;
    int nUn = 0;
    rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn);
    if( rc==SQLITE_ERROR ){
      *pzErr = sqlite3_mprintf("failed to uncompress index");
    }
    sqlite3_free(aIdx);
    aIdx = aUn;
    nIdx = nUn;
  }
................................................................................

static void zonefileFree(void *p){
  sqlite3_free(p);
}

static int zonefileCtxUncompress(
  sqlite3_context *pCtx, 
  ZonefileCompress *pMethod, 
  void *pCmp, 
  u8 *aBuf, int nBuf,
  int iKeyOff, int nKey
){
  int rc;
  u8 *aUn = 0;
  int nUn = 0;
    
  rc = zonefileUncompress(pMethod, pCmp, aBuf, nBuf, &aUn, &nUn);
  if( rc==SQLITE_OK ){
    sqlite3_result_blob(pCtx, &aUn[iKeyOff], nKey, SQLITE_TRANSIENT);
  }else if( rc==SQLITE_ERROR ){
    zonefileCtxError(pCtx, "failed to uncompress frame");
  }

  sqlite3_free(aUn);
................................................................................
  FILE *pFd = 0;
  int rc = SQLITE_OK;
  u32 iOff = 0;                   /* Offset of frame in file */
  u32 szFrame = 0;                /* Size of frame in bytes */
  int iKeyOff = 0;                /* Offset of record within frame */
  int szKey = 0;                  /* Uncompressed size of record in bytes */
  ZonefileHeader hdr;
  ZonefileCompress *pCmpMethod = 0;
  void *pCmp = 0;

  if( pTab->pIdToName==0 ){
    rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, 
        "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?",
        pTab->zDb, pTab->zName
    );
    if( rc!=SQLITE_OK ){
................................................................................
  }

  /* Read the zonefile header */
  if( rc==SQLITE_OK ){
    rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr);
  }

  /* Find the compression method and open the compressor instance */
  if( rc==SQLITE_OK ){
    rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr);
  }
  if( pCmpMethod ){
    int nDict = 0;
    u8 *aDict = 0;
    assert( rc==SQLITE_OK );
    if( hdr.byteOffsetDictionary ){
      nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary;
      aDict = sqlite3_malloc(nDict);
      if( aDict==0 ){
        rc = SQLITE_NOMEM;
      }else{
        rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary);
      }
    }
    if( rc==SQLITE_OK ){
      rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
    }
    sqlite3_free(aDict);
  }

  /* Find the offset (iOff) and size (szFrame) of the frame that the
  ** record is stored in.  */
  if( rc==SQLITE_OK ){
    int iFrame = sqlite3_column_int(pCsr->pSelect, 2);
    u8 aSpace[8] = {0,0,0,0,0,0,0,0};
................................................................................
    sqlite3_free(aFree);
  }

  /* Read some data into memory. If the data is uncompressed, then just
  ** the required record is read. Otherwise, the entire frame is read
  ** into memory.  */
  if( rc==SQLITE_OK ){
    int sz = (pCmpMethod ? (int)szFrame : szKey);
    i64 ofst = iOff + (pCmpMethod ? 0 : iKeyOff);
    u8 *aBuf = sqlite3_malloc(sz);
    if( aBuf==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + ofst);
      if( rc==SQLITE_OK ){
        if( pCmpMethod==0 ){
          sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree);
          aBuf = 0;
        }else{
          rc = zonefileCtxUncompress(
              pCtx, pCmpMethod, pCmp, aBuf, szFrame, iKeyOff, szKey
          );
        }
      }else{
        zErr = sqlite3_mprintf(
            "failed to read %d bytes at offset %d from file \"%s\"", 
            sz, (int)(hdr.byteOffsetFrames+ofst), zFile
        );
      }
................................................................................
  sqlite3_reset(pTab->pIdToName);
  if( zErr ){
    assert( rc!=SQLITE_OK );
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
  }
  zonefileFileClose(pFd);
  if( pCmpMethod ) pCmpMethod->xClose(pCmp);
  return rc;
}

/* 
** zonefile virtual table module xColumn method.
*/
static int zonefileColumn(

} {}

do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0

do_execsql_test 1.7 { DROP TABLE z1 }

#-------------------------------------------------------------------------















foreach {tn cmp cmpidx} {
  1 none none
  2 zlib none
  3 none zlib
  4 zlib zlib

} {



























  reset_db
  load_static_extension db zonefile
  


  do_execsql_test 2.$tn.0 {
    CREATE TABLE zz(
      k INTEGER PRIMARY KEY, 
      frame INTEGER DEFAULT -1, 
      idx INTEGER DEFAULT -1, 
      v BLOB
    );
................................................................................
    WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
    ORDER BY 1
  } {
    test19.zonefile 1 1
    test20.zonefile 2 2
    test21.zonefile 7 4
  }
}


finish_test

} {}

do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0

do_execsql_test 1.7 { DROP TABLE z1 }

#-------------------------------------------------------------------------
# Figure out which compression algorithms, if any, are supported by
# this build. Populate the global list $COMPRESSION_METHODS with the
# result.
reset_db
load_static_extension db zonefile
do_execsql_test 2.0 {
  CREATE TABLE bb(
    k INTEGER PRIMARY KEY, 
    frame INTEGER DEFAULT -1, 
    idx INTEGER DEFAULT -1, 
    v BLOB
  );
  INSERT INTO bb(k, v) VALUES(1, randomblob(100));
}
set COMPRESSION_METHODS [list]
foreach cmp {




  none zlib zstd zstd_global_dict lz4 lz4hc brotli
} {
  set res [catchsql {
    WITH p(n,v) AS (
        VALUES('compressionTypeContent', $cmp)
    )
    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
  }]

  if {[lindex $res 0]==0} {
    lappend COMPRESSION_METHODS $cmp
  }
}

# Check that it is not possible to use zstd_global_dict to compress
# the zonefile index.
#
if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} {
  do_catchsql_test 2.1 {
    WITH p(n,v) AS (
        VALUES('compressionTypeIndexData', 'zstd_global_dict')
    )
    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
}
# puts $COMPRESSION_METHODS

foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
  if {$cmpidx == "zstd_global_dict"} continue
  reset_db
  load_static_extension db zonefile

  set tn "$cmp/$cmpidx"

  do_execsql_test 2.$tn.0 {
    CREATE TABLE zz(
      k INTEGER PRIMARY KEY, 
      frame INTEGER DEFAULT -1, 
      idx INTEGER DEFAULT -1, 
      v BLOB
    );
................................................................................
    WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile')
    ORDER BY 1
  } {
    test19.zonefile 1 1
    test20.zonefile 2 2
    test21.zonefile 7 4
  }
}}


finish_test