/ Check-in [55cf920c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for invoking encryption hooks to zonefile. And mock encryption method "xor" for testing.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256:55cf920c5a13473d04f0cb885117c04b2bc054bfed6ee549be84cb9485c104d2
User & Date: dan 2018-02-19 21:07:20
Context
2018-02-20
18:47
Instead of just the frame number, store frame sizes and offsets in zonefile shadow table %_shadow_idx. check-in: 56801c46 user: dan tags: zonefile
2018-02-19
21:07
Add support for invoking encryption hooks to zonefile. And mock encryption method "xor" for testing. check-in: 55cf920c user: dan tags: zonefile
16:28
Add support for the ExtendedHeaderSize header field to zonefile. check-in: 78267a09 user: dan tags: zonefile
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/zonefile/zonefile.c.

    30     30   # define ALWAYS(X)      ((X)?1:(assert(0),0))
    31     31   # define NEVER(X)       ((X)?(assert(0),1):0)
    32     32   #else
    33     33   # define ALWAYS(X)      (X)
    34     34   # define NEVER(X)       (X)
    35     35   #endif
    36     36   #endif   /* SQLITE_AMALGAMATION */
           37  +
           38  +#ifndef SQLITE_HAVE_ZONEFILE_CODEC
           39  +typedef struct ZonefileCodec ZonefileCodec;
           40  +
           41  +struct ZonefileCodec {
           42  +  u8 aKey[16];
           43  +};
           44  +
           45  +/* Create a new encryption module instance using algorithm iAlg.
           46  +**
           47  +**   iAlg==1   AES128 CTR
           48  +**   iAlg==2   AES128 CBC
           49  +**   iAlg==3   AES256 CTR
           50  +**   iAlg==4   AES256 CBC
           51  +**   iAlg==5   XOR          Testing use only
           52  +**
           53  +** If the requested algorithm is not available, the routine returns
           54  +** a NULL pointer.  NULL is also returned on a OOM error.
           55  +**
           56  +** Use zonefileCodecDestroy() to reclaim memory.
           57  +*/
           58  +static int zonefileCodecCreate(
           59  +  int iAlg, 
           60  +  unsigned char *pKey, int nKey, 
           61  +  ZonefileCodec **pp, 
           62  +  char **pzErr
           63  +){
           64  +  ZonefileCodec *pRet;
           65  +  int rc = SQLITE_OK;
           66  +  
           67  +  if( iAlg!=5 ){
           68  +    *pzErr = sqlite3_mprintf("unsupported encryption method: %d", iAlg);
           69  +    rc = SQLITE_ERROR;
           70  +  }else{
           71  +    *pp = pRet = (ZonefileCodec*)sqlite3_malloc(sizeof(ZonefileCodec));
           72  +    if( pRet==0 ){
           73  +      rc = SQLITE_NOMEM;
           74  +    }else{
           75  +      int i;
           76  +      for(i=0; i<sizeof(pRet->aKey); i++){
           77  +        pRet->aKey[i] = pKey[i % nKey];
           78  +      }
           79  +    }
           80  +  }
           81  +
           82  +  return rc;
           83  +}
           84  +
           85  +/* Return the size of the nonce used for the given encryption module */
           86  +static int zonefileCodecNonceSize(ZonefileCodec *pCodec){
           87  +  return 16;
           88  +}
           89  +
           90  +/* Encrypt in-place.
           91  +**
           92  +** The size of the content will grow by the nonce size.  Hence, the
           93  +** buffer must have at least nonce bytes of extra space available at
           94  +** the end to accommodate that growth.  When persisting results, be
           95  +** sure to include the extra bytes.
           96  +*/
           97  +static void zonefileCodecEncode(
           98  +  ZonefileCodec *pCodec, 
           99  +  unsigned char *pIn, int nIn
          100  +){
          101  +  int i;
          102  +  u8 *aNonce = &pIn[nIn];
          103  +  sqlite3_randomness(16, aNonce);
          104  +  for(i=0; i<nIn; i++){
          105  +    pIn[i] = pIn[i] ^ aNonce[i%16] ^ pCodec->aKey[i%16];
          106  +  }
          107  +}
          108  +
          109  +/* Decrypt in-place.
          110  +**
          111  +** The size of the decrypted text will be less than the input buffer
          112  +** by nonce-size bytes.
          113  +*/
          114  +static void zonefileCodecDecode(
          115  +  ZonefileCodec *pCodec, 
          116  +  unsigned char *pIn, int nIn
          117  +){
          118  +  int i;
          119  +  u8 *aNonce = &pIn[nIn-16];
          120  +  for(i=0; i<nIn-16; i++){
          121  +    pIn[i] = pIn[i] ^ aNonce[i%16] ^ pCodec->aKey[i%16];
          122  +  }
          123  +}
          124  +
          125  +/* Destroy an encryption module.
          126  +** It is harmless to pass in a NULL pointer.
          127  +*/
          128  +static void zonefileCodecDestroy(ZonefileCodec *pCodec){
          129  +  sqlite3_free(pCodec);
          130  +}
          131  +#endif                           /* SQLITE_HAVE_ZONEFILE_CODEC */
          132  +
          133  +typedef struct ZonefileGlobal ZonefileGlobal;
          134  +typedef struct ZonefileKey ZonefileKey;
          135  +struct ZonefileGlobal {
          136  +  ZonefileKey *pList;
          137  +};
          138  +struct ZonefileKey {
          139  +  const char *zName;              /* Table name */
          140  +  const char *zDb;                /* Database name */
          141  +  i64 iFileid;                    /* File id */
          142  +  const char *zKey;               /* Key buffer */
          143  +  int nKey;                       /* Size of zKey in bytes */
          144  +  ZonefileKey *pNext;             /* Next key on same db connection */
          145  +};
          146  +
    37    147   
    38    148   #define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024)
    39         -#define ZONEFILE_DEFAULT_ENCRYPTION       0
          149  +#define ZONEFILE_DEFAULT_ENCRYPTION       1
    40    150   #define ZONEFILE_DEFAULT_COMPRESSION      0
    41    151   #define ZONEFILE_DEFAULT_DICTSIZE         (64*1024)
    42    152   
    43    153   #define ZONEFILE_MAGIC_NUMBER 0x464B3138
    44    154   
    45    155   #define ZONEFILE_SZ_HEADER           32
    46    156   #define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20
................................................................................
   353    463       0,
   354    464       zfLz4CompressBound, zfLz4hcCompress, 
   355    465       zfGenericUncompressSize, zfLz4Uncompress
   356    466     },
   357    467   #endif /* SQLITE_HAVE_LZ4 */
   358    468   };
   359    469   
   360         -static ZonefileCompress *zonefileCompress(const char *zName){
          470  +static int zonefileCompress(
          471  +  const char *zName,              /* Name of requested compression method */
          472  +  ZonefileCompress **pp,          /* OUT: Pointer to compression object */
          473  +  char **pzErr                    /* OUT: Error message */
          474  +){
   361    475     int i;
          476  +  assert( *pzErr==0 );
   362    477     for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
   363    478       if( sqlite3_stricmp(aZonefileCompress[i].zName, zName)==0 ){
   364         -      return &aZonefileCompress[i];
          479  +      *pp = &aZonefileCompress[i];
          480  +      return SQLITE_OK;
   365    481       }
   366    482     }
   367         -  return 0;
          483  +  *pzErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName);
          484  +  return SQLITE_ERROR;
   368    485   }
   369    486   
   370    487   static ZonefileCompress *zonefileCompressByValue(int eType){
   371    488     int i;
   372    489     for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
   373    490       if( aZonefileCompress[i].eType==eType ){
   374    491         return &aZonefileCompress[i];
................................................................................
   401    518   
   402    519   
   403    520   
   404    521   /*
   405    522   ** A structure to store the parameters for a single zonefile_write()
   406    523   ** invocation. 
   407    524   */
   408         -typedef struct ZonefileWrite ZonefileWrite;
   409         -struct ZonefileWrite {
          525  +typedef struct ZonefileParam ZonefileParam;
          526  +struct ZonefileParam {
   410    527     ZonefileCompress *pCmpIdx;      /* For compressing the index */
   411    528     ZonefileCompress *pCmpData;     /* For compressing each frame */
   412    529     int encryptionType;
   413    530     int maxAutoFrameSize;
   414    531     int debugExtendedHeaderSize;    /* Size of extended header */
          532  +  char *encryptionKey;            /* Encryption key */
   415    533   };
   416    534   
   417    535   /*
   418    536   ** A structure to store a deserialized zonefile header in.
   419    537   */
   420    538   typedef struct ZonefileHeader ZonefileHeader;
   421    539   struct ZonefileHeader {
................................................................................
   543    661   
   544    662   int zonefileIsAutoFrame(sqlite3_value *pFrame){
   545    663     return (
   546    664         sqlite3_value_type(pFrame)==SQLITE_INTEGER 
   547    665      && sqlite3_value_int64(pFrame)==-1
   548    666     );
   549    667   }
          668  +
          669  +static int zonefileEncryption(const char *zName, int *peType, char **pzErr){
          670  +  struct Encryption {
          671  +    const char *zName;
          672  +    int eType;
          673  +  } a[] = {
          674  +    {"NONE", 0}, 
          675  +    {"AES_128_CTR", 1}, 
          676  +    {"AES_128_CBC", 2}, 
          677  +    {"AES_256_CTR", 3}, 
          678  +    {"AES_256_CBC", 4}, 
          679  +    {"XOR",         5}, 
          680  +  };
          681  +  int i;
          682  +
          683  +  for(i=0; i<sizeof(a)/sizeof(a[0]); i++){
          684  +    if( 0==sqlite3_stricmp(zName, a[i].zName) ){
          685  +      *peType = a[i].eType;
          686  +      return SQLITE_OK;
          687  +    }
          688  +  }
          689  +
          690  +  *pzErr = sqlite3_mprintf("unknown encryption type: %s", zName);
          691  +  return SQLITE_ERROR;
          692  +}
   550    693   
   551    694   static int zonefileGetParams(
   552    695     sqlite3_context *pCtx,          /* Leave any error message here */
   553    696     const char *zJson,              /* JSON configuration parameter (or NULL) */
   554         -  ZonefileWrite *p                /* Populate this object before returning */
          697  +  ZonefileParam *p                /* Populate this object before returning */
   555    698   ){
   556    699     sqlite3 *db = sqlite3_context_db_handle(pCtx);
   557    700     sqlite3_stmt *pStmt = 0;
   558    701     char *zErr = 0;
   559    702     int rc = SQLITE_OK;
   560    703     int rc2;
   561    704   
   562         -  memset(p, 0, sizeof(ZonefileWrite));
          705  +  memset(p, 0, sizeof(ZonefileParam));
   563    706     p->maxAutoFrameSize = ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE;
          707  +  p->encryptionType = ZONEFILE_DEFAULT_ENCRYPTION;
   564    708     p->pCmpData = p->pCmpIdx = zonefileCompressByValue(0);
   565    709   
   566    710     rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
   567    711     if( rc==SQLITE_OK ){
   568    712       sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
   569    713     }
   570    714     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
................................................................................
   580    724         p->debugExtendedHeaderSize = iVal;
   581    725       }else
   582    726       if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
   583    727         p->maxAutoFrameSize = iVal;
   584    728       }else
   585    729       if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
   586    730         const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
   587         -      p->pCmpIdx = zonefileCompress(zName);
   588         -      if( p->pCmpIdx==0 ){
   589         -        rc = SQLITE_ERROR;
   590         -        zErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName);
   591         -      }
          731  +      rc = zonefileCompress(zName, &p->pCmpIdx, &zErr);
   592    732       }else
   593    733       if( sqlite3_stricmp("compressionTypeContent", zKey)==0 ){
   594    734         const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
   595         -      p->pCmpData = zonefileCompress(zName);
   596         -      if( p->pCmpData==0 ){
   597         -        rc = SQLITE_ERROR;
   598         -        zErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName);
   599         -      }
          735  +      rc = zonefileCompress(zName, &p->pCmpData, &zErr);
          736  +    }else
          737  +    if( sqlite3_stricmp("encryptionKey", zKey)==0 ){
          738  +      const char *zKey = (const char*)sqlite3_column_text(pStmt, 1);
          739  +      p->encryptionKey = sqlite3_mprintf("%s", zKey);
          740  +      if( p->encryptionKey==0 ) rc = SQLITE_NOMEM;
   600    741       }else
   601    742       if( sqlite3_stricmp("encryptionType", zKey)==0 ){
   602         -      p->encryptionType = iVal;
          743  +      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
          744  +      rc = zonefileEncryption(zName, &p->encryptionType, &zErr);
   603    745       }else{
   604    746         rc = SQLITE_ERROR;
   605    747         zErr = sqlite3_mprintf("unknown parameter name: \"%s\"", zKey);
   606    748       }
   607    749     }
   608    750     rc2 = sqlite3_finalize(pStmt);
   609    751     if( rc==SQLITE_OK ) rc = rc2;
   610    752   
   611    753     if( zErr ){
   612    754       sqlite3_result_error(pCtx, zErr, -1);
   613    755       sqlite3_free(zErr);
          756  +  }else{
          757  +    if( p->encryptionKey==0 ){
          758  +      p->encryptionType = 0;
          759  +    }
   614    760     }
   615    761     return rc;
   616    762   }
   617    763   
   618    764   /*
   619    765   ** Check that there is room in buffer pBuf for at least nByte bytes more 
   620    766   ** data. If not, attempt to allocate more space. If the allocation attempt
................................................................................
   698    844     return pFd;
   699    845   }
   700    846   
   701    847   static void zonefileFileClose(FILE *pFd){
   702    848     if( pFd ) fclose(pFd);
   703    849   }
   704    850   
   705         -static int zonefileAppendCompressed(
          851  +static int zonefileAppendData(
   706    852     sqlite3_context *pCtx,          /* Leave any error message here */
   707    853     ZonefileCompress *pMethod,      /* Compression method object */
   708    854     void *pCmp,                     /* Compression handle */
          855  +  ZonefileCodec *pCodec,          /* Compression method, if any */
   709    856     ZonefileBuffer *pTo,            /* Append new data here */
   710    857     ZonefileBuffer *pFrom           /* Input buffer */
   711    858   ){
   712    859     int rc = SQLITE_OK;
   713    860     if( pFrom->n ){
          861  +    int nNonce = pCodec ? zonefileCodecNonceSize(pCodec) : 0;
          862  +    int nOrig = pTo->n;
   714    863       if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
   715         -      if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
          864  +      if( zonefileBufferGrow(pCtx, pTo, pFrom->n + nNonce) ){
   716    865           rc = SQLITE_ERROR;
   717    866         }else{
   718    867           zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
   719    868         }
   720    869       }else{
   721    870         int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
   722         -      if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
          871  +      if( zonefileBufferGrow(pCtx, pTo, nReq + nNonce) ) return SQLITE_ERROR;
   723    872         rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
   724    873         pTo->n += nReq;
   725    874         if( rc!=SQLITE_OK ){
   726    875           return rc;
   727    876         }
   728    877       }
          878  +
          879  +    /* Encrypt the data just appended to buffer pTo. */
          880  +    if( pCodec ){
          881  +      zonefileCodecEncode(pCodec, &pTo->a[nOrig], pTo->n - nOrig);
          882  +      pTo->n += nNonce;
          883  +    }
   729    884     }
   730    885     return SQLITE_OK;
   731    886   }
   732    887   
   733    888   /*
   734    889   ** Append nByte bytes of garbage data to the file opened with pFd. Return
   735    890   ** SQLITE_OK if successful, or SQLITE_ERROR if an error occurs.
................................................................................
   758    913     sqlite3_context *pCtx,       /* Context object */
   759    914     int objc,                       /* Number of SQL arguments */
   760    915     sqlite3_value **objv            /* Array of SQL arguments */
   761    916   ){
   762    917     const char *zFile = 0;          /* File to write to */
   763    918     const char *zTbl = 0;           /* Database object to read from */
   764    919     const char *zJson = 0;          /* JSON configuration parameters */
   765         -  ZonefileWrite sWrite;           /* Decoded JSON parameters */
          920  +  ZonefileParam sParam;           /* Decoded JSON parameters */
   766    921     int nKey = 0;                   /* Number of keys in new zonefile */
   767    922     int nFrame = 0;                 /* Number of frames in new zonefile */
   768    923     sqlite3_stmt *pStmt = 0;        /* SQL used to read data from source table */
   769    924     FILE *pFd = 0;
   770    925     int rc;
   771    926     sqlite3_value *pPrev = 0;
   772    927     char *zErr = 0;
   773    928     void *pCmp = 0;                 /* Data compressor handle */
   774    929     u32 iOff;
          930  +  ZonefileCodec *pCodec = 0;
   775    931   
   776    932     ZonefileBuffer sFrameIdx = {0, 0, 0};     /* Array of frame offsets */
   777    933     ZonefileBuffer sKeyIdx = {0, 0, 0};       /* Array of key locations */
   778    934     ZonefileBuffer sData = {0, 0, 0};         /* All completed frames so far */
   779    935     ZonefileBuffer sFrame = {0, 0, 0};        /* Current frame (uncompressed) */
   780    936     ZonefileBuffer sDict = {0, 0, 0};         /* Compression model (if any) */
   781    937     ZonefileBuffer sSample = {0,0,0};         /* Sample data for compressor */
................................................................................
   785    941   
   786    942     assert( objc==2 || objc==3 );
   787    943     zFile = (const char*)sqlite3_value_text(objv[0]);
   788    944     zTbl = (const char*)sqlite3_value_text(objv[1]);
   789    945     if( objc==3 ){
   790    946       zJson = (const char*)sqlite3_value_text(objv[2]);
   791    947     }
   792         -  if( zonefileGetParams(pCtx, zJson, &sWrite) ) return;
          948  +  if( zonefileGetParams(pCtx, zJson, &sParam) ) return;
          949  +
          950  +  if( sParam.encryptionType!=0 ){
          951  +    int n = strlen(sParam.encryptionKey);
          952  +    rc = zonefileCodecCreate(
          953  +        sParam.encryptionType, (u8*)sParam.encryptionKey, n, &pCodec, &zErr
          954  +    );
          955  +    if( rc!=SQLITE_OK ){
          956  +      sqlite3_result_error(pCtx, zErr, -1);
          957  +      sqlite3_free(zErr);
          958  +      goto zone_write_out;
          959  +    }
          960  +  }
   793    961   
   794    962     /* Check that the index-data compressor is not one that uses an external
   795    963     ** dictionary. This is not permitted as there is no sense in using a
   796    964     ** dictionary to compress a single blob of data, and the index-data
   797    965     ** compressor only ever compresses a single frame. Also, there is nowhere
   798    966     ** in the file-format to store such a dictionary for the index-data.  */
   799         -  if( sWrite.pCmpIdx->xTrain ){
          967  +  if( sParam.pCmpIdx->xTrain ){
   800    968       zonefileCtxError(pCtx, 
   801    969           "compressor \"%s\" may not be used to compress the zonefile index", 
   802         -        sWrite.pCmpIdx->zName
          970  +        sParam.pCmpIdx->zName
   803    971       );
   804    972       goto zone_write_out;
   805    973     }
   806    974   
   807         -  if( sWrite.pCmpData->xOpen ){
   808         -    rc = sWrite.pCmpData->xOpen(&pCmp, 0, 0);
          975  +  if( sParam.pCmpData->xOpen ){
          976  +    rc = sParam.pCmpData->xOpen(&pCmp, 0, 0);
   809    977       if( rc!=SQLITE_OK ){
   810    978         zonefileCtxError(pCtx, "error in compressor construction");
   811    979         goto zone_write_out;
   812    980       }
   813    981     }
   814    982   
   815    983     /* Prepare the SQL statement used to read data from the source table. This
................................................................................
   825    993       sqlite3_result_error(pCtx, zErr, -1);
   826    994       sqlite3_free(zErr);
   827    995       goto zone_write_out;
   828    996     }
   829    997   
   830    998     /* If the data compressor uses a global dictionary, create the dictionary
   831    999     ** and store it in buffer sDict.  */
   832         -  if( sWrite.pCmpData->xTrain ){
         1000  +  if( sParam.pCmpData->xTrain ){
   833   1001       int nSample = 0;
   834   1002   
   835   1003       while( SQLITE_ROW==sqlite3_step(pStmt) ){
   836   1004         int nByte = sqlite3_column_bytes(pStmt, 2);
   837   1005         const u8 *aByte = (const u8*)sqlite3_column_blob(pStmt, 2);
   838   1006         if( zonefileBufferGrow(pCtx, &sSample, nByte) ){
   839   1007           goto zone_write_out;
................................................................................
   858   1026       }
   859   1027   
   860   1028       if( zonefileBufferGrow(pCtx, &sDict, ZONEFILE_DEFAULT_DICTSIZE) ){
   861   1029         goto zone_write_out;
   862   1030       }
   863   1031       sDict.n = sDict.nAlloc;
   864   1032   
   865         -    rc = sWrite.pCmpData->xTrain(
         1033  +    rc = sParam.pCmpData->xTrain(
   866   1034           pCmp, sDict.a, &sDict.n, sSample.a, aSample, nSample
   867   1035       );
   868   1036       if( rc!=SQLITE_OK ){
   869   1037         zonefileCtxError(pCtx, "error generating dictionary");
   870   1038         goto zone_write_out;
   871   1039       }
   872   1040     }
................................................................................
   876   1044       sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1);
   877   1045       int nBlob = sqlite3_column_bytes(pStmt, 2);
   878   1046       const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);
   879   1047   
   880   1048       int bAuto = zonefileIsAutoFrame(pFrame);
   881   1049       if( sFrame.n>0 ){
   882   1050         if( zonefileCompareValue(pFrame, pPrev) 
   883         -       || (bAuto && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
         1051  +       || (bAuto && (sFrame.n+nBlob)>sParam.maxAutoFrameSize)
   884   1052         ){
   885   1053           /* Add new entry to sFrame */
   886   1054           if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
   887         -         || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp,&sData,&sFrame)
         1055  +         || zonefileAppendData(pCtx,sParam.pCmpData,pCmp,pCodec,&sData,&sFrame)
   888   1056           ){
   889   1057             goto zone_write_out;
   890   1058           }
   891   1059           sFrame.n = 0;
   892   1060           zonefileAppend32(&sFrameIdx, sData.n);
   893   1061           sqlite3_value_free(pPrev);
   894   1062           pPrev = 0;
................................................................................
   917   1085       if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out;
   918   1086       zonefileAppendBlob(&sFrame, pBlob, nBlob);
   919   1087       nKey++;
   920   1088     }
   921   1089   
   922   1090     if( sFrame.n>0 ){
   923   1091       if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
   924         -     || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame)
         1092  +     || zonefileAppendData(pCtx, sParam.pCmpData, pCmp, pCodec, &sData, &sFrame)
   925   1093       ){
   926   1094         goto zone_write_out;
   927   1095       }
   928   1096       zonefileAppend32(&sFrameIdx, sData.n);
   929   1097       nFrame++;
   930   1098     }
   931   1099   
   932   1100     /* If a compression method was specified, compress the key-index here */
   933         -  if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
         1101  +  if( sParam.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
   934   1102       if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out;
   935   1103       zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n);
   936   1104       zonefileBufferFree(&sKeyIdx);
   937         -    rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, 0, &sKeyIdx,&sFrameIdx);
         1105  +    rc = zonefileAppendData(pCtx, sParam.pCmpIdx, 0, 0, &sKeyIdx, &sFrameIdx);
   938   1106       sFrameIdx.n = 0;
   939   1107       if( rc ) goto zone_write_out;
   940   1108     }
   941   1109   
   942   1110     /* Create the zonefile header in the in-memory buffer */
   943   1111     memset(aHdr, 0, ZONEFILE_SZ_HEADER);
   944   1112     zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
   945         -  aHdr[4] = sWrite.pCmpIdx->eType;
   946         -  aHdr[5] = sWrite.pCmpData->eType;
         1113  +  aHdr[4] = sParam.pCmpIdx->eType;
         1114  +  aHdr[5] = sParam.pCmpData->eType;
   947   1115     iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
   948         -  zonefilePut32(&aHdr[6], sDict.n ? iOff+sWrite.debugExtendedHeaderSize : 0);
   949         -  zonefilePut32(&aHdr[10], iOff + sWrite.debugExtendedHeaderSize + sDict.n);
         1116  +  zonefilePut32(&aHdr[6], sDict.n ? iOff+sParam.debugExtendedHeaderSize : 0);
         1117  +  zonefilePut32(&aHdr[10], iOff + sParam.debugExtendedHeaderSize + sDict.n);
   950   1118     zonefilePut32(&aHdr[14], nFrame);
   951   1119     zonefilePut32(&aHdr[18], nKey);
   952         -  aHdr[22] = sWrite.encryptionType;
         1120  +  aHdr[22] = sParam.encryptionType;
   953   1121     aHdr[23] = 0;                   /* Encryption key index */
   954   1122     aHdr[24] = 0;                   /* extended header version */
   955         -  aHdr[25] = sWrite.debugExtendedHeaderSize;
         1123  +  aHdr[25] = sParam.debugExtendedHeaderSize;
   956   1124     assert( ZONEFILE_SZ_HEADER>=26 );
   957   1125   
   958   1126     rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
   959         -  if( rc==SQLITE_OK ) rc = zonefilePad(pFd, sWrite.debugExtendedHeaderSize);
         1127  +  if( rc==SQLITE_OK ) rc = zonefilePad(pFd, sParam.debugExtendedHeaderSize);
   960   1128     if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
   961   1129     if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
   962   1130     if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
   963   1131     if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
   964   1132     if( rc ){
   965   1133       zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
   966   1134       goto zone_write_out;
................................................................................
   968   1136   
   969   1137     if( fclose(pFd) ){
   970   1138       zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile);
   971   1139     }
   972   1140     pFd = 0;
   973   1141   
   974   1142    zone_write_out:
   975         -  if( pCmp ) sWrite.pCmpData->xClose(pCmp);
         1143  +  if( pCmp ) sParam.pCmpData->xClose(pCmp);
   976   1144     if( pFd ) fclose(pFd);
   977   1145     sqlite3_value_free(pPrev);
   978   1146     sqlite3_finalize(pStmt);
         1147  +  zonefileCodecDestroy(pCodec);
   979   1148     zonefileBufferFree(&sFrameIdx);
   980   1149     zonefileBufferFree(&sKeyIdx);
   981   1150     zonefileBufferFree(&sFrame);
   982   1151     zonefileBufferFree(&sDict);
   983   1152     zonefileBufferFree(&sData);
   984   1153     zonefileBufferFree(&sSample);
   985   1154     sqlite3_free(aSample);
         1155  +  sqlite3_free(sParam.encryptionKey);
   986   1156   }
   987   1157   
   988   1158   typedef struct ZonefileFilesTab ZonefileFilesTab;
   989   1159   struct ZonefileFilesTab {
   990   1160     sqlite3_vtab base;              /* Base class - must be first */
   991   1161     sqlite3 *db;
   992   1162     char *zBase;                    /* Name of this table */
   993   1163     char *zDb;                      /* Database containing this table */
         1164  +  ZonefileGlobal *pGlobal;        /* Global object */
   994   1165     sqlite3_stmt *pInsert;          /* Insert into the %_shadow_file table */
   995   1166     sqlite3_stmt *pInsertIdx;       /* Insert into the %_shadow_idx table */
   996   1167     sqlite3_stmt *pDeleteIdx;       /* Delete by fileid from %_shadow_idx table */
   997   1168     sqlite3_stmt *pDelete;          /* Delete by rowid from %_shadow_file table */
   998   1169   };
   999   1170   
  1000   1171   typedef struct ZonefileFilesCsr ZonefileFilesCsr;
................................................................................
  1005   1176   
  1006   1177   /*
  1007   1178   ** This function does the work of xCreate (if bCreate!=0) or xConnect
  1008   1179   ** (if bCreate==0) for the zonefile_files module.
  1009   1180   */
  1010   1181   static int zffCreateConnect(
  1011   1182     int bCreate,
         1183  +  void *pAux,
  1012   1184     sqlite3 *db,
  1013   1185     int argc, const char *const*argv,
  1014   1186     sqlite3_vtab **ppVtab,
  1015   1187     char **pzErr
  1016   1188   ){
  1017   1189     ZonefileFilesTab *p;
  1018   1190     const char *zName = argv[2];
................................................................................
  1034   1206       memset(p, 0, sizeof(ZonefileFilesTab));
  1035   1207       p->zBase = (char*)&p[1];
  1036   1208       memcpy(p->zBase, zName, nName-6);
  1037   1209       p->zBase[nName-6] = '\0';
  1038   1210       p->zDb = &p->zBase[nName+1];
  1039   1211       memcpy(p->zDb, zDb, nDb+1);
  1040   1212       p->db = db;
         1213  +    p->pGlobal = (ZonefileGlobal*)pAux;
  1041   1214       rc = sqlite3_declare_vtab(db, ZONEFILE_FILES_SCHEMA);
  1042   1215     }
  1043   1216   
  1044   1217     if( rc!=SQLITE_OK ){
  1045   1218       sqlite3_free(p);
  1046   1219       p = 0;
  1047   1220     }
................................................................................
  1055   1228   static int zffCreate(
  1056   1229     sqlite3 *db,
  1057   1230     void *pAux,
  1058   1231     int argc, const char *const*argv,
  1059   1232     sqlite3_vtab **ppVtab,
  1060   1233     char **pzErr
  1061   1234   ){
  1062         -  return zffCreateConnect(1, db, argc, argv, ppVtab, pzErr);
         1235  +  return zffCreateConnect(1, pAux, db, argc, argv, ppVtab, pzErr);
  1063   1236   }
  1064   1237   
  1065   1238   /* 
  1066   1239   ** zonefile_files virtual table module xConnect method.
  1067   1240   */
  1068   1241   static int zffConnect(
  1069   1242     sqlite3 *db,
  1070   1243     void *pAux,
  1071   1244     int argc, const char *const*argv,
  1072   1245     sqlite3_vtab **ppVtab,
  1073   1246     char **pzErr
  1074   1247   ){
  1075         -  return zffCreateConnect(0, db, argc, argv, ppVtab, pzErr);
         1248  +  return zffCreateConnect(0, pAux, db, argc, argv, ppVtab, pzErr);
  1076   1249   }
  1077   1250   
  1078   1251   /* 
  1079   1252   ** zonefile_files virtual table module xDisconnect method.
  1080   1253   */
  1081   1254   static int zffDisconnect(sqlite3_vtab *pVtab){
  1082   1255     ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab;
................................................................................
  1243   1416   }
  1244   1417   
  1245   1418   /* 
  1246   1419   ** zonefile_files virtual table module xColumn method.
  1247   1420   */
  1248   1421   static int zffColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
  1249   1422     ZonefileFilesCsr *pCsr = (ZonefileFilesCsr*)cur;
         1423  +  if( sqlite3_vtab_nochange(ctx) ){
         1424  +    return SQLITE_OK;
         1425  +  }
  1250   1426     switch( i ){
  1251   1427       case 0: /* filename */
  1252   1428         sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pSelect, 0));
  1253   1429         break;
  1254   1430       case 1: /* ekey */
  1255   1431         break;
  1256   1432       case 2: { /* header */
................................................................................
  1414   1590     }
  1415   1591   
  1416   1592     if( rc==SQLITE_OK && hdr.numKeys>0 ){
  1417   1593       u8 *aKey;                     /* Entire KeyOffsets array */
  1418   1594       int nKey;                     /* Size of buffer aKey[] in bytes */
  1419   1595       int i;
  1420   1596   
  1421         -    assert( hdr.encryptionType==0 );
  1422   1597       rc = zonefileLoadIndex(&hdr, pFd, &aKey, &nKey, &pTab->base.zErrMsg);
  1423   1598   
  1424   1599       if( rc==SQLITE_OK && pTab->pInsertIdx==0 ){
  1425   1600         rc = zonefilePrepare(pTab->db, &pTab->pInsertIdx, &pTab->base.zErrMsg,
  1426   1601             "INSERT INTO %Q.'%q_shadow_idx'(k, fileid, frame, ofst, sz)"
  1427   1602             "VALUES(?,?,?,?,?)",
  1428   1603             pTab->zDb, pTab->zBase
................................................................................
  1446   1621       }
  1447   1622       sqlite3_free(aKey);
  1448   1623     }
  1449   1624   
  1450   1625     zonefileFileClose(pFd);
  1451   1626     return rc;
  1452   1627   }
         1628  +
         1629  +static int zonefileStoreKey(
         1630  +  ZonefileFilesTab *pTab, 
         1631  +  i64 iFileid, 
         1632  +  const char *zKey
         1633  +){
         1634  +  ZonefileKey **pp;
         1635  +
         1636  +  for(pp=&pTab->pGlobal->pList; *pp; pp=&((*pp)->pNext)){
         1637  +    ZonefileKey *pThis = *pp;
         1638  +    if( pThis->iFileid==iFileid 
         1639  +     && 0==sqlite3_stricmp(pTab->zBase, pThis->zName)
         1640  +     && 0==sqlite3_stricmp(pTab->zDb, pThis->zDb)
         1641  +    ){
         1642  +      *pp = pThis->pNext;
         1643  +      sqlite3_free(pThis);
         1644  +      break;
         1645  +    }
         1646  +  }
         1647  +
         1648  +  if( zKey ){
         1649  +    int nKey = strlen(zKey);
         1650  +    int nDb = strlen(pTab->zDb);
         1651  +    int nName = strlen(pTab->zBase);
         1652  +    ZonefileKey *pNew;
         1653  +    pNew = (ZonefileKey*)sqlite3_malloc(
         1654  +        sizeof(ZonefileKey) + nKey+1 + nDb+1 + nName+1
         1655  +        );
         1656  +    if( pNew==0 ) return SQLITE_NOMEM;
         1657  +    memset(pNew, 0, sizeof(ZonefileKey));
         1658  +    pNew->iFileid = iFileid;
         1659  +    pNew->zKey = (const char*)&pNew[1];
         1660  +    pNew->nKey = nKey;
         1661  +    pNew->zDb = &pNew->zKey[nKey+1];
         1662  +    pNew->zName = &pNew->zDb[nDb+1];
         1663  +    memcpy((char*)pNew->zKey, zKey, nKey+1);
         1664  +    memcpy((char*)pNew->zDb, pTab->zDb, nDb+1);
         1665  +    memcpy((char*)pNew->zName, pTab->zBase, nName+1);
         1666  +
         1667  +    pNew->pNext = pTab->pGlobal->pList;
         1668  +    pTab->pGlobal->pList = pNew;
         1669  +  }
         1670  +  return SQLITE_OK;
         1671  +}
  1453   1672   
  1454   1673   /*
  1455   1674   ** zonefile_files virtual table module xUpdate method.
  1456   1675   **
  1457   1676   ** A delete specifies a single argument - the rowid of the row to remove.
  1458   1677   ** 
  1459   1678   ** Update and insert operations pass:
................................................................................
  1468   1687     sqlite3_value **apVal, 
  1469   1688     sqlite_int64 *pRowid
  1470   1689   ){
  1471   1690     int rc = SQLITE_OK;
  1472   1691     ZonefileFilesTab *pTab = (ZonefileFilesTab*)pVtab;
  1473   1692   
  1474   1693     if( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ){
  1475         -    if( pTab->pDelete==0 ){
  1476         -      rc = zonefilePrepare(pTab->db, &pTab->pDelete, &pVtab->zErrMsg,
  1477         -          "DELETE FROM %Q.'%q_shadow_file' WHERE fileid=?",
  1478         -          pTab->zDb, pTab->zBase
  1479         -      );
  1480         -    }
  1481         -    if( rc==SQLITE_OK && pTab->pDeleteIdx==0 ){
  1482         -      rc = zonefilePrepare(pTab->db, &pTab->pDeleteIdx, &pVtab->zErrMsg,
  1483         -          "DELETE FROM %Q.'%q_shadow_idx' WHERE fileid=?",
  1484         -          pTab->zDb, pTab->zBase
  1485         -      );
  1486         -    }
  1487         -
  1488         -    if( rc==SQLITE_OK ){
  1489         -      sqlite3_bind_value(pTab->pDelete, 1, apVal[0]);
  1490         -      sqlite3_step(pTab->pDelete);
  1491         -      rc = sqlite3_reset(pTab->pDelete);
  1492         -    }
  1493         -    if( rc==SQLITE_OK ){
  1494         -      sqlite3_bind_value(pTab->pDeleteIdx, 1, apVal[0]);
  1495         -      sqlite3_step(pTab->pDeleteIdx);
  1496         -      rc = sqlite3_reset(pTab->pDeleteIdx);
         1694  +    if( nVal>1 && sqlite3_value_nochange(apVal[2]) ){
         1695  +      const char *zKey = (const char*)sqlite3_value_text(apVal[3]);
         1696  +      i64 iFileid = sqlite3_value_int64(apVal[0]);
         1697  +      return zonefileStoreKey(pTab, iFileid, zKey);
         1698  +    }else{
         1699  +      if( pTab->pDelete==0 ){
         1700  +        rc = zonefilePrepare(pTab->db, &pTab->pDelete, &pVtab->zErrMsg,
         1701  +            "DELETE FROM %Q.'%q_shadow_file' WHERE fileid=?",
         1702  +            pTab->zDb, pTab->zBase
         1703  +            );
         1704  +      }
         1705  +      if( rc==SQLITE_OK && pTab->pDeleteIdx==0 ){
         1706  +        rc = zonefilePrepare(pTab->db, &pTab->pDeleteIdx, &pVtab->zErrMsg,
         1707  +            "DELETE FROM %Q.'%q_shadow_idx' WHERE fileid=?",
         1708  +            pTab->zDb, pTab->zBase
         1709  +            );
         1710  +      }
         1711  +
         1712  +      if( rc==SQLITE_OK ){
         1713  +        sqlite3_bind_value(pTab->pDelete, 1, apVal[0]);
         1714  +        sqlite3_step(pTab->pDelete);
         1715  +        rc = sqlite3_reset(pTab->pDelete);
         1716  +      }
         1717  +      if( rc==SQLITE_OK ){
         1718  +        sqlite3_bind_value(pTab->pDeleteIdx, 1, apVal[0]);
         1719  +        sqlite3_step(pTab->pDeleteIdx);
         1720  +        rc = sqlite3_reset(pTab->pDeleteIdx);
         1721  +      }
  1497   1722       }
  1498   1723     }
  1499   1724     if( nVal>1 ){
         1725  +    i64 iFileid = 0;
  1500   1726       const char *zFile = (const char*)sqlite3_value_text(apVal[2]);
  1501   1727   
  1502   1728       if( pTab->pInsert==0 ){
  1503   1729         rc = zonefilePrepare(pTab->db, &pTab->pInsert, &pVtab->zErrMsg,
  1504   1730             "INSERT INTO %Q.'%q_shadow_file'(filename) VALUES(?)",
  1505   1731             pTab->zDb, pTab->zBase
  1506   1732         );
................................................................................
  1512   1738         sqlite3_step(pTab->pInsert);
  1513   1739         rc = sqlite3_reset(pTab->pInsert);
  1514   1740       }
  1515   1741   
  1516   1742       /* Populate the %_shadow_idx table with entries for all keys in
  1517   1743       ** the zonefile just added to %_shadow_file.  */
  1518   1744       if( rc==SQLITE_OK ){
  1519         -      i64 iFileid = sqlite3_last_insert_rowid(pTab->db);
         1745  +      iFileid = sqlite3_last_insert_rowid(pTab->db);
  1520   1746         rc = zonefilePopulateIndex(pTab, zFile, iFileid);
  1521   1747       }
         1748  +
         1749  +    if( rc==SQLITE_OK ){
         1750  +      const char *zKey = (const char*)sqlite3_value_text(apVal[3]);
         1751  +      rc = zonefileStoreKey(pTab, iFileid, zKey);
         1752  +    }
  1522   1753     }
  1523   1754   
  1524   1755     return rc;
  1525   1756   }
  1526   1757   
  1527   1758   typedef struct ZonefileTab ZonefileTab;
  1528   1759   struct ZonefileTab {
  1529   1760     sqlite3_vtab base;         /* Base class - must be first */
  1530   1761     sqlite3 *db;
  1531   1762     sqlite3_stmt *pIdToName;   /* Translate fileid to filename */
         1763  +  ZonefileGlobal *pGlobal;
  1532   1764     char *zName;               /* Name of this table */
  1533   1765     char *zDb;                 /* Name of db containing this table */
  1534   1766   };
  1535   1767   
  1536   1768   typedef struct ZonefileCsr ZonefileCsr;
  1537   1769   struct ZonefileCsr {
  1538   1770     sqlite3_vtab_cursor base;  /* Base class - must be first */
................................................................................
  1545   1777   **
  1546   1778   **   argv[0]   -> module name  ("zonefile")
  1547   1779   **   argv[1]   -> database name
  1548   1780   **   argv[2]   -> table name
  1549   1781   */
  1550   1782   static int zonefileCreateConnect(
  1551   1783     int bCreate,
         1784  +  void *pAux,
  1552   1785     sqlite3 *db,
  1553   1786     int argc, const char *const*argv,
  1554   1787     sqlite3_vtab **ppVtab,
  1555   1788     char **pzErr
  1556   1789   ){
  1557   1790     ZonefileTab *p;
  1558   1791     const char *zName = argv[2];
................................................................................
  1567   1800     }else{
  1568   1801       memset(p, 0, sizeof(ZonefileTab));
  1569   1802       p->zName = (char*)&p[1];
  1570   1803       memcpy(p->zName, zName, nName+1);
  1571   1804       p->zDb = &p->zName[nName+1];
  1572   1805       memcpy(p->zDb, zDb, nDb+1);
  1573   1806       p->db = db;
         1807  +    p->pGlobal = (ZonefileGlobal*)pAux;
  1574   1808     
  1575   1809       if( bCreate ){
  1576   1810         char *zSql = sqlite3_mprintf(
  1577   1811             "CREATE TABLE %Q.'%q_shadow_idx'(" 
  1578   1812             "  k INTEGER PRIMARY KEY,"
  1579   1813             "  fileid INTEGER,"
  1580   1814             "  frame INTEGER,"
................................................................................
  1616   1850   static int zonefileCreate(
  1617   1851     sqlite3 *db,
  1618   1852     void *pAux,
  1619   1853     int argc, const char *const*argv,
  1620   1854     sqlite3_vtab **ppVtab,
  1621   1855     char **pzErr
  1622   1856   ){
  1623         -  return zonefileCreateConnect(1, db, argc, argv, ppVtab, pzErr);
         1857  +  return zonefileCreateConnect(1, pAux, db, argc, argv, ppVtab, pzErr);
  1624   1858   }
  1625   1859   
  1626   1860   /* 
  1627   1861   ** zonefile virtual table module xConnect method.
  1628   1862   */
  1629   1863   static int zonefileConnect(
  1630   1864     sqlite3 *db,
  1631   1865     void *pAux,
  1632   1866     int argc, const char *const*argv,
  1633   1867     sqlite3_vtab **ppVtab,
  1634   1868     char **pzErr
  1635   1869   ){
  1636         -  return zonefileCreateConnect(0, db, argc, argv, ppVtab, pzErr);
         1870  +  return zonefileCreateConnect(0, pAux, db, argc, argv, ppVtab, pzErr);
  1637   1871   }
  1638   1872   
  1639   1873   /* 
  1640   1874   ** zonefile virtual table module xDisconnect method.
  1641   1875   */
  1642   1876   static int zonefileDisconnect(sqlite3_vtab *pVtab){
  1643   1877     ZonefileTab *pTab = (ZonefileTab*)pVtab;
................................................................................
  1864   2098     }else if( rc==SQLITE_ERROR ){
  1865   2099       zonefileCtxError(pCtx, "failed to uncompress frame");
  1866   2100     }
  1867   2101   
  1868   2102     sqlite3_free(aUn);
  1869   2103     return rc;
  1870   2104   }
         2105  +
         2106  +static int zonefileFindKey(ZonefileTab *pTab, i64 iFileid, const char **pzKey){
         2107  +  ZonefileKey *pKey;
         2108  +
         2109  +  for(pKey=pTab->pGlobal->pList; pKey; pKey=pKey->pNext){
         2110  +    if( pKey->iFileid==iFileid 
         2111  +     && 0==sqlite3_stricmp(pTab->zName, pKey->zName)
         2112  +     && 0==sqlite3_stricmp(pTab->zDb, pKey->zDb)
         2113  +    ){
         2114  +      *pzKey = pKey->zKey;
         2115  +      return pKey->nKey;
         2116  +    }
         2117  +  }
         2118  +
         2119  +  return 0;
         2120  +}
  1871   2121   
  1872   2122   static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){
  1873   2123     ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab;
  1874   2124     const char *zFile = 0;
  1875   2125     char *zErr = 0;
  1876   2126     FILE *pFd = 0;
  1877   2127     int rc = SQLITE_OK;
  1878   2128     u32 iOff = 0;                   /* Offset of frame in file */
  1879   2129     u32 szFrame = 0;                /* Size of frame in bytes */
  1880   2130     int iKeyOff = 0;                /* Offset of record within frame */
  1881   2131     int szKey = 0;                  /* Uncompressed size of record in bytes */
         2132  +  i64 iFileid = 0;
  1882   2133     ZonefileHeader hdr;
  1883   2134     ZonefileCompress *pCmpMethod = 0;
         2135  +  ZonefileCodec *pCodec = 0;
  1884   2136     void *pCmp = 0;
  1885   2137   
  1886   2138     if( pTab->pIdToName==0 ){
  1887   2139       rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, 
  1888   2140           "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?",
  1889   2141           pTab->zDb, pTab->zName
  1890   2142       );
................................................................................
  1894   2146       }
  1895   2147     }
  1896   2148   
  1897   2149     iKeyOff = sqlite3_column_int(pCsr->pSelect, 3);
  1898   2150     szKey = sqlite3_column_int(pCsr->pSelect, 4);
  1899   2151   
  1900   2152     /* Open the file to read the blob from */
  1901         -  sqlite3_bind_int64(pTab->pIdToName, 1, sqlite3_column_int64(pCsr->pSelect,1));
         2153  +  iFileid = sqlite3_column_int64(pCsr->pSelect,1);
         2154  +  sqlite3_bind_int64(pTab->pIdToName, 1, iFileid);
  1902   2155     if( SQLITE_ROW==sqlite3_step(pTab->pIdToName) ){
  1903   2156       zFile = (const char*)sqlite3_column_text(pTab->pIdToName, 0);
  1904   2157       pFd = zonefileFileOpen(zFile, 0, &zErr);
  1905   2158     }
  1906   2159     if( zFile==0 ){
  1907   2160       rc = sqlite3_reset(pTab->pIdToName);
  1908   2161       if( rc!=SQLITE_OK ){
................................................................................
  1937   2190         }
  1938   2191       }
  1939   2192       if( rc==SQLITE_OK ){
  1940   2193         rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
  1941   2194       }
  1942   2195       sqlite3_free(aDict);
  1943   2196     }
         2197  +
         2198  +  if( hdr.encryptionType ){
         2199  +    const char *zKey = 0;
         2200  +    int nKey = zonefileFindKey(pTab, iFileid, &zKey);
         2201  +    if( nKey==0 ){
         2202  +      zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
         2203  +      rc = SQLITE_ERROR;
         2204  +    }else{
         2205  +      rc = zonefileCodecCreate(hdr.encryptionType,(u8*)zKey,nKey,&pCodec,&zErr);
         2206  +    }
         2207  +  }
  1944   2208   
  1945   2209     /* Find the offset (iOff) and size (szFrame) of the frame that the
  1946   2210     ** record is stored in.  */
  1947   2211     if( rc==SQLITE_OK ){
  1948   2212       int iFrame = sqlite3_column_int(pCsr->pSelect, 2);
  1949   2213       u8 aSpace[8] = {0,0,0,0,0,0,0,0};
  1950   2214       u8 *aOff = aSpace;
................................................................................
  1966   2230       sqlite3_free(aFree);
  1967   2231     }
  1968   2232   
  1969   2233     /* Read some data into memory. If the data is uncompressed, then just
  1970   2234     ** the required record is read. Otherwise, the entire frame is read
  1971   2235     ** into memory.  */
  1972   2236     if( rc==SQLITE_OK ){
  1973         -    int sz = (pCmpMethod ? (int)szFrame : szKey);
  1974         -    i64 ofst = iOff + (pCmpMethod ? 0 : iKeyOff);
  1975         -    u8 *aBuf = sqlite3_malloc(sz);
         2237  +    int nRead;                       /* Bytes of data to read */
         2238  +    i64 iRead;                       /* Offset to read at */
         2239  +
         2240  +    if( pCmpMethod || pCodec ){
         2241  +      nRead = szFrame;
         2242  +      iRead = iOff;
         2243  +    }else{
         2244  +      nRead = szKey;
         2245  +      iRead = iOff + iKeyOff;
         2246  +    }
         2247  +    
         2248  +    u8 *aBuf = sqlite3_malloc(nRead);
  1976   2249       if( aBuf==0 ){
  1977   2250         rc = SQLITE_NOMEM;
  1978   2251       }else{
  1979         -      rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + ofst);
         2252  +      rc = zonefileFileRead(pFd, aBuf, nRead, hdr.byteOffsetFrames + iRead);
  1980   2253         if( rc==SQLITE_OK ){
         2254  +        if( pCodec ){
         2255  +          zonefileCodecDecode(pCodec, aBuf, nRead);
         2256  +        }
  1981   2257           if( pCmpMethod==0 ){
  1982         -          sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree);
  1983         -          aBuf = 0;
         2258  +          if( pCodec==0 ){
         2259  +            sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree);
         2260  +            aBuf = 0;
         2261  +          }else{
         2262  +            sqlite3_result_blob(pCtx, &aBuf[iKeyOff], szKey, SQLITE_TRANSIENT);
         2263  +          }
  1984   2264           }else{
  1985   2265             rc = zonefileCtxUncompress(
  1986   2266                 pCtx, pCmpMethod, pCmp, aBuf, szFrame, iKeyOff, szKey
  1987   2267             );
  1988   2268           }
  1989   2269         }else{
  1990   2270           zErr = sqlite3_mprintf(
  1991   2271               "failed to read %d bytes at offset %d from file \"%s\"", 
  1992         -            sz, (int)(hdr.byteOffsetFrames+ofst), zFile
         2272  +            nRead, (int)(hdr.byteOffsetFrames+iRead), zFile
  1993   2273           );
  1994   2274         }
  1995   2275         sqlite3_free(aBuf);
  1996   2276       }
  1997   2277     }
  1998   2278   
  1999   2279     sqlite3_reset(pTab->pIdToName);
  2000   2280     if( zErr ){
  2001   2281       assert( rc!=SQLITE_OK );
  2002   2282       sqlite3_result_error(pCtx, zErr, -1);
  2003   2283       sqlite3_free(zErr);
  2004   2284     }
  2005   2285     zonefileFileClose(pFd);
         2286  +  zonefileCodecDestroy(pCodec);
  2006   2287     if( pCmpMethod ) pCmpMethod->xClose(pCmp);
  2007   2288     return rc;
  2008   2289   }
  2009   2290   
  2010   2291   /* 
  2011   2292   ** zonefile virtual table module xColumn method.
  2012   2293   */
................................................................................
  2044   2325   ** zonefile virtual table module xRowid method.
  2045   2326   */
  2046   2327   static int zonefileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  2047   2328     ZonefileCsr *pCsr = (ZonefileCsr*)cur;
  2048   2329     *pRowid = sqlite3_column_int64(pCsr->pSelect, 0);
  2049   2330     return SQLITE_OK;
  2050   2331   }
         2332  +
         2333  +static void zonefileGlobalFree(void *p){
         2334  +  ZonefileGlobal *pGlobal = (ZonefileGlobal*)p;
         2335  +  ZonefileKey *pKey;
         2336  +  ZonefileKey *pNext;
         2337  +  for(pKey=pGlobal->pList; pKey; pKey=pNext){
         2338  +    pNext = pKey->pNext;
         2339  +    sqlite3_free(pKey);
         2340  +  }
         2341  +  sqlite3_free(pGlobal);
         2342  +}
  2051   2343   
  2052   2344   /*
  2053   2345   ** Register the "zonefile" extensions.
  2054   2346   */
  2055   2347   static int zonefileRegister(sqlite3 *db){
  2056   2348     static sqlite3_module filesModule = {
  2057   2349       0,                            /* iVersion */
................................................................................
  2108   2400       const char *z;
  2109   2401       int n;
  2110   2402       void (*x)(sqlite3_context*,int,sqlite3_value**);
  2111   2403     } aFunc[] = {
  2112   2404       { "zonefile_write", 2, zonefileWriteFunc },
  2113   2405       { "zonefile_write", 3, zonefileWriteFunc },
  2114   2406     };
         2407  +  ZonefileGlobal *pGlobal = 0;
  2115   2408   
  2116   2409     int rc = SQLITE_OK;
  2117   2410     int i;
  2118   2411   
  2119   2412     for(i=0; rc==SQLITE_OK && i<sizeof(aFunc)/sizeof(aFunc[0]); i++){
  2120   2413       struct Func *p = &aFunc[i];
  2121   2414       rc = sqlite3_create_function(db, p->z, p->n, SQLITE_ANY, 0, p->x, 0, 0);
  2122   2415     }
  2123   2416   
  2124   2417     if( rc==SQLITE_OK ){
  2125         -    rc = sqlite3_create_module(db, "zonefile_files", &filesModule, 0);
         2418  +    pGlobal = (ZonefileGlobal*)sqlite3_malloc(sizeof(ZonefileGlobal));
         2419  +    if( pGlobal==0 ){
         2420  +      rc = SQLITE_NOMEM;
         2421  +    }else{
         2422  +      memset(pGlobal, 0, sizeof(ZonefileGlobal));
         2423  +    }
         2424  +  }
         2425  +
         2426  +  if( rc==SQLITE_OK ){
         2427  +    rc = sqlite3_create_module(
         2428  +        db, "zonefile_files", &filesModule, (void*)pGlobal
         2429  +    );
  2126   2430     }
  2127   2431     if( rc==SQLITE_OK ){
  2128         -    rc = sqlite3_create_module(db, "zonefile", &zonefileModule, 0);
         2432  +    rc = sqlite3_create_module_v2(db, "zonefile", &zonefileModule, 
         2433  +        (void*)pGlobal, zonefileGlobalFree
         2434  +    );
         2435  +    pGlobal = 0;
  2129   2436     }
  2130   2437   
         2438  +  sqlite3_free(pGlobal);
  2131   2439     return rc;
  2132   2440   }
  2133   2441   
  2134   2442   #else         /* SQLITE_OMIT_VIRTUALTABLE */
  2135   2443   # define zonefileRegister(x) SQLITE_OK
  2136   2444   #endif
  2137   2445   

Changes to ext/zonefile/zonefile1.test.

   196    196       ORDER BY 1
   197    197     } {
   198    198       test19.zonefile 1 1
   199    199       test20.zonefile 2 2
   200    200       test21.zonefile 7 4
   201    201     }
   202    202   }}
          203  +
          204  +#--------------------------------------------------------------------------
          205  +#
          206  +reset_db
          207  +load_static_extension db zonefile
          208  +do_execsql_test 3.0 {
          209  +  CREATE TABLE dd(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
          210  +  INSERT INTO dd VALUES(1000, 1, -1, randomblob(44));
          211  +  INSERT INTO dd VALUES(1001, 1, -1, randomblob(55));
          212  +  INSERT INTO dd VALUES(1002, 2, -1, randomblob(66));
          213  +  WITH p(n,v) AS (
          214  +      VALUES('maxAutoFrameSize', 2000) UNION ALL
          215  +      VALUES('encryptionType', 'xor') UNION ALL
          216  +      VALUES('encryptionKey', '0123456789')
          217  +  )
          218  +  SELECT zonefile_write('test.zonefile', 'dd', json_group_object(n, v)) FROM p;
          219  +} {{}}
          220  +
          221  +do_execsql_test 3.1 {
          222  +  CREATE VIRTUAL TABLE cc USING zonefile;
          223  +  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','0123456789');
          224  +  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
          225  +} {1 1 1}
          226  +
          227  +do_execsql_test 3.2 {
          228  +  DELETE FROM cc_files;
          229  +  INSERT INTO cc_files(filename,ekey) VALUES('test.zonefile','abcdefghij');
          230  +  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
          231  +} {0 0 0}
          232  +
          233  +do_execsql_test 3.3 {
          234  +  UPDATE cc_files SET ekey = '0123456789';
          235  +  SELECT quote(dd.v)==quote(cc.v) FROM cc JOIN dd USING (k)
          236  +} {1 1 1}
          237  +
   203    238   
   204    239   
   205    240   finish_test
   206    241