/ Check-in [3eb25b3f]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for "brotli" compression to the zonefile module.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256: 3eb25b3fa5733b4418e7e2633be34b763e2c70342bb9c418a07c9f7d4b196fac
User & Date: dan 2018-02-17 20:22:23
Context
2018-02-19
14:27
Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details. check-in: 4dbe0cba user: dan tags: zonefile
2018-02-17
20:22
Add support for "brotli" compression to the zonefile module. check-in: 3eb25b3f user: dan tags: zonefile
19:38
Add support for compression types "lz4" and "lz4hc" to the zonefile module. check-in: bbe5b21f user: dan tags: zonefile
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/zonefile/zonefile.c.

    68     68          + (((u32)aBuf[3]) <<  0);
    69     69   }
    70     70   
    71     71   #include <stdio.h>
    72     72   #include <string.h>
    73     73   #include <assert.h>
    74     74   
    75         -#ifdef SQLITE_HAVE_ZLIB 
    76         -#include <zlib.h>
    77         -static int zfZlibOpen(void **pp, u8 *aDict, int nDict){
           75  +static int zfGenericOpen(void **pp, u8 *aDict, int nDict){
    78     76     *pp = 0;
    79     77     return SQLITE_OK;
    80     78   }
    81         -static void zfZlibClose(void *p){
           79  +static void zfGenericClose(void *p){
    82     80   }
           81  +static int zfGenericUncompressSize(
           82  +  void *p, 
           83  +  const u8 *aSrc, int nSrc
           84  +){
           85  +  return (int)zonefileGet32(aSrc);
           86  +}
           87  +
           88  +#ifdef SQLITE_HAVE_ZLIB 
           89  +#include <zlib.h>
    83     90   static int zfZlibCompressBound(void *p, int nSrc){
    84     91     return (int)compressBound((uLong)nSrc) + 4;
    85     92   }
    86     93   static int zfZlibCompress(
    87     94     void *p, 
    88     95     u8 *aDest, int *pnDest, 
    89     96     const u8 *aSrc, int nSrc
................................................................................
    90     97   ){
    91     98     uLongf destLen = (uLongf)(*pnDest)-4;
    92     99     int rc = compress(&aDest[4], &destLen, aSrc, (uLong)nSrc);
    93    100     *pnDest = (int)(destLen+4);
    94    101     zonefilePut32(aDest, nSrc);
    95    102     return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
    96    103   }
    97         -static int zfZlibUncompressSize(
    98         -  void *p, 
    99         -  const u8 *aSrc, int nSrc
   100         -){
   101         -  return (int)zonefileGet32(aSrc);
   102         -}
   103    104   static int zfZlibUncompress(
   104    105     void *p, 
   105    106     u8 *aDest, int nDest, 
   106    107     const u8 *aSrc, int nSrc
   107    108   ){
   108    109     uLongf destLen = (uLongf)nDest;
   109    110     int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4);
   110    111     return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR;
   111    112   }
   112    113   #endif
   113    114   #ifdef SQLITE_HAVE_ZSTD 
   114    115   #include <zstd.h>
   115         -static int zfZstdOpen(void **pp, u8 *aDict, int nDict){
   116         -  *pp = 0;
   117         -  return SQLITE_OK;
   118         -}
   119         -static void zfZstdClose(void *p){
   120         -}
   121    116   static int zfZstdCompressBound(void *p, int nSrc){
   122    117     return (int)ZSTD_compressBound((size_t)nSrc);
   123    118   }
   124    119   static int zfZstdCompress(
   125    120     void *p, 
   126    121     u8 *aDest, int *pnDest, 
   127    122     const u8 *aSrc, int nSrc
................................................................................
   193    188     if( ZDICT_isError(sz) ) return SQLITE_ERROR;
   194    189     pCmp->pCDict = ZSTD_createCDict(aDict, sz, 1);
   195    190     pCmp->pCCtx = ZSTD_createCCtx();
   196    191     if( pCmp->pCDict==0 || pCmp->pCCtx==0 ) return SQLITE_ERROR;
   197    192     *pnDict = (int)sz;
   198    193     return SQLITE_OK;
   199    194   }
   200         -static int zfZstddictCompressBound(void *p, int nSrc){
   201         -  return (int)ZSTD_compressBound((size_t)nSrc);
   202         -}
   203    195   static int zfZstddictCompress(
   204    196     void *p, 
   205    197     u8 *aDest, int *pnDest, 
   206    198     const u8 *aSrc, int nSrc
   207    199   ){
   208    200     ZfZstddict *pCmp = (ZfZstddict*)p;
   209    201     size_t szDest = (size_t)(*pnDest);
................................................................................
   212    204     rc = ZSTD_compress_usingCDict(
   213    205         pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict
   214    206     );
   215    207     if( ZSTD_isError(rc) ) return SQLITE_ERROR;
   216    208     *pnDest = (int)rc;
   217    209     return SQLITE_OK;
   218    210   }
   219         -static int zfZstddictUncompressSize(void *p, const u8 *aSrc, int nSrc){
   220         -  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
   221         -}
   222    211   static int zfZstddictUncompress(
   223    212     void *p, 
   224    213     u8 *aDest, int nDest, 
   225    214     const u8 *aSrc, int nSrc
   226    215   ){
   227    216     ZfZstddict *pCmp = (ZfZstddict*)p;
   228    217     size_t rc = ZSTD_decompress_usingDDict(
................................................................................
   232    221     return SQLITE_OK;
   233    222   }
   234    223   #endif
   235    224   
   236    225   #ifdef SQLITE_HAVE_LZ4 
   237    226   #include <lz4.h>
   238    227   #include <lz4hc.h>
   239         -static int zfLz4Open(void **pp, u8 *aDict, int nDict){
   240         -  *pp = 0;
   241         -  return SQLITE_OK;
   242         -}
   243         -static void zfLz4Close(void *p){
   244         -}
   245    228   static int zfLz4CompressBound(void *p, int nSrc){
   246    229     return (int)LZ4_compressBound((uLong)nSrc) + 4;
   247    230   }
   248    231   static int zfLz4Uncompress(
   249    232     void *p, 
   250    233     u8 *aDest, int nDest, 
   251    234     const u8 *aSrc, int nSrc
................................................................................
   263    246     int rc = LZ4_compress_default(
   264    247         (const char*)aSrc, (char*)&aDest[4], nSrc, (*pnDest - 4)
   265    248     );
   266    249     *pnDest = rc+4;
   267    250     zonefilePut32(aDest, nSrc);
   268    251     return rc==0 ? SQLITE_ERROR : SQLITE_OK;
   269    252   }
   270         -static int zfLz4UncompressSize(void *p, const u8 *aSrc, int nSrc){
   271         -  return (int)zonefileGet32(aSrc);
   272         -}
   273    253   static int zfLz4hcCompress(
   274    254     void *p, 
   275    255     u8 *aDest, int *pnDest, 
   276    256     const u8 *aSrc, int nSrc
   277    257   ){
   278    258     int rc = LZ4_compress_HC(
   279    259         (const char*)aSrc, (char*)&aDest[4], nSrc, *pnDest, 0
   280    260     );
   281    261     *pnDest = rc+4;
   282    262     zonefilePut32(aDest, nSrc);
   283    263     return rc==0 ? SQLITE_ERROR : SQLITE_OK;
   284    264   }
   285    265   #endif
          266  +
          267  +#ifdef SQLITE_HAVE_BROTLI 
          268  +#include <brotli/encode.h>
          269  +#include <brotli/decode.h>
          270  +
          271  +static int zfBrotliCompressBound(void *p, int nSrc){
          272  +  return (int)BrotliEncoderMaxCompressedSize((size_t)nSrc) + 4;
          273  +}
          274  +static int zfBrotliCompress(
          275  +  void *p, 
          276  +  u8 *aDest, int *pnDest, 
          277  +  const u8 *aSrc, int nSrc
          278  +){
          279  +  size_t nDest = (size_t)*pnDest - 4;
          280  +  BROTLI_BOOL rc = BrotliEncoderCompress(
          281  +      BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE,
          282  +      (size_t)nSrc, aSrc, &nDest, &aDest[4]
          283  +  );
          284  +  *pnDest = (int)nDest + 4;
          285  +  zonefilePut32(aDest, nSrc);
          286  +  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
          287  +}
          288  +static int zfBrotliUncompress(
          289  +  void *p, 
          290  +  u8 *aDest, int nDest, 
          291  +  const u8 *aSrc, int nSrc
          292  +){
          293  +  size_t n = nDest;
          294  +  BrotliDecoderResult rc = BrotliDecoderDecompress(nSrc-4, &aSrc[4], &n, aDest);
          295  +  return rc==BROTLI_DECODER_RESULT_SUCCESS ? SQLITE_OK : SQLITE_ERROR;
          296  +}
          297  +#endif
   286    298   
   287    299   typedef struct ZonefileCompress ZonefileCompress;
   288    300   static struct ZonefileCompress {
   289    301     int eType;
   290    302     const char *zName;
   291    303     int (*xOpen)(void**, u8 *aDict, int nDict);
   292    304     void (*xClose)(void*);
................................................................................
   297    309     int (*xUncompress)(void*, u8 *aDest, int nDest, const u8 *aSrc, int nSrc);
   298    310   } aZonefileCompress[] = {
   299    311     { ZONEFILE_COMPRESSION_NONE,             "none",
   300    312       0, 0, 0, 0, 0, 0, 0
   301    313     },
   302    314   #ifdef SQLITE_HAVE_ZSTD
   303    315     { ZONEFILE_COMPRESSION_ZSTD,             "zstd",
   304         -    zfZstdOpen, zfZstdClose, 
          316  +    zfGenericOpen, zfGenericClose, 
   305    317       0,
   306    318       zfZstdCompressBound, zfZstdCompress, 
   307    319       zfZstdUncompressSize, zfZstdUncompress
   308    320     },
   309    321     { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict",
   310    322       zfZstddictOpen, zfZstddictClose, 
   311    323       zfZstddictTrain,
   312         -    zfZstddictCompressBound, zfZstddictCompress, 
   313         -    zfZstddictUncompressSize, zfZstddictUncompress
          324  +    zfZstdCompressBound, zfZstddictCompress, 
          325  +    zfZstdUncompressSize, zfZstddictUncompress
   314    326     },
   315    327   #endif /* SQLITE_HAVE_ZSTD */
   316    328   #ifdef SQLITE_HAVE_ZLIB
   317    329     { ZONEFILE_COMPRESSION_ZLIB,             "zlib",
   318         -    zfZlibOpen, zfZlibClose, 
          330  +    zfGenericOpen, zfGenericClose, 
   319    331       0,
   320    332       zfZlibCompressBound, zfZlibCompress, 
   321         -    zfZlibUncompressSize, zfZlibUncompress
          333  +    zfGenericUncompressSize, zfZlibUncompress
   322    334     },
   323    335   #endif /* SQLITE_HAVE_ZLIB */
   324    336   #ifdef SQLITE_HAVE_BROTLI
   325    337     { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
   326         -    0, 0, 0, 0, 0, 0, 0
          338  +    zfGenericOpen, zfGenericClose, 
          339  +    0,
          340  +    zfBrotliCompressBound, zfBrotliCompress, 
          341  +    zfGenericUncompressSize, zfBrotliUncompress
   327    342     },
   328    343   #endif /* SQLITE_HAVE_BROTLI */
   329    344   #ifdef SQLITE_HAVE_LZ4
   330    345     { ZONEFILE_COMPRESSION_LZ4,              "lz4",
   331         -    zfLz4Open, zfLz4Close, 
          346  +    zfGenericOpen, zfGenericClose, 
   332    347       0,
   333    348       zfLz4CompressBound, zfLz4Compress, 
   334         -    zfLz4UncompressSize, zfLz4Uncompress
          349  +    zfGenericUncompressSize, zfLz4Uncompress
   335    350     },
   336    351     { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",
   337         -    zfLz4Open, zfLz4Close, 
          352  +    zfGenericOpen, zfGenericClose, 
   338    353       0,
   339    354       zfLz4CompressBound, zfLz4hcCompress, 
   340         -    zfLz4UncompressSize, zfLz4Uncompress
          355  +    zfGenericUncompressSize, zfLz4Uncompress
   341    356     },
   342    357   #endif /* SQLITE_HAVE_LZ4 */
   343    358   };
   344    359   
   345    360   static ZonefileCompress *zonefileCompress(const char *zName){
   346    361     int i;
   347    362     for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){