Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add support for "brotli" compression to the zonefile module. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | zonefile |
Files: | files | file ages | folders |
SHA3-256: |
3eb25b3fa5733b4418e7e2633be34b76 |
User & Date: | dan 2018-02-17 20:22:23.498 |
Context
2018-02-19
| ||
14:27 | Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details. (check-in: 4dbe0cba3f user: dan tags: zonefile) | |
2018-02-17
| ||
20:22 | Add support for "brotli" compression to the zonefile module. (check-in: 3eb25b3fa5 user: dan tags: zonefile) | |
19:38 | Add support for compression types "lz4" and "lz4hc" to the zonefile module. (check-in: bbe5b21ffa user: dan tags: zonefile) | |
Changes
Changes to ext/zonefile/zonefile.c.
︙ | ︙ | |||
68 69 70 71 72 73 74 | + (((u32)aBuf[3]) << 0); } #include <stdio.h> #include <string.h> #include <assert.h> | < < | | > > > > > > > > > < < < < < < < < < < < < | 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | + (((u32)aBuf[3]) << 0); } #include <stdio.h> #include <string.h> #include <assert.h> static int zfGenericOpen(void **pp, u8 *aDict, int nDict){ *pp = 0; return SQLITE_OK; } static void zfGenericClose(void *p){ } static int zfGenericUncompressSize( void *p, const u8 *aSrc, int nSrc ){ return (int)zonefileGet32(aSrc); } #ifdef SQLITE_HAVE_ZLIB #include <zlib.h> static int zfZlibCompressBound(void *p, int nSrc){ return (int)compressBound((uLong)nSrc) + 4; } static int zfZlibCompress( void *p, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc ){ uLongf destLen = (uLongf)(*pnDest)-4; int rc = compress(&aDest[4], &destLen, aSrc, (uLong)nSrc); *pnDest = (int)(destLen+4); zonefilePut32(aDest, nSrc); return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR; } static int zfZlibUncompress( void *p, u8 *aDest, int nDest, const u8 *aSrc, int nSrc ){ uLongf destLen = (uLongf)nDest; int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4); return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR; } #endif #ifdef SQLITE_HAVE_ZSTD #include <zstd.h> static int zfZstdCompressBound(void *p, int nSrc){ return (int)ZSTD_compressBound((size_t)nSrc); } static int zfZstdCompress( void *p, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc |
︙ | ︙ | |||
193 194 195 196 197 198 199 | if( ZDICT_isError(sz) ) return SQLITE_ERROR; pCmp->pCDict = ZSTD_createCDict(aDict, sz, 1); pCmp->pCCtx = ZSTD_createCCtx(); if( pCmp->pCDict==0 || pCmp->pCCtx==0 ) return SQLITE_ERROR; *pnDict = (int)sz; return SQLITE_OK; } | < < < < < < < < < < < < | 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 | if( ZDICT_isError(sz) ) return SQLITE_ERROR; pCmp->pCDict = ZSTD_createCDict(aDict, sz, 1); pCmp->pCCtx = ZSTD_createCCtx(); if( pCmp->pCDict==0 || pCmp->pCCtx==0 ) return SQLITE_ERROR; *pnDict = (int)sz; return SQLITE_OK; } static int zfZstddictCompress( void *p, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc ){ ZfZstddict *pCmp = (ZfZstddict*)p; size_t szDest = (size_t)(*pnDest); size_t rc; assert( pCmp && pCmp->pCDict && pCmp->pCCtx ); rc = ZSTD_compress_usingCDict( pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict ); if( ZSTD_isError(rc) ) return SQLITE_ERROR; *pnDest = (int)rc; return SQLITE_OK; } static int zfZstddictUncompress( void *p, u8 *aDest, int nDest, const u8 *aSrc, int nSrc ){ ZfZstddict *pCmp = (ZfZstddict*)p; size_t rc = ZSTD_decompress_usingDDict( pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict ); if( rc!=(size_t)nDest ) return SQLITE_ERROR; return SQLITE_OK; } #endif #ifdef SQLITE_HAVE_LZ4 #include <lz4.h> #include <lz4hc.h> static int zfLz4CompressBound(void *p, int nSrc){ return (int)LZ4_compressBound((uLong)nSrc) + 4; } static int zfLz4Uncompress( void *p, u8 *aDest, int nDest, const u8 *aSrc, int nSrc |
︙ | ︙ | |||
263 264 265 266 267 268 269 | int rc = LZ4_compress_default( (const char*)aSrc, (char*)&aDest[4], nSrc, (*pnDest - 4) ); *pnDest = rc+4; zonefilePut32(aDest, nSrc); return rc==0 ? SQLITE_ERROR : SQLITE_OK; } | < < < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | > | > > | | | | | 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 | int rc = LZ4_compress_default( (const char*)aSrc, (char*)&aDest[4], nSrc, (*pnDest - 4) ); *pnDest = rc+4; zonefilePut32(aDest, nSrc); return rc==0 ? SQLITE_ERROR : SQLITE_OK; } static int zfLz4hcCompress( void *p, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc ){ int rc = LZ4_compress_HC( (const char*)aSrc, (char*)&aDest[4], nSrc, *pnDest, 0 ); *pnDest = rc+4; zonefilePut32(aDest, nSrc); return rc==0 ? SQLITE_ERROR : SQLITE_OK; } #endif #ifdef SQLITE_HAVE_BROTLI #include <brotli/encode.h> #include <brotli/decode.h> static int zfBrotliCompressBound(void *p, int nSrc){ return (int)BrotliEncoderMaxCompressedSize((size_t)nSrc) + 4; } static int zfBrotliCompress( void *p, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc ){ size_t nDest = (size_t)*pnDest - 4; BROTLI_BOOL rc = BrotliEncoderCompress( BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, (size_t)nSrc, aSrc, &nDest, &aDest[4] ); *pnDest = (int)nDest + 4; zonefilePut32(aDest, nSrc); return rc==0 ? SQLITE_ERROR : SQLITE_OK; } static int zfBrotliUncompress( void *p, u8 *aDest, int nDest, const u8 *aSrc, int nSrc ){ size_t n = nDest; BrotliDecoderResult rc = BrotliDecoderDecompress(nSrc-4, &aSrc[4], &n, aDest); return rc==BROTLI_DECODER_RESULT_SUCCESS ? SQLITE_OK : SQLITE_ERROR; } #endif typedef struct ZonefileCompress ZonefileCompress; static struct ZonefileCompress { int eType; const char *zName; int (*xOpen)(void**, u8 *aDict, int nDict); void (*xClose)(void*); int (*xTrain)(void*, u8 *aDict, int *pnDict, u8 *a, size_t *aSz, int n); int (*xCompressBound)(void*, int nSrc); int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc); int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc); int (*xUncompress)(void*, u8 *aDest, int nDest, const u8 *aSrc, int nSrc); } aZonefileCompress[] = { { ZONEFILE_COMPRESSION_NONE, "none", 0, 0, 0, 0, 0, 0, 0 }, #ifdef SQLITE_HAVE_ZSTD { ZONEFILE_COMPRESSION_ZSTD, "zstd", zfGenericOpen, zfGenericClose, 0, zfZstdCompressBound, zfZstdCompress, zfZstdUncompressSize, zfZstdUncompress }, { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict", zfZstddictOpen, zfZstddictClose, zfZstddictTrain, zfZstdCompressBound, zfZstddictCompress, zfZstdUncompressSize, zfZstddictUncompress }, #endif /* SQLITE_HAVE_ZSTD */ #ifdef SQLITE_HAVE_ZLIB { ZONEFILE_COMPRESSION_ZLIB, "zlib", zfGenericOpen, zfGenericClose, 0, zfZlibCompressBound, zfZlibCompress, zfGenericUncompressSize, zfZlibUncompress }, #endif /* SQLITE_HAVE_ZLIB */ #ifdef SQLITE_HAVE_BROTLI { ZONEFILE_COMPRESSION_BROTLI, "brotli", zfGenericOpen, zfGenericClose, 0, zfBrotliCompressBound, zfBrotliCompress, zfGenericUncompressSize, zfBrotliUncompress }, #endif /* SQLITE_HAVE_BROTLI */ #ifdef SQLITE_HAVE_LZ4 { ZONEFILE_COMPRESSION_LZ4, "lz4", zfGenericOpen, zfGenericClose, 0, zfLz4CompressBound, zfLz4Compress, zfGenericUncompressSize, zfLz4Uncompress }, { ZONEFILE_COMPRESSION_LZ4HC, "lz4hc", zfGenericOpen, zfGenericClose, 0, zfLz4CompressBound, zfLz4hcCompress, zfGenericUncompressSize, zfLz4Uncompress }, #endif /* SQLITE_HAVE_LZ4 */ }; static ZonefileCompress *zonefileCompress(const char *zName){ int i; for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){ |
︙ | ︙ |