SQLite

Check-in [bbe5b21ffa]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for compression types "lz4" and "lz4hc" to the zonefile module.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256: bbe5b21ffab3cd312680ca9f179c5847790c17fb91d4174985153c6c398d48e3
User & Date: dan 2018-02-17 19:38:02.757
Context
2018-02-17
20:22
Add support for "brotli" compression to the zonefile module. (check-in: 3eb25b3fa5 user: dan tags: zonefile)
19:38
Add support for compression types "lz4" and "lz4hc" to the zonefile module. (check-in: bbe5b21ffa user: dan tags: zonefile)
18:33
Add support for compression methods "zstd" and "zstd_global_dict". (check-in: a993a50bb8 user: dan tags: zonefile)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/zonefile/zonefile.c.
︙︙
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

){
  size_t szDest = (size_t)(*pnDest);
  size_t rc = ZSTD_compress(aDest, szDest, aSrc, (size_t)nSrc, 1);
  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
  *pnDest = (int)rc;
  return SQLITE_OK;
}
static int zfZstdUncompressSize(
  void *p, 
  const u8 *aSrc, int nSrc
){
  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
}
static int zfZstdUncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){








|
<
<
<








128
129
130
131
132
133
134
135



136
137
138
139
140
141
142

){
  size_t szDest = (size_t)(*pnDest);
  size_t rc = ZSTD_compress(aDest, szDest, aSrc, (size_t)nSrc, 1);
  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
  *pnDest = (int)rc;
  return SQLITE_OK;
}
static int zfZstdUncompressSize(void *p, const u8 *aSrc, int nSrc){



  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
}
static int zfZstdUncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){
︙︙
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240



















































241
242
243
244
245
246
247

  rc = ZSTD_compress_usingCDict(
      pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict
  );
  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
  *pnDest = (int)rc;
  return SQLITE_OK;
}
static int zfZstddictUncompressSize(
  void *p, 
  const u8 *aSrc, int nSrc
){
  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
}
static int zfZstddictUncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){
  ZfZstddict *pCmp = (ZfZstddict*)p;
  size_t rc = ZSTD_decompress_usingDDict(
      pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict
  );
  if( rc!=(size_t)nDest ) return SQLITE_ERROR;
  return SQLITE_OK;
}
#endif




















































typedef struct ZonefileCompress ZonefileCompress;
static struct ZonefileCompress {
  int eType;
  const char *zName;
  int (*xOpen)(void**, u8 *aDict, int nDict);
  void (*xClose)(void*);








|
<
<
<















>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>








212
213
214
215
216
217
218
219



220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292

  rc = ZSTD_compress_usingCDict(
      pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict
  );
  if( ZSTD_isError(rc) ) return SQLITE_ERROR;
  *pnDest = (int)rc;
  return SQLITE_OK;
}
static int zfZstddictUncompressSize(void *p, const u8 *aSrc, int nSrc){



  return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc);
}
static int zfZstddictUncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){
  ZfZstddict *pCmp = (ZfZstddict*)p;
  size_t rc = ZSTD_decompress_usingDDict(
      pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict
  );
  if( rc!=(size_t)nDest ) return SQLITE_ERROR;
  return SQLITE_OK;
}
#endif

#ifdef SQLITE_HAVE_LZ4 
#include <lz4.h>
#include <lz4hc.h>
static int zfLz4Open(void **pp, u8 *aDict, int nDict){
  *pp = 0;
  return SQLITE_OK;
}
static void zfLz4Close(void *p){
}
static int zfLz4CompressBound(void *p, int nSrc){
  return (int)LZ4_compressBound((uLong)nSrc) + 4;
}
static int zfLz4Uncompress(
  void *p, 
  u8 *aDest, int nDest, 
  const u8 *aSrc, int nSrc
){
  int rc = LZ4_decompress_safe(
      (const char*)&aSrc[4], (char*)aDest, nSrc-4, nDest
  );
  return rc==nDest ? SQLITE_OK : SQLITE_ERROR;
}
static int zfLz4Compress(
  void *p, 
  u8 *aDest, int *pnDest, 
  const u8 *aSrc, int nSrc
){
  int rc = LZ4_compress_default(
      (const char*)aSrc, (char*)&aDest[4], nSrc, (*pnDest - 4)
  );
  *pnDest = rc+4;
  zonefilePut32(aDest, nSrc);
  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
}
static int zfLz4UncompressSize(void *p, const u8 *aSrc, int nSrc){
  return (int)zonefileGet32(aSrc);
}
static int zfLz4hcCompress(
  void *p, 
  u8 *aDest, int *pnDest, 
  const u8 *aSrc, int nSrc
){
  int rc = LZ4_compress_HC(
      (const char*)aSrc, (char*)&aDest[4], nSrc, *pnDest, 0
  );
  *pnDest = rc+4;
  zonefilePut32(aDest, nSrc);
  return rc==0 ? SQLITE_ERROR : SQLITE_OK;
}
#endif

typedef struct ZonefileCompress ZonefileCompress;
static struct ZonefileCompress {
  int eType;
  const char *zName;
  int (*xOpen)(void**, u8 *aDict, int nDict);
  void (*xClose)(void*);
︙︙
279
280
281
282
283
284
285

286


287
288

289


290
291
292
293
294
295
296

#ifdef SQLITE_HAVE_BROTLI
  { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
    0, 0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_BROTLI */
#ifdef SQLITE_HAVE_LZ4
  { ZONEFILE_COMPRESSION_LZ4,              "lz4",

    0, 0, 0, 0, 0, 0, 0


  },
  { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",

    0, 0, 0, 0, 0, 0, 0


  },
#endif /* SQLITE_HAVE_LZ4 */
};

static ZonefileCompress *zonefileCompress(const char *zName){
  int i;
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){








>
|
>
>


>
|
>
>








324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347

#ifdef SQLITE_HAVE_BROTLI
  { ZONEFILE_COMPRESSION_BROTLI,           "brotli",
    0, 0, 0, 0, 0, 0, 0
  },
#endif /* SQLITE_HAVE_BROTLI */
#ifdef SQLITE_HAVE_LZ4
  { ZONEFILE_COMPRESSION_LZ4,              "lz4",
    zfLz4Open, zfLz4Close, 
    0,
    zfLz4CompressBound, zfLz4Compress, 
    zfLz4UncompressSize, zfLz4Uncompress
  },
  { ZONEFILE_COMPRESSION_LZ4HC,            "lz4hc",
    zfLz4Open, zfLz4Close, 
    0,
    zfLz4CompressBound, zfLz4hcCompress, 
    zfLz4UncompressSize, zfLz4Uncompress
  },
#endif /* SQLITE_HAVE_LZ4 */
};

static ZonefileCompress *zonefileCompress(const char *zName){
  int i;
  for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){
︙︙
630
631
632
633
634
635
636

637
638
639
640
641
642
643
644
645
646
647
648
649

650
651
652
653
654
655
656

  sqlite3_context *pCtx,          /* Leave any error message here */
  ZonefileCompress *pMethod,      /* Compression method object */
  void *pCmp,                     /* Compression handle */
  ZonefileBuffer *pTo,            /* Append new data here */
  ZonefileBuffer *pFrom           /* Input buffer */
){
  int rc = SQLITE_OK;

  if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
    if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
      rc = SQLITE_ERROR;
    }else{
      zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
    }
  }else{
    int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
    if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
    rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
    pTo->n += nReq;
    if( rc!=SQLITE_OK ){
      return rc;

    }
  }
  return SQLITE_OK;
}

/*
** Function:     zonefile_write(F,T[,J])








>
|
|
|
|
|
|
|
|
|
|
|
|
|
>








681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709

  sqlite3_context *pCtx,          /* Leave any error message here */
  ZonefileCompress *pMethod,      /* Compression method object */
  void *pCmp,                     /* Compression handle */
  ZonefileBuffer *pTo,            /* Append new data here */
  ZonefileBuffer *pFrom           /* Input buffer */
){
  int rc = SQLITE_OK;
  if( pFrom->n ){
    if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){
      if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){
        rc = SQLITE_ERROR;
      }else{
        zonefileAppendBlob(pTo, pFrom->a, pFrom->n);
      }
    }else{
      int nReq = pMethod->xCompressBound(pCmp, pFrom->n);
      if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
      rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
      pTo->n += nReq;
      if( rc!=SQLITE_OK ){
        return rc;
      }
    }
  }
  return SQLITE_OK;
}

/*
** Function:     zonefile_write(F,T[,J])
︙︙
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292

    sqlite3_free(aIdx);
    aIdx = 0;
    nIdx = 0;
  }

  *paIdx = aIdx;
  *pnIdx = nIdx;
  return SQLITE_OK;
}


static int zonefilePopulateIndex(
  ZonefileFilesTab *pTab,
  const char *zFile,
  i64 iFileid








|








1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345

    sqlite3_free(aIdx);
    aIdx = 0;
    nIdx = 0;
  }

  *paIdx = aIdx;
  *pnIdx = nIdx;
  return rc;
}


static int zonefilePopulateIndex(
  ZonefileFilesTab *pTab,
  const char *zFile,
  i64 iFileid
︙︙