Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add support for compression methods "zstd" and "zstd_global_dict". |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | zonefile |
Files: | files | file ages | folders |
SHA3-256: |
a993a50bb8d5a3bf7cf79e09204814e1 |
User & Date: | dan 2018-02-17 18:33:43.145 |
Context
2018-02-17
| ||
19:38 | Add support for compression types "lz4" and "lz4hc" to the zonefile module. (check-in: bbe5b21ffa user: dan tags: zonefile) | |
18:33 | Add support for compression methods "zstd" and "zstd_global_dict". (check-in: a993a50bb8 user: dan tags: zonefile) | |
2018-02-15
| ||
20:37 | Add support for zlib compression to the zonefile module. (check-in: 72b8a7ef98 user: dan tags: zonefile) | |
Changes
Changes to ext/zonefile/zonefile.c.
︙ | ︙ | |||
31 32 33 34 35 36 37 38 39 40 41 42 | # define NEVER(X) ((X)?(assert(0),1):0) #else # define ALWAYS(X) (X) # define NEVER(X) (X) #endif #endif /* SQLITE_AMALGAMATION */ #define ZONEFILE_MAGIC_NUMBER 0x464B3138 #define ZONEFILE_SZ_HEADER 32 #define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20 | > > > > > < < < | 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 | # define NEVER(X) ((X)?(assert(0),1):0) #else # define ALWAYS(X) (X) # define NEVER(X) (X) #endif #endif /* SQLITE_AMALGAMATION */ #define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024) #define ZONEFILE_DEFAULT_ENCRYPTION 0 #define ZONEFILE_DEFAULT_COMPRESSION 0 #define ZONEFILE_DEFAULT_DICTSIZE (64*1024) #define ZONEFILE_MAGIC_NUMBER 0x464B3138 #define ZONEFILE_SZ_HEADER 32 #define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20 #define ZONEFILE_COMPRESSION_NONE 0 #define ZONEFILE_COMPRESSION_ZSTD 1 #define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2 #define ZONEFILE_COMPRESSION_ZLIB 3 #define ZONEFILE_COMPRESSION_BROTLI 4 #define ZONEFILE_COMPRESSION_LZ4 5 |
︙ | ︙ | |||
62 63 64 65 66 67 68 | static u32 zonefileGet32(const u8 *aBuf){ return (((u32)aBuf[0]) << 24) + (((u32)aBuf[1]) << 16) + (((u32)aBuf[2]) << 8) + (((u32)aBuf[3]) << 0); } | | | | > > | | 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | static u32 zonefileGet32(const u8 *aBuf){ return (((u32)aBuf[0]) << 24) + (((u32)aBuf[1]) << 16) + (((u32)aBuf[2]) << 8) + (((u32)aBuf[3]) << 0); } #include <stdio.h> #include <string.h> #include <assert.h> #ifdef SQLITE_HAVE_ZLIB #include <zlib.h> static int zfZlibOpen(void **pp, u8 *aDict, int nDict){ *pp = 0; return SQLITE_OK; } static void zfZlibClose(void *p){ } static int zfZlibCompressBound(void *p, int nSrc){ return (int)compressBound((uLong)nSrc) + 4; |
︙ | ︙ | |||
94 95 96 97 98 99 100 | void *p, const u8 *aSrc, int nSrc ){ return (int)zonefileGet32(aSrc); } static int zfZlibUncompress( void *p, | | | > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > | | > | > > < > > > > > | | | | 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 | void *p, const u8 *aSrc, int nSrc ){ return (int)zonefileGet32(aSrc); } static int zfZlibUncompress( void *p, u8 *aDest, int nDest, const u8 *aSrc, int nSrc ){ uLongf destLen = (uLongf)nDest; int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4); return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR; } #endif #ifdef SQLITE_HAVE_ZSTD #include <zstd.h> static int zfZstdOpen(void **pp, u8 *aDict, int nDict){ *pp = 0; return SQLITE_OK; } static void zfZstdClose(void *p){ } static int zfZstdCompressBound(void *p, int nSrc){ return (int)ZSTD_compressBound((size_t)nSrc); } static int zfZstdCompress( void *p, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc ){ size_t szDest = (size_t)(*pnDest); size_t rc = ZSTD_compress(aDest, szDest, aSrc, (size_t)nSrc, 1); if( ZSTD_isError(rc) ) return SQLITE_ERROR; *pnDest = (int)rc; return SQLITE_OK; } static int zfZstdUncompressSize( void *p, const u8 *aSrc, int nSrc ){ return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc); } static int zfZstdUncompress( void *p, u8 *aDest, int nDest, const u8 *aSrc, int nSrc ){ size_t rc = ZSTD_decompress(aDest, (size_t)nDest, aSrc, (size_t)nSrc); if( rc!=(size_t)nDest ) return SQLITE_ERROR; return SQLITE_OK; } #include <zdict.h> typedef struct ZfZstddict ZfZstddict; struct ZfZstddict { ZSTD_CDict *pCDict; ZSTD_CCtx *pCCtx; ZSTD_DDict *pDDict; ZSTD_DCtx *pDCtx; }; static void zfZstddictClose(void *p){ if( p ){ ZfZstddict *pCmp = (ZfZstddict*)p; if( pCmp->pCDict ) ZSTD_freeCDict(pCmp->pCDict); if( pCmp->pCCtx ) ZSTD_freeCCtx(pCmp->pCCtx); if( pCmp->pDCtx ) ZSTD_freeDCtx(pCmp->pDCtx); sqlite3_free(pCmp); } } static int zfZstddictOpen(void **pp, u8 *aDict, int nDict){ int rc = SQLITE_OK; ZfZstddict *pDict = (ZfZstddict*)sqlite3_malloc(sizeof(ZfZstddict)); if( pDict==0 ){ rc = SQLITE_NOMEM; }else{ memset(pDict, 0, sizeof(ZfZstddict)); if( aDict ){ pDict->pDDict = ZSTD_createDDict(aDict, nDict); pDict->pDCtx = ZSTD_createDCtx(); if( pDict->pDDict==0 || pDict->pDCtx==0 ){ zfZstddictClose((void*)pDict); pDict = 0; rc = SQLITE_ERROR; } } } *pp = (void*)pDict; return rc; } static int zfZstddictTrain( void *p, /* Compressor handle */ u8 *aDict, int *pnDict, /* OUT: Dictionary buffer */ u8 *aSamp, size_t *aSz, int nSamp /* IN: Training samples */ ){ ZfZstddict *pCmp = (ZfZstddict*)p; size_t sz = ZDICT_trainFromBuffer(aDict, (size_t)*pnDict, aSamp, aSz, nSamp); if( ZDICT_isError(sz) ) return SQLITE_ERROR; pCmp->pCDict = ZSTD_createCDict(aDict, sz, 1); pCmp->pCCtx = ZSTD_createCCtx(); if( pCmp->pCDict==0 || pCmp->pCCtx==0 ) return SQLITE_ERROR; *pnDict = (int)sz; return SQLITE_OK; } static int zfZstddictCompressBound(void *p, int nSrc){ return (int)ZSTD_compressBound((size_t)nSrc); } static int zfZstddictCompress( void *p, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc ){ ZfZstddict *pCmp = (ZfZstddict*)p; size_t szDest = (size_t)(*pnDest); size_t rc; assert( pCmp && pCmp->pCDict && pCmp->pCCtx ); rc = ZSTD_compress_usingCDict( pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict ); if( ZSTD_isError(rc) ) return SQLITE_ERROR; *pnDest = (int)rc; return SQLITE_OK; } static int zfZstddictUncompressSize( void *p, const u8 *aSrc, int nSrc ){ return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc); } static int zfZstddictUncompress( void *p, u8 *aDest, int nDest, const u8 *aSrc, int nSrc ){ ZfZstddict *pCmp = (ZfZstddict*)p; size_t rc = ZSTD_decompress_usingDDict( pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict ); if( rc!=(size_t)nDest ) return SQLITE_ERROR; return SQLITE_OK; } #endif typedef struct ZonefileCompress ZonefileCompress; static struct ZonefileCompress { int eType; const char *zName; int (*xOpen)(void**, u8 *aDict, int nDict); void (*xClose)(void*); int (*xTrain)(void*, u8 *aDict, int *pnDict, u8 *a, size_t *aSz, int n); int (*xCompressBound)(void*, int nSrc); int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc); int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc); int (*xUncompress)(void*, u8 *aDest, int nDest, const u8 *aSrc, int nSrc); } aZonefileCompress[] = { { ZONEFILE_COMPRESSION_NONE, "none", 0, 0, 0, 0, 0, 0, 0 }, #ifdef SQLITE_HAVE_ZSTD { ZONEFILE_COMPRESSION_ZSTD, "zstd", zfZstdOpen, zfZstdClose, 0, zfZstdCompressBound, zfZstdCompress, zfZstdUncompressSize, zfZstdUncompress }, { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict", zfZstddictOpen, zfZstddictClose, zfZstddictTrain, zfZstddictCompressBound, zfZstddictCompress, zfZstddictUncompressSize, zfZstddictUncompress }, #endif /* SQLITE_HAVE_ZSTD */ #ifdef SQLITE_HAVE_ZLIB { ZONEFILE_COMPRESSION_ZLIB, "zlib", zfZlibOpen, zfZlibClose, 0, zfZlibCompressBound, zfZlibCompress, zfZlibUncompressSize, zfZlibUncompress }, #endif /* SQLITE_HAVE_ZLIB */ #ifdef SQLITE_HAVE_BROTLI { ZONEFILE_COMPRESSION_BROTLI, "brotli", 0, 0, 0, 0, 0, 0, 0 }, #endif /* SQLITE_HAVE_BROTLI */ #ifdef SQLITE_HAVE_LZ4 { ZONEFILE_COMPRESSION_LZ4, "lz4", 0, 0, 0, 0, 0, 0, 0 }, { ZONEFILE_COMPRESSION_LZ4HC, "lz4hc", 0, 0, 0, 0, 0, 0, 0 }, #endif /* SQLITE_HAVE_LZ4 */ }; static ZonefileCompress *zonefileCompress(const char *zName){ int i; for(i=0; i<sizeof(aZonefileCompress)/sizeof(aZonefileCompress[0]); i++){ |
︙ | ︙ | |||
191 192 193 194 195 196 197 | "CREATE TABLE z2(" \ " filename TEXT," \ " ekey BLOB," \ " header JSON HIDDEN" \ ")" | < < < | 330 331 332 333 334 335 336 337 338 339 340 341 342 343 | "CREATE TABLE z2(" \ " filename TEXT," \ " ekey BLOB," \ " header JSON HIDDEN" \ ")" /* ** A structure to store the parameters for a single zonefile_write() ** invocation. */ typedef struct ZonefileWrite ZonefileWrite; struct ZonefileWrite { |
︙ | ︙ | |||
488 489 490 491 492 493 494 | static void zonefileFileClose(FILE *pFd){ if( pFd ) fclose(pFd); } static int zonefileAppendCompressed( sqlite3_context *pCtx, /* Leave any error message here */ | | > | | | | 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 | static void zonefileFileClose(FILE *pFd){ if( pFd ) fclose(pFd); } static int zonefileAppendCompressed( sqlite3_context *pCtx, /* Leave any error message here */ ZonefileCompress *pMethod, /* Compression method object */ void *pCmp, /* Compression handle */ ZonefileBuffer *pTo, /* Append new data here */ ZonefileBuffer *pFrom /* Input buffer */ ){ int rc = SQLITE_OK; if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){ if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){ rc = SQLITE_ERROR; }else{ zonefileAppendBlob(pTo, pFrom->a, pFrom->n); } }else{ int nReq = pMethod->xCompressBound(pCmp, pFrom->n); if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR; rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n); pTo->n += nReq; if( rc!=SQLITE_OK ){ return rc; } } return SQLITE_OK; } |
︙ | ︙ | |||
530 531 532 533 534 535 536 537 | int nKey = 0; /* Number of keys in new zonefile */ int nFrame = 0; /* Number of frames in new zonefile */ sqlite3_stmt *pStmt = 0; /* SQL used to read data from source table */ FILE *pFd = 0; int rc; sqlite3_value *pPrev = 0; char *zErr = 0; | > > | | > > > > > > > > > > > > > > > > > > > > > > > > | < > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 | int nKey = 0; /* Number of keys in new zonefile */ int nFrame = 0; /* Number of frames in new zonefile */ sqlite3_stmt *pStmt = 0; /* SQL used to read data from source table */ FILE *pFd = 0; int rc; sqlite3_value *pPrev = 0; char *zErr = 0; void *pCmp = 0; /* Data compressor handle */ u32 iOff; ZonefileBuffer sFrameIdx = {0, 0, 0}; /* Array of frame offsets */ ZonefileBuffer sKeyIdx = {0, 0, 0}; /* Array of key locations */ ZonefileBuffer sData = {0, 0, 0}; /* All completed frames so far */ ZonefileBuffer sFrame = {0, 0, 0}; /* Current frame (uncompressed) */ ZonefileBuffer sDict = {0, 0, 0}; /* Compression model (if any) */ ZonefileBuffer sSample = {0,0,0}; /* Sample data for compressor */ size_t *aSample = 0; /* Array of sample sizes */ u8 aHdr[ZONEFILE_SZ_HEADER]; /* Space to assemble zonefile header */ assert( objc==2 || objc==3 ); zFile = (const char*)sqlite3_value_text(objv[0]); zTbl = (const char*)sqlite3_value_text(objv[1]); if( objc==3 ){ zJson = (const char*)sqlite3_value_text(objv[2]); } if( zonefileGetParams(pCtx, zJson, &sWrite) ) return; /* Check that the index-data compressor is not one that uses an external ** dictionary. This is not permitted as there is no sense in using a ** dictionary to compress a single blob of data, and the index-data ** compressor only ever compresses a single frame. Also, there is nowhere ** in the file-format to store such a dictionary for the index-data. */ if( sWrite.pCmpIdx->xTrain ){ zonefileCtxError(pCtx, "compressor \"%s\" may not be used to compress the zonefile index", sWrite.pCmpIdx->zName ); goto zone_write_out; } if( sWrite.pCmpData->xOpen ){ rc = sWrite.pCmpData->xOpen(&pCmp, 0, 0); if( rc!=SQLITE_OK ){ zonefileCtxError(pCtx, "error in compressor construction"); goto zone_write_out; } } /* Prepare the SQL statement used to read data from the source table. This ** also serves to verify the suitability of the source table schema. */ pStmt = zonefileCtxPrepare(pCtx, "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl ); if( pStmt==0 ) goto zone_write_out; /* Open a file-handle used to write out the zonefile */ pFd = zonefileFileOpen(zFile, 1, &zErr); if( pFd==0 ){ sqlite3_result_error(pCtx, zErr, -1); sqlite3_free(zErr); goto zone_write_out; } /* If the data compressor uses a global dictionary, create the dictionary ** now. */ if( sWrite.pCmpData->xTrain ){ int nSample = 0; while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nByte = sqlite3_column_bytes(pStmt, 2); const u8 *aByte = (const u8*)sqlite3_column_blob(pStmt, 2); if( zonefileBufferGrow(pCtx, &sSample, nByte) ){ goto zone_write_out; } if( (nSample & (nSample-1))==0 ){ int nNew = nSample ? nSample*2 : 8; size_t *aNew = (size_t*)sqlite3_realloc(aSample, sizeof(size_t) * nNew); if( aNew==0 ){ sqlite3_result_error_nomem(pCtx); goto zone_write_out; } aSample = aNew; } aSample[nSample] = nByte; zonefileAppendBlob(&sSample, aByte, nByte); nSample++; } rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK ){ zonefileTransferError(pCtx); goto zone_write_out; } if( zonefileBufferGrow(pCtx, &sDict, ZONEFILE_DEFAULT_DICTSIZE) ){ goto zone_write_out; } sDict.n = sDict.nAlloc; rc = sWrite.pCmpData->xTrain( pCmp, sDict.a, &sDict.n, sSample.a, aSample, nSample ); if( rc!=SQLITE_OK ){ zonefileCtxError(pCtx, "error generating dictionary"); goto zone_write_out; } } while( SQLITE_ROW==sqlite3_step(pStmt) ){ sqlite3_int64 k = sqlite3_column_int64(pStmt, 0); sqlite3_value *pFrame = sqlite3_column_value(pStmt, 1); int nBlob = sqlite3_column_bytes(pStmt, 2); const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2); int bAuto = zonefileIsAutoFrame(pFrame); if( zonefileCompareValue(pFrame, pPrev) || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize) ){ /* Add new entry to sFrame */ if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) ){ goto zone_write_out; } sFrame.n = 0; zonefileAppend32(&sFrameIdx, sData.n); sqlite3_value_free(pPrev); pPrev = sqlite3_value_dup(pFrame); |
︙ | ︙ | |||
603 604 605 606 607 608 609 | zonefileAppend32(&sKeyIdx, nBlob); /* Add uncompressed data for new entry to sFrame */ if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out; zonefileAppendBlob(&sFrame, pBlob, nBlob); nKey++; } | | | > | < | > | | > > > > > | 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 | zonefileAppend32(&sKeyIdx, nBlob); /* Add uncompressed data for new entry to sFrame */ if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out; zonefileAppendBlob(&sFrame, pBlob, nBlob); nKey++; } if( sFrame.n>0 && zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) ){ goto zone_write_out; } sqlite3_value_free(pPrev); pPrev = 0; /* If a compression method was specified, compress the key-index here */ if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){ if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out; zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n); zonefileBufferFree(&sKeyIdx); rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, 0, &sKeyIdx,&sFrameIdx); sFrameIdx.n = 0; if( rc ) goto zone_write_out; } /* Create the zonefile header in the in-memory buffer */ memset(aHdr, 0, ZONEFILE_SZ_HEADER); zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER); aHdr[4] = sWrite.pCmpIdx->eType; aHdr[5] = sWrite.pCmpData->eType; iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n; zonefilePut32(&aHdr[6], sDict.n ? iOff : 0); zonefilePut32(&aHdr[10], iOff + sDict.n); zonefilePut32(&aHdr[14], nFrame); zonefilePut32(&aHdr[18], nKey); aHdr[22] = sWrite.encryptionType; aHdr[23] = 0; /* Encryption key index */ aHdr[24] = 0; /* extended header version */ aHdr[25] = 0; /* extended header size */ assert( ZONEFILE_SZ_HEADER>=26 ); rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n); if( rc ){ zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile); goto zone_write_out; } if( fclose(pFd) ){ zonefileCtxError(pCtx, "error writing file \"%s\" (fclose())", zFile); } pFd = 0; zone_write_out: if( pCmp ) sWrite.pCmpData->xClose(pCmp); if( pFd ) fclose(pFd); sqlite3_finalize(pStmt); zonefileBufferFree(&sFrameIdx); zonefileBufferFree(&sKeyIdx); zonefileBufferFree(&sFrame); zonefileBufferFree(&sDict); zonefileBufferFree(&sData); zonefileBufferFree(&sSample); sqlite3_free(aSample); } typedef struct ZonefileFilesTab ZonefileFilesTab; struct ZonefileFilesTab { sqlite3_vtab base; /* Base class - must be first */ sqlite3 *db; char *zBase; /* Name of this table */ |
︙ | ︙ | |||
980 981 982 983 984 985 986 | ); } return rc; } static int zonefileUncompress( | | > | | | | 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 | ); } return rc; } static int zonefileUncompress( ZonefileCompress *pMethod, void *pCmp, u8 *aIn, int nIn, u8 **paOut, int *pnOut ){ int rc; int nOut = pMethod->xUncompressSize(pCmp, aIn, nIn); u8 *aOut = sqlite3_malloc(nOut); assert( pMethod->eType!=ZONEFILE_COMPRESSION_NONE ); if( aOut==0 ){ rc = SQLITE_NOMEM; }else{ rc = pMethod->xUncompress(pCmp, aOut, nOut, aIn, nIn); if( rc ){ sqlite3_free(aOut); aOut = 0; } } *paOut = aOut; |
︙ | ︙ | |||
1048 1049 1050 1051 1052 1053 1054 | rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER); } } if( rc==SQLITE_OK && pCmp ){ u8 *aUn = 0; int nUn = 0; | | | 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 | rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER); } } if( rc==SQLITE_OK && pCmp ){ u8 *aUn = 0; int nUn = 0; rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn); if( rc==SQLITE_ERROR ){ *pzErr = sqlite3_mprintf("failed to uncompress index"); } sqlite3_free(aIdx); aIdx = aUn; nIdx = nUn; } |
︙ | ︙ | |||
1520 1521 1522 1523 1524 1525 1526 | static void zonefileFree(void *p){ sqlite3_free(p); } static int zonefileCtxUncompress( sqlite3_context *pCtx, | | > | | 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 | static void zonefileFree(void *p){ sqlite3_free(p); } static int zonefileCtxUncompress( sqlite3_context *pCtx, ZonefileCompress *pMethod, void *pCmp, u8 *aBuf, int nBuf, int iKeyOff, int nKey ){ int rc; u8 *aUn = 0; int nUn = 0; rc = zonefileUncompress(pMethod, pCmp, aBuf, nBuf, &aUn, &nUn); if( rc==SQLITE_OK ){ sqlite3_result_blob(pCtx, &aUn[iKeyOff], nKey, SQLITE_TRANSIENT); }else if( rc==SQLITE_ERROR ){ zonefileCtxError(pCtx, "failed to uncompress frame"); } sqlite3_free(aUn); |
︙ | ︙ | |||
1550 1551 1552 1553 1554 1555 1556 | FILE *pFd = 0; int rc = SQLITE_OK; u32 iOff = 0; /* Offset of frame in file */ u32 szFrame = 0; /* Size of frame in bytes */ int iKeyOff = 0; /* Offset of record within frame */ int szKey = 0; /* Uncompressed size of record in bytes */ ZonefileHeader hdr; | | > | 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 | FILE *pFd = 0; int rc = SQLITE_OK; u32 iOff = 0; /* Offset of frame in file */ u32 szFrame = 0; /* Size of frame in bytes */ int iKeyOff = 0; /* Offset of record within frame */ int szKey = 0; /* Uncompressed size of record in bytes */ ZonefileHeader hdr; ZonefileCompress *pCmpMethod = 0; void *pCmp = 0; if( pTab->pIdToName==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, "SELECT filename FROM %Q.'%q_shadow_file' WHERE fileid=?", pTab->zDb, pTab->zName ); if( rc!=SQLITE_OK ){ |
︙ | ︙ | |||
1588 1589 1590 1591 1592 1593 1594 | } /* Read the zonefile header */ if( rc==SQLITE_OK ){ rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr); } | | | > > > > > > > > > > > > > > > > > > | 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 | } /* Read the zonefile header */ if( rc==SQLITE_OK ){ rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr); } /* Find the compression method and open the compressor instance */ if( rc==SQLITE_OK ){ rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr); } if( pCmpMethod ){ int nDict = 0; u8 *aDict = 0; assert( rc==SQLITE_OK ); if( hdr.byteOffsetDictionary ){ nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary; aDict = sqlite3_malloc(nDict); if( aDict==0 ){ rc = SQLITE_NOMEM; }else{ rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary); } } if( rc==SQLITE_OK ){ rc = pCmpMethod->xOpen(&pCmp, aDict, nDict); } sqlite3_free(aDict); } /* Find the offset (iOff) and size (szFrame) of the frame that the ** record is stored in. */ if( rc==SQLITE_OK ){ int iFrame = sqlite3_column_int(pCsr->pSelect, 2); u8 aSpace[8] = {0,0,0,0,0,0,0,0}; |
︙ | ︙ | |||
1621 1622 1623 1624 1625 1626 1627 | sqlite3_free(aFree); } /* Read some data into memory. If the data is uncompressed, then just ** the required record is read. Otherwise, the entire frame is read ** into memory. */ if( rc==SQLITE_OK ){ | | | | | > > > | 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 | sqlite3_free(aFree); } /* Read some data into memory. If the data is uncompressed, then just ** the required record is read. Otherwise, the entire frame is read ** into memory. */ if( rc==SQLITE_OK ){ int sz = (pCmpMethod ? (int)szFrame : szKey); i64 ofst = iOff + (pCmpMethod ? 0 : iKeyOff); u8 *aBuf = sqlite3_malloc(sz); if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + ofst); if( rc==SQLITE_OK ){ if( pCmpMethod==0 ){ sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree); aBuf = 0; }else{ rc = zonefileCtxUncompress( pCtx, pCmpMethod, pCmp, aBuf, szFrame, iKeyOff, szKey ); } }else{ zErr = sqlite3_mprintf( "failed to read %d bytes at offset %d from file \"%s\"", sz, (int)(hdr.byteOffsetFrames+ofst), zFile ); } sqlite3_free(aBuf); } } sqlite3_reset(pTab->pIdToName); if( zErr ){ assert( rc!=SQLITE_OK ); sqlite3_result_error(pCtx, zErr, -1); sqlite3_free(zErr); } zonefileFileClose(pFd); if( pCmpMethod ) pCmpMethod->xClose(pCmp); return rc; } /* ** zonefile virtual table module xColumn method. */ static int zonefileColumn( |
︙ | ︙ |
Changes to ext/zonefile/zonefile1.test.
︙ | ︙ | |||
61 62 63 64 65 66 67 | } {} do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0 do_execsql_test 1.7 { DROP TABLE z1 } #------------------------------------------------------------------------- | > > > > > > > > > > > > > | > | < < | < > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > | 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | } {} do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0 do_execsql_test 1.7 { DROP TABLE z1 } #------------------------------------------------------------------------- # Figure out which compression algorithms, if any, are supported by # this build. Populate the global list $COMPRESSION_METHODS with the # result. reset_db load_static_extension db zonefile do_execsql_test 2.0 { CREATE TABLE bb( k INTEGER PRIMARY KEY, frame INTEGER DEFAULT -1, idx INTEGER DEFAULT -1, v BLOB ); INSERT INTO bb(k, v) VALUES(1, randomblob(100)); } set COMPRESSION_METHODS [list] foreach cmp { none zlib zstd zstd_global_dict lz4 lz4hc brotli } { set res [catchsql { WITH p(n,v) AS ( VALUES('compressionTypeContent', $cmp) ) SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p; }] if {[lindex $res 0]==0} { lappend COMPRESSION_METHODS $cmp } } # Check that it is not possible to use zstd_global_dict to compress # the zonefile index. # if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} { do_catchsql_test 2.1 { WITH p(n,v) AS ( VALUES('compressionTypeIndexData', 'zstd_global_dict') ) SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p; } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}} } # puts $COMPRESSION_METHODS foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS { if {$cmpidx == "zstd_global_dict"} continue reset_db load_static_extension db zonefile set tn "$cmp/$cmpidx" do_execsql_test 2.$tn.0 { CREATE TABLE zz( k INTEGER PRIMARY KEY, frame INTEGER DEFAULT -1, idx INTEGER DEFAULT -1, v BLOB ); |
︙ | ︙ | |||
152 153 154 155 156 157 158 | WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile') ORDER BY 1 } { test19.zonefile 1 1 test20.zonefile 2 2 test21.zonefile 7 4 } | | | 192 193 194 195 196 197 198 199 200 201 202 203 | WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile') ORDER BY 1 } { test19.zonefile 1 1 test20.zonefile 2 2 test21.zonefile 7 4 } }} finish_test |