Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Add support for the ExtendedHeaderSize header field to zonefile. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | zonefile |
| Files: | files | file ages | folders |
| SHA3-256: |
78267a091307e2c29a4fb1606fa9c799 |
| User & Date: | dan 2018-02-19 16:28:42 |
Context
|
2018-02-19
| ||
| 21:07 | Add support for invoking encryption hooks to zonefile. And mock encryption method "xor" for testing. check-in: 55cf920c user: dan tags: zonefile | |
| 16:28 | Add support for the ExtendedHeaderSize header field to zonefile. check-in: 78267a09 user: dan tags: zonefile | |
| 14:27 | Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details. check-in: 4dbe0cba user: dan tags: zonefile | |
Changes
Changes to ext/zonefile/zonefile.c.
407 408 409 410 411 412 413 414 415 416 417 418 419 420 ... 565 566 567 568 569 570 571 572 573 574 575 576 577 578 ... 712 713 714 715 716 717 718 719 720 721 722 723 724 725 ... 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 .... 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 .... 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 |
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
ZonefileCompress *pCmpIdx; /* For compressing the index */
ZonefileCompress *pCmpData; /* For compressing each frame */
int encryptionType;
int maxAutoFrameSize;
};
/*
** A structure to store a deserialized zonefile header in.
*/
typedef struct ZonefileHeader ZonefileHeader;
struct ZonefileHeader {
................................................................................
rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
if( rc==SQLITE_OK ){
sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
int iVal = sqlite3_column_int(pStmt, 1);
if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
p->maxAutoFrameSize = iVal;
}else
if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
p->pCmpIdx = zonefileCompress(zName);
if( p->pCmpIdx==0 ){
................................................................................
if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
pTo->n += nReq;
if( rc!=SQLITE_OK ){
return rc;
}
}
}
return SQLITE_OK;
}
/*
** Function: zonefile_write(F,T[,J])
*/
................................................................................
/* Create the zonefile header in the in-memory buffer */
memset(aHdr, 0, ZONEFILE_SZ_HEADER);
zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
aHdr[4] = sWrite.pCmpIdx->eType;
aHdr[5] = sWrite.pCmpData->eType;
iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
zonefilePut32(&aHdr[6], sDict.n ? iOff : 0);
zonefilePut32(&aHdr[10], iOff + sDict.n);
zonefilePut32(&aHdr[14], nFrame);
zonefilePut32(&aHdr[18], nKey);
aHdr[22] = sWrite.encryptionType;
aHdr[23] = 0; /* Encryption key index */
aHdr[24] = 0; /* extended header version */
aHdr[25] = 0; /* extended header size */
assert( ZONEFILE_SZ_HEADER>=26 );
rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
if( rc ){
zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
goto zone_write_out;
................................................................................
rc = zfFindCompress(pHdr->compressionTypeIndexData, &pCmp, pzErr);
if( rc==SQLITE_OK ){
if( pHdr->byteOffsetDictionary ){
nIdx = pHdr->byteOffsetDictionary - ZONEFILE_SZ_HEADER;
}else{
nIdx = pHdr->byteOffsetFrames - ZONEFILE_SZ_HEADER;
}
aIdx = (u8*)sqlite3_malloc(nIdx);
if( aIdx==0 ){
rc = SQLITE_NOMEM;
}else{
rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER);
}
}
if( rc==SQLITE_OK && pCmp ){
u8 *aUn = 0;
int nUn = 0;
rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn);
................................................................................
u8 *aOff = aSpace;
u8 *aFree = 0;
if( hdr.compressionTypeIndexData ){
int nFree = 0;
rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr);
if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)];
}else{
rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * (iFrame-1));
}
szFrame = zonefileGet32(&aOff[4]);
if( iFrame>0 ){
iOff = zonefileGet32(aOff);
szFrame = szFrame - iOff;
}
sqlite3_free(aFree);
|
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | > > > | | > > |
407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 ... 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 ... 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 ... 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 .... 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 .... 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 |
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
ZonefileCompress *pCmpIdx; /* For compressing the index */
ZonefileCompress *pCmpData; /* For compressing each frame */
int encryptionType;
int maxAutoFrameSize;
int debugExtendedHeaderSize; /* Size of extended header */
};
/*
** A structure to store a deserialized zonefile header in.
*/
typedef struct ZonefileHeader ZonefileHeader;
struct ZonefileHeader {
................................................................................
rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
if( rc==SQLITE_OK ){
sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
}
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
int iVal = sqlite3_column_int(pStmt, 1);
if( sqlite3_stricmp("debugExtendedHeaderSize", zKey)==0 ){
if( iVal<0 || iVal>255 ){
zErr = sqlite3_mprintf(
"debugExtendedHeaderSize value out of range: %d", iVal
);
rc = SQLITE_ERROR;
}
p->debugExtendedHeaderSize = iVal;
}else
if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
p->maxAutoFrameSize = iVal;
}else
if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
p->pCmpIdx = zonefileCompress(zName);
if( p->pCmpIdx==0 ){
................................................................................
if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
pTo->n += nReq;
if( rc!=SQLITE_OK ){
return rc;
}
}
}
return SQLITE_OK;
}
/*
** Append nByte bytes of garbage data to the file opened with pFd. Return
** SQLITE_OK if successful, or SQLITE_ERROR if an error occurs.
**
** Parameter nByte is only ever non-zero when running tests. So it doesn't
** matter if this function is inefficient in those cases.
*/
static int zonefilePad(FILE *pFd, int nByte){
assert( nByte>=0 && nByte<256 );
if( nByte ){
int nRem = nByte;
u8 buf[16] = "0123456789ABCDEF";
while( nRem>0 ){
int n = MIN(nRem, sizeof(buf));
if( zonefileFileWrite(pFd, buf, n) ) return SQLITE_ERROR;
nRem -= n;
}
}
return SQLITE_OK;
}
/*
** Function: zonefile_write(F,T[,J])
*/
................................................................................
/* Create the zonefile header in the in-memory buffer */
memset(aHdr, 0, ZONEFILE_SZ_HEADER);
zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
aHdr[4] = sWrite.pCmpIdx->eType;
aHdr[5] = sWrite.pCmpData->eType;
iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
zonefilePut32(&aHdr[6], sDict.n ? iOff+sWrite.debugExtendedHeaderSize : 0);
zonefilePut32(&aHdr[10], iOff + sWrite.debugExtendedHeaderSize + sDict.n);
zonefilePut32(&aHdr[14], nFrame);
zonefilePut32(&aHdr[18], nKey);
aHdr[22] = sWrite.encryptionType;
aHdr[23] = 0; /* Encryption key index */
aHdr[24] = 0; /* extended header version */
aHdr[25] = sWrite.debugExtendedHeaderSize;
assert( ZONEFILE_SZ_HEADER>=26 );
rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
if( rc==SQLITE_OK ) rc = zonefilePad(pFd, sWrite.debugExtendedHeaderSize);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
if( rc ){
zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
goto zone_write_out;
................................................................................
rc = zfFindCompress(pHdr->compressionTypeIndexData, &pCmp, pzErr);
if( rc==SQLITE_OK ){
if( pHdr->byteOffsetDictionary ){
nIdx = pHdr->byteOffsetDictionary - ZONEFILE_SZ_HEADER;
}else{
nIdx = pHdr->byteOffsetFrames - ZONEFILE_SZ_HEADER;
}
nIdx -= pHdr->extendedHeaderSize;
aIdx = (u8*)sqlite3_malloc(nIdx);
if( aIdx==0 ){
rc = SQLITE_NOMEM;
}else{
i64 iOff = ZONEFILE_SZ_HEADER + pHdr->extendedHeaderSize;
rc = zonefileFileRead(pFd, aIdx, nIdx, iOff);
}
}
if( rc==SQLITE_OK && pCmp ){
u8 *aUn = 0;
int nUn = 0;
rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn);
................................................................................
u8 *aOff = aSpace;
u8 *aFree = 0;
if( hdr.compressionTypeIndexData ){
int nFree = 0;
rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr);
if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)];
}else{
rc = zonefileFileRead(pFd, aOff, 8,
ZONEFILE_SZ_HEADER + hdr.extendedHeaderSize + 4 * (iFrame-1)
);
}
szFrame = zonefileGet32(&aOff[4]);
if( iFrame>0 ){
iOff = zonefileGet32(aOff);
szFrame = szFrame - iOff;
}
sqlite3_free(aFree);
|
Changes to ext/zonefile/zonefile1.test.
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
...
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
VALUES('compressionTypeIndexData', 'zstd_global_dict')
)
SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
} {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
}
# puts $COMPRESSION_METHODS
foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
if {$cmpidx == "zstd_global_dict"} continue
reset_db
load_static_extension db zonefile
set tn "$cmp/$cmpidx"
do_execsql_test 2.$tn.0 {
CREATE TABLE zz(
k INTEGER PRIMARY KEY,
frame INTEGER DEFAULT -1,
idx INTEGER DEFAULT -1,
v BLOB
................................................................................
execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
}
execsql { INSERT INTO rt SELECT k, v FROM zz }
execsql {
WITH p(n,v) AS (
VALUES('maxAutoFrameSize', 2000) UNION ALL
VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
VALUES('compressionTypeContent', $cmp)
)
SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
INSERT INTO zone_files(filename) VALUES($zonefile);
}
}
do_execsql_test 2.$tn.1 {
|
>
>
|
>
|
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
...
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
VALUES('compressionTypeIndexData', 'zstd_global_dict')
)
SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
} {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
}
# puts $COMPRESSION_METHODS
set extra_header 0
foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
if {$cmpidx == "zstd_global_dict"} continue
reset_db
load_static_extension db zonefile
set tn "$cmp/$cmpidx"
set extra_header [expr {$extra_header ? 0 : 100}]
do_execsql_test 2.$tn.0 {
CREATE TABLE zz(
k INTEGER PRIMARY KEY,
frame INTEGER DEFAULT -1,
idx INTEGER DEFAULT -1,
v BLOB
................................................................................
execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
}
execsql { INSERT INTO rt SELECT k, v FROM zz }
execsql {
WITH p(n,v) AS (
VALUES('maxAutoFrameSize', 2000) UNION ALL
VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
VALUES('compressionTypeContent', $cmp) UNION ALL
VALUES('debugExtendedHeaderSize', $extra_header)
)
SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
INSERT INTO zone_files(filename) VALUES($zonefile);
}
}
do_execsql_test 2.$tn.1 {
|