/ Check-in [78267a09]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for the ExtendedHeaderSize header field to zonefile.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256:78267a091307e2c29a4fb1606fa9c79939fe010b801749614f4c48dc8715810e
User & Date: dan 2018-02-19 16:28:42
Context
2018-02-19
21:07
Add support for invoking encryption hooks to zonefile. And mock encryption method "xor" for testing. check-in: 55cf920c user: dan tags: zonefile
16:28
Add support for the ExtendedHeaderSize header field to zonefile. check-in: 78267a09 user: dan tags: zonefile
14:27
Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details. check-in: 4dbe0cba user: dan tags: zonefile
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/zonefile/zonefile.c.

407
408
409
410
411
412
413

414
415
416
417
418
419
420
...
565
566
567
568
569
570
571









572
573
574
575
576
577
578
...
712
713
714
715
716
717
718





















719
720
721
722
723
724
725
...
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927

928
929
930
931
932
933
934
....
1327
1328
1329
1330
1331
1332
1333

1334
1335
1336
1337
1338

1339
1340
1341
1342
1343
1344
1345
1346
....
1916
1917
1918
1919
1920
1921
1922
1923


1924
1925
1926
1927
1928
1929
1930
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
  ZonefileCompress *pCmpIdx;      /* For compressing the index */
  ZonefileCompress *pCmpData;     /* For compressing each frame */
  int encryptionType;
  int maxAutoFrameSize;

};

/*
** A structure to store a deserialized zonefile header in.
*/
typedef struct ZonefileHeader ZonefileHeader;
struct ZonefileHeader {
................................................................................
  rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
  }
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
    const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
    int iVal = sqlite3_column_int(pStmt, 1);









    if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
      p->maxAutoFrameSize = iVal;
    }else
    if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
      p->pCmpIdx = zonefileCompress(zName);
      if( p->pCmpIdx==0 ){
................................................................................
      if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
      rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
      pTo->n += nReq;
      if( rc!=SQLITE_OK ){
        return rc;
      }
    }





















  }
  return SQLITE_OK;
}

/*
** Function:     zonefile_write(F,T[,J])
*/
................................................................................

  /* Create the zonefile header in the in-memory buffer */
  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
  aHdr[4] = sWrite.pCmpIdx->eType;
  aHdr[5] = sWrite.pCmpData->eType;
  iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
  zonefilePut32(&aHdr[6], sDict.n ? iOff : 0);
  zonefilePut32(&aHdr[10], iOff + sDict.n);
  zonefilePut32(&aHdr[14], nFrame);
  zonefilePut32(&aHdr[18], nKey);
  aHdr[22] = sWrite.encryptionType;
  aHdr[23] = 0;                   /* Encryption key index */
  aHdr[24] = 0;                   /* extended header version */
  aHdr[25] = 0;                   /* extended header size */
  assert( ZONEFILE_SZ_HEADER>=26 );

  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);

  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
  if( rc ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
    goto zone_write_out;
................................................................................
  rc = zfFindCompress(pHdr->compressionTypeIndexData, &pCmp, pzErr);
  if( rc==SQLITE_OK ){
    if( pHdr->byteOffsetDictionary ){
      nIdx = pHdr->byteOffsetDictionary - ZONEFILE_SZ_HEADER;
    }else{
      nIdx = pHdr->byteOffsetFrames - ZONEFILE_SZ_HEADER;
    }

    aIdx = (u8*)sqlite3_malloc(nIdx);

    if( aIdx==0 ){
      rc = SQLITE_NOMEM;
    }else{

      rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER);
    }
  }

  if( rc==SQLITE_OK && pCmp ){
    u8 *aUn = 0;
    int nUn = 0;
    rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn);
................................................................................
    u8 *aOff = aSpace;
    u8 *aFree = 0;
    if( hdr.compressionTypeIndexData ){
      int nFree = 0;
      rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr);
      if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)];
    }else{
      rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * (iFrame-1));


    }
    szFrame = zonefileGet32(&aOff[4]);
    if( iFrame>0 ){
      iOff = zonefileGet32(aOff);
      szFrame = szFrame - iOff;
    }
    sqlite3_free(aFree);







>







 







>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
|





|



>







 







>





>
|







 







|
>
>







407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
...
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
...
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
...
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
....
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
....
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
*/
typedef struct ZonefileWrite ZonefileWrite;
struct ZonefileWrite {
  ZonefileCompress *pCmpIdx;      /* For compressing the index */
  ZonefileCompress *pCmpData;     /* For compressing each frame */
  int encryptionType;
  int maxAutoFrameSize;
  int debugExtendedHeaderSize;    /* Size of extended header */
};

/*
** A structure to store a deserialized zonefile header in.
*/
typedef struct ZonefileHeader ZonefileHeader;
struct ZonefileHeader {
................................................................................
  rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)");
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pStmt, 1, zJson, -1, SQLITE_STATIC);
  }
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
    const char *zKey = (const char*)sqlite3_column_text(pStmt, 0);
    int iVal = sqlite3_column_int(pStmt, 1);
    if( sqlite3_stricmp("debugExtendedHeaderSize", zKey)==0 ){
      if( iVal<0 || iVal>255 ){
        zErr = sqlite3_mprintf(
            "debugExtendedHeaderSize value out of range: %d", iVal
        );
        rc = SQLITE_ERROR;
      }
      p->debugExtendedHeaderSize = iVal;
    }else
    if( sqlite3_stricmp("maxAutoFrameSize", zKey)==0 ){
      p->maxAutoFrameSize = iVal;
    }else
    if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){
      const char *zName = (const char*)sqlite3_column_text(pStmt, 1);
      p->pCmpIdx = zonefileCompress(zName);
      if( p->pCmpIdx==0 ){
................................................................................
      if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR;
      rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n);
      pTo->n += nReq;
      if( rc!=SQLITE_OK ){
        return rc;
      }
    }
  }
  return SQLITE_OK;
}

/*
** Append nByte bytes of garbage data to the file opened with pFd. Return
** SQLITE_OK if successful, or SQLITE_ERROR if an error occurs.
**
** Parameter nByte is only ever non-zero when running tests. So it doesn't
** matter if this function is inefficient in those cases.
*/
static int zonefilePad(FILE *pFd, int nByte){
  assert( nByte>=0 && nByte<256 );
  if( nByte ){
    int nRem = nByte;
    u8 buf[16] = "0123456789ABCDEF";
    while( nRem>0 ){
      int n = MIN(nRem, sizeof(buf));
      if( zonefileFileWrite(pFd, buf, n) ) return SQLITE_ERROR;
      nRem -= n;
    }
  }
  return SQLITE_OK;
}

/*
** Function:     zonefile_write(F,T[,J])
*/
................................................................................

  /* Create the zonefile header in the in-memory buffer */
  memset(aHdr, 0, ZONEFILE_SZ_HEADER);
  zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER);
  aHdr[4] = sWrite.pCmpIdx->eType;
  aHdr[5] = sWrite.pCmpData->eType;
  iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n;
  zonefilePut32(&aHdr[6], sDict.n ? iOff+sWrite.debugExtendedHeaderSize : 0);
  zonefilePut32(&aHdr[10], iOff + sWrite.debugExtendedHeaderSize + sDict.n);
  zonefilePut32(&aHdr[14], nFrame);
  zonefilePut32(&aHdr[18], nKey);
  aHdr[22] = sWrite.encryptionType;
  aHdr[23] = 0;                   /* Encryption key index */
  aHdr[24] = 0;                   /* extended header version */
  aHdr[25] = sWrite.debugExtendedHeaderSize;
  assert( ZONEFILE_SZ_HEADER>=26 );

  rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER);
  if( rc==SQLITE_OK ) rc = zonefilePad(pFd, sWrite.debugExtendedHeaderSize);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n);
  if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n);
  if( rc ){
    zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile);
    goto zone_write_out;
................................................................................
  rc = zfFindCompress(pHdr->compressionTypeIndexData, &pCmp, pzErr);
  if( rc==SQLITE_OK ){
    if( pHdr->byteOffsetDictionary ){
      nIdx = pHdr->byteOffsetDictionary - ZONEFILE_SZ_HEADER;
    }else{
      nIdx = pHdr->byteOffsetFrames - ZONEFILE_SZ_HEADER;
    }
    nIdx -= pHdr->extendedHeaderSize;
    aIdx = (u8*)sqlite3_malloc(nIdx);

    if( aIdx==0 ){
      rc = SQLITE_NOMEM;
    }else{
      i64 iOff = ZONEFILE_SZ_HEADER + pHdr->extendedHeaderSize;
      rc = zonefileFileRead(pFd, aIdx, nIdx, iOff);
    }
  }

  if( rc==SQLITE_OK && pCmp ){
    u8 *aUn = 0;
    int nUn = 0;
    rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn);
................................................................................
    u8 *aOff = aSpace;
    u8 *aFree = 0;
    if( hdr.compressionTypeIndexData ){
      int nFree = 0;
      rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr);
      if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)];
    }else{
      rc = zonefileFileRead(pFd, aOff, 8, 
          ZONEFILE_SZ_HEADER + hdr.extendedHeaderSize + 4 * (iFrame-1)
      );
    }
    szFrame = zonefileGet32(&aOff[4]);
    if( iFrame>0 ){
      iOff = zonefileGet32(aOff);
      szFrame = szFrame - iOff;
    }
    sqlite3_free(aFree);

Changes to ext/zonefile/zonefile1.test.

104
105
106
107
108
109
110

111
112
113
114
115
116

117
118
119
120
121
122
123
...
167
168
169
170
171
172
173
174

175
176
177
178
179
180
181
        VALUES('compressionTypeIndexData', 'zstd_global_dict')
    )
    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
}
# puts $COMPRESSION_METHODS


foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
  if {$cmpidx == "zstd_global_dict"} continue
  reset_db
  load_static_extension db zonefile

  set tn "$cmp/$cmpidx"


  do_execsql_test 2.$tn.0 {
    CREATE TABLE zz(
      k INTEGER PRIMARY KEY, 
      frame INTEGER DEFAULT -1, 
      idx INTEGER DEFAULT -1, 
      v BLOB
................................................................................
      execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
    }
    execsql { INSERT INTO rt SELECT k, v FROM zz }
    execsql { 
      WITH p(n,v) AS (
        VALUES('maxAutoFrameSize', 2000) UNION ALL
        VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
        VALUES('compressionTypeContent', $cmp)

      )
      SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
      INSERT INTO zone_files(filename) VALUES($zonefile);
    }
  }
  
  do_execsql_test 2.$tn.1 {







>






>







 







|
>







104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
...
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
        VALUES('compressionTypeIndexData', 'zstd_global_dict')
    )
    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
  } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}}
}
# puts $COMPRESSION_METHODS

set extra_header 0
foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS {
  if {$cmpidx == "zstd_global_dict"} continue
  reset_db
  load_static_extension db zonefile

  set tn "$cmp/$cmpidx"
  set extra_header [expr {$extra_header ? 0 : 100}]

  do_execsql_test 2.$tn.0 {
    CREATE TABLE zz(
      k INTEGER PRIMARY KEY, 
      frame INTEGER DEFAULT -1, 
      idx INTEGER DEFAULT -1, 
      v BLOB
................................................................................
      execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) }
    }
    execsql { INSERT INTO rt SELECT k, v FROM zz }
    execsql { 
      WITH p(n,v) AS (
        VALUES('maxAutoFrameSize', 2000) UNION ALL
        VALUES('compressionTypeIndexData', $cmpidx) UNION ALL
        VALUES('compressionTypeContent', $cmp) UNION ALL
        VALUES('debugExtendedHeaderSize', $extra_header)
      )
      SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p;
      INSERT INTO zone_files(filename) VALUES($zonefile);
    }
  }
  
  do_execsql_test 2.$tn.1 {