/ Check-in [f4d42162]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add tests cases and fix some minor zonefile problems.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | zonefile
Files: files | file ages | folders
SHA3-256:f4d42162fa2196078cea4db9c2247a08f5439d29e49f5e8dc23db317b8101865
User & Date: dan 2018-02-22 21:06:08
Context
2018-02-23
13:45
Merge updates from trunk. check-in: 53f2100a user: mistachkin tags: zonefile
2018-02-22
21:06
Add tests cases and fix some minor zonefile problems. check-in: f4d42162 user: dan tags: zonefile
16:46
Add an LRU cache of uncompressed frame content to the zonefile virtual table implementation. check-in: 883e7e75 user: dan tags: zonefile
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/zonefile/zonefile.c.

315
316
317
318
319
320
321

322
323
324
325
326
327
328
...
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
...
926
927
928
929
930
931
932


933
934
935
936
937
938
939
....
1126
1127
1128
1129
1130
1131
1132

1133



1134
1135
1136
1137
1138
1139
1140
....
1327
1328
1329
1330
1331
1332
1333



1334
1335
1336
1337
1338
1339
1340
....
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
    pNew->zName = &pNew->zDb[nDb+1];
    memcpy((char*)pNew->zKey, zKey, nKey+1);
    memcpy((char*)pNew->zDb, zDb, nDb+1);
    memcpy((char*)pNew->zName, zTab, nTab+1);

    pNew->pHashNext = pGlobal->aHash[iHash % pGlobal->nHash];
    pGlobal->aHash[iHash % pGlobal->nHash] = pNew;

  }

  return SQLITE_OK;
}

/*
** Search the key-store passed as the first argument for an encryption
................................................................................
  int rc;
  va_list ap;
  char *zSql;
  va_start(ap, zFmt);
  zSql = sqlite3_vmprintf(zFmt, ap);
  *ppStmt = 0;
  if( zSql ){
    rc = sqlite3_prepare(db, zSql, -1, ppStmt, 0);
    if( rc!=SQLITE_OK ){
      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
    }
    sqlite3_free(zSql);
  }else{
    rc = SQLITE_NOMEM;
  }
................................................................................
  }
  rc2 = sqlite3_finalize(pStmt);
  if( rc==SQLITE_OK ) rc = rc2;

  if( zErr ){
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);


  }else{
    if( p->encryptionKey==0 ){
      p->encryptionType = 0;
    }
  }
  return rc;
}
................................................................................

  if( sParam.encryptionType!=0 ){
    int n = strlen(sParam.encryptionKey);
    rc = zonefileCodecCreate(
        sParam.encryptionType, (u8*)sParam.encryptionKey, n, &pCodec, &zErr
    );
    if( rc!=SQLITE_OK ){

      sqlite3_result_error(pCtx, zErr, -1);



      sqlite3_free(zErr);
      goto zone_write_out;
    }
  }

  /* Check that the index-data compressor is not one that uses an external
  ** dictionary. This is not permitted as there is no sense in using a
................................................................................
  zonefileBufferFree(&sKeyIdx);
  zonefileBufferFree(&sFrame);
  zonefileBufferFree(&sDict);
  zonefileBufferFree(&sData);
  zonefileBufferFree(&sSample);
  sqlite3_free(aSample);
  sqlite3_free(sParam.encryptionKey);



}

typedef struct ZonefileFilesTab ZonefileFilesTab;
struct ZonefileFilesTab {
  sqlite3_vtab base;              /* Base class - must be first */
  sqlite3 *db;
  char *zBase;                    /* Name of this table */
................................................................................
      if( rc==SQLITE_OK ){
        rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
      }
      sqlite3_free(aDict);
    }

    /* Find the encryption method and key. */
    if( hdr.encryptionType ){
      const char *z = 0;
      int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFile, &z);
      if( n==0 ){
        zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
        rc = SQLITE_ERROR;
      }else{
        rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr);







>







 







|







 







>
>







 







>
|
>
>
>







 







>
>
>







 







|







315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
...
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
...
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
....
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
....
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
....
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
    pNew->zName = &pNew->zDb[nDb+1];
    memcpy((char*)pNew->zKey, zKey, nKey+1);
    memcpy((char*)pNew->zDb, zDb, nDb+1);
    memcpy((char*)pNew->zName, zTab, nTab+1);

    pNew->pHashNext = pGlobal->aHash[iHash % pGlobal->nHash];
    pGlobal->aHash[iHash % pGlobal->nHash] = pNew;
    pGlobal->nEntry++;
  }

  return SQLITE_OK;
}

/*
** Search the key-store passed as the first argument for an encryption
................................................................................
  int rc;
  va_list ap;
  char *zSql;
  va_start(ap, zFmt);
  zSql = sqlite3_vmprintf(zFmt, ap);
  *ppStmt = 0;
  if( zSql ){
    rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
    if( rc!=SQLITE_OK ){
      *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
    }
    sqlite3_free(zSql);
  }else{
    rc = SQLITE_NOMEM;
  }
................................................................................
  }
  rc2 = sqlite3_finalize(pStmt);
  if( rc==SQLITE_OK ) rc = rc2;

  if( zErr ){
    sqlite3_result_error(pCtx, zErr, -1);
    sqlite3_free(zErr);
  }else if( rc ){
    sqlite3_result_error_code(pCtx, rc);
  }else{
    if( p->encryptionKey==0 ){
      p->encryptionType = 0;
    }
  }
  return rc;
}
................................................................................

  if( sParam.encryptionType!=0 ){
    int n = strlen(sParam.encryptionKey);
    rc = zonefileCodecCreate(
        sParam.encryptionType, (u8*)sParam.encryptionKey, n, &pCodec, &zErr
    );
    if( rc!=SQLITE_OK ){
      if( zErr ){
        sqlite3_result_error(pCtx, zErr, -1);
      }else{
        sqlite3_result_error_code(pCtx, rc);
      }
      sqlite3_free(zErr);
      goto zone_write_out;
    }
  }

  /* Check that the index-data compressor is not one that uses an external
  ** dictionary. This is not permitted as there is no sense in using a
................................................................................
  zonefileBufferFree(&sKeyIdx);
  zonefileBufferFree(&sFrame);
  zonefileBufferFree(&sDict);
  zonefileBufferFree(&sData);
  zonefileBufferFree(&sSample);
  sqlite3_free(aSample);
  sqlite3_free(sParam.encryptionKey);
  if( rc==SQLITE_NOMEM ){
    sqlite3_result_error_nomem(pCtx);
  }
}

typedef struct ZonefileFilesTab ZonefileFilesTab;
struct ZonefileFilesTab {
  sqlite3_vtab base;              /* Base class - must be first */
  sqlite3 *db;
  char *zBase;                    /* Name of this table */
................................................................................
      if( rc==SQLITE_OK ){
        rc = pCmpMethod->xOpen(&pCmp, aDict, nDict);
      }
      sqlite3_free(aDict);
    }

    /* Find the encryption method and key. */
    if( rc==SQLITE_OK && hdr.encryptionType ){
      const char *z = 0;
      int n = zonefileKeyFind(pTab->pGlobal, pTab->zDb, pTab->zName, iFile, &z);
      if( n==0 ){
        zErr = sqlite3_mprintf("missing encryption key for file \"%s\"", zFile);
        rc = SQLITE_ERROR;
      }else{
        rc = zonefileCodecCreate(hdr.encryptionType, (u8*)z, n, &pCodec, &zErr);

Changes to ext/zonefile/zonefile1.test.

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    idx INTEGER DEFAULT -1, 
    v BLOB
  );
  INSERT INTO bb(k, v) VALUES(1, randomblob(100));
}
set COMPRESSION_METHODS [list]
foreach cmp {
  none zlib zstd zstd_global_dict lz4 lz4hc brotli
} {
  set res [catchsql {
    WITH p(n,v) AS (
        VALUES('compressionTypeContent', $cmp)
    )
    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
  }]







|







76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
    idx INTEGER DEFAULT -1, 
    v BLOB
  );
  INSERT INTO bb(k, v) VALUES(1, randomblob(100));
}
set COMPRESSION_METHODS [list]
foreach cmp {
  none zlib zstd zstd_global_dict lz4 lz4hc brotli nosuchcmp
} {
  set res [catchsql {
    WITH p(n,v) AS (
        VALUES('compressionTypeContent', $cmp)
    )
    SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p;
  }]

Changes to ext/zonefile/zonefileenc.test.

83
84
85
86
87
88
89





90




































91
92
do_execsql_test 1.7 {
  SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
} {0}
do_execsql_test 1.8 {
  SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
} {300}











































finish_test








>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
do_execsql_test 1.7 {
  SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v!=gg.v;
} {0}
do_execsql_test 1.8 {
  SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
} {300}

forcedelete test.db2
do_execsql_test 1.9.1 {
  ATTACH 'test.db2' AS maing;
  CREATE VIRTUAL TABLE maing.g USING zonefile;
  INSERT INTO g_files(filename) SELECT filename FROM gg_files;
}
do_catchsql_test 1.9.2 {
  SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v!=g.v;
} {1 {missing encryption key for file "test0.zonefile"}}
do_execsql_test 1.9.3 {
  UPDATE g_files SET ekey = k(rowid-1);
  SELECT count(*) FROM rr JOIN g USING(k) WHERE rr.v==g.v;
} {300}

do_execsql_test 1.10 {
  SELECT count(*) FROM rr JOIN gg USING(k) WHERE rr.v==gg.v;
} {300}
#-------------------------------------------------------------------------

reset_db
load_static_extension db zonefile

do_execsql_test 2.0 {
  CREATE TABLE zz(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
}
foreach {tn alg id} {
  1 aes_128_ctr 1
  2 aes_128_cbc 2
  3 AES_256_CTR 3
  4 Aes_256_CBC 4
} {
  do_catchsql_test 2.$tn {
    WITH p(n,v) AS (
        VALUES('encryptionType', $alg) UNION ALL
        VALUES('encryptionKey', 'secret')
    )
    SELECT zonefile_write('test' || $i || '.zonefile', 'zz', 
          json_group_object(n, v)
    ) FROM p;
  } "1 {unsupported encryption method: $id}"
}

finish_test

Added ext/zonefile/zonefilefault.test.



























































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
# 2018 Feb 11
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The focus of this file is testing the zonefile extension.
#

if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source [file join $testdir tester.tcl]
source [file join $testdir malloc_common.tcl]
set testprefix zonefilefault
load_static_extension db zonefile

do_execsql_test 1.0 {
  CREATE TABLE tt(k INTEGER PRIMARY KEY, frame INTEGER, idx INTEGER, v BLOB);
  INSERT INTO tt VALUES(1, -1, -1, randomblob(100));
  INSERT INTO tt VALUES(2, -1, -1, randomblob(200));
  INSERT INTO tt VALUES(3,  1, -1, randomblob(300));
  INSERT INTO tt VALUES(4,  2, -1, randomblob(400));
}

do_faultsim_test 1.1 -faults oom* -prep {
  sqlite3 db test.db
  load_static_extension db zonefile
} -body {
  execsql {
    SELECT zonefile_write('test.zonefile', 'tt',
        '{"encryptionType":"xor", "encryptionKey":"secret"}'
    );
  }
} -test {
  faultsim_test_result {0 {{}}}
}

do_faultsim_test 1.2 -faults oom* -prep {
  sqlite3 db test.db
  load_static_extension db zonefile
} -body {
  execsql {
    SELECT zonefile_write('test.zonefile', 'tt',
        '{"compressionTypeContent":"zstd_global_dict"}'
    );
  }
} -test {
  faultsim_test_result {0 {{}}}
}

do_execsql_test 2.0 {
  SELECT zonefile_write('test.zonefile', 'tt',
      '{"encryptionType":"xor", "encryptionKey":"secret"}'
  );
  CREATE VIRTUAL TABLE zz USING zonefile;
} {{}}

faultsim_save_and_close
do_faultsim_test 2 -faults oom* -prep {
  faultsim_restore_and_reopen
  load_static_extension db zonefile
} -body {
  execsql {
    INSERT INTO zz_files(filename, ekey) VALUES('test.zonefile', 'secret')
  }
} -test {
  faultsim_test_result {0 {}}
}

finish_test