SQLite4
Check-in [b942b91a3d]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add some human readable text to the bt file header. Refuse to open a database (SQLITE_NOTADB) if a valid header cannot be located in the database or wal files.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: b942b91a3deb3e19b17cbc9145bb575e8201bcc6
User & Date: dan 2014-02-15 17:04:56
Context
2014-02-15
20:29
Defer opening a bt database until it is first read. check-in: d9560b73de user: dan tags: trunk
17:04
Add some human readable text to the bt file header. Refuse to open a database (SQLITE_NOTADB) if a valid header cannot be located in the database or wal files. check-in: b942b91a3d user: dan tags: trunk
2014-02-14
18:53
Fix memory leaks in test suite. check-in: f4d0f55571 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btInt.h.

48
49
50
51
52
53
54


55
56
57
58
59
60
61
..
69
70
71
72
73
74
75

76

77
78
79
80
81
82
83
/* By default pages are 1024 bytes in size. */
#define BT_DEFAULT_PGSZ 1024

/* By default blocks are 512K bytes in size. */
#define BT_DEFAULT_BLKSZ (512*1024)

/*


**
** pgsz, blksz:
**   Byte offset 0 of the database file is the first byte of both page 1
**   and block 1. Each page is pHdr->pgsz bytes in size. Each block is
**   pHdr->blksz bytes in size. It is guaranteed that the block-size is
**   an integer multiple of the page-size.
**
................................................................................
**   sub-tree, not including any overflow pages. Pages (except overflow 
**   pages) are always allocated contiguously within the block.
**
**   The reason these are likely a stop-gap is that the write-magnification
**   caused by using a b-tree for to populate level-0 sub-trees is too 
**   expensive.
*/

struct BtDbHdr {

  u32 pgsz;                       /* Page size in bytes */
  u32 blksz;                      /* Block size in bytes */
  u32 nPg;                        /* Size of database file in pages */

  u32 iRoot;                      /* B-tree root page */
  u32 iMRoot;                     /* Root page of meta-tree */
  u32 iSRoot;                     /* Root page of schedule-tree */







>
>







 







>

>







48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
..
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
/* By default pages are 1024 bytes in size. */
#define BT_DEFAULT_PGSZ 1024

/* By default blocks are 512K bytes in size. */
#define BT_DEFAULT_BLKSZ (512*1024)

/*
** This structure is the in-memory representation of all data stored in
** the database header at the start of the db file.
**
** pgsz, blksz:
**   Byte offset 0 of the database file is the first byte of both page 1
**   and block 1. Each page is pHdr->pgsz bytes in size. Each block is
**   pHdr->blksz bytes in size. It is guaranteed that the block-size is
**   an integer multiple of the page-size.
**
................................................................................
**   sub-tree, not including any overflow pages. Pages (except overflow 
**   pages) are always allocated contiguously within the block.
**
**   The reason these are likely a stop-gap is that the write-magnification
**   caused by using a b-tree for to populate level-0 sub-trees is too 
**   expensive.
*/
#define BT_DBHDR_STRING "SQLite4 bt database 0001"
struct BtDbHdr {
  char azStr[24];                 /* Copy of BT_DBHDR_STRING */
  u32 pgsz;                       /* Page size in bytes */
  u32 blksz;                      /* Block size in bytes */
  u32 nPg;                        /* Size of database file in pages */

  u32 iRoot;                      /* B-tree root page */
  u32 iMRoot;                     /* Root page of meta-tree */
  u32 iSRoot;                     /* Root page of schedule-tree */

Changes to src/bt_log.c.

872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896












897
898
899
900
901
902
903
904
905
906
907
908
909
910
...
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
....
1008
1009
1010
1011
1012
1013
1014

1015
1016
1017
1018
1019

1020

1021
1022
1023
1024
1025
1026
1027















1028
1029
1030
1031
1032
1033
1034
....
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
....
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
....
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
....
2022
2023
2024
2025
2026
2027
2028
2029
2030
    }
  }

  aLog[5] = iLast;
  return btLogHashRollback(pLog, btLogFrameHash(pLog, iLast), iLast);
}

static void btLogDecodeDbhdr(BtLog *pLog, u8 *aData, BtDbHdr *pHdr){
  BtDbHdrCksum hdr;
  u32 aCksum[2] = {0,0};

  if( aData ){
    memcpy(&hdr, aData, sizeof(BtDbHdrCksum));
    btLogChecksum32(1, (u8*)&hdr, offsetof(BtDbHdrCksum, aCksum), 0, aCksum);
  }

  if( aData==0 || aCksum[0]!=hdr.aCksum[0] || aCksum[1]!=hdr.aCksum[1] ){
    memset(&hdr, 0, sizeof(BtDbHdrCksum));
    hdr.hdr.pgsz = pLog->pLock->nPgsz;
    hdr.hdr.blksz = pLog->pLock->nBlksz;
    hdr.hdr.nPg = 2;
    hdr.hdr.iRoot = 2;
  }

  memcpy(pHdr, &hdr, sizeof(BtDbHdr));












}

static int btLogReadDbhdr(BtLog *pLog, BtDbHdr *pHdr, u32 iFrame){
  BtLock *p = pLog->pLock;        /* BtLock handle */
  int rc;                         /* Return code */
  i64 nByte;                      /* Size of database file in byte */
  u8 *aBuffer[sizeof(BtDbHdrCksum)];
  u8 *aData = 0;

  if( iFrame==0 ){
    rc = p->pVfs->xSize(p->pFd, &nByte);
    if( rc==SQLITE4_OK && nByte>0 ){
      rc = p->pVfs->xRead(p->pFd, 0, aBuffer, sizeof(BtDbHdrCksum));
      aData = aBuffer;
................................................................................
    i64 iOff = btLogFrameOffset(pLog, pLog->snapshot.dbhdr.pgsz, iFrame);
    iOff += sizeof(BtFrameHdr);
    rc = p->pVfs->xRead(pLog->pFd, iOff, aBuffer, sizeof(BtDbHdrCksum));
    aData = aBuffer;
  }

  if( rc==SQLITE4_OK ){
    btLogDecodeDbhdr(pLog, aData, pHdr);
  }
  return rc;
}

static int btLogUpdateDbhdr(BtLog *pLog, u8 *aData){
  BtDbHdrCksum hdr;

................................................................................
      pShm->ckpt.iFirstRead = pHdr->iFirstFrame;
      pShm->ckpt.iFirstRecover = pHdr->iFirstFrame;
      rc = btLogRollbackRecovery(pLog, &ctx);
      pLog->snapshot.iNextFrame = ctx.iNextFrame;
      pLog->snapshot.dbhdr.pgsz = pHdr->nPgsz;
      assert( pShm->ckpt.iFirstRead>0 );
    }


    /* Based on the wal-header, the page-size and number of pages in the
    ** database are now known and stored in snapshot.dbhdr. But the other
    ** header field values (iCookie, iRoot etc.) are still unknown. Read
    ** them from page 1 of the database file now.  */

    rc = btLogReadDbhdr(pLog, &pLog->snapshot.dbhdr, ctx.iPageOneFrame);


  }else if( rc==SQLITE4_OK ){
    /* There is no data in the log file. Read the database header directly
    ** from offset 0 of the database file.  */
    btLogZeroSnapshot(pLog);
    rc = btLogReadDbhdr(pLog, &pLog->snapshot.dbhdr, 0);
  }
















  if( rc==SQLITE4_OK ){
    btDebugTopology(
        pLog->pLock, "recovered", pLog->snapshot.iHashSide, pLog->snapshot.aLog
    );

    btDebugDbhdr(pLog->pLock, "read", &pLog->snapshot.dbhdr);
................................................................................
    if( rc==SQLITE4_OK ){
      rc = btLogUpdateSharedHdr(pLog);
    }
  }

 open_out:
  if( rc!=SQLITE4_OK ){
    sqlite4_free(pEnv, pLog);
    pLog = 0;
  }
  *ppLog = pLog;
  return rc;
}

/*
................................................................................
*/
int sqlite4BtLogClose(BtLog *pLog, int bCleanup){
  int rc = SQLITE4_OK;
  if( pLog ){
    sqlite4_env *pEnv = pLog->pLock->pEnv;
    bt_env *pVfs = pLog->pLock->pVfs;

    pVfs->xClose(pLog->pFd);
    if( bCleanup ){
      BtPager *pPager = (BtPager*)pLog->pLock;
      const char *zWal = sqlite4BtPagerFilename(pPager, BT_PAGERFILE_LOG);
      rc = pVfs->xUnlink(pEnv, pVfs, zWal);
    }

    sqlite4_free(pEnv, pLog->apShm);
................................................................................
}

static int btLogMerge(BtLog *pLog, u8 *aBuf){
  bt_db *db = (bt_db*)sqlite4BtPagerExtra((BtPager*)pLog->pLock);
  return sqlite4BtMerge(db, &pLog->snapshot.dbhdr, aBuf);
}


int sqlite4BtLogCheckpoint(BtLog *pLog, int nFrameBuffer){
  BtLock *pLock = pLog->pLock;
  int rc;

  /* Take the CHECKPOINTER lock. */
  rc = sqlite4BtLockCkpt(pLock);
  if( rc==SQLITE4_OK ){
................................................................................
    pLog->snapshot.dbhdr.iCookie = iCookie;
    btLogUpdateDbhdr(pLog, sqlite4BtPageData(pOne));
  }
  sqlite4BtPageRelease(pOne);

  return rc;
}









|









|
<
<
<
<



>
>
>
>
>
>
>
>
>
>
>
>






|







 







|







 







>





>
|
>







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







 







|







 







<







 








<
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889




890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
...
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
....
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
....
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
....
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
....
1849
1850
1851
1852
1853
1854
1855

1856
1857
1858
1859
1860
1861
1862
....
2047
2048
2049
2050
2051
2052
2053
2054

    }
  }

  aLog[5] = iLast;
  return btLogHashRollback(pLog, btLogFrameHash(pLog, iLast), iLast);
}

static int btLogDecodeDbhdr(BtLog *pLog, u8 *aData, BtDbHdr *pHdr){
  BtDbHdrCksum hdr;
  u32 aCksum[2] = {0,0};

  if( aData ){
    memcpy(&hdr, aData, sizeof(BtDbHdrCksum));
    btLogChecksum32(1, (u8*)&hdr, offsetof(BtDbHdrCksum, aCksum), 0, aCksum);
  }

  if( aData==0 || aCksum[0]!=hdr.aCksum[0] || aCksum[1]!=hdr.aCksum[1] ){
    return SQLITE4_NOTFOUND;




  }

  memcpy(pHdr, &hdr, sizeof(BtDbHdr));
  return SQLITE4_OK;
}

static void btLogZeroDbhdr(BtLog *pLog, BtDbHdr *pHdr){
  assert( sizeof(pHdr->azStr)==strlen(BT_DBHDR_STRING) );

  memset(pHdr, 0, sizeof(BtDbHdr));
  memcpy(pHdr->azStr, BT_DBHDR_STRING, strlen(BT_DBHDR_STRING));
  pHdr->pgsz = pLog->pLock->nPgsz;
  pHdr->blksz = pLog->pLock->nBlksz;
  pHdr->nPg = 2;
  pHdr->iRoot = 2;
}

static int btLogReadDbhdr(BtLog *pLog, BtDbHdr *pHdr, u32 iFrame){
  BtLock *p = pLog->pLock;        /* BtLock handle */
  int rc;                         /* Return code */
  i64 nByte;                      /* Size of database file in byte */
  u8 aBuffer[sizeof(BtDbHdrCksum)];
  u8 *aData = 0;

  if( iFrame==0 ){
    rc = p->pVfs->xSize(p->pFd, &nByte);
    if( rc==SQLITE4_OK && nByte>0 ){
      rc = p->pVfs->xRead(p->pFd, 0, aBuffer, sizeof(BtDbHdrCksum));
      aData = aBuffer;
................................................................................
    i64 iOff = btLogFrameOffset(pLog, pLog->snapshot.dbhdr.pgsz, iFrame);
    iOff += sizeof(BtFrameHdr);
    rc = p->pVfs->xRead(pLog->pFd, iOff, aBuffer, sizeof(BtDbHdrCksum));
    aData = aBuffer;
  }

  if( rc==SQLITE4_OK ){
    rc = btLogDecodeDbhdr(pLog, aData, pHdr);
  }
  return rc;
}

static int btLogUpdateDbhdr(BtLog *pLog, u8 *aData){
  BtDbHdrCksum hdr;

................................................................................
      pShm->ckpt.iFirstRead = pHdr->iFirstFrame;
      pShm->ckpt.iFirstRecover = pHdr->iFirstFrame;
      rc = btLogRollbackRecovery(pLog, &ctx);
      pLog->snapshot.iNextFrame = ctx.iNextFrame;
      pLog->snapshot.dbhdr.pgsz = pHdr->nPgsz;
      assert( pShm->ckpt.iFirstRead>0 );
    }
    assert( rc!=SQLITE4_NOTFOUND );

    /* Based on the wal-header, the page-size and number of pages in the
    ** database are now known and stored in snapshot.dbhdr. But the other
    ** header field values (iCookie, iRoot etc.) are still unknown. Read
    ** them from page 1 of the database file now.  */
    if( rc==SQLITE4_OK ){
      rc = btLogReadDbhdr(pLog, &pLog->snapshot.dbhdr, ctx.iPageOneFrame);
    }

  }else if( rc==SQLITE4_OK ){
    /* There is no data in the log file. Read the database header directly
    ** from offset 0 of the database file.  */
    btLogZeroSnapshot(pLog);
    rc = btLogReadDbhdr(pLog, &pLog->snapshot.dbhdr, 0);
  }

  if( rc==SQLITE4_NOTFOUND ){
    /* Check the size of the db file. If it is greater than zero bytes in
    ** size, refuse to open the file (as it is probably not a database
    ** file). Or, if it is exactly zero bytes in size, this is a brand
    ** new database.  */
    rc = pVfs->xSize(pLog->pLock->pFd, &nByte);
    if( rc==SQLITE4_OK ){
      if( nByte==0 ){
        btLogZeroDbhdr(pLog, &pLog->snapshot.dbhdr);
      }else{
        rc = btErrorBkpt(SQLITE4_NOTADB);
      }
    }
  }

  if( rc==SQLITE4_OK ){
    btDebugTopology(
        pLog->pLock, "recovered", pLog->snapshot.iHashSide, pLog->snapshot.aLog
    );

    btDebugDbhdr(pLog->pLock, "read", &pLog->snapshot.dbhdr);
................................................................................
    if( rc==SQLITE4_OK ){
      rc = btLogUpdateSharedHdr(pLog);
    }
  }

 open_out:
  if( rc!=SQLITE4_OK ){
    sqlite4BtLogClose(pLog, 0);
    pLog = 0;
  }
  *ppLog = pLog;
  return rc;
}

/*
................................................................................
*/
int sqlite4BtLogClose(BtLog *pLog, int bCleanup){
  int rc = SQLITE4_OK;
  if( pLog ){
    sqlite4_env *pEnv = pLog->pLock->pEnv;
    bt_env *pVfs = pLog->pLock->pVfs;

    if( pLog->pFd ) pVfs->xClose(pLog->pFd);
    if( bCleanup ){
      BtPager *pPager = (BtPager*)pLog->pLock;
      const char *zWal = sqlite4BtPagerFilename(pPager, BT_PAGERFILE_LOG);
      rc = pVfs->xUnlink(pEnv, pVfs, zWal);
    }

    sqlite4_free(pEnv, pLog->apShm);
................................................................................
}

static int btLogMerge(BtLog *pLog, u8 *aBuf){
  bt_db *db = (bt_db*)sqlite4BtPagerExtra((BtPager*)pLog->pLock);
  return sqlite4BtMerge(db, &pLog->snapshot.dbhdr, aBuf);
}


int sqlite4BtLogCheckpoint(BtLog *pLog, int nFrameBuffer){
  BtLock *pLock = pLog->pLock;
  int rc;

  /* Take the CHECKPOINTER lock. */
  rc = sqlite4BtLockCkpt(pLock);
  if( rc==SQLITE4_OK ){
................................................................................
    pLog->snapshot.dbhdr.iCookie = iCookie;
    btLogUpdateDbhdr(pLog, sqlite4BtPageData(pOne));
  }
  sqlite4BtPageRelease(pOne);

  return rc;
}


Changes to src/bt_pager.c.

251
252
253
254
255
256
257

258
259
260
261
262
263
264
    }
  }
  btHashClear(p);
}

static int btCheckpoint(BtLock *pLock){
  BtPager *p = (BtPager*)pLock;

  return sqlite4BtLogCheckpoint(p->pLog, 0);
}

static int btCleanup(BtLock *pLock){
  BtPager *p = (BtPager*)pLock;
  int rc = sqlite4BtLogClose(p->pLog, 1);
  p->pLog = 0;







>







251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
    }
  }
  btHashClear(p);
}

static int btCheckpoint(BtLock *pLock){
  BtPager *p = (BtPager*)pLock;
  if( p->pLog==0 ) return SQLITE4_BUSY;
  return sqlite4BtLogCheckpoint(p->pLog, 0);
}

static int btCleanup(BtLock *pLock){
  BtPager *p = (BtPager*)pLock;
  int rc = sqlite4BtLogClose(p->pLog, 1);
  p->pLog = 0;

Changes to test/attach.test.

780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
    }
  } {1 {no such table: AAAAAA}}
}

# Create a malformed file (a file that is not a valid database)
# and try to attach it
#
# UPDATE: This does not fail with LSM. 
#
do_test attach-8.1 {
  set fd [open test2.db w]
  puts $fd "This file is not a valid SQLite database"
  close $fd
  catchsql {
    ATTACH 'test2.db' AS t2;
  }
} {0 {}}
do_test attach-8.2 {
  db errorcode
} {0}
forcedelete test2.db
do_test attach-8.3 {
  sqlite4 db2 test2.db
  db2 eval {CREATE TABLE t1(x); BEGIN EXCLUSIVE}
  catchsql {
    ATTACH 'test2.db' AS t3;
  }







<
<







|


|







780
781
782
783
784
785
786


787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
    }
  } {1 {no such table: AAAAAA}}
}

# Create a malformed file (a file that is not a valid database)
# and try to attach it
#


do_test attach-8.1 {
  set fd [open test2.db w]
  puts $fd "This file is not a valid SQLite database"
  close $fd
  catchsql {
    ATTACH 'test2.db' AS t2;
  }
} {1 {unable to open database: test2.db}}
do_test attach-8.2 {
  db errorcode
} {26}
forcedelete test2.db
do_test attach-8.3 {
  sqlite4 db2 test2.db
  db2 eval {CREATE TABLE t1(x); BEGIN EXCLUSIVE}
  catchsql {
    ATTACH 'test2.db' AS t3;
  }

Added test/bt1.test.















































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# 2012 November 02
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix bt1

# Set tcl variable $str to the hex representation of the 24 byte string 
# located at the start of every bt database file.
set str ""
binary scan "SQLite4 bt database 0001" c24 I
foreach i $I {
  append str [format %02X $i ]
}

#-------------------------------------------------------------------------
# Test that, assuming there is no *-wal file, the bt module will refuse
# to open an existing database that does not contain a valid header.
#
do_test 1.0 {
  execsql {
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES('abcd');
  }

  db close
  list [file exists test.db] [file exists test.db-wal]
} {1 0}

do_test 1.1 { 
  hexio_read test.db 0 24 
} $str

do_test 1.2 { 
  sqlite4 db test.db
  execsql { SELECT * FROM t1 }
} {abcd}

do_test 1.3 { 
  db close
  hexio_write test.db  8 55555555

  list [catch {sqlite4 db test.db} msg] $msg
} {1 {file is encrypted or is not a database}}

#-------------------------------------------------------------------------
# Test that, if there is a *-wal file that contains a valid copy of page
# 1 (with the db header), it is possible to open the database even if
# the header at byte offset 0 is damaged.
#
reset_db
do_test 2.0 {
  execsql {
    CREATE TABLE t1(x);
    INSERT INTO t1 VALUES(randomblob(20000));
  }
  db close
  list [file exists test.db] [file exists test.db-wal]
} {1 0}

do_test 2.1 {
  sqlite4 db test.db
  # This moves pages to the free-list. Meaning page 1 must be 
  # updated to set the pointer to the first free-list page.
  execsql { UPDATE t1 SET x = 'abcd' }
  db_save
  db close
  db_restore
  list [file exists test.db] [file exists test.db-wal]
} {1 1}

do_test 2.2 { 
  hexio_write test.db  8 55555555
  sqlite4 db test.db
  execsql { SELECT * FROM t1 }
} {abcd}

finish_test

Changes to test/permutations.test.

130
131
132
133
134
135
136


137
138
139
140
141
142
143
#   quick
#   full
#
lappend ::testsuitelist xxx

test_suite "bt" -prefix "bt-" -description {
} -files {


aggerror.test
alter.test
alter3.test
alter4.test
analyze.test
analyze3.test
analyze4.test







>
>







130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#   quick
#   full
#
lappend ::testsuitelist xxx

test_suite "bt" -prefix "bt-" -description {
} -files {
bt1.test

aggerror.test
alter.test
alter3.test
alter4.test
analyze.test
analyze3.test
analyze4.test