SQLite

Check-in [60105c2253]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge the latest changes from trunk.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | lsm-vtab
Files: files | file ages | folders
SHA3-256: 60105c2253d0b617037e975b0d0b729bcb59b950aa2c33d8891394ad45aabb69
User & Date: drh 2017-06-15 15:06:26.353
Context
2017-06-26
06:53
Fix some compiler warnings in lsm_file.c. (check-in: 23f16382ed user: dan tags: lsm-vtab)
2017-06-15
15:06
Merge the latest changes from trunk. (check-in: 60105c2253 user: drh tags: lsm-vtab)
13:07
Fix harmless compiler warnings in the shell.c file (check-in: 9afd7a2ffd user: drh tags: trunk)
2017-06-07
19:46
Fix another lsmtest build problem. Add the "-trans BOOLEAN" option to "lsmtest speed2". (check-in: 61853bc171 user: dan tags: lsm-vtab)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/rbu/rbuvacuum.test.
294
295
296
297
298
299
300








301
302
303
304
305
306
307
  sqlite3rbu_vacuum rbu test.db state.db
  rbu step
} {SQLITE_ERROR}
do_test 2.1.2 {
  list [catch { rbu close } msg] $msg
} {1 {SQLITE_ERROR - cannot vacuum wal mode database}}









reset_db
do_execsql_test 2.2.0 {
  CREATE TABLE tx(a PRIMARY KEY, b BLOB);
  INSERT INTO tx VALUES(1, randomblob(900));
  INSERT INTO tx SELECT a+1, randomblob(900) FROM tx;
  INSERT INTO tx SELECT a+2, randomblob(900) FROM tx;
  INSERT INTO tx SELECT a+4, randomblob(900) FROM tx;







>
>
>
>
>
>
>
>







294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
  sqlite3rbu_vacuum rbu test.db state.db
  rbu step
} {SQLITE_ERROR}
do_test 2.1.2 {
  list [catch { rbu close } msg] $msg
} {1 {SQLITE_ERROR - cannot vacuum wal mode database}}

do_test 2.1.3 {
  sqlite3rbu_vacuum rbu test.db state.db
  rbu step
} {SQLITE_ERROR}
do_test 2.1.4 {
  list [catch { rbu close_no_error } msg] $msg
} {1 SQLITE_ERROR}

reset_db
do_execsql_test 2.2.0 {
  CREATE TABLE tx(a PRIMARY KEY, b BLOB);
  INSERT INTO tx VALUES(1, randomblob(900));
  INSERT INTO tx SELECT a+1, randomblob(900) FROM tx;
  INSERT INTO tx SELECT a+2, randomblob(900) FROM tx;
  INSERT INTO tx SELECT a+4, randomblob(900) FROM tx;
Changes to ext/rbu/sqlite3rbu.c.
3783
3784
3785
3786
3787
3788
3789

3790



3791
3792
3793
3794
3795
3796
3797
    sqlite3_close(p->dbMain);
    rbuDeleteVfs(p);
    sqlite3_free(p->aBuf);
    sqlite3_free(p->aFrame);

    rbuEditErrmsg(p);
    rc = p->rc;

    *pzErrmsg = p->zErrmsg;



    sqlite3_free(p->zState);
    sqlite3_free(p);
  }else{
    rc = SQLITE_NOMEM;
    *pzErrmsg = 0;
  }
  return rc;







>
|
>
>
>







3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
    sqlite3_close(p->dbMain);
    rbuDeleteVfs(p);
    sqlite3_free(p->aBuf);
    sqlite3_free(p->aFrame);

    rbuEditErrmsg(p);
    rc = p->rc;
    if( pzErrmsg ){
      *pzErrmsg = p->zErrmsg;
    }else{
      sqlite3_free(p->zErrmsg);
    }
    sqlite3_free(p->zState);
    sqlite3_free(p);
  }else{
    rc = SQLITE_NOMEM;
    *pzErrmsg = 0;
  }
  return rc;
Changes to ext/rbu/sqlite3rbu.h.
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
**
** If the RBU update has been completely applied, mark the RBU database
** as fully applied. Otherwise, assuming no error has occurred, save the
** current state of the RBU update appliation to the RBU database.
**
** If an error has already occurred as part of an sqlite3rbu_step()
** or sqlite3rbu_open() call, or if one occurs within this function, an
** SQLite error code is returned. Additionally, *pzErrmsg may be set to
** point to a buffer containing a utf-8 formatted English language error
** message. It is the responsibility of the caller to eventually free any 
** such buffer using sqlite3_free().
**
** Otherwise, if no error occurs, this function returns SQLITE_OK if the
** update has been partially applied, or SQLITE_DONE if it has been 
** completely applied.
*/
int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg);








|
|
|
|







416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
**
** If the RBU update has been completely applied, mark the RBU database
** as fully applied. Otherwise, assuming no error has occurred, save the
** current state of the RBU update appliation to the RBU database.
**
** If an error has already occurred as part of an sqlite3rbu_step()
** or sqlite3rbu_open() call, or if one occurs within this function, an
** SQLite error code is returned. Additionally, if pzErrmsg is not NULL,
** *pzErrmsg may be set to point to a buffer containing a utf-8 formatted
** English language error message. It is the responsibility of the caller to
** eventually free any such buffer using sqlite3_free().
**
** Otherwise, if no error occurs, this function returns SQLITE_OK if the
** update has been partially applied, or SQLITE_DONE if it has been 
** completely applied.
*/
int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg);

Changes to ext/rbu/test_rbu.c.
74
75
76
77
78
79
80

81
82
83
84
85
86
87
    {"create_rbu_delta", 2, ""},  /* 2 */
    {"savestate", 2, ""},         /* 3 */
    {"dbMain_eval", 3, "SQL"},    /* 4 */
    {"bp_progress", 2, ""},       /* 5 */
    {"db", 3, "RBU"},             /* 6 */
    {"state", 2, ""},             /* 7 */
    {"progress", 2, ""},          /* 8 */

    {0,0,0}
  };
  int iCmd;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
    return TCL_ERROR;







>







74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
    {"create_rbu_delta", 2, ""},  /* 2 */
    {"savestate", 2, ""},         /* 3 */
    {"dbMain_eval", 3, "SQL"},    /* 4 */
    {"bp_progress", 2, ""},       /* 5 */
    {"db", 3, "RBU"},             /* 6 */
    {"state", 2, ""},             /* 7 */
    {"progress", 2, ""},          /* 8 */
    {"close_no_error", 2, ""},    /* 9 */
    {0,0,0}
  };
  int iCmd;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "METHOD");
    return TCL_ERROR;
98
99
100
101
102
103
104

105
106
107
108

109



110
111
112
113
114
115
116
  switch( iCmd ){
    case 0: /* step */ {
      int rc = sqlite3rbu_step(pRbu);
      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
      break;
    }


    case 1: /* close */ {
      char *zErrmsg = 0;
      int rc;
      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));

      rc = sqlite3rbu_close(pRbu, &zErrmsg);



      if( rc==SQLITE_OK || rc==SQLITE_DONE ){
        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
        assert( zErrmsg==0 );
      }else{
        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
        if( zErrmsg ){
          Tcl_AppendResult(interp, " - ", zErrmsg, 0);







>




>
|
>
>
>







99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
  switch( iCmd ){
    case 0: /* step */ {
      int rc = sqlite3rbu_step(pRbu);
      Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
      break;
    }

    case 9: /* close_no_error */ 
    case 1: /* close */ {
      char *zErrmsg = 0;
      int rc;
      Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
      if( iCmd==1 ){
        rc = sqlite3rbu_close(pRbu, &zErrmsg);
      }else{
        rc = sqlite3rbu_close(pRbu, 0);
      }
      if( rc==SQLITE_OK || rc==SQLITE_DONE ){
        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
        assert( zErrmsg==0 );
      }else{
        Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
        if( zErrmsg ){
          Tcl_AppendResult(interp, " - ", zErrmsg, 0);
Changes to ext/session/sessiondiff.test.
12
13
14
15
16
17
18

19
20
21
22
23
24
25
# Tests for the [sqldiff --changeset] command.
#
#
if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl

set testprefix sessiondiff

set PROG [test_find_sqldiff]
db close

proc sqlesc {id} {
  set ret "'[string map {' ''} $id]'"







>







12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# Tests for the [sqldiff --changeset] command.
#
#
if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
ifcapable !session {finish_test; return}
set testprefix sessiondiff

set PROG [test_find_sqldiff]
db close

proc sqlesc {id} {
  set ret "'[string map {' ''} $id]'"
107
108
109
110
111
112
113
114
} {
  CREATE TABLE t2(a, b, c, PRIMARY KEY(b, c));
  INSERT INTO t2 VALUES(1, 2, 11);
  INSERT INTO t2 VALUES(7, 8, 9);
}

finish_test








<
108
109
110
111
112
113
114

} {
  CREATE TABLE t2(a, b, c, PRIMARY KEY(b, c));
  INSERT INTO t2 VALUES(1, 2, 11);
  INSERT INTO t2 VALUES(7, 8, 9);
}

finish_test

Changes to src/btree.c.
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781

  if( pKey ){
    assert( nKey==(i64)(int)nKey );
    pIdxKey = sqlite3VdbeAllocUnpackedRecord(pCur->pKeyInfo);
    if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT;
    sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey);
    if( pIdxKey->nField==0 ){
      rc = SQLITE_CORRUPT_BKPT;
      goto moveto_done;
    }
  }else{
    pIdxKey = 0;
  }
  rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes);
moveto_done:







|







767
768
769
770
771
772
773
774
775
776
777
778
779
780
781

  if( pKey ){
    assert( nKey==(i64)(int)nKey );
    pIdxKey = sqlite3VdbeAllocUnpackedRecord(pCur->pKeyInfo);
    if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT;
    sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey);
    if( pIdxKey->nField==0 ){
      rc = SQLITE_CORRUPT_PGNO(pCur->apPage[pCur->iPage]->pgno);
      goto moveto_done;
    }
  }else{
    pIdxKey = 0;
  }
  rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes);
moveto_done:
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
  }
  assert( offset <= (int)pBt->usableSize-5 );
  assert( pEType!=0 );
  *pEType = pPtrmap[offset];
  if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]);

  sqlite3PagerUnref(pDbPage);
  if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_BKPT;
  return SQLITE_OK;
}

#else /* if defined SQLITE_OMIT_AUTOVACUUM */
  #define ptrmapPut(w,x,y,z,rc)
  #define ptrmapGet(w,x,y,z) SQLITE_OK
  #define ptrmapPutOvflPtr(x, y, rc)







|







996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
  }
  assert( offset <= (int)pBt->usableSize-5 );
  assert( pEType!=0 );
  *pEType = pPtrmap[offset];
  if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]);

  sqlite3PagerUnref(pDbPage);
  if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_PGNO(iPtrmap);
  return SQLITE_OK;
}

#else /* if defined SQLITE_OMIT_AUTOVACUUM */
  #define ptrmapPut(w,x,y,z,rc)
  #define ptrmapGet(w,x,y,z) SQLITE_OK
  #define ptrmapPutOvflPtr(x, y, rc)
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
      if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){
        u8 *pEnd = &data[cellOffset + nCell*2];
        u8 *pAddr;
        int sz2 = 0;
        int sz = get2byte(&data[iFree+2]);
        int top = get2byte(&data[hdr+5]);
        if( iFree2 ){
          if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_BKPT;
          sz2 = get2byte(&data[iFree2+2]);
          assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize );
          memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz));
          sz += sz2;
        }
        cbrk = top+sz;
        assert( cbrk+(iFree-top) <= usableSize );







|







1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
      if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){
        u8 *pEnd = &data[cellOffset + nCell*2];
        u8 *pAddr;
        int sz2 = 0;
        int sz = get2byte(&data[iFree+2]);
        int top = get2byte(&data[hdr+5]);
        if( iFree2 ){
          if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
          sz2 = get2byte(&data[iFree2+2]);
          assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize );
          memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz));
          sz += sz2;
        }
        cbrk = top+sz;
        assert( cbrk+(iFree-top) <= usableSize );
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
    pc = get2byte(pAddr);
    testcase( pc==iCellFirst );
    testcase( pc==iCellLast );
    /* These conditions have already been verified in btreeInitPage()
    ** if PRAGMA cell_size_check=ON.
    */
    if( pc<iCellFirst || pc>iCellLast ){
      return SQLITE_CORRUPT_BKPT;
    }
    assert( pc>=iCellFirst && pc<=iCellLast );
    size = pPage->xCellSize(pPage, &src[pc]);
    cbrk -= size;
    if( cbrk<iCellFirst || pc+size>usableSize ){
      return SQLITE_CORRUPT_BKPT;
    }
    assert( cbrk+size<=usableSize && cbrk>=iCellFirst );
    testcase( cbrk+size==usableSize );
    testcase( pc+size==usableSize );
    put2byte(pAddr, cbrk);
    if( temp==0 ){
      int x;
      if( cbrk==pc ) continue;
      temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
      x = get2byte(&data[hdr+5]);
      memcpy(&temp[x], &data[x], (cbrk+size) - x);
      src = temp;
    }
    memcpy(&data[cbrk], &src[pc], size);
  }
  data[hdr+7] = 0;

 defragment_out:
  if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){
    return SQLITE_CORRUPT_BKPT;
  }
  assert( cbrk>=iCellFirst );
  put2byte(&data[hdr+5], cbrk);
  data[hdr+1] = 0;
  data[hdr+2] = 0;
  memset(&data[iCellFirst], 0, cbrk-iCellFirst);
  assert( sqlite3PagerIswriteable(pPage->pDbPage) );







|





|



















|







1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
    pc = get2byte(pAddr);
    testcase( pc==iCellFirst );
    testcase( pc==iCellLast );
    /* These conditions have already been verified in btreeInitPage()
    ** if PRAGMA cell_size_check=ON.
    */
    if( pc<iCellFirst || pc>iCellLast ){
      return SQLITE_CORRUPT_PGNO(pPage->pgno);
    }
    assert( pc>=iCellFirst && pc<=iCellLast );
    size = pPage->xCellSize(pPage, &src[pc]);
    cbrk -= size;
    if( cbrk<iCellFirst || pc+size>usableSize ){
      return SQLITE_CORRUPT_PGNO(pPage->pgno);
    }
    assert( cbrk+size<=usableSize && cbrk>=iCellFirst );
    testcase( cbrk+size==usableSize );
    testcase( pc+size==usableSize );
    put2byte(pAddr, cbrk);
    if( temp==0 ){
      int x;
      if( cbrk==pc ) continue;
      temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
      x = get2byte(&data[hdr+5]);
      memcpy(&temp[x], &data[x], (cbrk+size) - x);
      src = temp;
    }
    memcpy(&data[cbrk], &src[pc], size);
  }
  data[hdr+7] = 0;

 defragment_out:
  if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }
  assert( cbrk>=iCellFirst );
  put2byte(&data[hdr+5], cbrk);
  data[hdr+1] = 0;
  data[hdr+2] = 0;
  memset(&data[iCellFirst], 0, cbrk-iCellFirst);
  assert( sqlite3PagerIswriteable(pPage->pDbPage) );
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502

  assert( pc>0 );
  do{
    int size;            /* Size of the free slot */
    /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of
    ** increasing offset. */
    if( pc>usableSize-4 || pc<iAddr+4 ){
      *pRc = SQLITE_CORRUPT_BKPT;
      return 0;
    }
    /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each
    ** freeblock form a big-endian integer which is the size of the freeblock
    ** in bytes, including the 4-byte header. */
    size = get2byte(&aData[pc+2]);
    if( (x = size - nByte)>=0 ){
      testcase( x==4 );
      testcase( x==3 );
      if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){
        *pRc = SQLITE_CORRUPT_BKPT;
        return 0;
      }else if( x<4 ){
        /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total
        ** number of bytes in fragments may not exceed 60. */
        if( aData[hdr+7]>57 ) return 0;

        /* Remove the slot from the free-list. Update the number of







|










|







1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502

  assert( pc>0 );
  do{
    int size;            /* Size of the free slot */
    /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of
    ** increasing offset. */
    if( pc>usableSize-4 || pc<iAddr+4 ){
      *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno);
      return 0;
    }
    /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each
    ** freeblock form a big-endian integer which is the size of the freeblock
    ** in bytes, including the 4-byte header. */
    size = get2byte(&aData[pc+2]);
    if( (x = size - nByte)>=0 ){
      testcase( x==4 );
      testcase( x==3 );
      if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){
        *pRc = SQLITE_CORRUPT_PGNO(pPg->pgno);
        return 0;
      }else if( x<4 ){
        /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total
        ** number of bytes in fragments may not exceed 60. */
        if( aData[hdr+7]>57 ) return 0;

        /* Remove the slot from the free-list. Update the number of
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
  ** integer, so a value of 0 is used in its place. */
  top = get2byte(&data[hdr+5]);
  assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */
  if( gap>top ){
    if( top==0 && pPage->pBt->usableSize==65536 ){
      top = 65536;
    }else{
      return SQLITE_CORRUPT_BKPT;
    }
  }

  /* If there is enough space between gap and top for one more cell pointer
  ** array entry offset, and if the freelist is not empty, then search the
  ** freelist looking for a free slot big enough to satisfy the request.
  */







|







1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
  ** integer, so a value of 0 is used in its place. */
  top = get2byte(&data[hdr+5]);
  assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */
  if( gap>top ){
    if( top==0 && pPage->pBt->usableSize==65536 ){
      top = 65536;
    }else{
      return SQLITE_CORRUPT_PGNO(pPage->pgno);
    }
  }

  /* If there is enough space between gap and top for one more cell pointer
  ** array entry offset, and if the freelist is not empty, then search the
  ** freelist looking for a free slot big enough to satisfy the request.
  */
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675


1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
  iPtr = hdr + 1;
  if( data[iPtr+1]==0 && data[iPtr]==0 ){
    iFreeBlk = 0;  /* Shortcut for the case when the freelist is empty */
  }else{
    while( (iFreeBlk = get2byte(&data[iPtr]))<iStart ){
      if( iFreeBlk<iPtr+4 ){
        if( iFreeBlk==0 ) break;
        return SQLITE_CORRUPT_BKPT;
      }
      iPtr = iFreeBlk;
    }
    if( iFreeBlk>iLast ) return SQLITE_CORRUPT_BKPT;
    assert( iFreeBlk>iPtr || iFreeBlk==0 );
  
    /* At this point:
    **    iFreeBlk:   First freeblock after iStart, or zero if none
    **    iPtr:       The address of a pointer to iFreeBlk
    **
    ** Check to see if iFreeBlk should be coalesced onto the end of iStart.
    */
    if( iFreeBlk && iEnd+3>=iFreeBlk ){
      nFrag = iFreeBlk - iEnd;
      if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT;
      iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]);
      if( iEnd > pPage->pBt->usableSize ) return SQLITE_CORRUPT_BKPT;


      iSize = iEnd - iStart;
      iFreeBlk = get2byte(&data[iFreeBlk]);
    }
  
    /* If iPtr is another freeblock (that is, if iPtr is not the freelist
    ** pointer in the page header) then check to see if iStart should be
    ** coalesced onto the end of iPtr.
    */
    if( iPtr>hdr+1 ){
      int iPtrEnd = iPtr + get2byte(&data[iPtr+2]);
      if( iPtrEnd+3>=iStart ){
        if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT;
        nFrag += iStart - iPtrEnd;
        iSize = iEnd - iPtr;
        iStart = iPtr;
      }
    }
    if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT;
    data[hdr+7] -= nFrag;
  }
  if( iStart==get2byte(&data[hdr+5]) ){
    /* The new freeblock is at the beginning of the cell content area,
    ** so just extend the cell content area rather than create another
    ** freelist entry */
    if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_BKPT;
    put2byte(&data[hdr+1], iFreeBlk);
    put2byte(&data[hdr+5], iEnd);
  }else{
    /* Insert the new freeblock into the freelist */
    put2byte(&data[iPtr], iStart);
    put2byte(&data[iStart], iFreeBlk);
    put2byte(&data[iStart+2], iSize);







|



|










|

|
>
>











|





|






|







1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
  iPtr = hdr + 1;
  if( data[iPtr+1]==0 && data[iPtr]==0 ){
    iFreeBlk = 0;  /* Shortcut for the case when the freelist is empty */
  }else{
    while( (iFreeBlk = get2byte(&data[iPtr]))<iStart ){
      if( iFreeBlk<iPtr+4 ){
        if( iFreeBlk==0 ) break;
        return SQLITE_CORRUPT_PGNO(pPage->pgno);
      }
      iPtr = iFreeBlk;
    }
    if( iFreeBlk>iLast ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
    assert( iFreeBlk>iPtr || iFreeBlk==0 );
  
    /* At this point:
    **    iFreeBlk:   First freeblock after iStart, or zero if none
    **    iPtr:       The address of a pointer to iFreeBlk
    **
    ** Check to see if iFreeBlk should be coalesced onto the end of iStart.
    */
    if( iFreeBlk && iEnd+3>=iFreeBlk ){
      nFrag = iFreeBlk - iEnd;
      if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
      iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]);
      if( iEnd > pPage->pBt->usableSize ){
        return SQLITE_CORRUPT_PGNO(pPage->pgno);
      }
      iSize = iEnd - iStart;
      iFreeBlk = get2byte(&data[iFreeBlk]);
    }
  
    /* If iPtr is another freeblock (that is, if iPtr is not the freelist
    ** pointer in the page header) then check to see if iStart should be
    ** coalesced onto the end of iPtr.
    */
    if( iPtr>hdr+1 ){
      int iPtrEnd = iPtr + get2byte(&data[iPtr+2]);
      if( iPtrEnd+3>=iStart ){
        if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
        nFrag += iStart - iPtrEnd;
        iSize = iEnd - iPtr;
        iStart = iPtr;
      }
    }
    if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
    data[hdr+7] -= nFrag;
  }
  if( iStart==get2byte(&data[hdr+5]) ){
    /* The new freeblock is at the beginning of the cell content area,
    ** so just extend the cell content area rather than create another
    ** freelist entry */
    if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_PGNO(pPage->pgno);
    put2byte(&data[hdr+1], iFreeBlk);
    put2byte(&data[hdr+5], iEnd);
  }else{
    /* Insert the new freeblock into the freelist */
    put2byte(&data[iPtr], iStart);
    put2byte(&data[iStart], iFreeBlk);
    put2byte(&data[iStart+2], iSize);
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
    pPage->intKeyLeaf = 0;
    pPage->xParseCell = btreeParseCellPtrIndex;
    pPage->maxLocal = pBt->maxLocal;
    pPage->minLocal = pBt->minLocal;
  }else{
    /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is
    ** an error. */
    return SQLITE_CORRUPT_BKPT;
  }
  pPage->max1bytePayload = pBt->max1bytePayload;
  return SQLITE_OK;
}

/*
** Initialize the auxiliary information for a disk block.







|







1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
    pPage->intKeyLeaf = 0;
    pPage->xParseCell = btreeParseCellPtrIndex;
    pPage->maxLocal = pBt->maxLocal;
    pPage->minLocal = pBt->minLocal;
  }else{
    /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is
    ** an error. */
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }
  pPage->max1bytePayload = pBt->max1bytePayload;
  return SQLITE_OK;
}

/*
** Initialize the auxiliary information for a disk block.
1801
1802
1803
1804
1805
1806
1807
1808


1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
  assert( pPage->isInit==0 );

  pBt = pPage->pBt;
  hdr = pPage->hdrOffset;
  data = pPage->aData;
  /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating
  ** the b-tree page type. */
  if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT;


  assert( pBt->pageSize>=512 && pBt->pageSize<=65536 );
  pPage->maskPage = (u16)(pBt->pageSize - 1);
  pPage->nOverflow = 0;
  usableSize = pBt->usableSize;
  pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize;
  pPage->aDataEnd = &data[usableSize];
  pPage->aCellIdx = &data[cellOffset];
  pPage->aDataOfst = &data[pPage->childPtrSize];
  /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates
  ** the start of the cell content area. A zero value for this integer is
  ** interpreted as 65536. */
  top = get2byteNotZero(&data[hdr+5]);
  /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
  ** number of cells on the page. */
  pPage->nCell = get2byte(&data[hdr+3]);
  if( pPage->nCell>MX_CELL(pBt) ){
    /* To many cells for a single page.  The page must be corrupt */
    return SQLITE_CORRUPT_BKPT;
  }
  testcase( pPage->nCell==MX_CELL(pBt) );
  /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only
  ** possible for a root page of a table that contains no rows) then the
  ** offset to the cell content area will equal the page size minus the
  ** bytes of reserved space. */
  assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB );







|
>
>

















|







1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
  assert( pPage->isInit==0 );

  pBt = pPage->pBt;
  hdr = pPage->hdrOffset;
  data = pPage->aData;
  /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating
  ** the b-tree page type. */
  if( decodeFlags(pPage, data[hdr]) ){
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }
  assert( pBt->pageSize>=512 && pBt->pageSize<=65536 );
  pPage->maskPage = (u16)(pBt->pageSize - 1);
  pPage->nOverflow = 0;
  usableSize = pBt->usableSize;
  pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize;
  pPage->aDataEnd = &data[usableSize];
  pPage->aCellIdx = &data[cellOffset];
  pPage->aDataOfst = &data[pPage->childPtrSize];
  /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates
  ** the start of the cell content area. A zero value for this integer is
  ** interpreted as 65536. */
  top = get2byteNotZero(&data[hdr+5]);
  /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
  ** number of cells on the page. */
  pPage->nCell = get2byte(&data[hdr+3]);
  if( pPage->nCell>MX_CELL(pBt) ){
    /* To many cells for a single page.  The page must be corrupt */
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }
  testcase( pPage->nCell==MX_CELL(pBt) );
  /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only
  ** possible for a root page of a table that contains no rows) then the
  ** offset to the cell content area will equal the page size minus the
  ** bytes of reserved space. */
  assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB );
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881

1882
1883
1884
1885
1886
1887
1888
1889
1890

1891
1892
1893

1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912

    if( !pPage->leaf ) iCellLast--;
    for(i=0; i<pPage->nCell; i++){
      pc = get2byteAligned(&data[cellOffset+i*2]);
      testcase( pc==iCellFirst );
      testcase( pc==iCellLast );
      if( pc<iCellFirst || pc>iCellLast ){
        return SQLITE_CORRUPT_BKPT;
      }
      sz = pPage->xCellSize(pPage, &data[pc]);
      testcase( pc+sz==usableSize );
      if( pc+sz>usableSize ){
        return SQLITE_CORRUPT_BKPT;
      }
    }
    if( !pPage->leaf ) iCellLast++;
  }  

  /* Compute the total free space on the page
  ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the
  ** start of the first freeblock on the page, or is zero if there are no
  ** freeblocks. */
  pc = get2byte(&data[hdr+1]);
  nFree = data[hdr+7] + top;  /* Init nFree to non-freeblock free space */
  if( pc>0 ){
    u32 next, size;
    if( pc<iCellFirst ){
      /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will
      ** always be at least one cell before the first freeblock.
      */
      return SQLITE_CORRUPT_BKPT; 
    }
    while( 1 ){
      if( pc>iCellLast ){
        return SQLITE_CORRUPT_BKPT; /* Freeblock off the end of the page */

      }
      next = get2byte(&data[pc]);
      size = get2byte(&data[pc+2]);
      nFree = nFree + size;
      if( next<=pc+size+3 ) break;
      pc = next;
    }
    if( next>0 ){
      return SQLITE_CORRUPT_BKPT;  /* Freeblock not in ascending order */

    }
    if( pc+size>(unsigned int)usableSize ){
      return SQLITE_CORRUPT_BKPT;  /* Last freeblock extends past page end */

    }
  }

  /* At this point, nFree contains the sum of the offset to the start
  ** of the cell-content area plus the number of free bytes within
  ** the cell-content area. If this is greater than the usable-size
  ** of the page, then the page must be corrupted. This check also
  ** serves to verify that the offset to the start of the cell-content
  ** area, according to the page header, lies within the page.
  */
  if( nFree>usableSize ){
    return SQLITE_CORRUPT_BKPT; 
  }
  pPage->nFree = (u16)(nFree - iCellFirst);
  pPage->isInit = 1;
  return SQLITE_OK;
}

/*







|




|

















|



|
>








|
>


|
>











|







1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919

    if( !pPage->leaf ) iCellLast--;
    for(i=0; i<pPage->nCell; i++){
      pc = get2byteAligned(&data[cellOffset+i*2]);
      testcase( pc==iCellFirst );
      testcase( pc==iCellLast );
      if( pc<iCellFirst || pc>iCellLast ){
        return SQLITE_CORRUPT_PGNO(pPage->pgno);
      }
      sz = pPage->xCellSize(pPage, &data[pc]);
      testcase( pc+sz==usableSize );
      if( pc+sz>usableSize ){
        return SQLITE_CORRUPT_PGNO(pPage->pgno);
      }
    }
    if( !pPage->leaf ) iCellLast++;
  }  

  /* Compute the total free space on the page
  ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the
  ** start of the first freeblock on the page, or is zero if there are no
  ** freeblocks. */
  pc = get2byte(&data[hdr+1]);
  nFree = data[hdr+7] + top;  /* Init nFree to non-freeblock free space */
  if( pc>0 ){
    u32 next, size;
    if( pc<iCellFirst ){
      /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will
      ** always be at least one cell before the first freeblock.
      */
      return SQLITE_CORRUPT_PGNO(pPage->pgno); 
    }
    while( 1 ){
      if( pc>iCellLast ){
        /* Freeblock off the end of the page */
        return SQLITE_CORRUPT_PGNO(pPage->pgno);
      }
      next = get2byte(&data[pc]);
      size = get2byte(&data[pc+2]);
      nFree = nFree + size;
      if( next<=pc+size+3 ) break;
      pc = next;
    }
    if( next>0 ){
      /* Freeblock not in ascending order */
      return SQLITE_CORRUPT_PGNO(pPage->pgno);
    }
    if( pc+size>(unsigned int)usableSize ){
      /* Last freeblock extends past page end */
      return SQLITE_CORRUPT_PGNO(pPage->pgno);
    }
  }

  /* At this point, nFree contains the sum of the offset to the start
  ** of the cell-content area plus the number of free bytes within
  ** the cell-content area. If this is greater than the usable-size
  ** of the page, then the page must be corrupted. This check also
  ** serves to verify that the offset to the start of the cell-content
  ** area, according to the page header, lies within the page.
  */
  if( nFree>usableSize ){
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }
  pPage->nFree = (u16)(nFree - iCellFirst);
  pPage->isInit = 1;
  return SQLITE_OK;
}

/*
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
  }
  assert( (*ppPage)->pgno==pgno );
  assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) );

  /* If obtaining a child page for a cursor, we must verify that the page is
  ** compatible with the root page. */
  if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){
    rc = SQLITE_CORRUPT_BKPT;
    releasePage(*ppPage);
    goto getAndInitPage_error;
  }
  return SQLITE_OK;

getAndInitPage_error:
  if( pCur ) pCur->iPage--;







|







2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
  }
  assert( (*ppPage)->pgno==pgno );
  assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) );

  /* If obtaining a child page for a cursor, we must verify that the page is
  ** compatible with the root page. */
  if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){
    rc = SQLITE_CORRUPT_PGNO(pgno);
    releasePage(*ppPage);
    goto getAndInitPage_error;
  }
  return SQLITE_OK;

getAndInitPage_error:
  if( pCur ) pCur->iPage--;
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
*/
static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){
  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  if( eType==PTRMAP_OVERFLOW2 ){
    /* The pointer is always the first 4 bytes of the page in this case.  */
    if( get4byte(pPage->aData)!=iFrom ){
      return SQLITE_CORRUPT_BKPT;
    }
    put4byte(pPage->aData, iTo);
  }else{
    int i;
    int nCell;
    int rc;

    rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage);
    if( rc ) return rc;
    nCell = pPage->nCell;

    for(i=0; i<nCell; i++){
      u8 *pCell = findCell(pPage, i);
      if( eType==PTRMAP_OVERFLOW1 ){
        CellInfo info;
        pPage->xParseCell(pPage, pCell, &info);
        if( info.nLocal<info.nPayload ){
          if( pCell+info.nSize > pPage->aData+pPage->pBt->usableSize ){
            return SQLITE_CORRUPT_BKPT;
          }
          if( iFrom==get4byte(pCell+info.nSize-4) ){
            put4byte(pCell+info.nSize-4, iTo);
            break;
          }
        }
      }else{
        if( get4byte(pCell)==iFrom ){
          put4byte(pCell, iTo);
          break;
        }
      }
    }
  
    if( i==nCell ){
      if( eType!=PTRMAP_BTREE || 
          get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){
        return SQLITE_CORRUPT_BKPT;
      }
      put4byte(&pPage->aData[pPage->hdrOffset+8], iTo);
    }
  }
  return SQLITE_OK;
}








|


















|

















|







3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
*/
static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){
  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  assert( sqlite3PagerIswriteable(pPage->pDbPage) );
  if( eType==PTRMAP_OVERFLOW2 ){
    /* The pointer is always the first 4 bytes of the page in this case.  */
    if( get4byte(pPage->aData)!=iFrom ){
      return SQLITE_CORRUPT_PGNO(pPage->pgno);
    }
    put4byte(pPage->aData, iTo);
  }else{
    int i;
    int nCell;
    int rc;

    rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage);
    if( rc ) return rc;
    nCell = pPage->nCell;

    for(i=0; i<nCell; i++){
      u8 *pCell = findCell(pPage, i);
      if( eType==PTRMAP_OVERFLOW1 ){
        CellInfo info;
        pPage->xParseCell(pPage, pCell, &info);
        if( info.nLocal<info.nPayload ){
          if( pCell+info.nSize > pPage->aData+pPage->pBt->usableSize ){
            return SQLITE_CORRUPT_PGNO(pPage->pgno);
          }
          if( iFrom==get4byte(pCell+info.nSize-4) ){
            put4byte(pCell+info.nSize-4, iTo);
            break;
          }
        }
      }else{
        if( get4byte(pCell)==iFrom ){
          put4byte(pCell, iTo);
          break;
        }
      }
    }
  
    if( i==nCell ){
      if( eType!=PTRMAP_BTREE || 
          get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){
        return SQLITE_CORRUPT_PGNO(pPage->pgno);
      }
      put4byte(&pPage->aData[pPage->hdrOffset+8], iTo);
    }
  }
  return SQLITE_OK;
}

4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
  assert( aPayload > pPage->aData );
  if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){
    /* Trying to read or write past the end of the data is an error.  The
    ** conditional above is really:
    **    &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize]
    ** but is recast into its current form to avoid integer overflow problems
    */
    return SQLITE_CORRUPT_BKPT;
  }

  /* Check if data must be read/written to/from the btree page itself. */
  if( offset<pCur->info.nLocal ){
    int a = amt;
    if( a+offset>pCur->info.nLocal ){
      a = pCur->info.nLocal - offset;







|







4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
  assert( aPayload > pPage->aData );
  if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){
    /* Trying to read or write past the end of the data is an error.  The
    ** conditional above is really:
    **    &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize]
    ** but is recast into its current form to avoid integer overflow problems
    */
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }

  /* Check if data must be read/written to/from the btree page itself. */
  if( offset<pCur->info.nLocal ){
    int a = amt;
    if( a+offset>pCur->info.nLocal ){
      a = pCur->info.nLocal - offset;
4688
4689
4690
4691
4692
4693
4694
4695

4696
4697
4698
4699
4700
4701
4702
      }
      if( rc ) break;
      iIdx++;
    }
  }

  if( rc==SQLITE_OK && amt>0 ){
    return SQLITE_CORRUPT_BKPT; /* Overflow chain ends prematurely */

  }
  return rc;
}

/*
** Read part of the payload for the row at which that cursor pCur is currently
** pointing.  "amt" bytes will be transferred into pBuf[].  The transfer







|
>







4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
      }
      if( rc ) break;
      iIdx++;
    }
  }

  if( rc==SQLITE_OK && amt>0 ){
    /* Overflow chain ends prematurely */
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }
  return rc;
}

/*
** Read part of the payload for the row at which that cursor pCur is currently
** pointing.  "amt" bytes will be transferred into pBuf[].  The transfer
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
  ** Earlier versions of SQLite assumed that this test could not fail
  ** if the root page was already loaded when this function was called (i.e.
  ** if pCur->iPage>=0). But this is not so if the database is corrupted 
  ** in such a way that page pRoot is linked into a second b-tree table 
  ** (or the freelist).  */
  assert( pRoot->intKey==1 || pRoot->intKey==0 );
  if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){
    return SQLITE_CORRUPT_BKPT;
  }

skip_init:  
  pCur->ix = 0;
  pCur->info.nSize = 0;
  pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl);








|







4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
  ** Earlier versions of SQLite assumed that this test could not fail
  ** if the root page was already loaded when this function was called (i.e.
  ** if pCur->iPage>=0). But this is not so if the database is corrupted 
  ** in such a way that page pRoot is linked into a second b-tree table 
  ** (or the freelist).  */
  assert( pRoot->intKey==1 || pRoot->intKey==0 );
  if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){
    return SQLITE_CORRUPT_PGNO(pCur->apPage[pCur->iPage]->pgno);
  }

skip_init:  
  pCur->ix = 0;
  pCur->info.nSize = 0;
  pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl);

5227
5228
5229
5230
5231
5232
5233
5234


5235
5236
5237
5238
5239
5240
5241
    pCur->ix = (u16)idx;
    if( xRecordCompare==0 ){
      for(;;){
        i64 nCellKey;
        pCell = findCellPastPtr(pPage, idx);
        if( pPage->intKeyLeaf ){
          while( 0x80 <= *(pCell++) ){
            if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT;


          }
        }
        getVarint(pCell, (u64*)&nCellKey);
        if( nCellKey<intKey ){
          lwr = idx+1;
          if( lwr>upr ){ c = -1; break; }
        }else if( nCellKey>intKey ){







|
>
>







5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
    pCur->ix = (u16)idx;
    if( xRecordCompare==0 ){
      for(;;){
        i64 nCellKey;
        pCell = findCellPastPtr(pPage, idx);
        if( pPage->intKeyLeaf ){
          while( 0x80 <= *(pCell++) ){
            if( pCell>=pPage->aDataEnd ){
              return SQLITE_CORRUPT_PGNO(pPage->pgno);
            }
          }
        }
        getVarint(pCell, (u64*)&nCellKey);
        if( nCellKey<intKey ){
          lwr = idx+1;
          if( lwr>upr ){ c = -1; break; }
        }else if( nCellKey>intKey ){
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
          pPage->xParseCell(pPage, pCellBody, &pCur->info);
          nCell = (int)pCur->info.nKey;
          testcase( nCell<0 );   /* True if key size is 2^32 or more */
          testcase( nCell==0 );  /* Invalid key size:  0x80 0x80 0x00 */
          testcase( nCell==1 );  /* Invalid key size:  0x80 0x80 0x01 */
          testcase( nCell==2 );  /* Minimum legal index key size */
          if( nCell<2 ){
            rc = SQLITE_CORRUPT_BKPT;
            goto moveto_finish;
          }
          pCellKey = sqlite3Malloc( nCell+18 );
          if( pCellKey==0 ){
            rc = SQLITE_NOMEM_BKPT;
            goto moveto_finish;
          }







|







5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
          pPage->xParseCell(pPage, pCellBody, &pCur->info);
          nCell = (int)pCur->info.nKey;
          testcase( nCell<0 );   /* True if key size is 2^32 or more */
          testcase( nCell==0 );  /* Invalid key size:  0x80 0x80 0x00 */
          testcase( nCell==1 );  /* Invalid key size:  0x80 0x80 0x01 */
          testcase( nCell==2 );  /* Minimum legal index key size */
          if( nCell<2 ){
            rc = SQLITE_CORRUPT_PGNO(pPage->pgno);
            goto moveto_finish;
          }
          pCellKey = sqlite3Malloc( nCell+18 );
          if( pCellKey==0 ){
            rc = SQLITE_NOMEM_BKPT;
            goto moveto_finish;
          }
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
        /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32
        ** stores the page number of the first page of the freelist, or zero if
        ** the freelist is empty. */
        iTrunk = get4byte(&pPage1->aData[32]);
      }
      testcase( iTrunk==mxPage );
      if( iTrunk>mxPage || nSearch++ > n ){
        rc = SQLITE_CORRUPT_BKPT;
      }else{
        rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0);
      }
      if( rc ){
        pTrunk = 0;
        goto end_allocate_page;
      }







|







5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
        /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32
        ** stores the page number of the first page of the freelist, or zero if
        ** the freelist is empty. */
        iTrunk = get4byte(&pPage1->aData[32]);
      }
      testcase( iTrunk==mxPage );
      if( iTrunk>mxPage || nSearch++ > n ){
        rc = SQLITE_CORRUPT_PGNO(pPrevTrunk ? pPrevTrunk->pgno : 1);
      }else{
        rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0);
      }
      if( rc ){
        pTrunk = 0;
        goto end_allocate_page;
      }
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
        *pPgno = iTrunk;
        memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4);
        *ppPage = pTrunk;
        pTrunk = 0;
        TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1));
      }else if( k>(u32)(pBt->usableSize/4 - 2) ){
        /* Value of k is out of range.  Database corruption */
        rc = SQLITE_CORRUPT_BKPT;
        goto end_allocate_page;
#ifndef SQLITE_OMIT_AUTOVACUUM
      }else if( searchList 
            && (nearby==iTrunk || (iTrunk<nearby && eMode==BTALLOC_LE)) 
      ){
        /* The list is being searched and this trunk page is the page
        ** to allocate, regardless of whether it has leaves.







|







5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
        *pPgno = iTrunk;
        memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4);
        *ppPage = pTrunk;
        pTrunk = 0;
        TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1));
      }else if( k>(u32)(pBt->usableSize/4 - 2) ){
        /* Value of k is out of range.  Database corruption */
        rc = SQLITE_CORRUPT_PGNO(iTrunk);
        goto end_allocate_page;
#ifndef SQLITE_OMIT_AUTOVACUUM
      }else if( searchList 
            && (nearby==iTrunk || (iTrunk<nearby && eMode==BTALLOC_LE)) 
      ){
        /* The list is being searched and this trunk page is the page
        ** to allocate, regardless of whether it has leaves.
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
          /* The trunk page is required by the caller but it contains 
          ** pointers to free-list leaves. The first leaf becomes a trunk
          ** page in this case.
          */
          MemPage *pNewTrunk;
          Pgno iNewTrunk = get4byte(&pTrunk->aData[8]);
          if( iNewTrunk>mxPage ){ 
            rc = SQLITE_CORRUPT_BKPT;
            goto end_allocate_page;
          }
          testcase( iNewTrunk==mxPage );
          rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0);
          if( rc!=SQLITE_OK ){
            goto end_allocate_page;
          }







|







5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
          /* The trunk page is required by the caller but it contains 
          ** pointers to free-list leaves. The first leaf becomes a trunk
          ** page in this case.
          */
          MemPage *pNewTrunk;
          Pgno iNewTrunk = get4byte(&pTrunk->aData[8]);
          if( iNewTrunk>mxPage ){ 
            rc = SQLITE_CORRUPT_PGNO(iTrunk);
            goto end_allocate_page;
          }
          testcase( iNewTrunk==mxPage );
          rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0);
          if( rc!=SQLITE_OK ){
            goto end_allocate_page;
          }
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
        }else{
          closest = 0;
        }

        iPage = get4byte(&aData[8+closest*4]);
        testcase( iPage==mxPage );
        if( iPage>mxPage ){
          rc = SQLITE_CORRUPT_BKPT;
          goto end_allocate_page;
        }
        testcase( iPage==mxPage );
        if( !searchList 
         || (iPage==nearby || (iPage<nearby && eMode==BTALLOC_LE)) 
        ){
          int noContent;







|







5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
        }else{
          closest = 0;
        }

        iPage = get4byte(&aData[8+closest*4]);
        testcase( iPage==mxPage );
        if( iPage>mxPage ){
          rc = SQLITE_CORRUPT_PGNO(iTrunk);
          goto end_allocate_page;
        }
        testcase( iPage==mxPage );
        if( !searchList 
         || (iPage==nearby || (iPage<nearby && eMode==BTALLOC_LE)) 
        ){
          int noContent;
6100
6101
6102
6103
6104
6105
6106
6107

6108
6109
6110
6111
6112
6113
6114

  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  pPage->xParseCell(pPage, pCell, pInfo);
  if( pInfo->nLocal==pInfo->nPayload ){
    return SQLITE_OK;  /* No overflow pages. Return without doing anything */
  }
  if( pCell+pInfo->nSize-1 > pPage->aData+pPage->maskPage ){
    return SQLITE_CORRUPT_BKPT;  /* Cell extends past end of page */

  }
  ovflPgno = get4byte(pCell + pInfo->nSize - 4);
  assert( pBt->usableSize > 4 );
  ovflPageSize = pBt->usableSize - 4;
  nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize;
  assert( nOvfl>0 || 
    (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)<ovflPageSize)







|
>







6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125

  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
  pPage->xParseCell(pPage, pCell, pInfo);
  if( pInfo->nLocal==pInfo->nPayload ){
    return SQLITE_OK;  /* No overflow pages. Return without doing anything */
  }
  if( pCell+pInfo->nSize-1 > pPage->aData+pPage->maskPage ){
    /* Cell extends past end of page */
    return SQLITE_CORRUPT_PGNO(pPage->pgno);
  }
  ovflPgno = get4byte(pCell + pInfo->nSize - 4);
  assert( pBt->usableSize > 4 );
  ovflPageSize = pBt->usableSize - 4;
  nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize;
  assert( nOvfl>0 || 
    (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)<ovflPageSize)
8158
8159
8160
8161
8162
8163
8164
8165


8166
8167
8168
8169
8170




8171
8172
8173
8174
8175
8176
8177
      goto end_insert;
    }
    oldCell = findCell(pPage, idx);
    if( !pPage->leaf ){
      memcpy(newCell, oldCell, 4);
    }
    rc = clearCell(pPage, oldCell, &info);
    if( info.nSize==szNew && info.nLocal==info.nPayload ){


      /* Overwrite the old cell with the new if they are the same size.
      ** We could also try to do this if the old cell is smaller, then add
      ** the leftover space to the free list.  But experiments show that
      ** doing that is no faster then skipping this optimization and just
      ** calling dropCell() and insertCell(). */




      assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */
      if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT;
      memcpy(oldCell, newCell, szNew);
      return SQLITE_OK;
    }
    dropCell(pPage, idx, info.nSize, &rc);
    if( rc ) goto end_insert;







|
>
>




|
>
>
>
>







8169
8170
8171
8172
8173
8174
8175
8176
8177
8178
8179
8180
8181
8182
8183
8184
8185
8186
8187
8188
8189
8190
8191
8192
8193
8194
      goto end_insert;
    }
    oldCell = findCell(pPage, idx);
    if( !pPage->leaf ){
      memcpy(newCell, oldCell, 4);
    }
    rc = clearCell(pPage, oldCell, &info);
    if( info.nSize==szNew && info.nLocal==info.nPayload 
     && (!ISAUTOVACUUM || szNew<pPage->minLocal)
    ){
      /* Overwrite the old cell with the new if they are the same size.
      ** We could also try to do this if the old cell is smaller, then add
      ** the leftover space to the free list.  But experiments show that
      ** doing that is no faster then skipping this optimization and just
      ** calling dropCell() and insertCell(). 
      **
      ** This optimization cannot be used on an autovacuum database if the
      ** new entry uses overflow pages, as the insertCell() call below is
      ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry.  */
      assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */
      if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT;
      memcpy(oldCell, newCell, szNew);
      return SQLITE_OK;
    }
    dropCell(pPage, idx, info.nSize, &rc);
    if( rc ) goto end_insert;
Changes to src/build.c.
935
936
937
938
939
940
941



942

943
944
945
946
947
948
949
    pParse->nErr++;
    goto begin_table_error;
  }
  pTable->zName = zName;
  pTable->iPKey = -1;
  pTable->pSchema = db->aDb[iDb].pSchema;
  pTable->nTabRef = 1;



  pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );

  assert( pParse->pNewTable==0 );
  pParse->pNewTable = pTable;

  /* If this is the magic sqlite_sequence table used by autoincrement,
  ** then record a pointer to this table in the main database structure
  ** so that INSERT can find the table easily.
  */







>
>
>

>







935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
    pParse->nErr++;
    goto begin_table_error;
  }
  pTable->zName = zName;
  pTable->iPKey = -1;
  pTable->pSchema = db->aDb[iDb].pSchema;
  pTable->nTabRef = 1;
#ifdef SQLITE_DEFAULT_ROWEST
  pTable->nRowLogEst = sqlite3LogEst(SQLITE_DEFAULT_ROWEST);
#else
  pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) );
#endif
  assert( pParse->pNewTable==0 );
  pParse->pNewTable = pTable;

  /* If this is the magic sqlite_sequence table used by autoincrement,
  ** then record a pointer to this table in the main database structure
  ** so that INSERT can find the table easily.
  */
Changes to src/insert.c.
520
521
522
523
524
525
526

527
528
529
530
531
532
533
  int tmask;                  /* Mask of trigger times */
#endif

  db = pParse->db;
  if( pParse->nErr || db->mallocFailed ){
    goto insert_cleanup;
  }


  /* If the Select object is really just a simple VALUES() list with a
  ** single row (the common case) then keep that one row of values
  ** and discard the other (unused) parts of the pSelect object
  */
  if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){
    pList = pSelect->pEList;







>







520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
  int tmask;                  /* Mask of trigger times */
#endif

  db = pParse->db;
  if( pParse->nErr || db->mallocFailed ){
    goto insert_cleanup;
  }
  dest.iSDParm = 0;  /* Suppress a harmless compiler warning */

  /* If the Select object is really just a simple VALUES() list with a
  ** single row (the common case) then keep that one row of values
  ** and discard the other (unused) parts of the pSelect object
  */
  if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){
    pList = pSelect->pEList;
Changes to src/main.c.
3334
3335
3336
3337
3338
3339
3340






3341
3342
3343
3344
3345
3346
3347
  return reportError(SQLITE_MISUSE, lineno, "misuse");
}
int sqlite3CantopenError(int lineno){
  testcase( sqlite3GlobalConfig.xLog!=0 );
  return reportError(SQLITE_CANTOPEN, lineno, "cannot open file");
}
#ifdef SQLITE_DEBUG






int sqlite3NomemError(int lineno){
  testcase( sqlite3GlobalConfig.xLog!=0 );
  return reportError(SQLITE_NOMEM, lineno, "OOM");
}
int sqlite3IoerrnomemError(int lineno){
  testcase( sqlite3GlobalConfig.xLog!=0 );
  return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error");







>
>
>
>
>
>







3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
  return reportError(SQLITE_MISUSE, lineno, "misuse");
}
int sqlite3CantopenError(int lineno){
  testcase( sqlite3GlobalConfig.xLog!=0 );
  return reportError(SQLITE_CANTOPEN, lineno, "cannot open file");
}
#ifdef SQLITE_DEBUG
int sqlite3CorruptPgnoError(int lineno, Pgno pgno){
  char zMsg[100];
  sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno);
  testcase( sqlite3GlobalConfig.xLog!=0 );
  return reportError(SQLITE_CORRUPT, lineno, zMsg);
}
int sqlite3NomemError(int lineno){
  testcase( sqlite3GlobalConfig.xLog!=0 );
  return reportError(SQLITE_NOMEM, lineno, "OOM");
}
int sqlite3IoerrnomemError(int lineno){
  testcase( sqlite3GlobalConfig.xLog!=0 );
  return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error");
Changes to src/select.c.
5286
5287
5288
5289
5290
5291
5292


5293
5294
5295
5296
5297
5298
5299
      }else{
        VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName));
      }
      pPrior = isSelfJoinView(pTabList, pItem);
      if( pPrior ){
        sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor);
        explainSetInteger(pItem->iSelectId, pPrior->iSelectId);


      }else{
        sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
        explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
        sqlite3Select(pParse, pSub, &dest);
      }
      pItem->pTab->nRowLogEst = pSub->nSelectRow;
      if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);







>
>







5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
      }else{
        VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName));
      }
      pPrior = isSelfJoinView(pTabList, pItem);
      if( pPrior ){
        sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor);
        explainSetInteger(pItem->iSelectId, pPrior->iSelectId);
        assert( pPrior->pSelect!=0 );
        pSub->nSelectRow = pPrior->pSelect->nSelectRow;
      }else{
        sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor);
        explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
        sqlite3Select(pParse, pSub, &dest);
      }
      pItem->pTab->nRowLogEst = pSub->nSelectRow;
      if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr);
Changes to src/shell.c.
504
505
506
507
508
509
510












511
512
513
514
515
516
517
** lower 30 bits of a 32-bit signed integer.
*/
static int strlen30(const char *z){
  const char *z2 = z;
  while( *z2 ){ z2++; }
  return 0x3fffffff & (int)(z2 - z);
}













/*
** This routine reads a line of text from FILE in, stores
** the text in memory obtained from malloc() and returns a pointer
** to the text.  NULL is returned at end of file, or if malloc()
** fails.
**







>
>
>
>
>
>
>
>
>
>
>
>







504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
** lower 30 bits of a 32-bit signed integer.
*/
static int strlen30(const char *z){
  const char *z2 = z;
  while( *z2 ){ z2++; }
  return 0x3fffffff & (int)(z2 - z);
}

/*
** Return the length of a string in characters.  Multibyte UTF8 characters
** count as a single character.
*/
static int strlenChar(const char *z){
  int n = 0;
  while( *z ){
    if( (0xc0&*(z++))!=0x80 ) n++;
  }
  return n;
}

/*
** This routine reads a line of text from FILE in, stores
** the text in memory obtained from malloc() and returns a pointer
** to the text.  NULL is returned at end of file, or if malloc()
** fails.
**
1168
1169
1170
1171
1172
1173
1174























































1175
1176
1177
1178
1179
1180
1181
    SHA3Update(p, &c3, 1);
  }
  for(i=0; i<p->nRate; i++){
    p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
  }
  return &p->u.x[p->nRate];
}
























































/*
** Implementation of the sha3(X,SIZE) function.
**
** Return a BLOB which is the SIZE-bit SHA3 hash of X.  The default
** size is 256.  If X is a BLOB, it is hashed as is.
** For all other non-NULL types of input, X is converted into a UTF-8 string







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
    SHA3Update(p, &c3, 1);
  }
  for(i=0; i<p->nRate; i++){
    p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
  }
  return &p->u.x[p->nRate];
}

/*
** SQL function:  shell_add_schema(S,X)
**
** Add the schema name X to the CREATE statement in S and return the result.
** Examples:
**
**    CREATE TABLE t1(x)   ->   CREATE TABLE xyz.t1(x);
**
** Also works on
**
**    CREATE INDEX
**    CREATE UNIQUE INDEX
**    CREATE VIEW
**    CREATE TRIGGER
**    CREATE VIRTUAL TABLE
**
** This UDF is used by the .schema command to insert the schema name of
** attached databases into the middle of the sqlite_master.sql field.
*/
static void shellAddSchemaName(
  sqlite3_context *pCtx,
  int nVal,
  sqlite3_value **apVal
){
  static const char *aPrefix[] = {
     "TABLE",
     "INDEX",
     "UNIQUE INDEX",
     "VIEW",
     "TRIGGER",
     "VIRTUAL TABLE"
  };
  int i = 0;
  const char *zIn = (const char*)sqlite3_value_text(apVal[0]);
  const char *zSchema = (const char*)sqlite3_value_text(apVal[1]);
  assert( nVal==2 );
  if( zIn!=0 && strncmp(zIn, "CREATE ", 7)==0 ){
    for(i=0; i<sizeof(aPrefix)/sizeof(aPrefix[0]); i++){
      int n = strlen30(aPrefix[i]);
      if( strncmp(zIn+7, aPrefix[i], n)==0 && zIn[n+7]==' ' ){
        char cQuote = quoteChar(zSchema);
        char *z;
        if( cQuote ){
         z = sqlite3_mprintf("%.*s \"%w\".%s", n+7, zIn, zSchema, zIn+n+8);
        }else{
          z = sqlite3_mprintf("%.*s %s.%s", n+7, zIn, zSchema, zIn+n+8);
        }
        sqlite3_result_text(pCtx, z, -1, sqlite3_free);
        return;
      }
    }
  }
  sqlite3_result_value(pCtx, apVal[0]);
}

/*
** Implementation of the sha3(X,SIZE) function.
**
** Return a BLOB which is the SIZE-bit SHA3 hash of X.  The default
** size is 256.  If X is a BLOB, it is hashed as is.
** For all other non-NULL types of input, X is converted into a UTF-8 string
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
          int w, n;
          if( i<ArraySize(p->colWidth) ){
            w = colWidth[i];
          }else{
            w = 0;
          }
          if( w==0 ){
            w = strlen30(azCol[i] ? azCol[i] : "");
            if( w<10 ) w = 10;
            n = strlen30(azArg && azArg[i] ? azArg[i] : p->nullValue);
            if( w<n ) w = n;
          }
          if( i<ArraySize(p->actualWidth) ){
            p->actualWidth[i] = w;
          }
          if( showHdr ){
            utf8_width_print(p->out, w, azCol[i]);







|

|







1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
          int w, n;
          if( i<ArraySize(p->colWidth) ){
            w = colWidth[i];
          }else{
            w = 0;
          }
          if( w==0 ){
            w = strlenChar(azCol[i] ? azCol[i] : "");
            if( w<10 ) w = 10;
            n = strlenChar(azArg && azArg[i] ? azArg[i] : p->nullValue);
            if( w<n ) w = n;
          }
          if( i<ArraySize(p->actualWidth) ){
            p->actualWidth[i] = w;
          }
          if( showHdr ){
            utf8_width_print(p->out, w, azCol[i]);
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
      for(i=0; i<nArg; i++){
        int w;
        if( i<ArraySize(p->actualWidth) ){
           w = p->actualWidth[i];
        }else{
           w = 10;
        }
        if( p->cMode==MODE_Explain && azArg[i] && strlen30(azArg[i])>w ){
          w = strlen30(azArg[i]);
        }
        if( i==1 && p->aiIndent && p->pStmt ){
          if( p->iIndent<p->nIndent ){
            utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], "");
          }
          p->iIndent++;
        }







|
|







2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
      for(i=0; i<nArg; i++){
        int w;
        if( i<ArraySize(p->actualWidth) ){
           w = p->actualWidth[i];
        }else{
           w = 10;
        }
        if( p->cMode==MODE_Explain && azArg[i] && strlenChar(azArg[i])>w ){
          w = strlenChar(azArg[i]);
        }
        if( i==1 && p->aiIndent && p->pStmt ){
          if( p->iIndent<p->nIndent ){
            utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], "");
          }
          p->iIndent++;
        }
3520
3521
3522
3523
3524
3525
3526



3527
3528
3529
3530
3531
3532
3533
                            sha3Func, 0, 0);
    sqlite3_create_function(p->db, "sha3", 2, SQLITE_UTF8, 0,
                            sha3Func, 0, 0);
    sqlite3_create_function(p->db, "sha3_query", 1, SQLITE_UTF8, 0,
                            sha3QueryFunc, 0, 0);
    sqlite3_create_function(p->db, "sha3_query", 2, SQLITE_UTF8, 0,
                            sha3QueryFunc, 0, 0);



  }
}

/*
** Do C-language style dequoting.
**
**    \a    -> alarm







>
>
>







3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
                            sha3Func, 0, 0);
    sqlite3_create_function(p->db, "sha3", 2, SQLITE_UTF8, 0,
                            sha3Func, 0, 0);
    sqlite3_create_function(p->db, "sha3_query", 1, SQLITE_UTF8, 0,
                            sha3QueryFunc, 0, 0);
    sqlite3_create_function(p->db, "sha3_query", 2, SQLITE_UTF8, 0,
                            sha3QueryFunc, 0, 0);
    sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                            shellAddSchemaName, 0, 0);
                            
  }
}

/*
** Do C-language style dequoting.
**
**    \a    -> alarm
5673
5674
5675
5676
5677
5678
5679

5680
5681



5682
5683
5684
5685

5686
5687
5688
5689
5690
5691
5692
    }else{
      raw_printf(stderr, "Usage: .scanstats on|off\n");
      rc = 1;
    }
  }else

  if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){

    ShellState data;
    char *zErrMsg = 0;



    open_db(p, 0);
    memcpy(&data, p, sizeof(data));
    data.showHeader = 0;
    data.cMode = data.mode = MODE_Semi;

    if( nArg>=2 && optionMatch(azArg[1], "indent") ){
      data.cMode = data.mode = MODE_Pretty;
      nArg--;
      if( nArg==2 ) azArg[1] = azArg[2];
    }
    if( nArg==2 && azArg[1][0]!='-' ){
      int i;







>


>
>
>




>







5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
    }else{
      raw_printf(stderr, "Usage: .scanstats on|off\n");
      rc = 1;
    }
  }else

  if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){
    ShellText sSelect;
    ShellState data;
    char *zErrMsg = 0;
    const char *zDiv = 0;
    int iSchema = 0;

    open_db(p, 0);
    memcpy(&data, p, sizeof(data));
    data.showHeader = 0;
    data.cMode = data.mode = MODE_Semi;
    initText(&sSelect);
    if( nArg>=2 && optionMatch(azArg[1], "indent") ){
      data.cMode = data.mode = MODE_Pretty;
      nArg--;
      if( nArg==2 ) azArg[1] = azArg[2];
    }
    if( nArg==2 && azArg[1][0]!='-' ){
      int i;
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749















































5750
5751
5752
5753
5754
5755
5756
                      ")";
        new_argv[1] = 0;
        new_colv[0] = "sql";
        new_colv[1] = 0;
        callback(&data, 1, new_argv, new_colv);
        rc = SQLITE_OK;
      }else{
        char *zSql;
        zSql = sqlite3_mprintf(
          "SELECT sql FROM "
          "  (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x"
          "     FROM sqlite_master UNION ALL"
          "   SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) "
          "WHERE lower(tbl_name) LIKE %Q"
          "  AND type!='meta' AND sql NOTNULL "
          "ORDER BY rowid", azArg[1]);
        rc = sqlite3_exec(p->db, zSql, callback, &data, &zErrMsg);
        sqlite3_free(zSql);
      }
    }else if( nArg==1 ){
      rc = sqlite3_exec(p->db,
         "SELECT sql FROM "
         "  (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x"
         "     FROM sqlite_master UNION ALL"
         "   SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) "
         "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' "
         "ORDER BY rowid",
         callback, &data, &zErrMsg
      );
    }else{
      raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n");
      rc = 1;
      goto meta_command_exit;
    }















































    if( zErrMsg ){
      utf8_printf(stderr,"Error: %s\n", zErrMsg);
      sqlite3_free(zErrMsg);
      rc = 1;
    }else if( rc != SQLITE_OK ){
      raw_printf(stderr,"Error: querying schema information\n");
      rc = 1;







<
|
<
<
<
<
<
<
<
<
<


|
<
<
<
<
<
<
<
<





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







5791
5792
5793
5794
5795
5796
5797

5798









5799
5800
5801








5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
                      ")";
        new_argv[1] = 0;
        new_colv[0] = "sql";
        new_colv[1] = 0;
        callback(&data, 1, new_argv, new_colv);
        rc = SQLITE_OK;
      }else{

        zDiv = "(";









      }
    }else if( nArg==1 ){
      zDiv = "(";








    }else{
      raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n");
      rc = 1;
      goto meta_command_exit;
    }
    if( zDiv ){
      sqlite3_stmt *pStmt = 0;
      sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list",
                         -1, &pStmt, 0);
      appendText(&sSelect, "SELECT sql FROM", 0);
      iSchema = 0;
      while( sqlite3_step(pStmt)==SQLITE_ROW ){
        const char *zDb = (const char*)sqlite3_column_text(pStmt, 0);
        char zScNum[30];
        sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema);
        appendText(&sSelect, zDiv, 0);
        zDiv = " UNION ALL ";
        if( strcmp(zDb, "main")!=0 ){
          appendText(&sSelect, "SELECT shell_add_schema(sql,", 0);
          appendText(&sSelect, zDb, '"');
          appendText(&sSelect, ") AS sql, type, tbl_name, name, rowid,", 0);
          appendText(&sSelect, zScNum, 0);
          appendText(&sSelect, " AS snum, ", 0);
          appendText(&sSelect, zDb, '\'');
          appendText(&sSelect, " AS sname FROM ", 0);
          appendText(&sSelect, zDb, '"');
          appendText(&sSelect, ".sqlite_master", 0);
        }else{
          appendText(&sSelect, "SELECT sql, type, tbl_name, name, rowid, ", 0);
          appendText(&sSelect, zScNum, 0);
          appendText(&sSelect, " AS snum, 'main' AS sname FROM sqlite_master",0);
        }
      }
      sqlite3_finalize(pStmt);
      appendText(&sSelect, ") WHERE ", 0);
      if( nArg>1 ){
        char *zQarg = sqlite3_mprintf("%Q", azArg[1]);
        if( strchr(azArg[1], '.') ){
          appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0);
        }else{
          appendText(&sSelect, "lower(tbl_name)", 0);
        }
        appendText(&sSelect, strchr(azArg[1], '*') ? " GLOB " : " LIKE ", 0);
        appendText(&sSelect, zQarg, 0);
        appendText(&sSelect, " AND ", 0);
        sqlite3_free(zQarg);
      }
      appendText(&sSelect, "type!='meta' AND sql IS NOT NULL"
                           " ORDER BY snum, rowid", 0);
      rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg);
      freeText(&sSelect);
    }
    if( zErrMsg ){
      utf8_printf(stderr,"Error: %s\n", zErrMsg);
      sqlite3_free(zErrMsg);
      rc = 1;
    }else if( rc != SQLITE_OK ){
      raw_printf(stderr,"Error: querying schema information\n");
      rc = 1;
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
  }else
#endif

  if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){
    int bIsInit = 0;         /* True to initialize the SELFTEST table */
    int bVerbose = 0;        /* Verbose output */
    int bSelftestExists;     /* True if SELFTEST already exists */
    char **azTest = 0;       /* Content of the SELFTEST table */
    int nRow = 0;            /* Number of rows in the SELFTEST table */
    int nCol = 4;            /* Number of columns in the SELFTEST table */
    int i;                   /* Loop counter */
    int nTest = 0;           /* Number of tests runs */
    int nErr = 0;            /* Number of errors seen */
    ShellText str;           /* Answer for a query */
    static char *azDefaultTest[] = {
       0, 0, 0, 0,
       "0", "memo", "Missing SELFTEST table - default checks only", "",
       "1", "run", "PRAGMA integrity_check", "ok"
    };
    static const int nDefaultRow = 2;

    open_db(p,0);
    for(i=1; i<nArg; i++){
      const char *z = azArg[i];
      if( z[0]=='-' && z[1]=='-' ) z++;
      if( strcmp(z,"-init")==0 ){
        bIsInit = 1;







<
<
<
|



<
<
|
<
<
<







6088
6089
6090
6091
6092
6093
6094



6095
6096
6097
6098


6099



6100
6101
6102
6103
6104
6105
6106
  }else
#endif

  if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){
    int bIsInit = 0;         /* True to initialize the SELFTEST table */
    int bVerbose = 0;        /* Verbose output */
    int bSelftestExists;     /* True if SELFTEST already exists */



    int i, k;                /* Loop counters */
    int nTest = 0;           /* Number of tests runs */
    int nErr = 0;            /* Number of errors seen */
    ShellText str;           /* Answer for a query */


    sqlite3_stmt *pStmt = 0; /* Query against the SELFTEST table */




    open_db(p,0);
    for(i=1; i<nArg; i++){
      const char *z = azArg[i];
      if( z[0]=='-' && z[1]=='-' ) z++;
      if( strcmp(z,"-init")==0 ){
        bIsInit = 1;
6026
6027
6028
6029
6030
6031
6032


6033

6034
6035
6036






6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058

6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094


6095
6096
6097
6098
6099
6100
6101
6102
6103
    }else{
      bSelftestExists = 1;
    }
    if( bIsInit ){
      createSelftestTable(p);
      bSelftestExists = 1;
    }


    if( bSelftestExists ){

      rc = sqlite3_get_table(p->db,
          "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno",
          &azTest, &nRow, &nCol, 0);






      if( rc ){
        raw_printf(stderr, "Error querying the selftest table\n");
        rc = 1;
        sqlite3_free_table(azTest);
        goto meta_command_exit;
      }else if( nRow==0 ){
        sqlite3_free_table(azTest);
        azTest = azDefaultTest;
        nRow = nDefaultRow;
      }
    }else{
      azTest = azDefaultTest;
      nRow = nDefaultRow;
    }
    initText(&str);
    appendText(&str, "x", 0);
    for(i=1; i<=nRow; i++){
      int tno = atoi(azTest[i*nCol]);
      const char *zOp = azTest[i*nCol+1];
      const char *zSql = azTest[i*nCol+2];
      const char *zAns = azTest[i*nCol+3];


      if( bVerbose>0 ){
        char *zQuote = sqlite3_mprintf("%q", zSql);
        printf("%d: %s %s\n", tno, zOp, zSql);
        sqlite3_free(zQuote);
      }
      if( strcmp(zOp,"memo")==0 ){
        utf8_printf(p->out, "%s\n", zSql);
      }else
      if( strcmp(zOp,"run")==0 ){
        char *zErrMsg = 0;
        str.n = 0;
        str.z[0] = 0;
        rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg);
        nTest++;
        if( bVerbose ){
          utf8_printf(p->out, "Result: %s\n", str.z);
        }
        if( rc || zErrMsg ){
          nErr++;
          rc = 1;
          utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg);
          sqlite3_free(zErrMsg);
        }else if( strcmp(zAns,str.z)!=0 ){
          nErr++;
          rc = 1;
          utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns);
          utf8_printf(p->out, "%d:      Got: [%s]\n", tno, str.z);
        }
      }else
      {
        utf8_printf(stderr,
          "Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
        rc = 1;
        break;
      }
    }


    freeText(&str);
    if( azTest!=azDefaultTest ) sqlite3_free_table(azTest);
    utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest);
  }else

  if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){
    if( nArg<2 || nArg>3 ){
      raw_printf(stderr, "Usage: .separator COL ?ROW?\n");
      rc = 1;







>
>
|
>
|
|
|
>
>
>
>
>
>



|

<
<
<
<

<
<
<
<
<
<
|
|
|
|
|

>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
>

<







6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146




6147






6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193

6194
6195
6196
6197
6198
6199
6200
    }else{
      bSelftestExists = 1;
    }
    if( bIsInit ){
      createSelftestTable(p);
      bSelftestExists = 1;
    }
    initText(&str);
    appendText(&str, "x", 0);
    for(k=bSelftestExists; k>=0; k--){
      if( k==1 ){
        rc = sqlite3_prepare_v2(p->db,
            "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno",
            -1, &pStmt, 0);
      }else{
        rc = sqlite3_prepare_v2(p->db,
          "VALUES(0,'memo','Missing SELFTEST table - default checks only',''),"
          "      (1,'run','PRAGMA integrity_check','ok')",
          -1, &pStmt, 0);
      }
      if( rc ){
        raw_printf(stderr, "Error querying the selftest table\n");
        rc = 1;
        sqlite3_finalize(pStmt);
        goto meta_command_exit;




      }






      for(i=1; sqlite3_step(pStmt)==SQLITE_ROW; i++){
        int tno = sqlite3_column_int(pStmt, 0);
        const char *zOp = (const char*)sqlite3_column_text(pStmt, 1);
        const char *zSql = (const char*)sqlite3_column_text(pStmt, 2);
        const char *zAns = (const char*)sqlite3_column_text(pStmt, 3);

        k = 0;
        if( bVerbose>0 ){
          char *zQuote = sqlite3_mprintf("%q", zSql);
          printf("%d: %s %s\n", tno, zOp, zSql);
          sqlite3_free(zQuote);
        }
        if( strcmp(zOp,"memo")==0 ){
          utf8_printf(p->out, "%s\n", zSql);
        }else
        if( strcmp(zOp,"run")==0 ){
          char *zErrMsg = 0;
          str.n = 0;
          str.z[0] = 0;
          rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg);
          nTest++;
          if( bVerbose ){
            utf8_printf(p->out, "Result: %s\n", str.z);
          }
          if( rc || zErrMsg ){
            nErr++;
            rc = 1;
            utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg);
            sqlite3_free(zErrMsg);
          }else if( strcmp(zAns,str.z)!=0 ){
            nErr++;
            rc = 1;
            utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns);
            utf8_printf(p->out, "%d:      Got: [%s]\n", tno, str.z);
          }
        }else
        {
          utf8_printf(stderr,
            "Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
          rc = 1;
          break;
        }
      } /* End loop over rows of content from SELFTEST */
      sqlite3_finalize(pStmt);
    } /* End loop over k */
    freeText(&str);

    utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest);
  }else

  if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){
    if( nArg<2 || nArg>3 ){
      raw_printf(stderr, "Usage: .separator COL ?ROW?\n");
      rc = 1;
6298
6299
6300
6301
6302
6303
6304
6305
6306


6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335










6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
  if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0)
   || (c=='i' && (strncmp(azArg[0], "indices", n)==0
                 || strncmp(azArg[0], "indexes", n)==0) )
  ){
    sqlite3_stmt *pStmt;
    char **azResult;
    int nRow, nAlloc;
    char *zSql = 0;
    int ii;


    open_db(p, 0);
    rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
    if( rc ) return shellDatabaseError(p->db);

    /* Create an SQL statement to query for the list of tables in the
    ** main and all attached databases where the table name matches the
    ** LIKE pattern bound to variable "?1". */
    if( c=='t' ){
      zSql = sqlite3_mprintf(
          "SELECT name FROM sqlite_master"
          " WHERE type IN ('table','view')"
          "   AND name NOT LIKE 'sqlite_%%'"
          "   AND name LIKE ?1");
    }else if( nArg>2 ){
      /* It is an historical accident that the .indexes command shows an error
      ** when called with the wrong number of arguments whereas the .tables
      ** command does not. */
      raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n");
      rc = 1;
      goto meta_command_exit;
    }else{
      zSql = sqlite3_mprintf(
          "SELECT name FROM sqlite_master"
          " WHERE type='index'"
          "   AND tbl_name LIKE ?1");
    }
    for(ii=0; zSql && sqlite3_step(pStmt)==SQLITE_ROW; ii++){
      const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1);
      if( zDbName==0 || ii==0 ) continue;










      if( c=='t' ){
        zSql = sqlite3_mprintf(
                 "%z UNION ALL "
                 "SELECT '%q.' || name FROM \"%w\".sqlite_master"
                 " WHERE type IN ('table','view')"
                 "   AND name NOT LIKE 'sqlite_%%'"
                 "   AND name LIKE ?1", zSql, zDbName, zDbName);
      }else{
        zSql = sqlite3_mprintf(
                 "%z UNION ALL "
                 "SELECT '%q.' || name FROM \"%w\".sqlite_master"
                 " WHERE type='index'"
                 "   AND tbl_name LIKE ?1", zSql, zDbName, zDbName);
      }
    }
    rc = sqlite3_finalize(pStmt);
    if( zSql && rc==SQLITE_OK ){
      zSql = sqlite3_mprintf("%z ORDER BY 1", zSql);
      if( zSql ) rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
    }
    sqlite3_free(zSql);
    if( !zSql ) return shellNomemError();
    if( rc ) return shellDatabaseError(p->db);

    /* Run the SQL statement prepared by the above block. Store the results
    ** as an array of nul-terminated strings in azResult[].  */
    nRow = nAlloc = 0;
    azResult = 0;
    if( nArg>1 ){







<

>
>




<
<
<
<
<
<
<
<
<
|






<
<
<
<
<

|

|
>
>
>
>
>
>
>
>
>
>

<
<
<
|
|
|

<
<
<
|
|



<
|
|
<
|
<







6395
6396
6397
6398
6399
6400
6401

6402
6403
6404
6405
6406
6407
6408









6409
6410
6411
6412
6413
6414
6415





6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430



6431
6432
6433
6434



6435
6436
6437
6438
6439

6440
6441

6442

6443
6444
6445
6446
6447
6448
6449
  if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0)
   || (c=='i' && (strncmp(azArg[0], "indices", n)==0
                 || strncmp(azArg[0], "indexes", n)==0) )
  ){
    sqlite3_stmt *pStmt;
    char **azResult;
    int nRow, nAlloc;

    int ii;
    ShellText s;
    initText(&s);
    open_db(p, 0);
    rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
    if( rc ) return shellDatabaseError(p->db);










    if( nArg>2 && c=='i' ){
      /* It is an historical accident that the .indexes command shows an error
      ** when called with the wrong number of arguments whereas the .tables
      ** command does not. */
      raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n");
      rc = 1;
      goto meta_command_exit;





    }
    for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){
      const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1);
      if( zDbName==0 ) continue;
      if( s.z && s.z[0] ) appendText(&s, " UNION ALL ", 0);
      if( sqlite3_stricmp(zDbName, "main")==0 ){
        appendText(&s, "SELECT name FROM ", 0);
      }else{
        appendText(&s, "SELECT ", 0);
        appendText(&s, zDbName, '\'');
        appendText(&s, "||'.'||name FROM ", 0);
      }
      appendText(&s, zDbName, '"');
      appendText(&s, ".sqlite_master ", 0);
      if( c=='t' ){



        appendText(&s," WHERE type IN ('table','view')"
                      "   AND name NOT LIKE 'sqlite_%'"
                      "   AND name LIKE ?1", 0);
      }else{



        appendText(&s," WHERE type='index'"
                      "   AND tbl_name LIKE ?1", 0);
      }
    }
    rc = sqlite3_finalize(pStmt);

    appendText(&s, " ORDER BY 1", 0);
    rc = sqlite3_prepare_v2(p->db, s.z, -1, &pStmt, 0);

    freeText(&s);

    if( rc ) return shellDatabaseError(p->db);

    /* Run the SQL statement prepared by the above block. Store the results
    ** as an array of nul-terminated strings in azResult[].  */
    nRow = nAlloc = 0;
    azResult = 0;
    if( nArg>1 ){
Changes to src/sqlite.h.in.
5616
5617
5618
5619
5620
5621
5622
5623


5624
5625
5626
5627
5628
5629
5630
** interface returns SQLITE_OK and fills in the non-NULL pointers in
** the final five arguments with appropriate values if the specified
** column exists.  ^The sqlite3_table_column_metadata() interface returns
** SQLITE_ERROR and if the specified column does not exist.
** ^If the column-name parameter to sqlite3_table_column_metadata() is a
** NULL pointer, then this routine simply checks for the existence of the
** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it
** does not.


**
** ^The column is identified by the second, third and fourth parameters to
** this function. ^(The second parameter is either the name of the database
** (i.e. "main", "temp", or an attached database) containing the specified
** table or NULL.)^ ^If it is NULL, then all attached databases are searched
** for the table using the same algorithm used by the database engine to
** resolve unqualified table references.







|
>
>







5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
** interface returns SQLITE_OK and fills in the non-NULL pointers in
** the final five arguments with appropriate values if the specified
** column exists.  ^The sqlite3_table_column_metadata() interface returns
** SQLITE_ERROR and if the specified column does not exist.
** ^If the column-name parameter to sqlite3_table_column_metadata() is a
** NULL pointer, then this routine simply checks for the existence of the
** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it
** does not.  If the table name parameter T in a call to
** sqlite3_table_column_metadata(X,D,T,C,...) is NULL then the result is
** undefined behavior.
**
** ^The column is identified by the second, third and fourth parameters to
** this function. ^(The second parameter is either the name of the database
** (i.e. "main", "temp", or an attached database) containing the specified
** table or NULL.)^ ^If it is NULL, then all attached databases are searched
** for the table using the same algorithm used by the database engine to
** resolve unqualified table references.
Changes to src/sqliteInt.h.
3398
3399
3400
3401
3402
3403
3404

3405
3406

3407
3408
3409

3410
3411
3412
3413
3414
3415
3416
int sqlite3CantopenError(int);
#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__)
#define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__)
#define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__)
#ifdef SQLITE_DEBUG
  int sqlite3NomemError(int);
  int sqlite3IoerrnomemError(int);

# define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__)
# define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__)

#else
# define SQLITE_NOMEM_BKPT SQLITE_NOMEM
# define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM

#endif

/*
** FTS3 and FTS4 both require virtual table support
*/
#if defined(SQLITE_OMIT_VIRTUALTABLE)
# undef SQLITE_ENABLE_FTS3







>


>



>







3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
int sqlite3CantopenError(int);
#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__)
#define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__)
#define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__)
#ifdef SQLITE_DEBUG
  int sqlite3NomemError(int);
  int sqlite3IoerrnomemError(int);
  int sqlite3CorruptPgnoError(int,Pgno);
# define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__)
# define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__)
# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P))
#else
# define SQLITE_NOMEM_BKPT SQLITE_NOMEM
# define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM
# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptError(__LINE__)
#endif

/*
** FTS3 and FTS4 both require virtual table support
*/
#if defined(SQLITE_OMIT_VIRTUALTABLE)
# undef SQLITE_ENABLE_FTS3
Changes to src/test_fs.c.
541
542
543
544
545
546
547

548
549
550
551
552
553
554
      for(i=nPrefix; zQuery[i]; i++){
        if( zQuery[i]==aWild[0] || zQuery[i]==aWild[1] ) break;
        if( zQuery[i]=='/' ) nDir = i;
      }
      zDir = zQuery;
    }
  }


  sqlite3_bind_text(pCsr->pStmt, 1, zDir, nDir, SQLITE_TRANSIENT);
  sqlite3_bind_text(pCsr->pStmt, 2, zRoot, nRoot, SQLITE_TRANSIENT);
  sqlite3_bind_text(pCsr->pStmt, 3, zPrefix, nPrefix, SQLITE_TRANSIENT);

#if SQLITE_OS_WIN
  sqlite3_free(zPrefix);







>







541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
      for(i=nPrefix; zQuery[i]; i++){
        if( zQuery[i]==aWild[0] || zQuery[i]==aWild[1] ) break;
        if( zQuery[i]=='/' ) nDir = i;
      }
      zDir = zQuery;
    }
  }
  if( nDir==0 ) nDir = 1;

  sqlite3_bind_text(pCsr->pStmt, 1, zDir, nDir, SQLITE_TRANSIENT);
  sqlite3_bind_text(pCsr->pStmt, 2, zRoot, nRoot, SQLITE_TRANSIENT);
  sqlite3_bind_text(pCsr->pStmt, 3, zPrefix, nPrefix, SQLITE_TRANSIENT);

#if SQLITE_OS_WIN
  sqlite3_free(zPrefix);
Changes to src/trigger.c.
302
303
304
305
306
307
308

309
310
311
312
313
314
315
    char *z;

    /* Make an entry in the sqlite_master table */
    v = sqlite3GetVdbe(pParse);
    if( v==0 ) goto triggerfinish_cleanup;
    sqlite3BeginWriteOperation(pParse, 0, iDb);
    z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n);

    sqlite3NestedParse(pParse,
       "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
       db->aDb[iDb].zDbSName, MASTER_NAME, zName,
       pTrig->table, z);
    sqlite3DbFree(db, z);
    sqlite3ChangeCookie(pParse, iDb);
    sqlite3VdbeAddParseSchemaOp(v, iDb,







>







302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
    char *z;

    /* Make an entry in the sqlite_master table */
    v = sqlite3GetVdbe(pParse);
    if( v==0 ) goto triggerfinish_cleanup;
    sqlite3BeginWriteOperation(pParse, 0, iDb);
    z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n);
    testcase( z==0 );
    sqlite3NestedParse(pParse,
       "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
       db->aDb[iDb].zDbSName, MASTER_NAME, zName,
       pTrig->table, z);
    sqlite3DbFree(db, z);
    sqlite3ChangeCookie(pParse, iDb);
    sqlite3VdbeAddParseSchemaOp(v, iDb,
Changes to src/vdbemem.c.
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
    }
    if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str;
    if( enc!=SQLITE_UTF8 ){
      rc = sqlite3VdbeChangeEncoding(pVal, enc);
    }
  }else if( op==TK_UMINUS ) {
    /* This branch happens for multiple negative signs.  Ex: -(-5) */
    if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) 
     && pVal!=0
    ){
      sqlite3VdbeMemNumerify(pVal);
      if( pVal->flags & MEM_Real ){
        pVal->u.r = -pVal->u.r;
      }else if( pVal->u.i==SMALLEST_INT64 ){
        pVal->u.r = -(double)SMALLEST_INT64;







|







1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
    }
    if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str;
    if( enc!=SQLITE_UTF8 ){
      rc = sqlite3VdbeChangeEncoding(pVal, enc);
    }
  }else if( op==TK_UMINUS ) {
    /* This branch happens for multiple negative signs.  Ex: -(-5) */
    if( SQLITE_OK==valueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal,pCtx) 
     && pVal!=0
    ){
      sqlite3VdbeMemNumerify(pVal);
      if( pVal->flags & MEM_Real ){
        pVal->u.r = -pVal->u.r;
      }else if( pVal->u.i==SMALLEST_INT64 ){
        pVal->u.r = -(double)SMALLEST_INT64;
Changes to src/where.c.
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
#ifdef WHERETRACE_ENABLED
    if( sqlite3WhereTrace ){    /* Display all of the WhereLoop objects */
      WhereLoop *p;
      int i;
      static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
                                             "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
      for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
        p->cId = zLabel[i%sizeof(zLabel)];
        whereLoopPrint(p, sWLB.pWC);
      }
    }
#endif
  
    wherePathSolver(pWInfo, 0);
    if( db->mallocFailed ) goto whereBeginError;







|







4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
#ifdef WHERETRACE_ENABLED
    if( sqlite3WhereTrace ){    /* Display all of the WhereLoop objects */
      WhereLoop *p;
      int i;
      static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
                                             "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
      for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
        p->cId = zLabel[i%(sizeof(zLabel)-1)];
        whereLoopPrint(p, sWLB.pWC);
      }
    }
#endif
  
    wherePathSolver(pWInfo, 0);
    if( db->mallocFailed ) goto whereBeginError;
Changes to test/autovacuum.test.
701
702
703
704
705
706
707







708
709
  execsql { INSERT INTO t1 SELECT NULL, randstr(50,50) FROM t1 }
} {}
do_test autovacuum-9.5 {
  execsql { DELETE FROM t1 WHERE rowid > (SELECT max(a)/2 FROM t1) }
  file size test.db
} $::sqlite_pending_byte
 








finish_test







>
>
>
>
>
>
>


701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
  execsql { INSERT INTO t1 SELECT NULL, randstr(50,50) FROM t1 }
} {}
do_test autovacuum-9.5 {
  execsql { DELETE FROM t1 WHERE rowid > (SELECT max(a)/2 FROM t1) }
  file size test.db
} $::sqlite_pending_byte
 
do_execsql_test autovacuum-10.1 {
  DROP TABLE t1;
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
  INSERT INTO t1 VALUES(25, randomblob(104));
  REPLACE INTO t1 VALUES(25, randomblob(1117));
  PRAGMA integrity_check;
} {ok}

finish_test
Changes to test/kvtest.c.
67
68
69
70
71
72
73
74
75
76
77





78
79
80
81


82
83
84
85
86
87
88
89
90
91
92
93
94
95
96


97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
"\n"
"        Generate a new test database file named DBFILE containing N\n"
"        BLOBs each of size M bytes.  The page size of the new database\n"
"        file will be X.  Additional options:\n"
"\n"
"           --variance V           Randomly vary M by plus or minus V\n"
"\n"
"   kvtest export DBFILE DIRECTORY\n"
"\n"
"        Export all the blobs in the kv table of DBFILE into separate\n"
"        files in DIRECTORY.\n"





"\n"
"   kvtest stat DBFILE\n"
"\n"
"        Display summary information about DBFILE\n"


"\n"
"   kvtest run DBFILE [options]\n"
"\n"
"        Run a performance test.  DBFILE can be either the name of a\n"
"        database or a directory containing sample files.  Options:\n"
"\n"
"           --asc                  Read blobs in ascending order\n"
"           --blob-api             Use the BLOB API\n"
"           --cache-size N         Database cache size\n"
"           --count N              Read N blobs\n"
"           --desc                 Read blobs in descending order\n"
"           --fsync                Synchronous file writes\n"
"           --integrity-check      Run \"PRAGMA integrity_check\" after test\n"
"           --max-id N             Maximum blob key to use\n"
"           --mmap N               Mmap as much as N bytes of DBFILE\n"


"           --nosync               Set \"PRAGMA synchronous=OFF\"\n"
"           --jmode MODE           Set MODE journal mode prior to starting\n"
"           --random               Read blobs in a random order\n"
"           --start N              Start reading with this blob key\n"
"           --stats                Output operating stats before exiting\n"
"           --update               To an overwrite test\n"
;

/* Reference resources used */
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <assert.h>
#include <string.h>
#include "sqlite3.h"

#ifndef _WIN32
# include <unistd.h>
#else
  /* Provide Windows equivalent for the needed parts of unistd.h */

# include <io.h>
# define R_OK 2
# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
# define access _access
#endif

#include <stdint.h>
#include <inttypes.h>

/*
** The following macros are used to cast pointers to integers and
** integers to pointers.  The way you do this varies from one compiler
** to the next, so we have developed the following set of #if statements
** to generate appropriate macros for a wide range of compilers.
**
** The correct "ANSI" way to do this is to use the intptr_t type.
** Unfortunately, that typedef is not available on all compilers, or
** if it is available, it requires an #include of specific headers
** that vary from one machine to the next.
**
** Ticket #3860:  The llvm-gcc-4.2 compiler from Apple chokes on
** the ((void*)&((char*)0)[X]) construct.  But MSVC chokes on ((void*)(X)).
** So we have to define the macros in different ways depending on the
** compiler.
*/
#if defined(__PTRDIFF_TYPE__)  /* This case should work for GCC */
# define SQLITE_INT_TO_PTR(X)  ((void*)(__PTRDIFF_TYPE__)(X))
# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(__PTRDIFF_TYPE__)(X))
#elif !defined(__GNUC__)       /* Works for compilers other than LLVM */
# define SQLITE_INT_TO_PTR(X)  ((void*)&((char*)0)[X])
# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(((char*)X)-(char*)0))
#elif defined(HAVE_STDINT_H)   /* Use this case if we have ANSI headers */
# define SQLITE_INT_TO_PTR(X)  ((void*)(intptr_t)(X))
# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(intptr_t)(X))
#else                          /* Generates a warning - but it always works */
# define SQLITE_INT_TO_PTR(X)  ((void*)(X))
# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(X))
#endif

/*
** Show thqe help text and quit.
*/
static void showHelp(void){
  fprintf(stdout, "%s", zHelp);







|


|
>
>
>
>
>

|

|
>
>















>
>





|















>








<




















|
<
<
<


<
<
<







67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156



157
158



159
160
161
162
163
164
165
"\n"
"        Generate a new test database file named DBFILE containing N\n"
"        BLOBs each of size M bytes.  The page size of the new database\n"
"        file will be X.  Additional options:\n"
"\n"
"           --variance V           Randomly vary M by plus or minus V\n"
"\n"
"   kvtest export DBFILE DIRECTORY [--tree]\n"
"\n"
"        Export all the blobs in the kv table of DBFILE into separate\n"
"        files in DIRECTORY.  DIRECTORY is created if it does not previously\n"
"        exist.  If the --tree option is used, then the blobs are written\n"
"        into a hierarchy of directories, using names like 00/00/00,\n"
"        00/00/01, 00/00/02, and so forth.  Without the --tree option, all\n"
"        files are in the top-level directory with names like 000000, 000001,\n"
"        000002, and so forth.\n"
"\n"
"   kvtest stat DBFILE [options]\n"
"\n"
"        Display summary information about DBFILE.  Options:\n"
"\n"
"           --vacuum               Run VACUUM on the database file\n"
"\n"
"   kvtest run DBFILE [options]\n"
"\n"
"        Run a performance test.  DBFILE can be either the name of a\n"
"        database or a directory containing sample files.  Options:\n"
"\n"
"           --asc                  Read blobs in ascending order\n"
"           --blob-api             Use the BLOB API\n"
"           --cache-size N         Database cache size\n"
"           --count N              Read N blobs\n"
"           --desc                 Read blobs in descending order\n"
"           --fsync                Synchronous file writes\n"
"           --integrity-check      Run \"PRAGMA integrity_check\" after test\n"
"           --max-id N             Maximum blob key to use\n"
"           --mmap N               Mmap as much as N bytes of DBFILE\n"
"           --multitrans           Each read or write in its own transaction\n"
"           --nocheckpoint         Omit the checkpoint on WAL mode writes\n"
"           --nosync               Set \"PRAGMA synchronous=OFF\"\n"
"           --jmode MODE           Set MODE journal mode prior to starting\n"
"           --random               Read blobs in a random order\n"
"           --start N              Start reading with this blob key\n"
"           --stats                Output operating stats before exiting\n"
"           --update               Do an overwrite test\n"
;

/* Reference resources used */
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <assert.h>
#include <string.h>
#include "sqlite3.h"

#ifndef _WIN32
# include <unistd.h>
#else
  /* Provide Windows equivalent for the needed parts of unistd.h */
# include <direct.h>
# include <io.h>
# define R_OK 2
# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
# define access _access
#endif

#include <stdint.h>


/*
** The following macros are used to cast pointers to integers and
** integers to pointers.  The way you do this varies from one compiler
** to the next, so we have developed the following set of #if statements
** to generate appropriate macros for a wide range of compilers.
**
** The correct "ANSI" way to do this is to use the intptr_t type.
** Unfortunately, that typedef is not available on all compilers, or
** if it is available, it requires an #include of specific headers
** that vary from one machine to the next.
**
** Ticket #3860:  The llvm-gcc-4.2 compiler from Apple chokes on
** the ((void*)&((char*)0)[X]) construct.  But MSVC chokes on ((void*)(X)).
** So we have to define the macros in different ways depending on the
** compiler.
*/
#if defined(__PTRDIFF_TYPE__)  /* This case should work for GCC */
# define SQLITE_INT_TO_PTR(X)  ((void*)(__PTRDIFF_TYPE__)(X))
# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(__PTRDIFF_TYPE__)(X))
#else



# define SQLITE_INT_TO_PTR(X)  ((void*)(intptr_t)(X))
# define SQLITE_PTR_TO_INT(X)  ((sqlite3_int64)(intptr_t)(X))



#endif

/*
** Show thqe help text and quit.
*/
static void showHelp(void){
  fprintf(stdout, "%s", zHelp);
233
234
235
236
237
238
239
240

241
242
243








244
245

246
247
248
249
250
251
252
253
254
255






256


257
258
259
260
261
262
263
  return isNeg? -v : v;
}


/*
** Check the filesystem object zPath.  Determine what it is:
**
**    PATH_DIR     A directory

**    PATH_DB      An SQLite database
**    PATH_NEXIST  Does not exist
**    PATH_OTHER   Something else








*/
#define PATH_DIR     1

#define PATH_DB      2
#define PATH_NEXIST  0
#define PATH_OTHER   99
static int pathType(const char *zPath){
  struct stat x;
  int rc;
  if( access(zPath,R_OK) ) return PATH_NEXIST;
  memset(&x, 0, sizeof(x));
  rc = stat(zPath, &x);
  if( rc<0 ) return PATH_OTHER;






  if( S_ISDIR(x.st_mode) ) return PATH_DIR;


  if( (x.st_size%512)==0 ) return PATH_DB;
  return PATH_OTHER;
}

/*
** Return the size of a file in bytes.  Or return -1 if the
** named object is not a regular file or does not exist.







|
>



>
>
>
>
>
>
>
>


>
|









>
>
>
>
>
>
|
>
>







236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
  return isNeg? -v : v;
}


/*
** Check the filesystem object zPath.  Determine what it is:
**
**    PATH_DIR     A single directory holding many files
**    PATH_TREE    A directory hierarchy with files at the leaves
**    PATH_DB      An SQLite database
**    PATH_NEXIST  Does not exist
**    PATH_OTHER   Something else
**
** PATH_DIR means all of the separate files are grouped together
** into a single directory with names like 000000, 000001, 000002, and
** so forth.  PATH_TREE means there is a hierarchy of directories so
** that no single directory has too many entries.  The files have names
** like 00/00/00, 00/00/01, 00/00/02 and so forth.  The decision between
** PATH_DIR and PATH_TREE is determined by the presence of a subdirectory
** named "00" at the top-level.
*/
#define PATH_DIR     1
#define PATH_TREE    2
#define PATH_DB      3
#define PATH_NEXIST  0
#define PATH_OTHER   99
static int pathType(const char *zPath){
  struct stat x;
  int rc;
  if( access(zPath,R_OK) ) return PATH_NEXIST;
  memset(&x, 0, sizeof(x));
  rc = stat(zPath, &x);
  if( rc<0 ) return PATH_OTHER;
  if( S_ISDIR(x.st_mode) ){
    char *zLayer1 = sqlite3_mprintf("%s/00", zPath);
    memset(&x, 0, sizeof(x));
    rc = stat(zLayer1, &x);
    sqlite3_free(zLayer1);
    if( rc<0 ) return PATH_DIR;
    if( S_ISDIR(x.st_mode) ) return PATH_TREE;
    return PATH_DIR;
  }
  if( (x.st_size%512)==0 ) return PATH_DB;
  return PATH_OTHER;
}

/*
** Return the size of a file in bytes.  Or return -1 if the
** named object is not a regular file or does not exist.
360
361
362
363
364
365
366

367
368
369
370
371
372
373
374




375
376
377
378
379
380





381
382
383
384
385
386
387
*/
static int statMain(int argc, char **argv){
  char *zDb;
  int i, rc;
  sqlite3 *db;
  char *zSql;
  sqlite3_stmt *pStmt;


  assert( strcmp(argv[1],"stat")==0 );
  assert( argc>=3 );
  zDb = argv[2];
  for(i=3; i<argc; i++){
    char *z = argv[i];
    if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
    if( z[1]=='-' ) z++;




    fatalError("unknown option: \"%s\"", argv[i]);
  }
  rc = sqlite3_open(zDb, &db);
  if( rc ){
    fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
  }





  zSql = sqlite3_mprintf(
    "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))"
    "  FROM kv"
  );
  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
  sqlite3_free(zSql);







>








>
>
>
>






>
>
>
>
>







381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
*/
static int statMain(int argc, char **argv){
  char *zDb;
  int i, rc;
  sqlite3 *db;
  char *zSql;
  sqlite3_stmt *pStmt;
  int doVacuum = 0;

  assert( strcmp(argv[1],"stat")==0 );
  assert( argc>=3 );
  zDb = argv[2];
  for(i=3; i<argc; i++){
    char *z = argv[i];
    if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
    if( z[1]=='-' ) z++;
    if( strcmp(z, "-vacuum")==0 ){
      doVacuum = 1;
      continue;
    }
    fatalError("unknown option: \"%s\"", argv[i]);
  }
  rc = sqlite3_open(zDb, &db);
  if( rc ){
    fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
  }
  if( doVacuum ){
    printf("Vacuuming...."); fflush(stdout);
    sqlite3_exec(db, "VACUUM", 0, 0, 0);
    printf("       done\n");
  }
  zSql = sqlite3_mprintf(
    "SELECT count(*), min(length(v)), max(length(v)), avg(length(v))"
    "  FROM kv"
  );
  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
  sqlite3_free(zSql);
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436


437
438
439
440
441
442

443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466











467
468
469
470
471
472
473
474
475
476


477



478
479
480

481
482
483












484

485
486
487
488
489
490










491
492

493


494


495

496
497







498



499
500

501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
  zSql = sqlite3_mprintf("PRAGMA page_count");
  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
  sqlite3_free(zSql);
  if( sqlite3_step(pStmt)==SQLITE_ROW ){
    printf("Page-count:         %8d\n", sqlite3_column_int(pStmt, 0));
  }
  sqlite3_finalize(pStmt);
  sqlite3_close(db);
  return 0;
}

/*
** Implementation of the "writefile(X,Y)" SQL function.  The argument Y
** is written into file X.  The number of bytes written is returned.  Or
** NULL is returned if something goes wrong, such as being unable to open
** file X for writing.
*/
static void writefileFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  FILE *out;
  const char *z;
  sqlite3_int64 rc;
  const char *zFile;

  zFile = (const char*)sqlite3_value_text(argv[0]);
  if( zFile==0 ) return;
  out = fopen(zFile, "wb");
  if( out==0 ) return;


  z = (const char*)sqlite3_value_blob(argv[1]);
  if( z==0 ){
    rc = 0;
  }else{
    rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
  }

  fclose(out);
  printf("\r%s   ", zFile); fflush(stdout);
  sqlite3_result_int64(context, rc);
}

/*
**      remember(V,PTR)
**
** Return the integer value V.  Also save the value of V in a
** C-language variable whose address is PTR.
*/
static void rememberFunc(
  sqlite3_context *pCtx,
  int argc,
  sqlite3_value **argv
){
  sqlite3_int64 v;
  sqlite3_int64 ptr;
  assert( argc==2 );
  v = sqlite3_value_int64(argv[0]);
  ptr = sqlite3_value_int64(argv[1]);
  *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v;
  sqlite3_result_int64(pCtx, v);
}












/*
** Export the kv table to individual files in the filesystem
*/
static int exportMain(int argc, char **argv){
  char *zDb;
  char *zDir;
  sqlite3 *db;
  char *zSql;
  int rc;


  char *zErrMsg = 0;




  assert( strcmp(argv[1],"export")==0 );
  assert( argc>=3 );

  zDb = argv[2];
  if( argc!=4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY");
  zDir = argv[3];












  if( pathType(zDir)!=PATH_DIR ){

    fatalError("object \"%s\" is not a directory", zDir);
  }
  rc = sqlite3_open(zDb, &db);
  if( rc ){
    fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
  }










  sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
                          writefileFunc, 0, 0);

  zSql = sqlite3_mprintf(


    "SELECT writefile(printf('%s/%%06d',k),v) FROM kv;",


    zDir

  );
  rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);







  if( rc ) fatalError("database create failed: %s", zErrMsg);



  sqlite3_free(zSql);
  sqlite3_close(db);

  printf("\n");
  return 0;
}

/*
** Read the content of file zName into memory obtained from sqlite3_malloc64()
** and return a pointer to the buffer. The caller is responsible for freeing 
** the memory. 
**
** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
** read.
**
** For convenience, a nul-terminator byte is always appended to the data read
** from the file before the buffer is returned. This byte is not included in
** the final value of (*pnByte), if applicable.
**
** NULL is returned if any error is encountered. The final value of *pnByte
** is undefined in this case.
*/
static unsigned char *readFile(const char *zName, int *pnByte){
  FILE *in;               /* FILE from which to read content of zName */
  sqlite3_int64 nIn;      /* Size of zName in bytes */
  size_t nRead;           /* Number of bytes actually read */
  unsigned char *pBuf;    /* Content read from disk */

  nIn = fileSize(zName);
  if( nIn<0 ) return 0;
  in = fopen(zName, "rb");
  if( in==0 ) return 0;
  pBuf = sqlite3_malloc64( nIn );
  if( pBuf==0 ) return 0;
  nRead = fread(pBuf, (size_t)nIn, 1, in);
  fclose(in);
  if( nRead!=1 ){
    sqlite3_free(pBuf);
    return 0;
  }
  if( pnByte ) *pnByte = (int)nIn;
  return pBuf;
}

/*
** Overwrite a file with randomness.  Do not change the size of the
** file.
*/
static void updateFile(const char *zName, int *pnByte, int doFsync){
  FILE *out;              /* FILE from which to read content of zName */
  sqlite3_int64 sz;       /* Size of zName in bytes */
  size_t nWritten;        /* Number of bytes actually read */
  unsigned char *pBuf;    /* Content to store on disk */
  const char *zMode = "wb";   /* Mode for fopen() */

  sz = fileSize(zName);
  if( sz<0 ){
    fatalError("No such file: \"%s\"", zName);
  }
  *pnByte = (int)sz;
  if( sz==0 ) return;
  pBuf = sqlite3_malloc64( sz );
  if( pBuf==0 ){
    fatalError("Cannot allocate %lld bytes\n", sz);
  }
  sqlite3_randomness((int)sz, pBuf); 
#if defined(_WIN32)








|
|
<
|
<
<
<
<
<
<
<
|
<
|
<
<
<
|
<
|
|
<
<
<
>
>
|
<
<
<
|

>
|
<
|





















>
>
>
>
>
>
>
>
>
>
>








|

>
>
|
>
>
>



>

<

>
>
>
>
>
>
>
>
>
>
>
>
|
>






>
>
>
>
>
>
>
>
>
>
|
|
>
|
>
>
|
>
>
|
>
|
|
>
>
>
>
>
>
>
|
>
>
>
|

>



















|

















|







|










|







436
437
438
439
440
441
442
443
444
445

446







447

448



449

450
451



452
453
454



455
456
457
458

459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512

513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
  zSql = sqlite3_mprintf("PRAGMA page_count");
  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));
  sqlite3_free(zSql);
  if( sqlite3_step(pStmt)==SQLITE_ROW ){
    printf("Page-count:         %8d\n", sqlite3_column_int(pStmt, 0));
  }
  sqlite3_finalize(pStmt);
  zSql = sqlite3_mprintf("PRAGMA freelist_count");
  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);

  if( rc ) fatalError("cannot prepare SQL [%s]: %s", zSql, sqlite3_errmsg(db));







  sqlite3_free(zSql);

  if( sqlite3_step(pStmt)==SQLITE_ROW ){



    printf("Freelist-count:     %8d\n", sqlite3_column_int(pStmt, 0));

  }
  sqlite3_finalize(pStmt);



  rc = sqlite3_prepare_v2(db, "PRAGMA integrity_check(10)", -1, &pStmt, 0);
  if( rc ) fatalError("cannot prepare integrity check: %s", sqlite3_errmsg(db));
  while( sqlite3_step(pStmt)==SQLITE_ROW ){



    printf("Integrity-check:    %s\n", sqlite3_column_text(pStmt, 0));
  }
  sqlite3_finalize(pStmt);
  sqlite3_close(db);

  return 0;
}

/*
**      remember(V,PTR)
**
** Return the integer value V.  Also save the value of V in a
** C-language variable whose address is PTR.
*/
static void rememberFunc(
  sqlite3_context *pCtx,
  int argc,
  sqlite3_value **argv
){
  sqlite3_int64 v;
  sqlite3_int64 ptr;
  assert( argc==2 );
  v = sqlite3_value_int64(argv[0]);
  ptr = sqlite3_value_int64(argv[1]);
  *(sqlite3_int64*)SQLITE_INT_TO_PTR(ptr) = v;
  sqlite3_result_int64(pCtx, v);
}

/*
** Make sure a directory named zDir exists.
*/
static void kvtest_mkdir(const char *zDir){
#if defined(_WIN32)
  (void)mkdir(zDir);
#else
  (void)mkdir(zDir, 0755);
#endif
}

/*
** Export the kv table to individual files in the filesystem
*/
static int exportMain(int argc, char **argv){
  char *zDb;
  char *zDir;
  sqlite3 *db;
  sqlite3_stmt *pStmt;
  int rc;
  int ePathType;
  int nFN;
  char *zFN;
  char *zTail;
  size_t nWrote;
  int i;

  assert( strcmp(argv[1],"export")==0 );
  assert( argc>=3 );
  if( argc<4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY [OPTIONS]");
  zDb = argv[2];

  zDir = argv[3];
  kvtest_mkdir(zDir);
  for(i=4; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' && z[1]=='-' ) z++;
    if( strcmp(z,"-tree")==0 ){
      zFN = sqlite3_mprintf("%s/00", zDir);
      kvtest_mkdir(zFN);
      sqlite3_free(zFN);
      continue;
    }
    fatalError("unknown argument: \"%s\"\n", argv[i]);
  }
  ePathType = pathType(zDir);
  if( ePathType!=PATH_DIR && ePathType!=PATH_TREE ){
    fatalError("object \"%s\" is not a directory", zDir);
  }
  rc = sqlite3_open(zDb, &db);
  if( rc ){
    fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
  }
  rc = sqlite3_prepare_v2(db, "SELECT k, v FROM kv ORDER BY k", -1, &pStmt, 0);
  if( rc ){
    fatalError("prepare_v2 failed: %s\n", sqlite3_errmsg(db));
  }
  nFN = (int)strlen(zDir);
  zFN = sqlite3_mprintf("%s/00/00/00.extra---------------------", zDir);
  if( zFN==0 ){
    fatalError("malloc failed\n");
  }
  zTail = zFN + nFN + 1;
  while( sqlite3_step(pStmt)==SQLITE_ROW ){
    int iKey = sqlite3_column_int(pStmt, 0);
    sqlite3_int64 nData = sqlite3_column_bytes(pStmt, 1);
    const void *pData = sqlite3_column_blob(pStmt, 1);
    FILE *out;
    if( ePathType==PATH_DIR ){
      sqlite3_snprintf(20, zTail, "%06d", iKey);
    }else{
      sqlite3_snprintf(20, zTail, "%02d", iKey/10000);
      kvtest_mkdir(zFN);
      sqlite3_snprintf(20, zTail, "%02d/%02d", iKey/10000, (iKey/100)%100);
      kvtest_mkdir(zFN);
      sqlite3_snprintf(20, zTail, "%02d/%02d/%02d",
                       iKey/10000, (iKey/100)%100, iKey%100);
    }
    out = fopen(zFN, "wb");      
    nWrote = fwrite(pData, 1, nData, out);
    fclose(out);
    printf("\r%s   ", zTail); fflush(stdout);
    if( nWrote!=nData ){
      fatalError("Wrote only %d of %d bytes to %s\n",
                  (int)nWrote, nData, zFN);
    }
  }
  sqlite3_finalize(pStmt);
  sqlite3_close(db);
  sqlite3_free(zFN);
  printf("\n");
  return 0;
}

/*
** Read the content of file zName into memory obtained from sqlite3_malloc64()
** and return a pointer to the buffer. The caller is responsible for freeing 
** the memory. 
**
** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
** read.
**
** For convenience, a nul-terminator byte is always appended to the data read
** from the file before the buffer is returned. This byte is not included in
** the final value of (*pnByte), if applicable.
**
** NULL is returned if any error is encountered. The final value of *pnByte
** is undefined in this case.
*/
static unsigned char *readFile(const char *zName, sqlite3_int64 *pnByte){
  FILE *in;               /* FILE from which to read content of zName */
  sqlite3_int64 nIn;      /* Size of zName in bytes */
  size_t nRead;           /* Number of bytes actually read */
  unsigned char *pBuf;    /* Content read from disk */

  nIn = fileSize(zName);
  if( nIn<0 ) return 0;
  in = fopen(zName, "rb");
  if( in==0 ) return 0;
  pBuf = sqlite3_malloc64( nIn );
  if( pBuf==0 ) return 0;
  nRead = fread(pBuf, (size_t)nIn, 1, in);
  fclose(in);
  if( nRead!=1 ){
    sqlite3_free(pBuf);
    return 0;
  }
  if( pnByte ) *pnByte = nIn;
  return pBuf;
}

/*
** Overwrite a file with randomness.  Do not change the size of the
** file.
*/
static void updateFile(const char *zName, sqlite3_int64 *pnByte, int doFsync){
  FILE *out;              /* FILE from which to read content of zName */
  sqlite3_int64 sz;       /* Size of zName in bytes */
  size_t nWritten;        /* Number of bytes actually read */
  unsigned char *pBuf;    /* Content to store on disk */
  const char *zMode = "wb";   /* Mode for fopen() */

  sz = fileSize(zName);
  if( sz<0 ){
    fatalError("No such file: \"%s\"", zName);
  }
  *pnByte = sz;
  if( sz==0 ) return;
  pBuf = sqlite3_malloc64( sz );
  if( pBuf==0 ){
    fatalError("Cannot allocate %lld bytes\n", sz);
  }
  sqlite3_randomness((int)sz, pBuf); 
#if defined(_WIN32)
738
739
740
741
742
743
744


745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767













768
769
770
771
772

















773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843

844
845
846
847
848
849
850
  int bBlobApi = 0;           /* Use the incremental blob I/O API */
  int bStats = 0;             /* Print stats before exiting */
  int eOrder = ORDER_ASC;     /* Access order */
  int isUpdateTest = 0;       /* Do in-place updates rather than reads */
  int doIntegrityCk = 0;      /* Run PRAGMA integrity_check after the test */
  int noSync = 0;             /* Disable synchronous mode */
  int doFsync = 0;            /* Update disk files synchronously */


  sqlite3 *db = 0;            /* Database connection */
  sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
  sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
  sqlite3_int64 tmStart;      /* Start time */
  sqlite3_int64 tmElapsed;    /* Elapsed time */
  int mmapSize = 0;           /* --mmap N argument */
  int nData = 0;              /* Bytes of data */
  sqlite3_int64 nTotal = 0;   /* Total data read */
  unsigned char *pData = 0;   /* Content of the blob */
  int nAlloc = 0;             /* Space allocated for pData[] */
  const char *zJMode = 0;     /* Journal mode */
  

  assert( strcmp(argv[1],"run")==0 );
  assert( argc>=3 );
  zDb = argv[2];
  eType = pathType(zDb);
  if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb);
  if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb);
  for(i=3; i<argc; i++){
    char *z = argv[i];
    if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
    if( z[1]=='-' ) z++;













    if( strcmp(z, "-count")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      nCount = integerValue(argv[++i]);
      if( nCount<1 ) fatalError("the --count must be positive");
      continue;

















    }
    if( strcmp(z, "-mmap")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      mmapSize = integerValue(argv[++i]);
      if( nCount<0 ) fatalError("the --mmap must be non-negative");
      continue;
    }
    if( strcmp(z, "-max-id")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iMax = integerValue(argv[++i]);
      continue;
    }
    if( strcmp(z, "-start")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iKey = integerValue(argv[++i]);
      if( iKey<1 ) fatalError("the --start must be positive");
      continue;
    }
    if( strcmp(z, "-cache-size")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iCache = integerValue(argv[++i]);
      continue;
    }
    if( strcmp(z, "-jmode")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      zJMode = argv[++i];
      continue;
    }
    if( strcmp(z, "-random")==0 ){
      eOrder = ORDER_RANDOM;
      continue;
    }
    if( strcmp(z, "-asc")==0 ){
      eOrder = ORDER_ASC;
      continue;
    }
    if( strcmp(z, "-desc")==0 ){
      eOrder = ORDER_DESC;
      continue;
    }
    if( strcmp(z, "-blob-api")==0 ){
      bBlobApi = 1;
      continue;
    }
    if( strcmp(z, "-stats")==0 ){
      bStats = 1;
      continue;
    }
    if( strcmp(z, "-update")==0 ){
      isUpdateTest = 1;
      continue;
    }
    if( strcmp(z, "-integrity-check")==0 ){
      doIntegrityCk = 1;
      continue;
    }
    if( strcmp(z, "-nosync")==0 ){
      noSync = 1;
      continue;
    }
    if( strcmp(z, "-fsync")==0 ){
      doFsync = 1;
      continue;
    }
    fatalError("unknown option: \"%s\"", argv[i]);
  }
  if( eType==PATH_DB ){
    /* Recover any prior crashes prior to starting the timer */
    sqlite3_open(zDb, &db);
    sqlite3_exec(db, "SELECT rowid FROM sqlite_master LIMIT 1", 0, 0, 0);
    sqlite3_close(db);

  }
  tmStart = timeOfDay();
  if( eType==PATH_DB ){
    char *zSql;
    rc = sqlite3_open(zDb, &db);
    if( rc ){
      fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));







>
>






|


|













>
>
>
>
>
>
>
>
>
>
>
>
>





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>












|
<
|
<


|
<
|


|
<
|






|
|
|
<
|
<
<
<
<
<










<
<
<
<
<
<
<
<
<
<
<
<







>







808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887

888

889
890
891

892
893
894
895

896
897
898
899
900
901
902
903
904
905

906





907
908
909
910
911
912
913
914
915
916












917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
  int bBlobApi = 0;           /* Use the incremental blob I/O API */
  int bStats = 0;             /* Print stats before exiting */
  int eOrder = ORDER_ASC;     /* Access order */
  int isUpdateTest = 0;       /* Do in-place updates rather than reads */
  int doIntegrityCk = 0;      /* Run PRAGMA integrity_check after the test */
  int noSync = 0;             /* Disable synchronous mode */
  int doFsync = 0;            /* Update disk files synchronously */
  int doMultiTrans = 0;       /* Each operation in its own transaction */
  int noCheckpoint = 0;       /* Omit the checkpoint in WAL mode */
  sqlite3 *db = 0;            /* Database connection */
  sqlite3_stmt *pStmt = 0;    /* Prepared statement for SQL access */
  sqlite3_blob *pBlob = 0;    /* Handle for incremental Blob I/O */
  sqlite3_int64 tmStart;      /* Start time */
  sqlite3_int64 tmElapsed;    /* Elapsed time */
  int mmapSize = 0;           /* --mmap N argument */
  sqlite3_int64 nData = 0;    /* Bytes of data */
  sqlite3_int64 nTotal = 0;   /* Total data read */
  unsigned char *pData = 0;   /* Content of the blob */
  sqlite3_int64 nAlloc = 0;   /* Space allocated for pData[] */
  const char *zJMode = 0;     /* Journal mode */
  

  assert( strcmp(argv[1],"run")==0 );
  assert( argc>=3 );
  zDb = argv[2];
  eType = pathType(zDb);
  if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb);
  if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb);
  for(i=3; i<argc; i++){
    char *z = argv[i];
    if( z[0]!='-' ) fatalError("unknown argument: \"%s\"", z);
    if( z[1]=='-' ) z++;
    if( strcmp(z, "-asc")==0 ){
      eOrder = ORDER_ASC;
      continue;
    }
    if( strcmp(z, "-blob-api")==0 ){
      bBlobApi = 1;
      continue;
    }
    if( strcmp(z, "-cache-size")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iCache = integerValue(argv[++i]);
      continue;
    }
    if( strcmp(z, "-count")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      nCount = integerValue(argv[++i]);
      if( nCount<1 ) fatalError("the --count must be positive");
      continue;
    }
    if( strcmp(z, "-desc")==0 ){
      eOrder = ORDER_DESC;
      continue;
    }
    if( strcmp(z, "-fsync")==0 ){
      doFsync = 1;
      continue;
    }
    if( strcmp(z, "-integrity-check")==0 ){
      doIntegrityCk = 1;
      continue;
    }
    if( strcmp(z, "-jmode")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      zJMode = argv[++i];
      continue;
    }
    if( strcmp(z, "-mmap")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      mmapSize = integerValue(argv[++i]);
      if( nCount<0 ) fatalError("the --mmap must be non-negative");
      continue;
    }
    if( strcmp(z, "-max-id")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iMax = integerValue(argv[++i]);
      continue;
    }
    if( strcmp(z, "-multitrans")==0 ){

      doMultiTrans = 1;

      continue;
    }
    if( strcmp(z, "-nocheckpoint")==0 ){

      noCheckpoint = 1;
      continue;
    }
    if( strcmp(z, "-nosync")==0 ){

      noSync = 1;
      continue;
    }
    if( strcmp(z, "-random")==0 ){
      eOrder = ORDER_RANDOM;
      continue;
    }
    if( strcmp(z, "-start")==0 ){
      if( i==argc-1 ) fatalError("missing argument on \"%s\"", argv[i]);
      iKey = integerValue(argv[++i]);

      if( iKey<1 ) fatalError("the --start must be positive");





      continue;
    }
    if( strcmp(z, "-stats")==0 ){
      bStats = 1;
      continue;
    }
    if( strcmp(z, "-update")==0 ){
      isUpdateTest = 1;
      continue;
    }












    fatalError("unknown option: \"%s\"", argv[i]);
  }
  if( eType==PATH_DB ){
    /* Recover any prior crashes prior to starting the timer */
    sqlite3_open(zDb, &db);
    sqlite3_exec(db, "SELECT rowid FROM sqlite_master LIMIT 1", 0, 0, 0);
    sqlite3_close(db);
    db = 0;
  }
  tmStart = timeOfDay();
  if( eType==PATH_DB ){
    char *zSql;
    rc = sqlite3_open(zDb, &db);
    if( rc ){
      fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
872
873
874
875
876
877
878



879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901

902




903
904
905
906
907
908
909
    }
    sqlite3_finalize(pStmt);
    pStmt = 0;
    if( zJMode ){
      zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode);
      sqlite3_exec(db, zSql, 0, 0, 0);
      sqlite3_free(zSql);



    }
    sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0);
    if( sqlite3_step(pStmt)==SQLITE_ROW ){
      zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
    }else{
      zJMode = "???";
    }
    sqlite3_finalize(pStmt);
    if( iMax<=0 ){
      sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0);
      if( sqlite3_step(pStmt)==SQLITE_ROW ){
        iMax = sqlite3_column_int(pStmt, 0);
      }
      sqlite3_finalize(pStmt);
    }
    pStmt = 0;
    sqlite3_exec(db, "BEGIN", 0, 0, 0);
  }
  if( iMax<=0 ) iMax = 1000;
  for(i=0; i<nCount; i++){
    if( eType==PATH_DIR ){
      /* CASE 1: Reading blobs out of separate files */
      char *zKey;

      zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);




      nData = 0;
      if( isUpdateTest ){
        updateFile(zKey, &nData, doFsync);
      }else{
        pData = readFile(zKey, &nData);
        sqlite3_free(pData);
      }







>
>
>
















|



|
|

>
|
>
>
>
>







953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
    }
    sqlite3_finalize(pStmt);
    pStmt = 0;
    if( zJMode ){
      zSql = sqlite3_mprintf("PRAGMA journal_mode=%Q", zJMode);
      sqlite3_exec(db, zSql, 0, 0, 0);
      sqlite3_free(zSql);
      if( noCheckpoint ){
        sqlite3_exec(db, "PRAGMA wal_autocheckpoint=0", 0, 0, 0);
      }
    }
    sqlite3_prepare_v2(db, "PRAGMA journal_mode", -1, &pStmt, 0);
    if( sqlite3_step(pStmt)==SQLITE_ROW ){
      zJMode = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
    }else{
      zJMode = "???";
    }
    sqlite3_finalize(pStmt);
    if( iMax<=0 ){
      sqlite3_prepare_v2(db, "SELECT max(k) FROM kv", -1, &pStmt, 0);
      if( sqlite3_step(pStmt)==SQLITE_ROW ){
        iMax = sqlite3_column_int(pStmt, 0);
      }
      sqlite3_finalize(pStmt);
    }
    pStmt = 0;
    if( !doMultiTrans ) sqlite3_exec(db, "BEGIN", 0, 0, 0);
  }
  if( iMax<=0 ) iMax = 1000;
  for(i=0; i<nCount; i++){
    if( eType==PATH_DIR || eType==PATH_TREE ){
      /* CASE 1: Reading or writing blobs out of separate files */
      char *zKey;
      if( eType==PATH_DIR ){
        zKey = sqlite3_mprintf("%s/%06d", zDb, iKey);
      }else{
        zKey = sqlite3_mprintf("%s/%02d/%02d/%02d", zDb, iKey/10000,
                               (iKey/100)%100, iKey%100);
      }
      nData = 0;
      if( isUpdateTest ){
        updateFile(zKey, &nData, doFsync);
      }else{
        pData = readFile(zKey, &nData);
        sqlite3_free(pData);
      }
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
      }else{
        rc = sqlite3_blob_reopen(pBlob, iKey);
      }
      if( rc==SQLITE_OK ){
        nData = sqlite3_blob_bytes(pBlob);
        if( nAlloc<nData+1 ){
          nAlloc = nData+100;
          pData = sqlite3_realloc(pData, nAlloc);
        }
        if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1);
        if( isUpdateTest ){
          sqlite3_randomness((int)nData, pData);
          rc = sqlite3_blob_write(pBlob, pData, nData, 0);
          if( rc!=SQLITE_OK ){
            fatalError("could not write the blob at %d: %s", iKey,
                      sqlite3_errmsg(db));
          }
        }else{
          rc = sqlite3_blob_read(pBlob, pData, nData, 0);
          if( rc!=SQLITE_OK ){
            fatalError("could not read the blob at %d: %s", iKey,
                      sqlite3_errmsg(db));
          }
        }
      }
    }else{







|




|





|







1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
      }else{
        rc = sqlite3_blob_reopen(pBlob, iKey);
      }
      if( rc==SQLITE_OK ){
        nData = sqlite3_blob_bytes(pBlob);
        if( nAlloc<nData+1 ){
          nAlloc = nData+100;
          pData = sqlite3_realloc64(pData, nAlloc);
        }
        if( pData==0 ) fatalError("cannot allocate %d bytes", nData+1);
        if( isUpdateTest ){
          sqlite3_randomness((int)nData, pData);
          rc = sqlite3_blob_write(pBlob, pData, (int)nData, 0);
          if( rc!=SQLITE_OK ){
            fatalError("could not write the blob at %d: %s", iKey,
                      sqlite3_errmsg(db));
          }
        }else{
          rc = sqlite3_blob_read(pBlob, pData, (int)nData, 0);
          if( rc!=SQLITE_OK ){
            fatalError("could not read the blob at %d: %s", iKey,
                      sqlite3_errmsg(db));
          }
        }
      }
    }else{
986
987
988
989
990
991
992
993

994

995

996




997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009

1010
1011
1012
1013
1014
1015
1016
  if( nAlloc ) sqlite3_free(pData);
  if( pStmt ) sqlite3_finalize(pStmt);
  if( pBlob ) sqlite3_blob_close(pBlob);
  if( bStats ){
    display_stats(db, 0);
  }
  if( db ){
    sqlite3_exec(db, "COMMIT", 0, 0, 0);

    sqlite3_close(db);

  }

  tmElapsed = timeOfDay() - tmStart;




  if( nExtra ){
    printf("%d cycles due to %d misses\n", nCount, nExtra);
  }
  if( eType==PATH_DB ){
    printf("SQLite version: %s\n", sqlite3_libversion());
    if( doIntegrityCk ){
      sqlite3_open(zDb, &db);
      sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pStmt, 0);
      while( sqlite3_step(pStmt)==SQLITE_ROW ){
        printf("integrity-check: %s\n", sqlite3_column_text(pStmt, 0));
      }
      sqlite3_finalize(pStmt);
      sqlite3_close(db);

    }
  }
  printf("--count %d --max-id %d", nCount-nExtra, iMax);
  switch( eOrder ){
    case ORDER_RANDOM:  printf(" --random\n");  break;
    case ORDER_DESC:    printf(" --desc\n");    break;
    default:            printf(" --asc\n");     break;







|
>
|
>
|
>

>
>
>
>













>







1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
  if( nAlloc ) sqlite3_free(pData);
  if( pStmt ) sqlite3_finalize(pStmt);
  if( pBlob ) sqlite3_blob_close(pBlob);
  if( bStats ){
    display_stats(db, 0);
  }
  if( db ){
    if( !doMultiTrans ) sqlite3_exec(db, "COMMIT", 0, 0, 0);
    if( !noCheckpoint ){
      sqlite3_close(db);
      db = 0;
    }
  }
  tmElapsed = timeOfDay() - tmStart;
  if( db && noCheckpoint ){
    sqlite3_close(db);
    db = 0;
  }
  if( nExtra ){
    printf("%d cycles due to %d misses\n", nCount, nExtra);
  }
  if( eType==PATH_DB ){
    printf("SQLite version: %s\n", sqlite3_libversion());
    if( doIntegrityCk ){
      sqlite3_open(zDb, &db);
      sqlite3_prepare_v2(db, "PRAGMA integrity_check", -1, &pStmt, 0);
      while( sqlite3_step(pStmt)==SQLITE_ROW ){
        printf("integrity-check: %s\n", sqlite3_column_text(pStmt, 0));
      }
      sqlite3_finalize(pStmt);
      sqlite3_close(db);
      db = 0;
    }
  }
  printf("--count %d --max-id %d", nCount-nExtra, iMax);
  switch( eOrder ){
    case ORDER_RANDOM:  printf(" --random\n");  break;
    case ORDER_DESC:    printf(" --desc\n");    break;
    default:            printf(" --asc\n");     break;
Changes to test/vtabH.test.
212
213
214
215
216
217
218

219
220

221
222
223
224
225
226

227
228
229
230
231
232
233
234
235
236

237
238
239
240
241
      set fd [open $path w]
      puts -nonewline $fd [string repeat 1 $sz]
      close $fd
    }
  } {}

  set pwd [pwd]

  do_execsql_test 3.5 {
    SELECT path, size FROM fstree WHERE path GLOB $pwd || '/subdir/*' ORDER BY 1

  } [list \
    "$pwd/subdir/x1.txt" 143 \
    "$pwd/subdir/x2.txt" 153 \
  ]
  do_execsql_test 3.6 {
    SELECT path, size FROM fstree WHERE path LIKE $pwd || '/subdir/%' ORDER BY 1

  } [list \
    "$pwd/subdir/x1.txt" 143 \
    "$pwd/subdir/x2.txt" 153 \
  ]
  do_execsql_test 3.7 {
    SELECT sum(size) FROM fstree WHERE path LIKE $pwd || '/subdir/%'
  } 296
  do_execsql_test 3.8 {
    SELECT size FROM fstree WHERE path = $pwd || '/subdir/x1.txt'
  } 143


}


finish_test







>
|
|
>
|
|
|
|
|
|
>
|
|
|
|
|
|
|
|
|
|
>





212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
      set fd [open $path w]
      puts -nonewline $fd [string repeat 1 $sz]
      close $fd
    }
  } {}

  set pwd [pwd]
  if {![string match {*[_%]*} $pwd]} {
    do_execsql_test 3.5 {
      SELECT path, size FROM fstree 
       WHERE path GLOB $pwd || '/subdir/*' ORDER BY 1
    } [list \
      "$pwd/subdir/x1.txt" 143 \
      "$pwd/subdir/x2.txt" 153 \
    ]
    do_execsql_test 3.6 {
      SELECT path, size FROM fstree
       WHERE path LIKE $pwd || '/subdir/%' ORDER BY 1
    } [list \
      "$pwd/subdir/x1.txt" 143 \
      "$pwd/subdir/x2.txt" 153 \
    ]
    do_execsql_test 3.7 {
      SELECT sum(size) FROM fstree WHERE path LIKE $pwd || '/subdir/%'
    } 296
    do_execsql_test 3.8 {
      SELECT size FROM fstree WHERE path = $pwd || '/subdir/x1.txt'
    } 143
  }

}


finish_test
Changes to tool/spaceanal.tcl.
420
421
422
423
424
425
426

427
428
429
430
431
432
433
434
435




436
437
438
439
440
441
442
  # total_pages_percent: Pages consumed as a percentage of the file.
  # storage: Bytes consumed.
  # payload_percent: Payload bytes used as a percentage of $storage.
  # total_unused: Unused bytes on pages.
  # avg_payload: Average payload per btree entry.
  # avg_fanout: Average fanout for internal pages.
  # avg_unused: Average unused bytes per btree entry.

  # ovfl_cnt_percent: Percentage of btree entries that use overflow pages.
  #
  set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}]
  set total_pages_percent [percent $total_pages $file_pgcnt]
  set storage [expr {$total_pages*$pageSize}]
  set payload_percent [percent $payload $storage {of storage consumed}]
  set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}]
  set avg_payload [divide $payload $nentry]
  set avg_unused [divide $total_unused $nentry]




  if {$int_pages>0} {
    # TODO: Is this formula correct?
    set nTab [mem eval "
      SELECT count(*) FROM (
          SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0
      )
    "]







>









>
>
>
>







420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
  # total_pages_percent: Pages consumed as a percentage of the file.
  # storage: Bytes consumed.
  # payload_percent: Payload bytes used as a percentage of $storage.
  # total_unused: Unused bytes on pages.
  # avg_payload: Average payload per btree entry.
  # avg_fanout: Average fanout for internal pages.
  # avg_unused: Average unused bytes per btree entry.
  # avg_meta: Average metadata overhead per entry.
  # ovfl_cnt_percent: Percentage of btree entries that use overflow pages.
  #
  set total_pages [expr {$leaf_pages+$int_pages+$ovfl_pages}]
  set total_pages_percent [percent $total_pages $file_pgcnt]
  set storage [expr {$total_pages*$pageSize}]
  set payload_percent [percent $payload $storage {of storage consumed}]
  set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}]
  set avg_payload [divide $payload $nentry]
  set avg_unused [divide $total_unused $nentry]
  set total_meta [expr {$storage - $payload - $total_unused}]
  set total_meta [expr {$total_meta + 4*($ovfl_pages - $ovfl_cnt)}]
  set meta_percent [percent $total_meta $storage {of metadata}]
  set avg_meta [divide $total_meta $nentry]
  if {$int_pages>0} {
    # TODO: Is this formula correct?
    set nTab [mem eval "
      SELECT count(*) FROM (
          SELECT DISTINCT tblname FROM space_used WHERE $where AND is_index=0
      )
    "]
456
457
458
459
460
461
462

463
464
465

466
467
468
469
470
471
472
  if {$compressed_size!=$storage} {
    set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}]
    set pct [expr {$compressed_size*100.0/$storage}]
    set pct [format {%5.1f%%} $pct]
    statline {Bytes used after compression} $compressed_size $pct
  }
  statline {Bytes of payload} $payload $payload_percent

  if {$cnt==1} {statline {B-tree depth} $depth}
  statline {Average payload per entry} $avg_payload
  statline {Average unused bytes per entry} $avg_unused

  if {[info exists avg_fanout]} {
    statline {Average fanout} $avg_fanout
  }
  if {$showFrag && $total_pages>1} {
    set fragmentation [percent $gap_cnt [expr {$total_pages-1}]]
    statline {Non-sequential pages} $gap_cnt $fragmentation
  }







>



>







461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
  if {$compressed_size!=$storage} {
    set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}]
    set pct [expr {$compressed_size*100.0/$storage}]
    set pct [format {%5.1f%%} $pct]
    statline {Bytes used after compression} $compressed_size $pct
  }
  statline {Bytes of payload} $payload $payload_percent
  statline {Bytes of metadata} $total_meta $meta_percent
  if {$cnt==1} {statline {B-tree depth} $depth}
  statline {Average payload per entry} $avg_payload
  statline {Average unused bytes per entry} $avg_unused
  statline {Average metadata per entry} $avg_meta
  if {[info exists avg_fanout]} {
    statline {Average fanout} $avg_fanout
  }
  if {$showFrag && $total_pages>1} {
    set fragmentation [percent $gap_cnt [expr {$total_pages-1}]]
    statline {Non-sequential pages} $gap_cnt $fragmentation
  }
753
754
755
756
757
758
759









760
761
762
763
764
765
766
Bytes of payload

    The amount of payload stored under this category.  Payload is the data
    part of table entries and the key part of index entries.  The percentage
    at the right is the bytes of payload divided by the bytes of storage 
    consumed.










Average payload per entry

    The average amount of payload on each entry.  This is just the bytes of
    payload divided by the number of entries.

Average unused bytes per entry








>
>
>
>
>
>
>
>
>







760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
Bytes of payload

    The amount of payload stored under this category.  Payload is the data
    part of table entries and the key part of index entries.  The percentage
    at the right is the bytes of payload divided by the bytes of storage 
    consumed.

Bytes of metadata

    The amount of formatting and structural information stored on for the
    table or index.  Metadata includes the btree page header, the cell pointer
    array, the size field for each cell, the left child pointer or non-leaf
    cells, the overflow pointers for overflow cells, and the rowid value for
    rowid table cells.  In other words, metadata is everything that is not
    unused space and that is not content.

Average payload per entry

    The average amount of payload on each entry.  This is just the bytes of
    payload divided by the number of entries.

Average unused bytes per entry