/ Check-in [0fd5ce4e]
Login
Overview
Comment:Modifications to reduce memory consumption. (CVS 2422)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:0fd5ce4eefdc429ce0493f15d0dba9e8a3a0b0e2
User & Date: danielk1977 2005-03-28 08:44:07
Context
2005-03-28
16:50
Changes to alter3.test to allow it to work with a codec. (CVS 2423) check-in: 9e856bab user: drh tags: trunk
08:44
Modifications to reduce memory consumption. (CVS 2422) check-in: 0fd5ce4e user: danielk1977 tags: trunk
03:39
Fix some memory leaks that occur after a malloc failure. (CVS 2421) check-in: bcb5d72e user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/btree.c.

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
3753
3754
3755
3756
3757
3758
3759
3760

3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
....
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
....
3906
3907
3908
3909
3910
3911
3912

3913
3914


























3915
3916
3917
3918
3919
3920
3921
....
3944
3945
3946
3947
3948
3949
3950

3951
3952
3953
3954
3955
3956
3957
....
3971
3972
3973
3974
3975
3976
3977

3978
3979
3980
3981
3982
3983
3984
....
4016
4017
4018
4019
4020
4021
4022

4023
4024
4025
4026
4027
4028
4029
....
4047
4048
4049
4050
4051
4052
4053


4054
4055
4056
4057
4058
4059
4060
....
4142
4143
4144
4145
4146
4147
4148

4149
4150
4151
4152
4153
4154
4155
....
4156
4157
4158
4159
4160
4161
4162

4163
4164
4165
4166
4167
4168
4169
....
4175
4176
4177
4178
4179
4180
4181


4182
4183
4184
4185
4186
4187
4188
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.253 2005/03/21 04:04:02 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
** If this routine fails for any reason, it might leave the database
** in a corrupted state.  So if this routine fails, the database should
** be rolled back.
*/
static int balance_nonroot(MemPage *pPage){
  MemPage *pParent;            /* The parent of pPage */
  Btree *pBt;                  /* The whole database */
  int nCell = 0;               /* Number of cells in aCell[] */

  int nOld;                    /* Number of pages in apOld[] */
  int nNew;                    /* Number of pages in apNew[] */
  int nDiv;                    /* Number of cells in apDiv[] */
  int i, j, k;                 /* Loop counters */
  int idx;                     /* Index of pPage in pParent->aCell[] */
  int nxDiv;                   /* Next divider slot in pParent->aCell[] */
  int rc;                      /* The return code */
  int leafCorrection;          /* 4 if pPage is a leaf.  0 if not */
  int leafData;                /* True if pPage is a leaf of a LEAFDATA tree */
  int usableSpace;             /* Bytes in pPage beyond the header */
  int pageFlags;               /* Value of pPage->aData[0] */
  int subtotal;                /* Subtotal of bytes in cells on one page */
  int iSpace = 0;              /* First unused byte of aSpace[] */
  int mxCellPerPage;           /* Maximum number of cells in one page */
  MemPage *apOld[NB];          /* pPage and up to two siblings */
  Pgno pgnoOld[NB];            /* Page numbers for each page in apOld[] */
  MemPage *apCopy[NB];         /* Private copies of apOld[] pages */
  MemPage *apNew[NB+2];        /* pPage and up to NB siblings after balancing */
  Pgno pgnoNew[NB+2];          /* Page numbers for each page in apNew[] */
  int idxDiv[NB];              /* Indices of divider cells in pParent */
  u8 *apDiv[NB];               /* Divider cells in pParent */
................................................................................
    ** TODO: Check the siblings to the left of pPage. It may be that
    ** they are not full and no new page is required.
    */
    return balance_quick(pPage, pParent);
  }
#endif

  /*
  ** Allocate space for memory structures
  */
  mxCellPerPage = MX_CELL(pBt);
  apCell = sqliteMallocRaw( 
       (mxCellPerPage+2)*NB*(sizeof(u8*)+sizeof(int))
     + sizeof(MemPage)*NB
     + pBt->psAligned*(5+NB)
     + (ISAUTOVACUUM ? (mxCellPerPage+2)*NN*2 : 0)
  );
  if( apCell==0 ){
    return SQLITE_NOMEM;
  }
  szCell = (int*)&apCell[(mxCellPerPage+2)*NB];
  aCopy[0] = (u8*)&szCell[(mxCellPerPage+2)*NB];
  for(i=1; i<NB; i++){
    aCopy[i] = &aCopy[i-1][pBt->psAligned+sizeof(MemPage)];
  }
  aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)];
#ifndef SQLITE_OMIT_AUTOVACUUM
  if( pBt->autoVacuum ){
    aFrom = &aSpace[5*pBt->psAligned];
  }
#endif
  
  /*
  ** Find the cell in the parent page whose left child points back
  ** to pPage.  The "idx" variable is the index of that cell.  If pPage
  ** is the rightmost child of pParent then set idx to pParent->nCell 
  */
  if( pParent->idxShift ){
    Pgno pgno;
................................................................................
    }
    rc = getAndInitPage(pBt, pgnoOld[i], &apOld[i], pParent);
    if( rc ) goto balance_cleanup;
    apOld[i]->idxParent = k;
    apCopy[i] = 0;
    assert( i==nOld );
    nOld++;

  }



























  /*
  ** Make copies of the content of pPage and its siblings into aOld[].
  ** The rest of this function will use data from the copies rather
  ** that the original pages since the original pages will be in the
  ** process of being overwritten.
  */
  for(i=0; i<nOld; i++){
................................................................................
  nCell = 0;
  leafCorrection = pPage->leaf*4;
  leafData = pPage->leafData && pPage->leaf;
  for(i=0; i<nOld; i++){
    MemPage *pOld = apCopy[i];
    int limit = pOld->nCell+pOld->nOverflow;
    for(j=0; j<limit; j++){

      apCell[nCell] = findOverflowCell(pOld, j);
      szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( pBt->autoVacuum ){
        int a;
        aFrom[nCell] = i;
        for(a=0; a<pOld->nOverflow; a++){
................................................................................
        ** are duplicates of keys on the child pages.  We need to remove
        ** the divider cells from pParent, but the dividers cells are not
        ** added to apCell[] because they are duplicates of child cells.
        */
        dropCell(pParent, nxDiv, sz);
      }else{
        u8 *pTemp;

        szCell[nCell] = sz;
        pTemp = &aSpace[iSpace];
        iSpace += sz;
        assert( iSpace<=pBt->psAligned*5 );
        memcpy(pTemp, apDiv[i], sz);
        apCell[nCell] = pTemp+leafCorrection;
#ifndef SQLITE_OMIT_AUTOVACUUM
................................................................................
  **   cntNew[i]: Index in apCell[] and szCell[] for the first cell to
  **              the right of the i-th sibling page.
  ** usableSpace: Number of bytes of space available on each sibling.
  ** 
  */
  usableSpace = pBt->usableSize - 12 + leafCorrection;
  for(subtotal=k=i=0; i<nCell; i++){

    subtotal += szCell[i] + 2;
    if( subtotal > usableSpace ){
      szNew[k] = subtotal - szCell[i];
      cntNew[k] = i;
      if( leafData ){ i--; }
      subtotal = 0;
      k++;
................................................................................
    int szRight = szNew[i];  /* Size of sibling on the right */
    int szLeft = szNew[i-1]; /* Size of sibling on the left */
    int r;              /* Index of right-most cell in left sibling */
    int d;              /* Index of first cell to the left of right sibling */

    r = cntNew[i-1] - 1;
    d = r + 1 - leafData;


    while( szRight==0 || szRight+szCell[d]+2<=szLeft-(szCell[r]+2) ){
      szRight += szCell[d] + 2;
      szLeft -= szCell[r] + 2;
      cntNew[i-1]--;
      r = cntNew[i-1] - 1;
      d = r + 1 - leafData;
    }
................................................................................
  /*
  ** Evenly distribute the data in apCell[] across the new pages.
  ** Insert divider cells into pParent as necessary.
  */
  j = 0;
  for(i=0; i<nNew; i++){
    /* Assemble the new sibling page. */

    MemPage *pNew = apNew[i];
    assert( pNew->pgno==pgnoNew[i] );
    assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
    assert( pNew->nCell>0 );
    assert( pNew->nOverflow==0 );

#ifndef SQLITE_OMIT_AUTOVACUUM
................................................................................
    /* If this is an auto-vacuum database, update the pointer map entries
    ** that point to the siblings that were rearranged. These can be: left
    ** children of cells, the right-child of the page, or overflow pages
    ** pointed to by cells.
    */
    if( pBt->autoVacuum ){
      for(k=j; k<cntNew[i]; k++){

        if( aFrom[k]==0xFF || apCopy[aFrom[k]]->pgno!=pNew->pgno ){
          rc = ptrmapPutOvfl(pNew, k-j);
          if( rc!=SQLITE_OK ){
            goto balance_cleanup;
          }
        }
      }
................................................................................
    /* If the sibling page assembled above was not the right-most sibling,
    ** insert a divider cell into the parent page.
    */
    if( i<nNew-1 && j<nCell ){
      u8 *pCell;
      u8 *pTemp;
      int sz;


      pCell = apCell[j];
      sz = szCell[j] + leafCorrection;
      if( !pNew->leaf ){
        memcpy(&pNew->aData[8], pCell, 4);
        pTemp = 0;
      }else if( leafData ){
	/* If the tree is a leaf-data tree, and the siblings are leaves, 







|







 







|
>













<







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







>


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>







 







>







 







>







 







>
>







 







>







 







>







 







>
>







5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
....
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774

3775
3776
3777
3778
3779
3780
3781
....
3821
3822
3823
3824
3825
3826
3827

























3828
3829
3830
3831
3832
3833
3834
....
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
....
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
....
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
....
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
....
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
....
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
....
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
....
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** $Id: btree.c,v 1.254 2005/03/28 08:44:07 danielk1977 Exp $
**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
**     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
**     "Sorting And Searching", pages 473-480. Addison-Wesley
**     Publishing Company, Reading, Massachusetts.
................................................................................
** If this routine fails for any reason, it might leave the database
** in a corrupted state.  So if this routine fails, the database should
** be rolled back.
*/
static int balance_nonroot(MemPage *pPage){
  MemPage *pParent;            /* The parent of pPage */
  Btree *pBt;                  /* The whole database */
  int nCell = 0;               /* Number of cells in apCell[] */
  int nMaxCells = 0;           /* Allocated size of apCell, szCell, aFrom. */
  int nOld;                    /* Number of pages in apOld[] */
  int nNew;                    /* Number of pages in apNew[] */
  int nDiv;                    /* Number of cells in apDiv[] */
  int i, j, k;                 /* Loop counters */
  int idx;                     /* Index of pPage in pParent->aCell[] */
  int nxDiv;                   /* Next divider slot in pParent->aCell[] */
  int rc;                      /* The return code */
  int leafCorrection;          /* 4 if pPage is a leaf.  0 if not */
  int leafData;                /* True if pPage is a leaf of a LEAFDATA tree */
  int usableSpace;             /* Bytes in pPage beyond the header */
  int pageFlags;               /* Value of pPage->aData[0] */
  int subtotal;                /* Subtotal of bytes in cells on one page */
  int iSpace = 0;              /* First unused byte of aSpace[] */

  MemPage *apOld[NB];          /* pPage and up to two siblings */
  Pgno pgnoOld[NB];            /* Page numbers for each page in apOld[] */
  MemPage *apCopy[NB];         /* Private copies of apOld[] pages */
  MemPage *apNew[NB+2];        /* pPage and up to NB siblings after balancing */
  Pgno pgnoNew[NB+2];          /* Page numbers for each page in apNew[] */
  int idxDiv[NB];              /* Indices of divider cells in pParent */
  u8 *apDiv[NB];               /* Divider cells in pParent */
................................................................................
    ** TODO: Check the siblings to the left of pPage. It may be that
    ** they are not full and no new page is required.
    */
    return balance_quick(pPage, pParent);
  }
#endif


























  /*
  ** Find the cell in the parent page whose left child points back
  ** to pPage.  The "idx" variable is the index of that cell.  If pPage
  ** is the rightmost child of pParent then set idx to pParent->nCell 
  */
  if( pParent->idxShift ){
    Pgno pgno;
................................................................................
    }
    rc = getAndInitPage(pBt, pgnoOld[i], &apOld[i], pParent);
    if( rc ) goto balance_cleanup;
    apOld[i]->idxParent = k;
    apCopy[i] = 0;
    assert( i==nOld );
    nOld++;
    nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow;
  }

  /*
  ** Allocate space for memory structures
  */
  apCell = sqliteMallocRaw( 
       nMaxCells*sizeof(u8*)                           /* apCell */
     + nMaxCells*sizeof(int)                           /* szCell */
     + sizeof(MemPage)*NB                              /* aCopy */
     + pBt->psAligned*(5+NB)                           /* aSpace */
     + (ISAUTOVACUUM ? nMaxCells : 0)     /* aFrom */
  );
  if( apCell==0 ){
    rc = SQLITE_NOMEM;
    goto balance_cleanup;
  }
  szCell = (int*)&apCell[nMaxCells];
  aCopy[0] = (u8*)&szCell[nMaxCells];
  for(i=1; i<NB; i++){
    aCopy[i] = &aCopy[i-1][pBt->psAligned+sizeof(MemPage)];
  }
  aSpace = &aCopy[NB-1][pBt->psAligned+sizeof(MemPage)];
#ifndef SQLITE_OMIT_AUTOVACUUM
  if( pBt->autoVacuum ){
    aFrom = &aSpace[5*pBt->psAligned];
  }
#endif
  
  /*
  ** Make copies of the content of pPage and its siblings into aOld[].
  ** The rest of this function will use data from the copies rather
  ** that the original pages since the original pages will be in the
  ** process of being overwritten.
  */
  for(i=0; i<nOld; i++){
................................................................................
  nCell = 0;
  leafCorrection = pPage->leaf*4;
  leafData = pPage->leafData && pPage->leaf;
  for(i=0; i<nOld; i++){
    MemPage *pOld = apCopy[i];
    int limit = pOld->nCell+pOld->nOverflow;
    for(j=0; j<limit; j++){
      assert( nCell<nMaxCells );
      apCell[nCell] = findOverflowCell(pOld, j);
      szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
#ifndef SQLITE_OMIT_AUTOVACUUM
      if( pBt->autoVacuum ){
        int a;
        aFrom[nCell] = i;
        for(a=0; a<pOld->nOverflow; a++){
................................................................................
        ** are duplicates of keys on the child pages.  We need to remove
        ** the divider cells from pParent, but the dividers cells are not
        ** added to apCell[] because they are duplicates of child cells.
        */
        dropCell(pParent, nxDiv, sz);
      }else{
        u8 *pTemp;
        assert( nCell<nMaxCells );
        szCell[nCell] = sz;
        pTemp = &aSpace[iSpace];
        iSpace += sz;
        assert( iSpace<=pBt->psAligned*5 );
        memcpy(pTemp, apDiv[i], sz);
        apCell[nCell] = pTemp+leafCorrection;
#ifndef SQLITE_OMIT_AUTOVACUUM
................................................................................
  **   cntNew[i]: Index in apCell[] and szCell[] for the first cell to
  **              the right of the i-th sibling page.
  ** usableSpace: Number of bytes of space available on each sibling.
  ** 
  */
  usableSpace = pBt->usableSize - 12 + leafCorrection;
  for(subtotal=k=i=0; i<nCell; i++){
    assert( i<nMaxCells );
    subtotal += szCell[i] + 2;
    if( subtotal > usableSpace ){
      szNew[k] = subtotal - szCell[i];
      cntNew[k] = i;
      if( leafData ){ i--; }
      subtotal = 0;
      k++;
................................................................................
    int szRight = szNew[i];  /* Size of sibling on the right */
    int szLeft = szNew[i-1]; /* Size of sibling on the left */
    int r;              /* Index of right-most cell in left sibling */
    int d;              /* Index of first cell to the left of right sibling */

    r = cntNew[i-1] - 1;
    d = r + 1 - leafData;
    assert( d<nMaxCells );
    assert( r<nMaxCells );
    while( szRight==0 || szRight+szCell[d]+2<=szLeft-(szCell[r]+2) ){
      szRight += szCell[d] + 2;
      szLeft -= szCell[r] + 2;
      cntNew[i-1]--;
      r = cntNew[i-1] - 1;
      d = r + 1 - leafData;
    }
................................................................................
  /*
  ** Evenly distribute the data in apCell[] across the new pages.
  ** Insert divider cells into pParent as necessary.
  */
  j = 0;
  for(i=0; i<nNew; i++){
    /* Assemble the new sibling page. */
    assert( j<nMaxCells );
    MemPage *pNew = apNew[i];
    assert( pNew->pgno==pgnoNew[i] );
    assemblePage(pNew, cntNew[i]-j, &apCell[j], &szCell[j]);
    assert( pNew->nCell>0 );
    assert( pNew->nOverflow==0 );

#ifndef SQLITE_OMIT_AUTOVACUUM
................................................................................
    /* If this is an auto-vacuum database, update the pointer map entries
    ** that point to the siblings that were rearranged. These can be: left
    ** children of cells, the right-child of the page, or overflow pages
    ** pointed to by cells.
    */
    if( pBt->autoVacuum ){
      for(k=j; k<cntNew[i]; k++){
        assert( k<nMaxCells );
        if( aFrom[k]==0xFF || apCopy[aFrom[k]]->pgno!=pNew->pgno ){
          rc = ptrmapPutOvfl(pNew, k-j);
          if( rc!=SQLITE_OK ){
            goto balance_cleanup;
          }
        }
      }
................................................................................
    /* If the sibling page assembled above was not the right-most sibling,
    ** insert a divider cell into the parent page.
    */
    if( i<nNew-1 && j<nCell ){
      u8 *pCell;
      u8 *pTemp;
      int sz;

      assert( j<nMaxCells );
      pCell = apCell[j];
      sz = szCell[j] + leafCorrection;
      if( !pNew->leaf ){
        memcpy(&pNew->aData[8], pCell, 4);
        pTemp = 0;
      }else if( leafData ){
	/* If the tree is a leaf-data tree, and the siblings are leaves, 

Changes to src/pager.c.

14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
209
210
211
212
213
214
215
216

217
218






219
220
221
222
223
224
225
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.199 2005/03/28 03:39:56 drh Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

................................................................................
#define DATA_TO_PGHDR(D)  (&((PgHdr*)(D))[-1])
#define PGHDR_TO_EXTRA(G,P) ((void*)&((char*)(&(G)[1]))[(P)->psAligned])
#define PGHDR_TO_HIST(P,PGR)  \
            ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->psAligned+(PGR)->nExtra])

/*
** How big to make the hash table used for locating in-memory pages
** by page number.

*/
#define N_PG_HASH 2048







/*
** Hash a page number
*/
#define pager_hash(PN)  ((PN)&(N_PG_HASH-1))

/*







|







 







|
>

|
>
>
>
>
>
>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
...
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
** The pager is used to access a database disk file.  It implements
** atomic commit and rollback through the use of a journal file that
** is separate from the database file.  The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
**
** @(#) $Id: pager.c,v 1.200 2005/03/28 08:44:07 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include "pager.h"
#include <assert.h>
#include <string.h>

................................................................................
#define DATA_TO_PGHDR(D)  (&((PgHdr*)(D))[-1])
#define PGHDR_TO_EXTRA(G,P) ((void*)&((char*)(&(G)[1]))[(P)->psAligned])
#define PGHDR_TO_HIST(P,PGR)  \
            ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->psAligned+(PGR)->nExtra])

/*
** How big to make the hash table used for locating in-memory pages
** by page number. This macro looks a little silly, but is evaluated
** at compile-time, not run-time (at least for gcc this is true).
*/
#define N_PG_HASH (\
  (MAX_PAGES>1024)?2048: \
  (MAX_PAGES>512)?1024: \
  (MAX_PAGES>256)?512: \
  (MAX_PAGES>128)?256: \
  (MAX_PAGES>64)?128:64 \
)

/*
** Hash a page number
*/
#define pager_hash(PN)  ((PN)&(N_PG_HASH-1))

/*

Changes to src/vdbeaux.c.

53
54
55
56
57
58
59


60
61
62




63
64
65
66
67
68
69
70
...
162
163
164
165
166
167
168




169
170
171

172
173
174
175










176
177
178
179
180
181

182
183
184
185
186
187
188
...
598
599
600
601
602
603
604







605
606
607
608
609
610
611
612
613

614

615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
*/
void sqlite3VdbeTrace(Vdbe *p, FILE *trace){
  p->trace = trace;
}

/*
** Resize the Vdbe.aOp array so that it contains at least N


** elements.
*/
static void resizeOpArray(Vdbe *p, int N){




  if( p->nOpAlloc<N ){
    int oldSize = p->nOpAlloc;
    p->nOpAlloc = N+100;
    p->aOp = sqliteRealloc(p->aOp, p->nOpAlloc*sizeof(Op));
    if( p->aOp ){
      memset(&p->aOp[oldSize], 0, (p->nOpAlloc-oldSize)*sizeof(Op));
    }
  }
................................................................................

/*
** Loop through the program looking for P2 values that are negative.
** Each such value is a label.  Resolve the label by setting the P2
** value to its correct non-zero value.
**
** This routine is called once after all opcodes have been inserted.




*/
static void resolveP2Values(Vdbe *p){
  int i;

  Op *pOp;
  int *aLabel = p->aLabel;
  if( aLabel==0 ) return;
  for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){










    if( pOp->p2>=0 ) continue;
    assert( -1-pOp->p2<p->nLabel );
    pOp->p2 = aLabel[-1-pOp->p2];
  }
  sqliteFree(p->aLabel);
  p->aLabel = 0;

}

/*
** Return the address of the next instruction to be inserted.
*/
int sqlite3VdbeCurrentAddr(Vdbe *p){
  assert( p->magic==VDBE_MAGIC_INIT );
................................................................................

  assert( p!=0 );
  assert( p->magic==VDBE_MAGIC_INIT );

  /* There should be at least one opcode.
  */
  assert( p->nOp>0 );








  /* No instruction ever pushes more than a single element onto the
  ** stack.  And the stack never grows on successive executions of the
  ** same loop.  So the total number of instructions is an upper bound
  ** on the maximum stack depth required.
  **
  ** Allocation all the stack space we will ever need.
  */
  if( p->aStack==0 ){

    resolveP2Values(p);

    assert( nVar>=0 );
    n = isExplain ? 10 : p->nOp;
    p->aStack = sqliteMalloc(
        n*sizeof(p->aStack[0])         /* aStack */
      + n*sizeof(Mem*)                 /* apArg */
      + nVar*sizeof(Mem)               /* aVar */
      + nVar*sizeof(char*)             /* azVar */
      + nMem*sizeof(Mem)               /* aMem */
      + nCursor*sizeof(Cursor*)        /* apCsr */
      + nAgg*sizeof(Agg)               /* Aggregate contexts */
    );
    if( !sqlite3_malloc_failed ){
      p->aMem = &p->aStack[n];
      p->nMem = nMem;
      p->aVar = &p->aMem[nMem];
      p->nVar = nVar;
      p->okVar = 0;
      p->apArg = (Mem**)&p->aVar[nVar];
      p->azVar = (char**)&p->apArg[n];
      p->apCsr = (Cursor**)&p->azVar[nVar];
      if( nAgg>0 ){
        p->nAgg = nAgg;
        p->apAgg = (Agg*)&p->apCsr[nCursor];
      }
      p->nCursor = nCursor;
      for(n=0; n<nVar; n++){







>
>



>
>
>
>
|







 







>
>
>
>

|

>




>
>
>
>
>
>
>
>
>
>






>







 







>
>
>
>
>
>
>









>
|
>




|













|







53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
...
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
...
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
*/
void sqlite3VdbeTrace(Vdbe *p, FILE *trace){
  p->trace = trace;
}

/*
** Resize the Vdbe.aOp array so that it contains at least N
** elements. If the Vdbe is in VDBE_MAGIC_RUN state, then
** the Vdbe.aOp array will be sized to contain exactly N 
** elements.
*/
static void resizeOpArray(Vdbe *p, int N){
  if( p->magic==VDBE_MAGIC_RUN ){
    assert( N==p->nOp );
    p->nOpAlloc = N;
    p->aOp = sqliteRealloc(p->aOp, N*sizeof(Op));
  }else if( p->nOpAlloc<N ){
    int oldSize = p->nOpAlloc;
    p->nOpAlloc = N+100;
    p->aOp = sqliteRealloc(p->aOp, p->nOpAlloc*sizeof(Op));
    if( p->aOp ){
      memset(&p->aOp[oldSize], 0, (p->nOpAlloc-oldSize)*sizeof(Op));
    }
  }
................................................................................

/*
** Loop through the program looking for P2 values that are negative.
** Each such value is a label.  Resolve the label by setting the P2
** value to its correct non-zero value.
**
** This routine is called once after all opcodes have been inserted.
**
** Variable *pMaxFuncArgs is set to the maximum value of any P1 argument 
** to an OP_Function or P2 to an OP_AggFunc opcode. This is used by 
** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.
*/
static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
  int i;
  int nMax = 0;
  Op *pOp;
  int *aLabel = p->aLabel;
  if( aLabel==0 ) return;
  for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
    u8 opcode = pOp->opcode;

    /* Todo: Maybe OP_AggFunc should change to use P1 in the same
     * way as OP_Function. */
    if( opcode==OP_Function ){
      if( pOp->p1>nMax ) nMax = pOp->p1;
    }else if( opcode==OP_AggFunc ){
      if( pOp->p2>nMax ) nMax = pOp->p2;
    }

    if( pOp->p2>=0 ) continue;
    assert( -1-pOp->p2<p->nLabel );
    pOp->p2 = aLabel[-1-pOp->p2];
  }
  sqliteFree(p->aLabel);
  p->aLabel = 0;
  *pMaxFuncArgs = nMax;
}

/*
** Return the address of the next instruction to be inserted.
*/
int sqlite3VdbeCurrentAddr(Vdbe *p){
  assert( p->magic==VDBE_MAGIC_INIT );
................................................................................

  assert( p!=0 );
  assert( p->magic==VDBE_MAGIC_INIT );

  /* There should be at least one opcode.
  */
  assert( p->nOp>0 );

  /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. This
   * is because the call to resizeOpArray() below may shrink the
   * p->aOp[] array to save memory if called when in VDBE_MAGIC_RUN 
   * state.
   */
  p->magic = VDBE_MAGIC_RUN;

  /* No instruction ever pushes more than a single element onto the
  ** stack.  And the stack never grows on successive executions of the
  ** same loop.  So the total number of instructions is an upper bound
  ** on the maximum stack depth required.
  **
  ** Allocation all the stack space we will ever need.
  */
  if( p->aStack==0 ){
    int nArg;       /* Maximum number of args passed to a user function. */
    resolveP2Values(p, &nArg);
    resizeOpArray(p, p->nOp);
    assert( nVar>=0 );
    n = isExplain ? 10 : p->nOp;
    p->aStack = sqliteMalloc(
        n*sizeof(p->aStack[0])         /* aStack */
      + nArg*sizeof(Mem*)              /* apArg */
      + nVar*sizeof(Mem)               /* aVar */
      + nVar*sizeof(char*)             /* azVar */
      + nMem*sizeof(Mem)               /* aMem */
      + nCursor*sizeof(Cursor*)        /* apCsr */
      + nAgg*sizeof(Agg)               /* Aggregate contexts */
    );
    if( !sqlite3_malloc_failed ){
      p->aMem = &p->aStack[n];
      p->nMem = nMem;
      p->aVar = &p->aMem[nMem];
      p->nVar = nVar;
      p->okVar = 0;
      p->apArg = (Mem**)&p->aVar[nVar];
      p->azVar = (char**)&p->apArg[nArg];
      p->apCsr = (Cursor**)&p->azVar[nVar];
      if( nAgg>0 ){
        p->nAgg = nAgg;
        p->apAgg = (Agg*)&p->apCsr[nCursor];
      }
      p->nCursor = nCursor;
      for(n=0; n<nVar; n++){

Changes to tool/memleak3.tcl.

21
22
23
24
25
26
27




28
29
30
31
32






33

34
35
36
37
38
39
40
41
42






















43









44
45
46
47
48
49
50
51
..
63
64
65
66
67
68
69











70
71
72
73
74
75
76
77
78
79
80



81
82


83


84

85









86
87
88


89
90
91
92
93
94
95


96
97
98
99
100
101
102
103
104
105
106
107
108








































application binary (or interpreter) and the second argument the name of the
text file that contains the collected stderr output.

If all goes well a summary of unfreed allocations is printed out. If the
GNU C library is in use and SQLITE_DEBUG is 3 or greater a stack trace is
printed out for each unmatched allocation.





Example:

$ ./testfixture ../sqlite/test/select1.test 2> memtrace.out
$ tclsh $argv0 ./testfixture memtrace.out
"








# If stack traces are enabled, the 'addr2line' program is called to
# translate a binary stack address into a human-readable form.
set addr2line addr2line

if { [llength $argv]!=2 } {
  puts "Usage: $argv0 <binary file> <mem trace file>"
  puts ""
  puts [string trim $doco]
  exit -1






















}











proc process_input {input_file array_name} {
  upvar $array_name mem 
  set input [open $input_file]

  set MALLOC {([[:digit:]]+) malloc ([[:digit:]]+) bytes at 0x([[:xdigit:]]+)}
  # set STACK {^[[:digit:]]+: STACK: (.*)$}
................................................................................

    } elseif { [regexp $MALLOC $line dummy mallocid bytes addr]  } {
      # If this is a 'malloc' line, set an entry in the mem array. Each entry
      # is a list of length three, the number of bytes allocated , the malloc
      # number and the stack dump when it was allocated.
      set mem($addr) [list $bytes "malloc $mallocid" $stack]
      set stack ""












    } elseif { [regexp $FREE $line dummy bytes addr] } {
      # If this is a 'free' line, remove the entry from the mem array. If the 
      # entry does not exist, or is the wrong number of bytes, announce a
      # problem. This is more likely a bug in the regular expressions for
      # this script than an SQLite defect.
      if { [lindex $mem($addr) 0] != $bytes } {
        error "byte count mismatch"
      }
      unset mem($addr) 




    } elseif { [regexp $REALLOC $line dummy mallocid ob b oa a] } {
      # If it is a realloc line, remove the old mem entry and add a new one.


      unset mem($oa);


      set mem($a) [list $b "realloc $mallocid" $stack]

      set stack ""









    } else {
      # puts "REJECT: $line"
    }


  }

  close $input
}

process_input [lindex $argv 1] mem
set exe [lindex $argv 0]



foreach key [array names mem] {
  set bytes [lindex $mem($key) 0]
  set mallocid [lindex $mem($key) 1]
  set stack [lindex $mem($key) 2]
  puts "Leaked $bytes bytes at 0x$key: $mallocid"
  foreach frame [lrange $stack 1 10] {
    foreach {f l} [split [exec $addr2line -f --exe=$exe $frame] \n] {}
    puts [format "%-30s %s" $f $l]
  }
  if {[llength $stack]>0 } {puts ""}
}
















































>
>
>
>



|

>
>
>
>
>
>
|
>




<
<
<
<
<
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
|







 







>
>
>
>
>
>
>
>
>
>
>











>
>
>

<
>
>

>
>

>

>
>
>
>
>
>
>
>
>



>
>





|
|
>
>
|
<
<
<
<
<
|
|





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48





49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
...
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164





165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
application binary (or interpreter) and the second argument the name of the
text file that contains the collected stderr output.

If all goes well a summary of unfreed allocations is printed out. If the
GNU C library is in use and SQLITE_DEBUG is 3 or greater a stack trace is
printed out for each unmatched allocation.

If the \"-r <n>\" option is passed, then the program stops and prints out
the state of the heap immediately after the <n>th call to malloc() or
realloc().

Example:

$ ./testfixture ../sqlite/test/select1.test 2> memtrace.out
$ tclsh $argv0 ?-r <malloc-number>? ./testfixture memtrace.out
"
if { [llength $argv]!=2 && [llength $argv]!=4 } {
  set prg [file tail $argv0]
  puts "Usage: $prg ?-r <malloc-number>? <binary file> <mem trace file>"
  puts ""
  puts [string trim $doco]
  exit -1
}

# If stack traces are enabled, the 'addr2line' program is called to
# translate a binary stack address into a human-readable form.
set addr2line addr2line






# When the SQLITE_MEMDEBUG is set as described above, SQLite prints
# out a line for each malloc(), realloc() or free() call that the
# library makes. If SQLITE_MEMDEBUG is 3, then a stack trace is printed
# out before each malloc() and realloc() line.
#
# This program parses each line the SQLite library outputs and updates
# the following global Tcl variables to reflect the "current" state of
# the heap used by SQLite.
#
set nBytes 0               ;# Total number of bytes currently allocated.
set nMalloc 0              ;# Total number of malloc()/realloc() calls.
set nPeak 0                ;# Peak of nBytes.
set iPeak 0                ;# nMalloc when nPeak was set.
#
# More detailed state information is stored in the $memmap array. 
# Each key in the memmap array is the address of a chunk of memory
# currently allocated from the heap. The value is a list of the 
# following form
# 
#     {<number-of-bytes> <malloc id> <stack trace>}
#
array unset memmap

# The executable program being analyzed.
if {[llength $argv]==2} {
  set exe [lindex $argv 0]
  set memfile [lindex $argv 1]
  set report_at -1
} else {
  set exe [lindex $argv 2]
  set memfile [lindex $argv 3]
  set report_at [lindex $argv 1]
}

proc process_input {input_file array_name} {
  upvar $array_name mem 
  set input [open $input_file]

  set MALLOC {([[:digit:]]+) malloc ([[:digit:]]+) bytes at 0x([[:xdigit:]]+)}
  # set STACK {^[[:digit:]]+: STACK: (.*)$}
................................................................................

    } elseif { [regexp $MALLOC $line dummy mallocid bytes addr]  } {
      # If this is a 'malloc' line, set an entry in the mem array. Each entry
      # is a list of length three, the number of bytes allocated , the malloc
      # number and the stack dump when it was allocated.
      set mem($addr) [list $bytes "malloc $mallocid" $stack]
      set stack ""

      # Increase the current heap usage
      incr ::nBytes $bytes

      # Increase the number of malloc() calls
      incr ::nMalloc

      if {$::nBytes > $::nPeak} {
        set ::nPeak $::nBytes
        set ::iPeak $::nMalloc
      }

    } elseif { [regexp $FREE $line dummy bytes addr] } {
      # If this is a 'free' line, remove the entry from the mem array. If the 
      # entry does not exist, or is the wrong number of bytes, announce a
      # problem. This is more likely a bug in the regular expressions for
      # this script than an SQLite defect.
      if { [lindex $mem($addr) 0] != $bytes } {
        error "byte count mismatch"
      }
      unset mem($addr) 

      # Decrease the current heap usage
      incr ::nBytes [expr -1 * $bytes]

    } elseif { [regexp $REALLOC $line dummy mallocid ob b oa a] } {

      # "free" the old allocation in the internal model:
      incr ::nBytes [expr -1 * $ob]
      unset mem($oa);

      # "malloc" the new allocation
      set mem($a) [list $b "realloc $mallocid" $stack]
      incr ::nBytes $b
      set stack ""

      # Increase the number of malloc() calls
      incr ::nMalloc

      if {$::nBytes > $::nPeak} {
        set ::nPeak $::nBytes
        set ::iPeak $::nMalloc
      }

    } else {
      # puts "REJECT: $line"
    }

    if {$::nMalloc==$::report_at} report
  }

  close $input
}

proc printstack {stack} {
  set fcount 10
  if {[llength $stack]<10} {
    set fcount [llength $stack]
  }





  foreach frame [lrange $stack 1 $fcount] {
    foreach {f l} [split [exec $::addr2line -f --exe=$::exe $frame] \n] {}
    puts [format "%-30s %s" $f $l]
  }
  if {[llength $stack]>0 } {puts ""}
}

proc report {} {

  foreach key [array names ::memmap] {
    set stack [lindex $::memmap($key) 2]
    set bytes [lindex $::memmap($key) 0]
    lappend summarymap($stack) $bytes
  }

  foreach stack [array names summarymap] {
    set allocs $summarymap($stack)
    set sum 0
    foreach a $allocs {
      incr sum $a
    }
    lappend sorted [list $sum $stack]
  }

  set sorted [lsort -integer -index 0 $sorted]
  foreach s $sorted {
    set sum [lindex $s 0]
    set stack [lindex $s 1]
    set allocs $summarymap($stack)
    puts "$sum bytes in [llength $allocs] chunks ($allocs)"
    printstack $stack
  }

  # Print out summary statistics
  puts "Total allocations            : $::nMalloc"
  puts "Total outstanding allocations: [array size ::memmap]" 
  puts "Current heap usage           : $::nBytes bytes"
  puts "Peak heap usage              : $::nPeak bytes (malloc #$::iPeak)"

  exit
}

process_input $memfile memmap
report