/ Check-in [6f5eb74f]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Keep correct track of the amount of outstanding memory even when the system memory allocator returns a different number of bytes than requested. Ticket #1660. (CVS 3057)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 6f5eb74fd93a78ccdc2ddee0936ad4c5b2813720
User & Date: drh 2006-02-06 21:22:31
Context
2006-02-06
21:34
Add "autoinc" and "collseq" columns to the table_info() pragma. (CVS 3058) check-in: 7940a590 user: drh tags: trunk
21:22
Keep correct track of the amount of outstanding memory even when the system memory allocator returns a different number of bytes than requested. Ticket #1660. (CVS 3057) check-in: 6f5eb74f user: drh tags: trunk
13:59
Update the per-thread bytes allocated counter with the number of bytes actually allocated, not the number requested. Ticket #1660. (CVS 3056) check-in: 3f0a0ff1 user: danielk1977 tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/util.c.

10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
...
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
...
401
402
403
404
405
406
407

408
409
410
411
412
413
414
...
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
...
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
...
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542

543
544
545
546
547
548
























549
550
551
552
553
554
555
556
557
558
559
560
561
562

563
564
565
566
567
568
569
570
571
572
573
574
575
576
577


578
579
580
581
582
583
584
585
586
587
588







589
590
591
592
593

594
595
596

597
598


599
600
601

602
603
604
605
606
607
608
609
610

611
612
613
614
615
616
617
**
*************************************************************************
** Utility functions used throughout sqlite.
**
** This file contains functions for allocating memory, comparing
** strings, and stuff like that.
**
** $Id: util.c,v 1.183 2006/02/06 13:59:43 danielk1977 Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <stdarg.h>
#include <ctype.h>

/*
................................................................................
** Size reserved for storing the user string. Each time a Malloc() or Realloc()
** call succeeds, up to TESTALLOC_USERSIZE bytes of the string pointed to by
** sqlite3_malloc_id are stored along with the other test system metadata.
*/
#define TESTALLOC_USERSIZE 64
const char *sqlite3_malloc_id = 0;

/*
** Always allocate blocks to be a multiple of the following size in bytes.
** For example, if TESTALLOC_QUANTA is 8 and a block of 21 bytes is 
** requested, return a pointer to a block of 24 bytes.
*/
#define TESTALLOC_QUANTA 8

/*
** Blocks used by the test layer have the following format:
**
**        <sizeof(void *) pNext pointer>
**        <sizeof(void *) pPrev pointer>
**        <TESTALLOC_NGUARD 32-bit guard words>
**            <The application level allocation>
................................................................................
*/
#if defined(TCLSH) && defined(SQLITE_DEBUG) && SQLITE_MEMDEBUG>1
#include <tcl.h>
int sqlite3OutstandingMallocs(Tcl_Interp *interp){
  void *p;
  Tcl_Obj *pRes = Tcl_NewObj();
  Tcl_IncrRefCount(pRes);


  for(p=sqlite3_pFirst; p; p=((void **)p)[1]){
    Tcl_Obj *pEntry = Tcl_NewObj();
    Tcl_Obj *pStack = Tcl_NewObj();
    char *z;
    u32 iLine;
    int nBytes = sqlite3OsAllocationSize(p) - TESTALLOC_OVERHEAD;
................................................................................
*/
static void * OSMALLOC(int n){
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
  sqlite3_nMaxAlloc = 
      MAX(sqlite3_nMaxAlloc, sqlite3ThreadDataReadOnly()->nAlloc);
#endif
  assert( !sqlite3_mallocDisallowed );
  n += (TESTALLOC_QUANTA - (n % TESTALLOC_QUANTA)) % TESTALLOC_QUANTA;
  if( !sqlite3TestMallocFail() ){
    u32 *p;
    p = (u32 *)sqlite3OsMalloc(n + TESTALLOC_OVERHEAD);
    assert(p);
    sqlite3_nMalloc++;
    applyGuards(p);
    linkAlloc(p);
................................................................................
** This is the test layer's wrapper around sqlite3OsRealloc().
*/
static void * OSREALLOC(void *pRealloc, int n){
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
  sqlite3_nMaxAlloc = 
      MAX(sqlite3_nMaxAlloc, sqlite3ThreadDataReadOnly()->nAlloc);
#endif
  n += (TESTALLOC_QUANTA - (n % TESTALLOC_QUANTA)) % TESTALLOC_QUANTA;
  assert( !sqlite3_mallocDisallowed );
  if( !sqlite3TestMallocFail() ){
    u32 *p = 0;
    if( pRealloc ){
      u32 *p = (u32 *)getOsPointer(pRealloc);
      checkGuards(p);
    }
    p = sqlite3OsRealloc(p, n + TESTALLOC_OVERHEAD);
    applyGuards(p);
    relinkAlloc(p);
    return (void *)(&p[TESTALLOC_NGUARD + 2*sizeof(void *)/sizeof(u32)]);
  }
  return 0;
}
................................................................................

#endif  /* SQLITE_MEMDEBUG */
/*
** End code for memory allocation system test layer.
**--------------------------------------------------------------------------*/

/*
** The handleSoftLimit() function is called before each call to 
** sqlite3OsMalloc() or xRealloc(). The parameter 'n' is the number of
** extra bytes about to be allocated (for Realloc() this means the size of the
** new allocation less the size of the old allocation). If the extra allocation
** means that the total memory allocated to SQLite in this thread would exceed
** the limit set by sqlite3_soft_heap_limit(), then sqlite3_release_memory() is
** called to try to avoid this. No indication of whether or not this is
** successful is returned to the caller.

**
** If SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined, this routine is
** a no-op
*/
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
static int handleSoftLimit(int n){
























  ThreadData *pTsd = sqlite3ThreadData();
  if( pTsd ){
    pTsd->nAlloc += n;
    assert( pTsd->nAlloc>=0 );
    if( n>0 && pTsd->nSoftHeapLimit>0 ){
      while( pTsd->nAlloc>pTsd->nSoftHeapLimit && sqlite3_release_memory(n) );
    }else if( pTsd->nAlloc==0 && pTsd->nSoftHeapLimit==0 ){
      sqlite3ReleaseThreadData();
    }
  }
  return (pTsd ? 0 : 1);
}
#else
#define handleSoftLimit(x) 0

#endif

/*
** Allocate and return N bytes of uninitialised memory by calling
** sqlite3OsMalloc(). If the Malloc() call fails, attempt to free memory 
** by calling sqlite3_release_memory().
*/
void *sqlite3MallocRaw(int n){
  void *p = 0;
  if( n>0 && !sqlite3MallocFailed() ){
    while( (p = OSMALLOC(n))==0 && sqlite3_release_memory(n) );
    if( !p || handleSoftLimit(OSSIZEOF(p)) ){
      OSFREE(p);
      sqlite3FailedMalloc();
      OSMALLOC_FAILED();


    }
  }
  return p;
}

/*
** Resize the allocation at p to n bytes by calling sqlite3OsRealloc(). The
** pointer to the new allocation is returned.  If the Realloc() call fails,
** attempt to free memory by calling sqlite3_release_memory().
*/
void *sqlite3Realloc(void *p, int n){







  void *np = 0;
  if( !sqlite3MallocFailed() ){
#ifndef SQLITE_ENABLE_MEMORY_MANAGEMENT
    int oldsize = OSSIZEOF(p);
#endif

    while( (np = OSREALLOC(p, n))==0 && sqlite3_release_memory(n) );
    if( !np || handleSoftLimit(OSSIZEOF(np) - oldsize) ){
      OSFREE(np);

      sqlite3FailedMalloc();
      OSMALLOC_FAILED();


    }
  }
  return np;

}

/*
** Free the memory pointed to by p. p must be either a NULL pointer or a 
** value returned by a previous call to sqlite3Malloc() or sqlite3Realloc().
*/
void sqlite3FreeX(void *p){
  (void)handleSoftLimit(0 - OSSIZEOF(p));
  if( p ){

    OSFREE(p);
  }
}

/*
** A version of sqliteMalloc() that is always a function, not a macro.
** Currently, this is used only to alloc to allocate the parser engine.







|







 







<
<
<
<
<
<
<







 







>







 







<







 







<


<
<
|
|
<







 







|
|
|
|
|
|
|
|
>



|

|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




<
<
|



<


<
>









|

|
<


>
>











>
>
>
>
>
>
>
|
<
|
|

>
|
<
<
>
|
|
>
>
|
|
|
>







<

>







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
...
138
139
140
141
142
143
144







145
146
147
148
149
150
151
...
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
...
444
445
446
447
448
449
450

451
452
453
454
455
456
457
...
485
486
487
488
489
490
491

492
493


494
495

496
497
498
499
500
501
502
...
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566


567
568
569
570

571
572

573
574
575
576
577
578
579
580
581
582
583
584
585

586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608

609
610
611
612
613


614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629

630
631
632
633
634
635
636
637
638
**
*************************************************************************
** Utility functions used throughout sqlite.
**
** This file contains functions for allocating memory, comparing
** strings, and stuff like that.
**
** $Id: util.c,v 1.184 2006/02/06 21:22:31 drh Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <stdarg.h>
#include <ctype.h>

/*
................................................................................
** Size reserved for storing the user string. Each time a Malloc() or Realloc()
** call succeeds, up to TESTALLOC_USERSIZE bytes of the string pointed to by
** sqlite3_malloc_id are stored along with the other test system metadata.
*/
#define TESTALLOC_USERSIZE 64
const char *sqlite3_malloc_id = 0;








/*
** Blocks used by the test layer have the following format:
**
**        <sizeof(void *) pNext pointer>
**        <sizeof(void *) pPrev pointer>
**        <TESTALLOC_NGUARD 32-bit guard words>
**            <The application level allocation>
................................................................................
*/
#if defined(TCLSH) && defined(SQLITE_DEBUG) && SQLITE_MEMDEBUG>1
#include <tcl.h>
int sqlite3OutstandingMallocs(Tcl_Interp *interp){
  void *p;
  Tcl_Obj *pRes = Tcl_NewObj();
  Tcl_IncrRefCount(pRes);


  for(p=sqlite3_pFirst; p; p=((void **)p)[1]){
    Tcl_Obj *pEntry = Tcl_NewObj();
    Tcl_Obj *pStack = Tcl_NewObj();
    char *z;
    u32 iLine;
    int nBytes = sqlite3OsAllocationSize(p) - TESTALLOC_OVERHEAD;
................................................................................
*/
static void * OSMALLOC(int n){
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
  sqlite3_nMaxAlloc = 
      MAX(sqlite3_nMaxAlloc, sqlite3ThreadDataReadOnly()->nAlloc);
#endif
  assert( !sqlite3_mallocDisallowed );

  if( !sqlite3TestMallocFail() ){
    u32 *p;
    p = (u32 *)sqlite3OsMalloc(n + TESTALLOC_OVERHEAD);
    assert(p);
    sqlite3_nMalloc++;
    applyGuards(p);
    linkAlloc(p);
................................................................................
** This is the test layer's wrapper around sqlite3OsRealloc().
*/
static void * OSREALLOC(void *pRealloc, int n){
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
  sqlite3_nMaxAlloc = 
      MAX(sqlite3_nMaxAlloc, sqlite3ThreadDataReadOnly()->nAlloc);
#endif

  assert( !sqlite3_mallocDisallowed );
  if( !sqlite3TestMallocFail() ){


    u32 *p = (u32 *)getOsPointer(pRealloc);
    checkGuards(p);

    p = sqlite3OsRealloc(p, n + TESTALLOC_OVERHEAD);
    applyGuards(p);
    relinkAlloc(p);
    return (void *)(&p[TESTALLOC_NGUARD + 2*sizeof(void *)/sizeof(u32)]);
  }
  return 0;
}
................................................................................

#endif  /* SQLITE_MEMDEBUG */
/*
** End code for memory allocation system test layer.
**--------------------------------------------------------------------------*/

/*
** This routine is called when we are about to allocate n additional bytes
** of memory.  If the new allocation will put is over the soft allocation
** limit, then invoke sqlite3_release_memory() to try to release some
** memory before continuing with the allocation.
**
** This routine also makes sure that the thread-specific-data (TSD) has
** be allocated.  If it has not and can not be allocated, then return
** false.  The updateMemoryUsedCount() routine below will deallocate
** the TSD if it ought to be.
**
** If SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined, this routine is
** a no-op
*/ 
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
static int enforceSoftLimit(int n){
  ThreadData *pTsd = sqlite3ThreadData();
  if( pTsd==0 ){
    return 0;
  }
  assert( pTsd->nAlloc>=0 );
  if( n>0 && pTsd->nSoftHeapLimit>0 ){
    while( pTsd->nAlloc+n>pTsd->nSoftHeapLimit && sqlite3_release_memory(n) );
  }
  return 1;
}
#else
# define enforceSoftLimit(X)  1
#endif

/*
** Update the count of total outstanding memory that is held in
** thread-specific-data (TSD).  If after this update the TSD is
** no longer being used, then deallocate it.
**
** If SQLITE_ENABLE_MEMORY_MANAGEMENT is not defined, this routine is
** a no-op
*/
#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
static void updateMemoryUsedCount(int n){
  ThreadData *pTsd = sqlite3ThreadData();
  if( pTsd ){
    pTsd->nAlloc += n;
    assert( pTsd->nAlloc>=0 );


    if( pTsd->nAlloc==0 && pTsd->nSoftHeapLimit==0 ){
      sqlite3ReleaseThreadData();
    }
  }

}
#else

#define updateMemoryUsedCount(x)  /* no-op */
#endif

/*
** Allocate and return N bytes of uninitialised memory by calling
** sqlite3OsMalloc(). If the Malloc() call fails, attempt to free memory 
** by calling sqlite3_release_memory().
*/
void *sqlite3MallocRaw(int n){
  void *p = 0;
  if( n>0 && !sqlite3MallocFailed() && enforceSoftLimit(n) ){
    while( (p = OSMALLOC(n))==0 && sqlite3_release_memory(n) );
    if( !p ){

      sqlite3FailedMalloc();
      OSMALLOC_FAILED();
    }else{
      updateMemoryUsedCount(OSSIZEOF(p));
    }
  }
  return p;
}

/*
** Resize the allocation at p to n bytes by calling sqlite3OsRealloc(). The
** pointer to the new allocation is returned.  If the Realloc() call fails,
** attempt to free memory by calling sqlite3_release_memory().
*/
void *sqlite3Realloc(void *p, int n){
  if( sqlite3MallocFailed() ){
    return 0;
  }

  if( !p ){
    return sqlite3Malloc(n);
  }else{
    void *np = 0;

#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
    int origSize = OSSIZEOF(p);
#endif
    if( enforceSoftLimit(n - origSize) ){
      while( (np = OSREALLOC(p, n))==0 && sqlite3_release_memory(n) );


      if( !np ){
        sqlite3FailedMalloc();
        OSMALLOC_FAILED();
      }else{
        updateMemoryUsedCount(OSSIZEOF(np) - origSize);
      }
    }
    return np;
  }
}

/*
** Free the memory pointed to by p. p must be either a NULL pointer or a 
** value returned by a previous call to sqlite3Malloc() or sqlite3Realloc().
*/
void sqlite3FreeX(void *p){

  if( p ){
    updateMemoryUsedCount(0 - OSSIZEOF(p));
    OSFREE(p);
  }
}

/*
** A version of sqliteMalloc() that is always a function, not a macro.
** Currently, this is used only to alloc to allocate the parser engine.