/ Changes On Branch branch-3.5.9
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch branch-3.5.9 Excluding Merge-Ins

This is equivalent to a diff from b6129f4c to bb18f578

2010-01-30
23:08
The iInUseDB and iInUseMM variables do not need to be volatile. Leaf check-in: bb18f578 user: drh tags: branch-3.5.9
22:28
Always hold the MEM2 mutex when initially marking a pager as in use by its database connection. check-in: 622c1717 user: drh tags: branch-3.5.9
2010-01-29
21:23
Fix a missing mutex on page cache truncation during vacuum and auto-vacuum when SQLITE_ENABLE_MEMORY_MANAGEMENT is engaged. check-in: ea3b941a user: drh tags: branch-3.5.9
19:46
Changes to 3.5.9 proposed by Rob Stoddard. check-in: 5ed168c4 user: drh tags: branch-3.5.9
2008-05-15
08:34
Do not write pages to disk to free memory after an IO error occurs. (CVS 5132) check-in: 10ea8287 user: danielk1977 tags: trunk
2008-05-14
16:18
Version 3.5.9 (CVS 5131) check-in: b6129f4c user: drh tags: trunk
2008-05-13
19:41
On instruction from DRH, only do malloc failure tests for O/S ops on non-Windows systems. Better test fixture code will be introduced in 3.6.0 to add this coverage back in for Windows. (CVS 5130) check-in: e4aab150 user: shane tags: trunk

Changes to src/pager.c.

   397    397     void *pCodecArg;            /* First argument to xCodec() */
   398    398   #endif
   399    399     int nHash;                  /* Size of the pager hash table */
   400    400     PgHdr **aHash;              /* Hash table to map page number to PgHdr */
   401    401   #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
   402    402     Pager *pNext;               /* Doubly linked list of pagers on which */
   403    403     Pager *pPrev;               /* sqlite3_release_memory() will work */
   404         -  int iInUseMM;               /* Non-zero if unavailable to MM */
   405    404     int iInUseDB;               /* Non-zero if in sqlite3_release_memory() */
          405  +  u8 iInUseMM;                /* Non-zero if unavailable to MM */
          406  +  u8 onPagerList;             /* True if part of the sqlite3PagerList */
   406    407   #endif
   407    408     char *pTmpSpace;            /* Pager.pageSize bytes of space for tmp use */
   408    409     char dbFileVers[16];        /* Changes whenever database file changes */
   409    410   };
   410    411   
   411    412   /*
   412    413   ** The following global variables hold counters used for
................................................................................
   507    508   ** a mutex on each pager.  The mutex is recursive.
   508    509   **
   509    510   ** This is a special-purpose mutex.  It only provides mutual exclusion
   510    511   ** between the Btree and the Memory Management sqlite3_release_memory()
   511    512   ** function.  It does not prevent, for example, two Btrees from accessing
   512    513   ** the same pager at the same time.  Other general-purpose mutexes in
   513    514   ** the btree layer handle that chore.
          515  +**
          516  +** The pagerMutexHeld(X) macro is for sanity checking.  This macro verifies
          517  +** that the database-connection mutex is held for pager X and asserts if it
          518  +** is not.
   514    519   */
   515    520   #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
   516    521     static void pagerEnter(Pager *p){
   517         -    p->iInUseDB++;
   518         -    if( p->iInUseMM && p->iInUseDB==1 ){
          522  +    if( p->iInUseDB>0 ){
          523  +      p->iInUseDB++;
          524  +    }else{
   519    525   #ifndef SQLITE_MUTEX_NOOP
   520    526         sqlite3_mutex *mutex;
   521    527         mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM2);
   522    528   #endif
   523         -      p->iInUseDB = 0;
   524    529         sqlite3_mutex_enter(mutex);
          530  +      assert( p->iInUseMM==0 );
   525    531         p->iInUseDB = 1;
   526    532         sqlite3_mutex_leave(mutex);
   527    533       }
   528         -    assert( p->iInUseMM==0 );
   529    534     }
   530    535     static void pagerLeave(Pager *p){
   531    536       p->iInUseDB--;
   532    537       assert( p->iInUseDB>=0 );
   533    538     }
          539  +# define pagerMutexHeld(X)  assert( (X)->iInUseDB>0 || !(X)->onPagerList )
   534    540   #else
   535    541   # define pagerEnter(X)
   536    542   # define pagerLeave(X)
          543  +# define pagerMutexHeld(X)
   537    544   #endif
   538    545   
   539    546   /*
   540    547   ** Add page pPg to the end of the linked list managed by structure
   541    548   ** pList (pPg becomes the last entry in the list - the most recently 
   542    549   ** used). Argument pLink should point to either pPg->free or pPg->gfree,
   543    550   ** depending on whether pPg is being added to the pager-specific or
................................................................................
   695    702     if( aHash==0 ){
   696    703       /* Failure to rehash is not an error.  It is only a performance hit. */
   697    704       return;
   698    705     }
   699    706     sqlite3_free(pPager->aHash);
   700    707     pPager->nHash = N;
   701    708     pPager->aHash = aHash;
          709  +  pagerMutexHeld(pPager);
   702    710     for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
   703    711       int h;
   704    712       if( pPg->pgno==0 ){
   705    713         assert( pPg->pNextHash==0 && pPg->pPrevHash==0 );
   706    714         continue;
   707    715       }
   708    716       h = pPg->pgno & (N-1);
................................................................................
  1269   1277   ** sets the state of the pager back to what it was when it was first
  1270   1278   ** opened.  Any outstanding pages are invalidated and subsequent attempts
  1271   1279   ** to access those pages will likely result in a coredump.
  1272   1280   */
  1273   1281   static void pager_reset(Pager *pPager){
  1274   1282     PgHdr *pPg, *pNext;
  1275   1283     if( pPager->errCode ) return;
         1284  +  pagerMutexHeld(pPager);
  1276   1285     for(pPg=pPager->pAll; pPg; pPg=pNext){
  1277   1286       IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno));
  1278   1287       PAGER_INCR(sqlite3_pager_pgfree_count);
  1279   1288       pNext = pPg->pNextAll;
  1280   1289       lruListRemove(pPg);
  1281   1290       sqlite3_free(pPg->pData);
  1282   1291       sqlite3_free(pPg);
................................................................................
  1412   1421         pPager->journalOpen = 0;
  1413   1422         if( rc==SQLITE_OK ){
  1414   1423           rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0);
  1415   1424         }
  1416   1425       }
  1417   1426       sqlite3BitvecDestroy(pPager->pInJournal);
  1418   1427       pPager->pInJournal = 0;
         1428  +    pagerMutexHeld(pPager);
  1419   1429       for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
  1420   1430         pPg->inJournal = 0;
  1421   1431         pPg->dirty = 0;
  1422   1432         pPg->needSync = 0;
  1423   1433         pPg->alwaysRollback = 0;
  1424   1434   #ifdef SQLITE_CHECK_PAGES
  1425   1435         pPg->pageHash = pager_pagehash(pPg);
................................................................................
  2369   2379       pPager->pNext = sqlite3PagerList;
  2370   2380       if( sqlite3PagerList ){
  2371   2381         assert( sqlite3PagerList->pPrev==0 );
  2372   2382         sqlite3PagerList->pPrev = pPager;
  2373   2383       }
  2374   2384       pPager->pPrev = 0;
  2375   2385       sqlite3PagerList = pPager;
         2386  +    pPager->onPagerList = 1;
  2376   2387       sqlite3_mutex_leave(mutex);
  2377   2388     }
  2378   2389   #endif
  2379   2390     return SQLITE_OK;
  2380   2391   }
  2381   2392   
  2382   2393   /*
................................................................................
  2633   2644   ** to zero it and hope that we error out sanely.
  2634   2645   */
  2635   2646   static void pager_truncate_cache(Pager *pPager){
  2636   2647     PgHdr *pPg;
  2637   2648     PgHdr **ppPg;
  2638   2649     int dbSize = pPager->dbSize;
  2639   2650   
         2651  +  pagerMutexHeld(pPager);
  2640   2652     ppPg = &pPager->pAll;
  2641   2653     while( (pPg = *ppPg)!=0 ){
  2642   2654       if( pPg->pgno<=dbSize ){
  2643   2655         ppPg = &pPg->pNextAll;
  2644   2656       }else if( pPg->nRef>0 ){
  2645   2657         memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize);
  2646   2658         ppPg = &pPg->pNextAll;
................................................................................
  2703   2715       return rc;
  2704   2716     }
  2705   2717     if( nPage>=(unsigned)pPager->dbSize ){
  2706   2718       return SQLITE_OK;
  2707   2719     }
  2708   2720     if( MEMDB ){
  2709   2721       pPager->dbSize = nPage;
         2722  +    pagerEnter(pPager);
  2710   2723       pager_truncate_cache(pPager);
         2724  +    pagerLeave(pPager);
  2711   2725       return SQLITE_OK;
  2712   2726     }
  2713   2727     pagerEnter(pPager);
  2714   2728     rc = syncJournal(pPager);
  2715   2729     pagerLeave(pPager);
  2716   2730     if( rc!=SQLITE_OK ){
  2717   2731       return rc;
  2718   2732     }
  2719   2733   
  2720   2734     /* Get an exclusive lock on the database before truncating. */
  2721   2735     pagerEnter(pPager);
  2722   2736     rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
         2737  +  if( rc==SQLITE_OK ){
         2738  +    rc = pager_truncate(pPager, nPage);
         2739  +  }
  2723   2740     pagerLeave(pPager);
  2724         -  if( rc!=SQLITE_OK ){
  2725         -    return rc;
  2726         -  }
  2727         -
  2728         -  rc = pager_truncate(pPager, nPage);
  2729   2741     return rc;
  2730   2742   }
  2731   2743   
  2732   2744   /*
  2733   2745   ** Shutdown the page cache.  Free all memory and close all files.
  2734   2746   **
  2735   2747   ** If a transaction was in progress when this routine is called, that
................................................................................
  2740   2752   **
  2741   2753   ** This function always succeeds. If a transaction is active an attempt
  2742   2754   ** is made to roll it back. If an error occurs during the rollback 
  2743   2755   ** a hot journal may be left in the filesystem but no error is returned
  2744   2756   ** to the caller.
  2745   2757   */
  2746   2758   int sqlite3PagerClose(Pager *pPager){
         2759  +  pagerEnter(pPager);
         2760  +  pPager->errCode = 0;
         2761  +  pager_reset(pPager);
         2762  +  pagerLeave(pPager);
  2747   2763   #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
  2748   2764     if( !MEMDB ){
  2749   2765   #ifndef SQLITE_MUTEX_NOOP
  2750   2766       sqlite3_mutex *mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM2);
  2751   2767   #endif
  2752   2768       sqlite3_mutex_enter(mutex);
  2753   2769       if( pPager->pPrev ){
................................................................................
  2755   2771       }else{
  2756   2772         sqlite3PagerList = pPager->pNext;
  2757   2773       }
  2758   2774       if( pPager->pNext ){
  2759   2775         pPager->pNext->pPrev = pPager->pPrev;
  2760   2776       }
  2761   2777       sqlite3_mutex_leave(mutex);
         2778  +    pPager->onPagerList = 0;
  2762   2779     }
  2763   2780   #endif
  2764         -
  2765   2781     disable_simulated_io_errors();
  2766   2782     sqlite3FaultBeginBenign(-1);
  2767         -  pPager->errCode = 0;
  2768   2783     pPager->exclusiveMode = 0;
  2769         -  pager_reset(pPager);
  2770   2784     pagerUnlockAndRollback(pPager);
  2771   2785     enable_simulated_io_errors();
  2772   2786     sqlite3FaultEndBenign(-1);
  2773   2787     PAGERTRACE2("CLOSE %d\n", PAGERID(pPager));
  2774   2788     IOTRACE(("CLOSE %p\n", pPager))
  2775   2789     if( pPager->journalOpen ){
  2776   2790       sqlite3OsClose(pPager->jfd);
................................................................................
  2926   2940   
  2927   2941   #ifndef NDEBUG
  2928   2942     /* If the Pager.needSync flag is clear then the PgHdr.needSync
  2929   2943     ** flag must also be clear for all pages.  Verify that this
  2930   2944     ** invariant is true.
  2931   2945     */
  2932   2946     else{
         2947  +    pagerMutexHeld(pPager);
  2933   2948       for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){
  2934   2949         assert( pPg->needSync==0 );
  2935   2950       }
  2936   2951       assert( pPager->lru.pFirstSynced==pPager->lru.pFirst );
  2937   2952     }
  2938   2953   #endif
  2939   2954   
................................................................................
  3164   3179   */
  3165   3180   static int pager_recycle(Pager *pPager, PgHdr **ppPg){
  3166   3181     PgHdr *pPg;
  3167   3182     *ppPg = 0;
  3168   3183   
  3169   3184     /* It is illegal to call this function unless the pager object
  3170   3185     ** pointed to by pPager has at least one free page (page with nRef==0).
  3171         -  */ 
         3186  +  */
  3172   3187     assert(!MEMDB);
  3173   3188     assert(pPager->lru.pFirst);
  3174   3189   
  3175   3190     /* Find a page to recycle.  Try to locate a page that does not
  3176   3191     ** require us to do an fsync() on the journal.
  3177   3192     */
  3178   3193     pPg = pPager->lru.pFirstSynced;
................................................................................
  3275   3290     */
  3276   3291     for(pPager=sqlite3PagerList; pPager; pPager=pPager->pNext){
  3277   3292        pPager->iInUseMM = 1;
  3278   3293     }
  3279   3294   
  3280   3295     while( rc==SQLITE_OK && (nReq<0 || nReleased<nReq) ){
  3281   3296       PgHdr *pPg;
  3282         -    PgHdr *pRecycled;
  3283   3297    
  3284   3298       /* Try to find a page to recycle that does not require a sync(). If
  3285   3299       ** this is not possible, find one that does require a sync().
  3286   3300       */
  3287   3301       sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU));
  3288   3302       pPg = sqlite3LruPageList.pFirstSynced;
  3289   3303       while( pPg && (pPg->needSync || pPg->pPager->iInUseDB) ){
................................................................................
  3300   3314       /* If pPg==0, then the block above has failed to find a page to
  3301   3315       ** recycle. In this case return early - no further memory will
  3302   3316       ** be released.
  3303   3317       */
  3304   3318       if( !pPg ) break;
  3305   3319   
  3306   3320       pPager = pPg->pPager;
  3307         -    assert(!pPg->needSync || pPg==pPager->lru.pFirst);
  3308         -    assert(pPg->needSync || pPg==pPager->lru.pFirstSynced);
  3309         -  
         3321  +    assert( pPager->iInUseDB==0 );
  3310   3322       savedBusy = pPager->pBusyHandler;
  3311   3323       pPager->pBusyHandler = 0;
  3312         -    rc = pager_recycle(pPager, &pRecycled);
         3324  +    rc = pager_recycle(pPager, &pPg);
         3325  +    assert( pPager->iInUseDB==0 );
  3313   3326       pPager->pBusyHandler = savedBusy;
  3314         -    assert(pRecycled==pPg || rc!=SQLITE_OK);
  3315   3327       if( rc==SQLITE_OK ){
  3316   3328         /* We've found a page to free. At this point the page has been 
  3317   3329         ** removed from the page hash-table, free-list and synced-list 
  3318   3330         ** (pFirstSynced). It is still in the all pages (pAll) list. 
  3319   3331         ** Remove it from this list before freeing.
  3320   3332         **
  3321   3333         ** Todo: Check the Pager.pStmt list to make sure this is Ok. It 
................................................................................
  3613   3625   **          to be synced to disk or else disk syncing is currently
  3614   3626   **          allowed.
  3615   3627   */
  3616   3628   static int pagerAllocatePage(Pager *pPager, PgHdr **ppPg){
  3617   3629     int rc = SQLITE_OK;
  3618   3630     PgHdr *pPg;
  3619   3631     int nByteHdr;
         3632  +
  3620   3633   
  3621   3634     /* Create a new PgHdr if any of the four conditions defined 
  3622         -  ** above are met: */
         3635  +  ** above are met: 
         3636  +  */
         3637  +  pagerMutexHeld(pPager);
  3623   3638     if( pPager->nPage<pPager->mxPage
  3624   3639      || pPager->lru.pFirst==0 
  3625   3640      || MEMDB
  3626   3641      || (pPager->lru.pFirstSynced==0 && pPager->doNotSync)
  3627   3642     ){
  3628   3643       void *pData;
  3629   3644       if( pPager->nPage>=pPager->nHash ){
................................................................................
  3649   3664       if( pPg==0 ){
  3650   3665         rc = SQLITE_NOMEM;
  3651   3666         goto pager_allocate_out;
  3652   3667       }
  3653   3668       memset(pPg, 0, nByteHdr);
  3654   3669       pPg->pData = pData;
  3655   3670       pPg->pPager = pPager;
         3671  +    pagerMutexHeld(pPager);
  3656   3672       pPg->pNextAll = pPager->pAll;
  3657   3673       pPager->pAll = pPg;
  3658   3674       pPager->nPage++;
  3659   3675     }else{
  3660   3676       /* Recycle an existing page with a zero ref-count. */
         3677  +    pagerMutexHeld(pPager);
  3661   3678       rc = pager_recycle(pPager, &pPg);
  3662   3679       if( rc==SQLITE_BUSY ){
  3663   3680         rc = SQLITE_IOERR_BLOCKED;
  3664   3681       }
  3665   3682       if( rc!=SQLITE_OK ){
  3666   3683         goto pager_allocate_out;
  3667   3684       }
................................................................................
  4700   4717               if( rc!=SQLITE_OK ) goto sync_exit;
  4701   4718             }
  4702   4719           } 
  4703   4720         }
  4704   4721   #endif
  4705   4722         rc = writeMasterJournal(pPager, zMaster);
  4706   4723         if( rc!=SQLITE_OK ) goto sync_exit;
         4724  +      pagerMutexHeld(pPager);
  4707   4725         rc = syncJournal(pPager);
  4708   4726       }
  4709   4727       if( rc!=SQLITE_OK ) goto sync_exit;
  4710   4728   
  4711   4729   #ifndef SQLITE_OMIT_AUTOVACUUM
  4712   4730       if( nTrunc!=0 ){
  4713   4731         rc = sqlite3PagerTruncate(pPager, nTrunc);
................................................................................
  4827   4845   ** SQLITE_OK is returned.
  4828   4846   */
  4829   4847   int sqlite3PagerRollback(Pager *pPager){
  4830   4848     int rc;
  4831   4849     PAGERTRACE2("ROLLBACK %d\n", PAGERID(pPager));
  4832   4850     if( MEMDB ){
  4833   4851       PgHdr *p;
         4852  +    pagerMutexHeld(pPager);
  4834   4853       for(p=pPager->pAll; p; p=p->pNextAll){
  4835   4854         PgHistory *pHist;
  4836   4855         assert( !p->alwaysRollback );
  4837   4856         if( !p->dirty ){
  4838   4857           assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pOrig );
  4839   4858           assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pStmt );
  4840   4859           continue;