/ Check-in [83d1eafb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix for 'truncate file' operations on in-memory databases. (CVS 6131)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:83d1eafbde556f56969c6f285b6767d2c658dbfc
User & Date: danielk1977 2009-01-07 15:18:21
Context
2009-01-07
15:33
Conjecture: a journal header with nRec==0 must be the last header in the journal. Add asserts to make this conjecture explicit. (CVS 6132) check-in: 15b5b5f9 user: drh tags: trunk
15:18
Fix for 'truncate file' operations on in-memory databases. (CVS 6131) check-in: 83d1eafb user: danielk1977 tags: trunk
10:52
Add a comment to the openSubjournal() function in pager.c. (CVS 6130) check-in: 04387ae1 user: danielk1977 tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/pager.c.

    14     14   ** The pager is used to access a database disk file.  It implements
    15     15   ** atomic commit and rollback through the use of a journal file that
    16     16   ** is separate from the database file.  The pager also implements file
    17     17   ** locking to prevent two processes from writing the same database
    18     18   ** file simultaneously, or one process from reading the database while
    19     19   ** another is writing.
    20     20   **
    21         -** @(#) $Id: pager.c,v 1.537 2009/01/07 10:52:56 danielk1977 Exp $
           21  +** @(#) $Id: pager.c,v 1.538 2009/01/07 15:18:21 danielk1977 Exp $
    22     22   */
    23     23   #ifndef SQLITE_OMIT_DISKIO
    24     24   #include "sqliteInt.h"
    25     25   
    26     26   /*
    27     27   ** Macros for troubleshooting.  Normally turned off
    28     28   */
................................................................................
  1082   1082     }else if( pPager->state==PAGER_SYNCED ){
  1083   1083       pPager->state = PAGER_EXCLUSIVE;
  1084   1084     }
  1085   1085     pPager->dbOrigSize = 0;
  1086   1086     pPager->setMaster = 0;
  1087   1087     pPager->needSync = 0;
  1088   1088     /* lruListSetFirstSynced(pPager); */
         1089  +  sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize);
  1089   1090     if( !MEMDB ){
  1090   1091       pPager->dbSizeValid = 0;
  1091   1092     }
  1092   1093     pPager->dbModified = 0;
  1093   1094   
  1094   1095     return (rc==SQLITE_OK?rc2:rc);
  1095   1096   }
................................................................................
  1458   1459     }
  1459   1460     sqlite3_free(pMaster);
  1460   1461     return rc;
  1461   1462   }
  1462   1463   
  1463   1464   
  1464   1465   /*
  1465         -** Truncate the main file of the given pager to the number of pages
  1466         -** indicated. Also truncate the cached representation of the file.
         1466  +** If the main database file is open and an exclusive lock is held, 
         1467  +** truncate the main file of the given pager to the specified number 
         1468  +** of pages.
  1467   1469   **
  1468         -** Might might be the case that the file on disk is smaller than nPage.
         1470  +** It might might be the case that the file on disk is smaller than nPage.
  1469   1471   ** This can happen, for example, if we are in the middle of a transaction
  1470   1472   ** which has extended the file size and the new pages are still all held
  1471   1473   ** in cache, then an INSERT or UPDATE does a statement rollback.  Some
  1472   1474   ** operating system implementations can get confused if you try to
  1473   1475   ** truncate a file to some size that is larger than it currently is,
  1474   1476   ** so detect this case and write a single zero byte to the end of the new
  1475   1477   ** file instead.
................................................................................
  1487   1489           rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1);
  1488   1490         }
  1489   1491         if( rc==SQLITE_OK ){
  1490   1492           pPager->dbFileSize = nPage;
  1491   1493         }
  1492   1494       }
  1493   1495     }
  1494         -  if( rc==SQLITE_OK ){
  1495         -    pPager->dbSize = nPage;
  1496         -    sqlite3PcacheTruncate(pPager->pPCache, nPage);
  1497         -  }
  1498   1496     return rc;
  1499   1497   }
  1500   1498   
  1501   1499   /*
  1502   1500   ** Set the sectorSize for the given pager.
  1503   1501   **
  1504   1502   ** The sector size is at least as big as the sector size reported
................................................................................
  1664   1662       ** database file back to its original size.
  1665   1663       */
  1666   1664       if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){
  1667   1665         rc = pager_truncate(pPager, mxPg);
  1668   1666         if( rc!=SQLITE_OK ){
  1669   1667           goto end_playback;
  1670   1668         }
         1669  +      pPager->dbSize = mxPg;
  1671   1670       }
  1672   1671   
  1673   1672       /* Copy original pages out of the journal and back into the database file.
  1674   1673       */
  1675   1674       for(u=0; u<nRec; u++){
  1676   1675         rc = pager_playback_one_page(pPager, 1, &pPager->journalOff, 0, 0);
  1677   1676         if( rc!=SQLITE_OK ){
................................................................................
  2345   2344         pPager->state = (u8)locktype;
  2346   2345         IOTRACE(("LOCK %p %d\n", pPager, locktype))
  2347   2346       }
  2348   2347     }
  2349   2348     return rc;
  2350   2349   }
  2351   2350   
  2352         -/*
  2353         -** Truncate the file to the number of pages specified. 
  2354         -**
  2355         -** Unless an IO error occurs, this function is guaranteed to modify the 
  2356         -** database file itself. If an exclusive lock is not held when this function
  2357         -** is called, one is obtained before truncating the file.
  2358         -*/
  2359         -int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){
  2360         -  int rc = SQLITE_OK;
  2361         -  assert( pPager->state>=PAGER_SHARED );
  2362         -
  2363         -  sqlite3PagerPagecount(pPager, 0);
  2364         -  if( pPager->errCode ){
  2365         -    rc = pPager->errCode;
  2366         -  }else if( nPage<pPager->dbFileSize ){
  2367         -    rc = syncJournal(pPager);
  2368         -    if( rc==SQLITE_OK ){
  2369         -      /* Get an exclusive lock on the database before truncating. */
  2370         -      rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK);
  2371         -    }
  2372         -    if( rc==SQLITE_OK ){
  2373         -      rc = pager_truncate(pPager, nPage);
  2374         -    }
  2375         -  }
  2376         -
  2377         -  return rc;
  2378         -}
  2379         -
  2380   2351   #ifndef SQLITE_OMIT_AUTOVACUUM
  2381   2352   /*
  2382         -** Truncate the in-memory database file image to nPage pages. Unlike
  2383         -** sqlite3PagerTruncate(), this function does not actually modify the
  2384         -** database file on disk. It just sets the internal state of the pager
  2385         -** object so that the truncation will be done when the current 
  2386         -** transaction is committed.
         2353  +** Truncate the in-memory database file image to nPage pages. This 
         2354  +** function does not actually modify the database file on disk. It 
         2355  +** just sets the internal state of the pager object so that the 
         2356  +** truncation will be done when the current transaction is committed.
  2387   2357   */
  2388   2358   void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){
  2389   2359     assert( pPager->dbSizeValid );
  2390   2360     assert( pPager->dbSize>=nPage );
  2391   2361     pPager->dbSize = nPage;
  2392   2362   }
  2393   2363   
................................................................................
  2602   2572       if( !pPager->fd->pMethods ){
  2603   2573         assert(pPager->tempFile);
  2604   2574         rc = sqlite3PagerOpentemp(pPager, pPager->fd, pPager->vfsFlags);
  2605   2575         if( rc ) return rc;
  2606   2576       }
  2607   2577   
  2608   2578       /* If there are dirty pages in the page cache with page numbers greater
  2609         -    ** than Pager.dbSize, this means sqlite3PagerTruncate() was called to
         2579  +    ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
  2610   2580       ** make the file smaller (presumably by auto-vacuum code). Do not write
  2611   2581       ** any such pages to the file.
  2612   2582       */
  2613   2583       if( pList->pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){
  2614   2584         i64 offset = (pList->pgno-1)*(i64)pPager->pageSize;
  2615   2585         char *pData = CODEC2(pPager, pList->pData, pList->pgno, 6);
  2616   2586   
................................................................................
  3959   3929           rc = writeMasterJournal(pPager, zMaster);
  3960   3930           if( rc!=SQLITE_OK ) goto sync_exit;
  3961   3931           rc = syncJournal(pPager);
  3962   3932         }
  3963   3933       }
  3964   3934       if( rc!=SQLITE_OK ) goto sync_exit;
  3965   3935   
  3966         -#ifndef SQLITE_OMIT_AUTOVACUUM
  3967         -    if( pPager->dbSize<pPager->dbFileSize ){
  3968         -      rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
  3969         -      if( rc!=SQLITE_OK ) goto sync_exit;
  3970         -    }
  3971         -#endif
  3972         -
  3973   3936       /* Write all dirty pages to the database file */
  3974   3937       pPg = sqlite3PcacheDirtyList(pPager->pPCache);
  3975   3938       rc = pager_write_pagelist(pPg);
  3976   3939       if( rc!=SQLITE_OK ){
  3977   3940         assert( rc!=SQLITE_IOERR_BLOCKED );
  3978   3941         /* The error might have left the dirty list all fouled up here,
  3979   3942         ** but that does not matter because if the if the dirty list did
................................................................................
  3981   3944         ** discard the dirty list.  There is an assert in
  3982   3945         ** pager_get_all_dirty_pages() that verifies that no attempt
  3983   3946         ** is made to use an invalid dirty list.
  3984   3947         */
  3985   3948         goto sync_exit;
  3986   3949       }
  3987   3950       sqlite3PcacheCleanAll(pPager->pPCache);
         3951  +
         3952  +    if( pPager->dbSize<pPager->dbFileSize ){
         3953  +      assert( pPager->state>=PAGER_EXCLUSIVE );
         3954  +      rc = pager_truncate(pPager, pPager->dbSize);
         3955  +      if( rc!=SQLITE_OK ) goto sync_exit;
         3956  +    }
  3988   3957   
  3989   3958       /* Sync the database file. */
  3990   3959       if( !pPager->noSync && !noSync ){
  3991   3960         rc = sqlite3OsSync(pPager->fd, pPager->sync_flags);
  3992   3961       }
  3993   3962       IOTRACE(("DBSYNC %p\n", pPager))
  3994   3963   
  3995   3964       pPager->state = PAGER_SYNCED;
  3996         -  }else if( MEMDB && pPager->dbSize<pPager->dbFileSize ){
  3997         -    rc = sqlite3PagerTruncate(pPager, pPager->dbSize);
  3998   3965     }
  3999   3966   
  4000   3967   sync_exit:
  4001   3968     if( rc==SQLITE_IOERR_BLOCKED ){
  4002   3969       /* pager_incr_changecounter() may attempt to obtain an exclusive
  4003   3970        * lock to spill the cache and return IOERR_BLOCKED. But since 
  4004   3971        * there is no chance the cache is inconsistent, it is

Changes to src/pager.h.

     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** This header file defines the interface that the sqlite page cache
    13     13   ** subsystem.  The page cache subsystem reads and writes a file a page
    14     14   ** at a time and provides a journal for rollback.
    15     15   **
    16         -** @(#) $Id: pager.h,v 1.92 2009/01/06 14:19:37 drh Exp $
           16  +** @(#) $Id: pager.h,v 1.93 2009/01/07 15:18:21 danielk1977 Exp $
    17     17   */
    18     18   
    19     19   #ifndef _PAGER_H_
    20     20   #define _PAGER_H_
    21     21   
    22     22   /*
    23     23   ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise
................................................................................
    84     84   #define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0)
    85     85   DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno);
    86     86   int sqlite3PagerPageRefcount(DbPage*);
    87     87   int sqlite3PagerRef(DbPage*);
    88     88   int sqlite3PagerUnref(DbPage*);
    89     89   int sqlite3PagerWrite(DbPage*);
    90     90   int sqlite3PagerPagecount(Pager*, int*);
    91         -int sqlite3PagerTruncate(Pager*,Pgno);
    92     91   int sqlite3PagerBegin(DbPage*, int exFlag);
    93     92   int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int);
    94     93   int sqlite3PagerCommitPhaseTwo(Pager*);
    95     94   int sqlite3PagerRollback(Pager*);
    96     95   u8 sqlite3PagerIsreadonly(Pager*);
    97     96   void sqlite3PagerDontRollback(DbPage*);
    98     97   int sqlite3PagerDontWrite(DbPage*);

Changes to src/pcache1.c.

    12     12   **
    13     13   ** This file implements the default page cache implementation (the
    14     14   ** sqlite3_pcache interface). It also contains part of the implementation
    15     15   ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
    16     16   ** If the default page cache implementation is overriden, then neither of
    17     17   ** these two features are available.
    18     18   **
    19         -** @(#) $Id: pcache1.c,v 1.6 2008/12/10 18:03:46 drh Exp $
           19  +** @(#) $Id: pcache1.c,v 1.7 2009/01/07 15:18:21 danielk1977 Exp $
    20     20   */
    21     21   
    22     22   #include "sqliteInt.h"
    23     23   
    24     24   typedef struct PCache1 PCache1;
    25     25   typedef struct PgHdr1 PgHdr1;
    26     26   typedef struct PgFreeslot PgFreeslot;
................................................................................
    43     43     ** when the accessor is holding the global mutex (see pcache1EnterMutex() 
    44     44     ** and pcache1LeaveMutex()).
    45     45     */
    46     46     unsigned int nRecyclable;           /* Number of pages in the LRU list */
    47     47     unsigned int nPage;                 /* Total number of pages in apHash */
    48     48     unsigned int nHash;                 /* Number of slots in apHash[] */
    49     49     PgHdr1 **apHash;                    /* Hash table for fast lookup by key */
           50  +
           51  +  unsigned int iMaxKey;               /* Largest key seen since xTruncate() */
    50     52   };
    51     53   
    52     54   /*
    53     55   ** Each cache entry is represented by an instance of the following 
    54     56   ** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated 
    55     57   ** directly after the structure in memory (see the PGHDR1_TO_PAGE() 
    56     58   ** macro below).
................................................................................
   553    555       pPage->iKey = iKey;
   554    556       pPage->pNext = pCache->apHash[h];
   555    557       pPage->pCache = pCache;
   556    558       pCache->apHash[h] = pPage;
   557    559     }
   558    560   
   559    561   fetch_out:
          562  +  if( pPage && iKey>pCache->iMaxKey ){
          563  +    pCache->iMaxKey = iKey;
          564  +  }
   560    565     if( createFlag==1 ) sqlite3EndBenignMalloc();
   561    566     pcache1LeaveMutex();
   562    567     return (pPage ? PGHDR1_TO_PAGE(pPage) : 0);
   563    568   }
   564    569   
   565    570   
   566    571   /*
................................................................................
   627    632     }
   628    633     *pp = pPage->pNext;
   629    634   
   630    635     h = iNew%pCache->nHash;
   631    636     pPage->iKey = iNew;
   632    637     pPage->pNext = pCache->apHash[h];
   633    638     pCache->apHash[h] = pPage;
          639  +
          640  +  if( iNew>pCache->iMaxKey ){
          641  +    pCache->iMaxKey = iNew;
          642  +  }
   634    643   
   635    644     pcache1LeaveMutex();
   636    645   }
   637    646   
   638    647   /*
   639    648   ** Implementation of the sqlite3_pcache.xTruncate method. 
   640    649   **
................................................................................
   641    650   ** Discard all unpinned pages in the cache with a page number equal to
   642    651   ** or greater than parameter iLimit. Any pinned pages with a page number
   643    652   ** equal to or greater than iLimit are implicitly unpinned.
   644    653   */
   645    654   static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){
   646    655     PCache1 *pCache = (PCache1 *)p;
   647    656     pcache1EnterMutex();
   648         -  pcache1TruncateUnsafe(pCache, iLimit);
          657  +  if( iLimit<=pCache->iMaxKey ){
          658  +    pcache1TruncateUnsafe(pCache, iLimit);
          659  +    pCache->iMaxKey = iLimit-1;
          660  +  }
   649    661     pcache1LeaveMutex();
   650    662   }
   651    663   
   652    664   /*
   653    665   ** Implementation of the sqlite3_pcache.xDestroy method. 
   654    666   **
   655    667   ** Destroy a cache allocated using pcache1Create().

Changes to src/test2.c.

     9      9   **    May you share freely, never taking more than you give.
    10     10   **
    11     11   *************************************************************************
    12     12   ** Code for testing the pager.c module in SQLite.  This code
    13     13   ** is not included in the SQLite library.  It is used for automated
    14     14   ** testing of the SQLite library.
    15     15   **
    16         -** $Id: test2.c,v 1.64 2009/01/02 18:10:42 drh Exp $
           16  +** $Id: test2.c,v 1.65 2009/01/07 15:18:21 danielk1977 Exp $
    17     17   */
    18     18   #include "sqliteInt.h"
    19     19   #include "tcl.h"
    20     20   #include <stdlib.h>
    21     21   #include <string.h>
    22     22   #include <ctype.h>
    23     23   
................................................................................
   389    389   static int pager_truncate(
   390    390     void *NotUsed,
   391    391     Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   392    392     int argc,              /* Number of arguments */
   393    393     const char **argv      /* Text of each argument */
   394    394   ){
   395    395     Pager *pPager;
   396         -  int rc;
   397    396     int pgno;
   398    397     if( argc!=3 ){
   399    398       Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
   400    399          " ID PGNO\"", 0);
   401    400       return TCL_ERROR;
   402    401     }
   403    402     pPager = sqlite3TestTextToPtr(argv[1]);
   404    403     if( Tcl_GetInt(interp, argv[2], &pgno) ) return TCL_ERROR;
   405         -  rc = sqlite3PagerTruncate(pPager, pgno);
   406         -  if( rc!=SQLITE_OK ){
   407         -    Tcl_AppendResult(interp, errorName(rc), 0);
   408         -    return TCL_ERROR;
   409         -  }
          404  +  sqlite3PagerTruncateImage(pPager, pgno);
   410    405     return TCL_OK;
   411    406   }
   412    407   
   413    408   
   414    409   /*
   415    410   ** Usage:   page_unref PAGE
   416    411   **

Changes to test/memdb.test.

     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is in-memory database backend.
    13     13   #
    14         -# $Id: memdb.test,v 1.15 2006/01/30 22:48:44 drh Exp $
           14  +# $Id: memdb.test,v 1.16 2009/01/07 15:18:21 danielk1977 Exp $
    15     15   
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   
    20     20   ifcapable memorydb {
    21     21   
................................................................................
   406    406   } 5
   407    407   do_test memdb-8.2 {
   408    408     execsql {
   409    409       DELETE FROM t1;
   410    410       SELECT count(*) FROM t1;
   411    411     }
   412    412   } 0
          413  +
          414  +# Test that auto-vacuum works with in-memory databases.
          415  +# 
          416  +do_test memdb-9.1 {
          417  +  db close
          418  +  sqlite3 db test.db
          419  +  db cache size 0
          420  +  execsql {
          421  +    PRAGMA auto_vacuum = full;
          422  +    CREATE TABLE t1(a);
          423  +    INSERT INTO t1 VALUES(randstr(1000,1000));
          424  +    INSERT INTO t1 VALUES(randstr(1000,1000));
          425  +    INSERT INTO t1 VALUES(randstr(1000,1000));
          426  +  }
          427  +  set memused [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1]
          428  +  execsql { DELETE FROM t1 }
          429  +  set memused2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1]
          430  +  expr {$memused2 + 2048 < $memused}
          431  +} {1}
   413    432   
   414    433   
   415    434   } ;# ifcapable memorydb
   416    435   
   417    436   finish_test

Changes to test/pager.test.

     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is page cache subsystem.
    13     13   #
    14         -# $Id: pager.test,v 1.33 2009/01/05 17:15:00 danielk1977 Exp $
           14  +# $Id: pager.test,v 1.34 2009/01/07 15:18:21 danielk1977 Exp $
    15     15   
    16     16   
    17     17   set testdir [file dirname $argv0]
    18     18   source $testdir/tester.tcl
    19     19   
    20     20   if {[info commands pager_open]!=""} {
    21     21   db close
................................................................................
   409    409     pager_close [pager_open ptf2.db -15]
   410    410   } {}
   411    411   
   412    412   # Test truncate on an in-memory database is Ok.
   413    413   ifcapable memorydb {
   414    414     do_test pager-4.6.2 {
   415    415       set ::p2 [pager_open :memory: 10]
   416         -    pager_truncate $::p2 5
          416  +    pager_truncate $::p2 0
   417    417     } {}
   418    418     do_test pager-4.6.3 {
   419    419       set page1 [page_get $::p2 1]
   420    420       for {set i 1} {$i<5} {incr i} {
   421    421         set p [page_get $::p2 $i]
   422    422         page_write $p "Page $i"
   423    423         pager_commit $::p2
   424    424         page_unref $p
   425    425       }
   426    426       page_unref $page1
   427         -    # pager_truncate $::p2 3
          427  +    pager_truncate $::p2 3
   428    428     } {}
   429    429     do_test pager-4.6.4 {
   430    430       pager_close $::p2
   431    431     } {}
   432    432   }
   433    433   
   434    434   do_test pager-4.99 {