/ Check-in [8a4314a3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Disable the use of memory-mapped I/O if the SQLITE_DISABLE_MMAP macro is defined. Automatically define this macro for OpenBSD and QNX. Other systems are likely to be added to the disabled list over time.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 8a4314a398c6c6efb25e6993c826b7e60d9bb1f9
User & Date: drh 2013-04-08 20:47:49
Context
2013-04-09
19:19
Check in the instrumented code for the mptest test case that appears to show a spontaneously disappearing journal file. Closed-Leaf check-in: 96f04d99 user: dan tags: missing-journal-file
16:19
Add extra #ifndef statements in os_unix.c and os_win.c to make sure the memory mapped I/O really is disabled when SQLITE_DISABLE_MMAP is set. check-in: c1e2523c user: drh tags: trunk
2013-04-08
20:47
Disable the use of memory-mapped I/O if the SQLITE_DISABLE_MMAP macro is defined. Automatically define this macro for OpenBSD and QNX. Other systems are likely to be added to the disabled list over time. check-in: 8a4314a3 user: drh tags: trunk
19:13
In mptester, handle startScript no-change commits using a retry rather than grabbing an exclusive transaction. check-in: ae8bb24b user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/ctime.c.

    61     61     "DEFAULT_MMAP_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_LIMIT),
    62     62   #endif
    63     63   #ifdef SQLITE_DISABLE_DIRSYNC
    64     64     "DISABLE_DIRSYNC",
    65     65   #endif
    66     66   #ifdef SQLITE_DISABLE_LFS
    67     67     "DISABLE_LFS",
           68  +#endif
           69  +#ifdef SQLITE_DISABLE_MMAP
           70  +  "DISABLE_MMAP",
    68     71   #endif
    69     72   #ifdef SQLITE_ENABLE_ATOMIC_WRITE
    70     73     "ENABLE_ATOMIC_WRITE",
    71     74   #endif
    72     75   #ifdef SQLITE_ENABLE_CEROD
    73     76     "ENABLE_CEROD",
    74     77   #endif

Changes to src/os.c.

   137    137     int bExtend,                    /* True to extend file if necessary */
   138    138     void volatile **pp              /* OUT: Pointer to mapping */
   139    139   ){
   140    140     DO_OS_MALLOC_TEST(id);
   141    141     return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp);
   142    142   }
   143    143   
          144  +#if !defined(SQLITE_DISABLE_MMAP)
          145  +/* The real implementation of xFetch and xUnfetch */
   144    146   int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){
   145    147     DO_OS_MALLOC_TEST(id);
   146    148     return id->pMethods->xFetch(id, iOff, iAmt, pp);
   147    149   }
   148    150   int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){
   149    151     return id->pMethods->xUnfetch(id, iOff, p);
   150    152   }
          153  +#else
          154  +/* No-op stubs to use when memory-mapped I/O is disabled */
          155  +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){
          156  +  *pp = 0;
          157  +  return SQLITE_OK;
          158  +}
          159  +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){
          160  +  return SQLITE_OK;
          161  +}
          162  +#endif
   151    163   
   152    164   /*
   153    165   ** The next group of routines are convenience wrappers around the
   154    166   ** VFS methods.
   155    167   */
   156    168   int sqlite3OsOpen(
   157    169     sqlite3_vfs *pVfs, 

Changes to src/pager.c.

   766    766   */
   767    767   #ifdef SQLITE_OMIT_MEMORYDB
   768    768   # define MEMDB 0
   769    769   #else
   770    770   # define MEMDB pPager->memDb
   771    771   #endif
   772    772   
          773  +/*
          774  +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch
          775  +** interfaces to access the database using memory-mapped I/O.
          776  +*/
          777  +#ifdef SQLITE_DISABLE_MMAP
          778  +# define USEFETCH(x) 0
          779  +#else
          780  +# define USEFETCH(x) ((x)->bUseFetch)
          781  +#endif
          782  +
   773    783   /*
   774    784   ** The maximum legal page number is (2^31 - 1).
   775    785   */
   776    786   #define PAGER_MAX_PGNO 2147483647
   777    787   
   778    788   /*
   779    789   ** The argument to this macro is a file descriptor (type sqlite3_file*).
................................................................................
  3086   3096     ** the duplicate call is harmless.
  3087   3097     */
  3088   3098     sqlite3WalEndReadTransaction(pPager->pWal);
  3089   3099   
  3090   3100     rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed);
  3091   3101     if( rc!=SQLITE_OK || changed ){
  3092   3102       pager_reset(pPager);
  3093         -    if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0, 0);
         3103  +    if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0);
  3094   3104     }
  3095   3105   
  3096   3106     return rc;
  3097   3107   }
  3098   3108   #endif
  3099   3109   
  3100   3110   /*
................................................................................
  3352   3362     sqlite3PcacheSetCachesize(pPager->pPCache, mxPage);
  3353   3363   }
  3354   3364   
  3355   3365   /*
  3356   3366   ** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap.
  3357   3367   */
  3358   3368   static void pagerFixMaplimit(Pager *pPager){
         3369  +#if !defined(SQLITE_DISABLE_MMAP)
  3359   3370     sqlite3_file *fd = pPager->fd;
  3360   3371     if( isOpen(fd) ){
  3361   3372       sqlite3_int64 mx;
  3362   3373       pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0;
  3363   3374       mx = pPager->mxMmap;
  3364   3375       sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx);
  3365   3376     }
         3377  +#endif
  3366   3378   }
  3367   3379   
  3368   3380   /*
  3369   3381   ** Change the maximum size of any memory mapping made of the database file.
  3370   3382   */
  3371   3383   void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){
  3372   3384     pPager->mxMmap = mxMmap;
................................................................................
  5038   5050              || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK)
  5039   5051         );
  5040   5052       }
  5041   5053   
  5042   5054       if( !pPager->tempFile && (
  5043   5055           pPager->pBackup 
  5044   5056        || sqlite3PcachePagecount(pPager->pPCache)>0 
  5045         -     || pPager->bUseFetch
         5057  +     || USEFETCH(pPager)
  5046   5058       )){
  5047   5059         /* The shared-lock has just been acquired on the database file
  5048   5060         ** and there are already pages in the cache (from a previous
  5049   5061         ** read or write transaction).  Check to see if the database
  5050   5062         ** has been modified.  If the database has changed, flush the
  5051   5063         ** cache.
  5052   5064         **
................................................................................
  5081   5093   
  5082   5094           /* Unmap the database file. It is possible that external processes
  5083   5095           ** may have truncated the database file and then extended it back
  5084   5096           ** to its original size while this process was not holding a lock.
  5085   5097           ** In this case there may exist a Pager.pMap mapping that appears
  5086   5098           ** to be the right size but is not actually valid. Avoid this
  5087   5099           ** possibility by unmapping the db here. */
  5088         -        if( pPager->bUseFetch ){
         5100  +        if( USEFETCH(pPager) ){
  5089   5101             sqlite3OsUnfetch(pPager->fd, 0, 0);
  5090   5102           }
  5091   5103         }
  5092   5104       }
  5093   5105   
  5094   5106       /* If there is a WAL file in the file-system, open this database in WAL
  5095   5107       ** mode. Otherwise, the following function call is a no-op.
................................................................................
  5195   5207     u32 iFrame = 0;                 /* Frame to read from WAL file */
  5196   5208     const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT);
  5197   5209   
  5198   5210     /* It is acceptable to use a read-only (mmap) page for any page except
  5199   5211     ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY
  5200   5212     ** flag was specified by the caller. And so long as the db is not a 
  5201   5213     ** temporary or in-memory database.  */
  5202         -  const int bMmapOk = (pgno!=1 && pPager->bUseFetch
         5214  +  const int bMmapOk = (pgno!=1 && USEFETCH(pPager)
  5203   5215      && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY))
  5204   5216     );
  5205   5217   
  5206   5218     assert( pPager->eState>=PAGER_READER );
  5207   5219     assert( assert_pager_state(pPager) );
  5208   5220     assert( noContent==0 || bMmapOk==0 );
  5209   5221   

Changes to src/pragma.c.

   770    770           if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){
   771    771             sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, mx);
   772    772           }
   773    773         }
   774    774       }
   775    775       mx = -1;
   776    776       if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){
          777  +#if defined(SQLITE_DISABLE_MMAP)
          778  +      mx = 0;
          779  +#endif
   777    780         returnSingleInt(pParse, "mmap_limit", mx);
   778    781       }
   779    782     }else
   780    783   
   781    784     /*
   782    785     **   PRAGMA temp_store
   783    786     **   PRAGMA temp_store = "default"|"memory"|"file"

Changes to src/sqliteInt.h.

   535    535   */
   536    536   #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC
   537    537   # define EIGHT_BYTE_ALIGNMENT(X)   ((((char*)(X) - (char*)0)&3)==0)
   538    538   #else
   539    539   # define EIGHT_BYTE_ALIGNMENT(X)   ((((char*)(X) - (char*)0)&7)==0)
   540    540   #endif
   541    541   
          542  +/*
          543  +** Disable MMAP on platforms where it is not supported
          544  +*/
          545  +#if defined(__OpenBSD__) || defined(__QNXNTO__)
          546  +# undef SQLITE_DISABLE_MMAP
          547  +# define SQLITE_DISABLE_MMAP 1
          548  +#endif
          549  +
   542    550   
   543    551   /*
   544    552   ** An instance of the following structure is used to store the busy-handler
   545    553   ** callback for a given sqlite handle. 
   546    554   **
   547    555   ** The sqlite.busyHandler member of the sqlite struct contains the busy
   548    556   ** callback for the database handle. Each pager opened via the sqlite

Changes to src/sqliteLimit.h.

   206    206   #ifndef SQLITE_MAX_TRIGGER_DEPTH
   207    207   # define SQLITE_MAX_TRIGGER_DEPTH 1000
   208    208   #endif
   209    209   
   210    210   /*
   211    211   ** Default maximum size of memory used by xFetch in the VFS.
   212    212   */
          213  +#ifdef SQLITE_DISABLE_MMAP
          214  +# undef SQLITE_DEFAULT_MMAP_LIMIT
          215  +# define SQLITE_DEFAULT_MMAP_LIMIT 0
          216  +#endif
   213    217   #ifdef __APPLE__
   214    218   # include <TargetConditionals.h>
   215    219   # if TARGET_OS_IPHONE
   216    220   #   define SQLITE_DEFAULT_MMAP_LIMIT 0
   217    221   # endif
   218    222   #endif
   219    223   #ifndef SQLITE_DEFAULT_MMAP_LIMIT

Changes to src/test_config.c.

    82     82   #endif
    83     83   
    84     84   #ifdef SQLITE_DISABLE_LFS
    85     85     Tcl_SetVar2(interp, "sqlite_options", "lfs", "0", TCL_GLOBAL_ONLY);
    86     86   #else
    87     87     Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY);
    88     88   #endif
           89  +
           90  +#ifdef SQLITE_DISABLE_MMAP
           91  +  Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY);
           92  +#else
           93  +  Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY);
           94  +#endif
    89     95   
    90     96   #if 1 /* def SQLITE_MEMDEBUG */
    91     97     Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY);
    92     98   #else
    93     99     Tcl_SetVar2(interp, "sqlite_options", "memdebug", "0", TCL_GLOBAL_ONLY);
    94    100   #endif
    95    101   

Changes to test/mmap1.test.

     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #***********************************************************************
    11     11   #
    12     12   
    13     13   set testdir [file dirname $argv0]
    14     14   source $testdir/tester.tcl
           15  +ifcapable !mmap {
           16  +  finish_test
           17  +  return
           18  +}
    15     19   source $testdir/lock_common.tcl
    16     20   set testprefix mmap1
    17     21   
    18     22   proc nRead {db} {
    19     23     set bt [btree_from_db $db]
    20     24     db_enter $db
    21     25     array set stats [btree_pager_stats $bt]
................................................................................
   307    311         set res 1
   308    312       } {1}
   309    313     }
   310    314   }
   311    315   
   312    316   
   313    317   finish_test
   314         -
   315         -

Changes to test/mmap2.test.

    20     20   set testdir [file dirname $argv0]
    21     21   source $testdir/tester.tcl
    22     22   set testprefix mmap2
    23     23   
    24     24   if {$::tcl_platform(platform)!="unix" || [test_syscall defaultvfs] != "unix"} {
    25     25     finish_test
    26     26     return
           27  +}
           28  +ifcapable !mmap {
           29  +  finish_test
           30  +  return
    27     31   }
    28     32   
    29     33   db close
    30     34   sqlite3_shutdown
    31     35   test_sqlite3_log xLog
    32     36   proc xLog {error_code msg} {
    33     37     if {[string match os_unix.c* $msg]} {

Changes to test/pageropt.test.

    83     83       SELECT hex(x) FROM t1
    84     84     }
    85     85   } [list 0 0 0 $blobcontent]
    86     86   
    87     87   # But if the other thread modifies the database, then the cache
    88     88   # must refill.
    89     89   #
           90  +ifcapable mmap {
           91  +  set x [expr {[permutation]=="nommap" ? 6 : 1}]
           92  +} else {
           93  +  set x 6
           94  +}
    90     95   do_test pageropt-1.5 {
    91     96     db2 eval {CREATE TABLE t2(y)}
    92     97     pagercount_sql {
    93     98       SELECT hex(x) FROM t1
    94     99     }
    95         -} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent]
          100  +} [list $x 0 0 $blobcontent]
    96    101   do_test pageropt-1.6 {
    97    102     pagercount_sql {
    98    103       SELECT hex(x) FROM t1
    99    104     }
   100    105   } [list 0 0 0 $blobcontent]
   101    106   
   102    107   # Verify that the last page of an overflow chain is not read from

Changes to test/wal.test.

   725    725   do_test wal-11.9 {
   726    726     db close
   727    727     list [expr [file size test.db]/1024] [log_deleted test.db-wal]
   728    728   } {37 1}
   729    729   sqlite3_wal db test.db
   730    730   set nWal 39
   731    731   if {[permutation]=="nommap"} {set nWal 37}
          732  +ifcapable !mmap {set nWal 37}
   732    733   do_test wal-11.10 {
   733    734     execsql {
   734    735       PRAGMA cache_size = 10;
   735    736       BEGIN;
   736    737         INSERT INTO t1 SELECT blob(900) FROM t1;   -- 32
   737    738         SELECT count(*) FROM t1;
   738    739     }

Changes to test/wal5.test.

   239    239       # The checkpoint above only writes page 1 of the db file. The other
   240    240       # page (page 2) is locked by the read-transaction opened by the
   241    241       # [sql2] commmand above. So normally, the db is 1 page in size here.
   242    242       # However, in mmap() mode, the db is pre-allocated to 2 pages at the
   243    243       # start of the checkpoint, even though page 2 cannot be written.
   244    244       set nDb 2
   245    245       if {[permutation]=="no-mmap"} {set nDb 1}
          246  +    ifcapable !mmap {set nDb 1}
   246    247       do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4]
   247    248     }
   248    249   
   249    250     # Check that checkpoints block on the correct locks. And respond correctly
   250    251     # if they cannot obtain those locks. There are three locks that a checkpoint
   251    252     # may block on (in the following order):
   252    253     #