SQLite

Check-in [3098a3c1e7]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add test cases for errors in mmap() or mremap() is os_unix.c.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | experimental-mmap
Files: files | file ages | folders
SHA1: 3098a3c1e7305033904a496ef534cb312a876fab
User & Date: dan 2013-04-02 12:04:09.729
Context
2013-04-02
14:37
Fix a faulty assert() in the os_win.c VFS. (check-in: fd6ee54969 user: drh tags: experimental-mmap)
12:04
Add test cases for errors in mmap() or mremap() is os_unix.c. (check-in: 3098a3c1e7 user: dan tags: experimental-mmap)
10:29
Proposed template preprocessor magic for activating mmap only on platforms where we know it works. (check-in: d96272f031 user: drh tags: experimental-mmap)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/os_unix.c.
4556
4557
4558
4559
4560
4561
4562

4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573

4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588

4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612


4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
** continue accessing the database using the xRead() and xWrite()
** methods.
*/
static void unixRemapfile(
  unixFile *pFd,                  /* File descriptor object */
  i64 nNew                        /* Required mapping size */
){

  int h = pFd->h;                      /* File descriptor open on db file */
  u8 *pOrig = (u8 *)pFd->pMapRegion;   /* Pointer to current file mapping */
  i64 nOrig = pFd->mmapOrigsize;       /* Size of pOrig region in bytes */
  u8 *pNew = 0;                        /* Location of new mapping */
  int flags = PROT_READ;               /* Flags to pass to mmap() */

  assert( pFd->nFetchOut==0 );
  assert( nNew>pFd->mmapSize );
  assert( nNew<=pFd->mmapLimit );
  assert( nNew>0 );
  assert( pFd->mmapOrigsize>=pFd->mmapSize );


  if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;

  if( pOrig ){
    const int szSyspage = unixGetPagesize();
    i64 nReuse = (pFd->mmapSize & ~(szSyspage-1));
    u8 *pReq = &pOrig[nReuse];

    /* Unmap any pages of the existing mapping that cannot be reused. */
    if( nReuse!=nOrig ){
      osMunmap(pReq, nOrig-nReuse);
    }

#if HAVE_MREMAP
    pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE);

#else
    pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse);
    if( pNew!=MAP_FAILED ){
      if( pNew!=pReq ){
        osMunmap(pNew, nNew - nReuse);
        pNew = MAP_FAILED;
      }else{
        pNew = pOrig;
      }
    }
#endif

    /* The attempt to extend the existing mapping failed. Free the existing
    ** mapping and set pNew to NULL so that the code below will create a
    ** new mapping from scratch.  */
    if( pNew==MAP_FAILED ){
      pNew = 0;
      osMunmap(pOrig, nReuse);
    }
  }

  /* If pNew is still NULL, try to create an entirely new mapping. */
  if( pNew==0 ){
    pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0);


    if( pNew==MAP_FAILED ){
      pNew = 0;
      nNew = 0;
      unixLogError(SQLITE_OK, "mmap", pFd->zPath);

      /* If the mmap() above failed, assume that all subsequent mmap() calls
      ** will probably fail too. Fall back to using xRead/xWrite exclusively
      ** in this case.  */
      pFd->mmapLimit = 0;
    }
  }

  pFd->pMapRegion = (void *)pNew;
  pFd->mmapSize = pFd->mmapOrigsize = nNew;
}

/*
** Memory map or remap the file opened by file-descriptor pFd (if the file
** is already mapped, the existing mapping is replaced by the new). Or, if 







>











>















>





|










<







>
>
|
|
|
|

|
|
|
|
|
<
<







4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607

4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626


4627
4628
4629
4630
4631
4632
4633
** continue accessing the database using the xRead() and xWrite()
** methods.
*/
static void unixRemapfile(
  unixFile *pFd,                  /* File descriptor object */
  i64 nNew                        /* Required mapping size */
){
  const char *zErr = "mmap";
  int h = pFd->h;                      /* File descriptor open on db file */
  u8 *pOrig = (u8 *)pFd->pMapRegion;   /* Pointer to current file mapping */
  i64 nOrig = pFd->mmapOrigsize;       /* Size of pOrig region in bytes */
  u8 *pNew = 0;                        /* Location of new mapping */
  int flags = PROT_READ;               /* Flags to pass to mmap() */

  assert( pFd->nFetchOut==0 );
  assert( nNew>pFd->mmapSize );
  assert( nNew<=pFd->mmapLimit );
  assert( nNew>0 );
  assert( pFd->mmapOrigsize>=pFd->mmapSize );
  assert( MAP_FAILED!=0 );

  if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;

  if( pOrig ){
    const int szSyspage = unixGetPagesize();
    i64 nReuse = (pFd->mmapSize & ~(szSyspage-1));
    u8 *pReq = &pOrig[nReuse];

    /* Unmap any pages of the existing mapping that cannot be reused. */
    if( nReuse!=nOrig ){
      osMunmap(pReq, nOrig-nReuse);
    }

#if HAVE_MREMAP
    pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE);
    zErr = "mremap";
#else
    pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse);
    if( pNew!=MAP_FAILED ){
      if( pNew!=pReq ){
        osMunmap(pNew, nNew - nReuse);
        pNew = 0;
      }else{
        pNew = pOrig;
      }
    }
#endif

    /* The attempt to extend the existing mapping failed. Free the existing
    ** mapping and set pNew to NULL so that the code below will create a
    ** new mapping from scratch.  */
    if( pNew==MAP_FAILED ){

      osMunmap(pOrig, nReuse);
    }
  }

  /* If pNew is still NULL, try to create an entirely new mapping. */
  if( pNew==0 ){
    pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0);
  }

  if( pNew==MAP_FAILED ){
    pNew = 0;
    nNew = 0;
    unixLogError(SQLITE_OK, zErr, pFd->zPath);

    /* If the mmap() above failed, assume that all subsequent mmap() calls
    ** will probably fail too. Fall back to using xRead/xWrite exclusively
    ** in this case.  */
    pFd->mmapLimit = 0;
  }


  pFd->pMapRegion = (void *)pNew;
  pFd->mmapSize = pFd->mmapOrigsize = nNew;
}

/*
** Memory map or remap the file opened by file-descriptor pFd (if the file
** is already mapped, the existing mapping is replaced by the new). Or, if 
Changes to src/test_syscall.c.
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
static int ts_pread64(int fd, void *aBuf, size_t nBuf, off_t off);
static int ts_write(int fd, const void *aBuf, size_t nBuf);
static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off);
static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off);
static int ts_fchmod(int fd, mode_t mode);
static int ts_fallocate(int fd, off_t off, off_t len);
static void *ts_mmap(void *, size_t, int, int, int, off_t);


struct TestSyscallArray {
  const char *zName;
  sqlite3_syscall_ptr xTest;
  sqlite3_syscall_ptr xOrig;
  int default_errno;              /* Default value for errno following errors */
  int custom_errno;               /* Current value for errno if error */







|







104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
static int ts_pread64(int fd, void *aBuf, size_t nBuf, off_t off);
static int ts_write(int fd, const void *aBuf, size_t nBuf);
static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off);
static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off);
static int ts_fchmod(int fd, mode_t mode);
static int ts_fallocate(int fd, off_t off, off_t len);
static void *ts_mmap(void *, size_t, int, int, int, off_t);
static void *ts_mremap(void*, size_t, size_t, int, ...);

struct TestSyscallArray {
  const char *zName;
  sqlite3_syscall_ptr xTest;
  sqlite3_syscall_ptr xOrig;
  int default_errno;              /* Default value for errno following errors */
  int custom_errno;               /* Current value for errno if error */
130
131
132
133
134
135
136

137
138
139
140
141
142
143
  /* 10 */ { "pread64",   (sqlite3_syscall_ptr)ts_pread64,   0, 0, 0 },
  /* 11 */ { "write",     (sqlite3_syscall_ptr)ts_write,     0, 0, 0 },
  /* 12 */ { "pwrite",    (sqlite3_syscall_ptr)ts_pwrite,    0, 0, 0 },
  /* 13 */ { "pwrite64",  (sqlite3_syscall_ptr)ts_pwrite64,  0, 0, 0 },
  /* 14 */ { "fchmod",    (sqlite3_syscall_ptr)ts_fchmod,    0, 0, 0 },
  /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 },
  /* 16 */ { "mmap",      (sqlite3_syscall_ptr)ts_mmap,      0, 0, 0 },

           { 0, 0, 0, 0, 0 }
};

#define orig_open      ((int(*)(const char *, int, int))aSyscall[0].xOrig)
#define orig_close     ((int(*)(int))aSyscall[1].xOrig)
#define orig_access    ((int(*)(const char*,int))aSyscall[2].xOrig)
#define orig_getcwd    ((char*(*)(char*,size_t))aSyscall[3].xOrig)







>







130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
  /* 10 */ { "pread64",   (sqlite3_syscall_ptr)ts_pread64,   0, 0, 0 },
  /* 11 */ { "write",     (sqlite3_syscall_ptr)ts_write,     0, 0, 0 },
  /* 12 */ { "pwrite",    (sqlite3_syscall_ptr)ts_pwrite,    0, 0, 0 },
  /* 13 */ { "pwrite64",  (sqlite3_syscall_ptr)ts_pwrite64,  0, 0, 0 },
  /* 14 */ { "fchmod",    (sqlite3_syscall_ptr)ts_fchmod,    0, 0, 0 },
  /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 },
  /* 16 */ { "mmap",      (sqlite3_syscall_ptr)ts_mmap,      0, 0, 0 },
  /* 17 */ { "mremap",    (sqlite3_syscall_ptr)ts_mremap,    0, 0, 0 },
           { 0, 0, 0, 0, 0 }
};

#define orig_open      ((int(*)(const char *, int, int))aSyscall[0].xOrig)
#define orig_close     ((int(*)(int))aSyscall[1].xOrig)
#define orig_access    ((int(*)(const char*,int))aSyscall[2].xOrig)
#define orig_getcwd    ((char*(*)(char*,size_t))aSyscall[3].xOrig)
152
153
154
155
156
157
158

159
160
161
162
163
164
165
#define orig_pwrite    ((ssize_t(*)(int,const void*,size_t,off_t))\
                       aSyscall[12].xOrig)
#define orig_pwrite64  ((ssize_t(*)(int,const void*,size_t,off_t))\
                       aSyscall[13].xOrig)
#define orig_fchmod    ((int(*)(int,mode_t))aSyscall[14].xOrig)
#define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig)
#define orig_mmap      ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig)


/*
** This function is called exactly once from within each invocation of a
** system call wrapper in this file. It returns 1 if the function should
** fail, or 0 if it should succeed.
*/
static int tsIsFail(void){







>







153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
#define orig_pwrite    ((ssize_t(*)(int,const void*,size_t,off_t))\
                       aSyscall[12].xOrig)
#define orig_pwrite64  ((ssize_t(*)(int,const void*,size_t,off_t))\
                       aSyscall[13].xOrig)
#define orig_fchmod    ((int(*)(int,mode_t))aSyscall[14].xOrig)
#define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig)
#define orig_mmap      ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig)
#define orig_mremap    ((void*(*)(void*,size_t,size_t,int,...))aSyscall[17].xOrig)

/*
** This function is called exactly once from within each invocation of a
** system call wrapper in this file. It returns 1 if the function should
** fail, or 0 if it should succeed.
*/
static int tsIsFail(void){
390
391
392
393
394
395
396











397
398
399
400
401
402
403
  off_t iOff
){
  if( tsIsFailErrno("mmap") ){
    return MAP_FAILED;
  }
  return orig_mmap(pAddr, nByte, prot, flags, fd, iOff);
}












static int test_syscall_install(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){







>
>
>
>
>
>
>
>
>
>
>







392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
  off_t iOff
){
  if( tsIsFailErrno("mmap") ){
    return MAP_FAILED;
  }
  return orig_mmap(pAddr, nByte, prot, flags, fd, iOff);
}

static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){
  va_list ap;
  void *pArg;
  if( tsIsFailErrno("mremap") ){
    return MAP_FAILED;
  }
  va_start(ap, d);
  pArg = va_arg(ap, void *);
  return orig_mremap(a, b, c, d, pArg);
}

static int test_syscall_install(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
Added test/mmap2.test.






























































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# 2013 March 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests the effect of the mmap() or mremap() system calls 
# returning an error on the library. 
#
# If either mmap() or mremap() fails, SQLite should log an error 
# message, then continue accessing the database using read() and 
# write() exclusively.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix mmap2

if {[test_syscall defaultvfs] != "unix"} {
  finish_test
  return
}

db close
sqlite3_shutdown
test_sqlite3_log xLog
proc xLog {error_code msg} {
  if {[string match os_unix.c* $msg]} {
    lappend ::log $msg 
  }
}

foreach syscall {mmap mremap} {
  test_syscall uninstall 
  if {[catch {test_syscall install $syscall}]} continue

  for {set i 1} {$i < 20} {incr i} {
    reset_db

    test_syscall fault $i 1
    test_syscall errno $syscall ENOMEM
    set ::log ""

    do_execsql_test 1.$syscall.$i.1 {
      CREATE TABLE t1(a, b, UNIQUE(a, b));
      INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000));
      INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
      INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
      INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
      INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
      INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
      INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1;
    }

    set nFail [test_syscall fault 0 0]

    do_execsql_test 1.$syscall.$i.2 {
      SELECT count(*) FROM t1;
      PRAGMA integrity_check;
    } {64 ok}

    do_test 1.$syscall.$i.3 {
      expr {$nFail==0 || $nFail==1}
    } {1}

    do_test 1.$syscall.$i.4.nFail=$nFail {
      regexp ".*${syscall}.*" $::log
    } [expr $nFail>0]
  }
}

test_syscall uninstall 
finish_test

Changes to test/speed1p.test.
20
21
22
23
24
25
26


27
28
29
30
31
32
33
#sqlite3_config_scratch 29000 1
set old_lookaside [sqlite3_config_lookaside 2048 300]
#sqlite3_config_pagecache 1024 11000
set testdir [file dirname $argv0]
source $testdir/tester.tcl
speed_trial_init speed1



# Set a uniform random seed
expr srand(0)

# The number_name procedure below converts its argment (an integer)
# into a string which is the English-language name for that number.
#
# Example:







>
>







20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#sqlite3_config_scratch 29000 1
set old_lookaside [sqlite3_config_lookaside 2048 300]
#sqlite3_config_pagecache 1024 11000
set testdir [file dirname $argv0]
source $testdir/tester.tcl
speed_trial_init speed1

sqlite3_memdebug_vfs_oom_test 0

# Set a uniform random seed
expr srand(0)

# The number_name procedure below converts its argment (an integer)
# into a string which is the English-language name for that number.
#
# Example:
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    CREATE INDEX i2a ON t2(a);
    CREATE INDEX i2b ON t2(b);
  }
  execsql {
    SELECT name FROM sqlite_master ORDER BY 1;
  }
} {i2a i2b t1 t2}


# 50000 INSERTs on an unindexed table
#
set list {}
for {set i 1} {$i<=50000} {incr i} {
  set r [expr {int(rand()*500000)}]
  set x [number_name $r]







<







75
76
77
78
79
80
81

82
83
84
85
86
87
88
    CREATE INDEX i2a ON t2(a);
    CREATE INDEX i2b ON t2(b);
  }
  execsql {
    SELECT name FROM sqlite_master ORDER BY 1;
  }
} {i2a i2b t1 t2}


# 50000 INSERTs on an unindexed table
#
set list {}
for {set i 1} {$i<=50000} {incr i} {
  set r [expr {int(rand()*500000)}]
  set x [number_name $r]