Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | When possible, use memory mapping when appending new pages to a database file. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | experimental-mmap |
| Files: | files | file ages | folders |
| SHA1: |
14135da3cdbafd699563a29608f32347 |
| User & Date: | dan 2013-03-20 14:26:59 |
Context
|
2013-03-20
| ||
| 18:25 | Optimize the xMremap method in os_unix.c some. (check-in: 9529ed88 user: dan tags: experimental-mmap) | |
| 14:26 | When possible, use memory mapping when appending new pages to a database file. (check-in: 14135da3 user: dan tags: experimental-mmap) | |
| 10:07 | Add test file mmap1.test. (check-in: aee1f53a user: dan tags: experimental-mmap) | |
Changes
Changes to src/os.c.
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
int pgsz,
int bExtend, /* True to extend file if necessary */
void volatile **pp /* OUT: Pointer to mapping */
){
DO_OS_MALLOC_TEST(id);
return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp);
}
int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){
return id->pMethods->xMremap(id, iOff, nOld, nNew, pp);
}
/*
** The next group of routines are convenience wrappers around the
** VFS methods.
*/
int sqlite3OsOpen(
|
| > > > > > > > | |
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 |
int pgsz,
int bExtend, /* True to extend file if necessary */
void volatile **pp /* OUT: Pointer to mapping */
){
DO_OS_MALLOC_TEST(id);
return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp);
}
int sqlite3OsMremap(
sqlite3_file *id, /* Database file handle */
int flags, /* SQLITE_MREMAP_XXX flags */
i64 iOff, /* Offset at which mapping(s) start */
i64 nOld, /* Size of old mapping */
i64 nNew, /* Size of requested mapping */
void **pp /* IN/OUT: Pointer to mapped region */
){
return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp);
}
/*
** The next group of routines are convenience wrappers around the
** VFS methods.
*/
int sqlite3OsOpen(
|
Changes to src/os.h.
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 |
#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 int sqlite3OsSectorSize(sqlite3_file *id); int sqlite3OsDeviceCharacteristics(sqlite3_file *id); int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); /* ** Functions for accessing sqlite3_vfs methods */ int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); int sqlite3OsDelete(sqlite3_vfs *, const char *, int); |
| |
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 |
#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0
int sqlite3OsSectorSize(sqlite3_file *id);
int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **);
int sqlite3OsShmLock(sqlite3_file *id, int, int, int);
void sqlite3OsShmBarrier(sqlite3_file *id);
int sqlite3OsShmUnmap(sqlite3_file *id, int);
int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **);
/*
** Functions for accessing sqlite3_vfs methods
*/
int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *);
int sqlite3OsDelete(sqlite3_vfs *, const char *, int);
|
Changes to src/os_unix.c.
4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 |
#endif /* #ifndef SQLITE_OMIT_WAL */
/*
** Map, remap or unmap part of the database file.
*/
static int unixMremap(
sqlite3_file *fd, /* Main database file */
sqlite3_int64 iOff, /* Offset to start mapping at */
sqlite3_int64 nOld, /* Size of old mapping, or zero */
sqlite3_int64 nNew, /* Size of new mapping, or zero */
void **ppMap /* IN/OUT: Old/new mappings */
){
unixFile *p = (unixFile *)fd; /* The underlying database file */
int rc = SQLITE_OK; /* Return code */
void *pNew = 0; /* New mapping */
assert( iOff==0 );
if( nOld!=0 ){
void *pOld = *ppMap;
munmap(pOld, nOld);
}
if( nNew>0 ){
int flags = PROT_READ;
if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;
nNew = (nNew+4095) & ~(i64)((1 << 12)-1);
pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff);
if( pNew==MAP_FAILED ){
pNew = 0;
rc = SQLITE_IOERR;
}
}
*ppMap = pNew;
return rc;
}
|
> > > > > > > > > > > > > > > > > > > > > > > > > > > > < | | |
4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 |
#endif /* #ifndef SQLITE_OMIT_WAL */ /* ** Map, remap or unmap part of the database file. */ static int unixMremap( sqlite3_file *fd, /* Main database file */ int flags, /* Mask of SQLITE_MREMAP_XXX flags */ sqlite3_int64 iOff, /* Offset to start mapping at */ sqlite3_int64 nOld, /* Size of old mapping, or zero */ sqlite3_int64 nNew, /* Size of new mapping, or zero */ void **ppMap /* IN/OUT: Old/new mappings */ ){ unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ i64 nRnd; /* nNew rounded up to 4096 */ assert( iOff==0 ); nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. ** If this is the case, extend the file on disk using ftruncate(). */ assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); if( flags & SQLITE_MREMAP_EXTEND ){ struct stat statbuf; /* Low-level file information */ rc = osFstat(p->h, &statbuf); if( rc==SQLITE_OK && nNew>statbuf.st_size ){ rc = robust_ftruncate(p->h, nNew); } if( rc!=SQLITE_OK ) return rc; } #if defined(_GNU_SOURCE) && defined(__linux__) if( nRnd && nOld ){ void *pOld = *ppMap; *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); if( pNew==MAP_FAILED ){ *ppMap = 0; return SQLITE_IOERR_MREMAP; } return SQLITE_OK; } #endif if( nOld!=0 ){ void *pOld = *ppMap; munmap(pOld, nOld); } if( nNew>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; rc = SQLITE_IOERR_MREMAP; } } *ppMap = pNew; return rc; } |
Changes to src/pager.c.
3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 .... 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 .... 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 .... 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 |
}
/*
** Unmap any memory mapping of the database file.
*/
static int pagerUnmap(Pager *pPager){
if( pPager->pMap ){
sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap);
pPager->nMap = 0;
pPager->nMapValid = 0;
}
return SQLITE_OK;
}
/*
** Create, or recreate, the memory mapping of the database file.
*/
static int pagerMap(Pager *pPager){
int rc = SQLITE_OK; /* Return code */
Pgno nPg; /* Size of mapping to request in pages */
i64 sz; /* Size of mapping to request in bytes */
assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 );
assert( pPager->pMap==0 || pPager->nMap>0 );
assert( pPager->eState>=1 );
................................................................................
/* Figure out how large a mapping to request. Set variable sz to this
** value in bytes. */
nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize;
sz = (i64)nPg * pPager->pageSize;
if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit;
if( sz!=pPager->nMapValid ){
rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap);
if( rc==SQLITE_OK ){
assert( pPager->pMap!=0 );
pPager->nMap = sz;
}else{
assert( pPager->pMap==0 );
pPager->nMap = 0;
}
................................................................................
rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags);
}
/* Before the first write, give the VFS a hint of what the final
** file size will be.
*/
assert( rc!=SQLITE_OK || isOpen(pPager->fd) );
if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){
sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize;
sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile);
pPager->dbHintSize = pPager->dbSize;
}
while( rc==SQLITE_OK && pList ){
Pgno pgno = pList->pgno;
/* If there are dirty pages in the page cache with page numbers greater
** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
................................................................................
** Otherwise, request the page from the PCache layer. */
if( pPager->errCode!=SQLITE_OK ){
rc = pPager->errCode;
}else{
if( bMmapOk ){
if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){
rc = pagerMap(pPager);
}
if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){
if( pPager->eState>PAGER_READER ){
(void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg);
}
if( pPg==0 ){
rc = pagerAcquireMapPage(pPager, pgno, &pPg);
|
| | > | | > > > > > > > | |
3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 .... 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 .... 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 .... 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 |
}
/*
** Unmap any memory mapping of the database file.
*/
static int pagerUnmap(Pager *pPager){
if( pPager->pMap ){
sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap);
pPager->nMap = 0;
pPager->nMapValid = 0;
}
return SQLITE_OK;
}
/*
** Create, or recreate, the memory mapping of the database file.
*/
static int pagerMap(Pager *pPager, int bExtend){
int rc = SQLITE_OK; /* Return code */
Pgno nPg; /* Size of mapping to request in pages */
i64 sz; /* Size of mapping to request in bytes */
assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 );
assert( pPager->pMap==0 || pPager->nMap>0 );
assert( pPager->eState>=1 );
................................................................................
/* Figure out how large a mapping to request. Set variable sz to this
** value in bytes. */
nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize;
sz = (i64)nPg * pPager->pageSize;
if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit;
if( sz!=pPager->nMapValid ){
int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0);
rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap);
if( rc==SQLITE_OK ){
assert( pPager->pMap!=0 );
pPager->nMap = sz;
}else{
assert( pPager->pMap==0 );
pPager->nMap = 0;
}
................................................................................
rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags);
}
/* Before the first write, give the VFS a hint of what the final
** file size will be.
*/
assert( rc!=SQLITE_OK || isOpen(pPager->fd) );
if( rc==SQLITE_OK
&& (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize
){
sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize;
sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile);
pPager->dbHintSize = pPager->dbSize;
if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){
pPager->dbFileSize = pPager->dbSize;
pagerMap(pPager, 1);
}
}
while( rc==SQLITE_OK && pList ){
Pgno pgno = pList->pgno;
/* If there are dirty pages in the page cache with page numbers greater
** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
................................................................................
** Otherwise, request the page from the PCache layer. */
if( pPager->errCode!=SQLITE_OK ){
rc = pPager->errCode;
}else{
if( bMmapOk ){
if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){
rc = pagerMap(pPager, 0);
}
if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){
if( pPager->eState>PAGER_READER ){
(void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg);
}
if( pPg==0 ){
rc = pagerAcquireMapPage(pPager, pgno, &pPg);
|
Changes to src/pager.h.
104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); int sqlite3PagerSetJournalMode(Pager *, int); int sqlite3PagerGetJournalMode(Pager*); int sqlite3PagerOkToChangeJournalMode(Pager*); i64 sqlite3PagerJournalSizeLimit(Pager *, i64); |
> |
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
int sqlite3PagerReadFileheader(Pager*, int, unsigned char*);
/* Functions used to configure a Pager object. */
void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *);
int sqlite3PagerSetPagesize(Pager*, u32*, int);
int sqlite3PagerMaxPageCount(Pager*, int);
void sqlite3PagerSetCachesize(Pager*, int);
void sqlite3PagerSetMmapsize(Pager *, int);
void sqlite3PagerShrink(Pager*);
void sqlite3PagerSetSafetyLevel(Pager*,int,int,int);
int sqlite3PagerLockingMode(Pager *, int);
int sqlite3PagerSetJournalMode(Pager *, int);
int sqlite3PagerGetJournalMode(Pager*);
int sqlite3PagerOkToChangeJournalMode(Pager*);
i64 sqlite3PagerJournalSizeLimit(Pager *, i64);
|
Changes to src/sqlite.h.in.
466
467
468
469
470
471
472
473
474
475
476
477
478
479
...
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
|
#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8))
#define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8))
#define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8))
#define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8))
#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8))
#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8))
#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8))
#define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8))
#define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8))
#define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8))
#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8))
#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8))
#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8))
#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8))
................................................................................
int (*xDeviceCharacteristics)(sqlite3_file*);
/* Methods above are valid for version 1 */
int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
int (*xShmLock)(sqlite3_file*, int offset, int n, int flags);
void (*xShmBarrier)(sqlite3_file*);
int (*xShmUnmap)(sqlite3_file*, int deleteFlag);
/* Methods above are valid for version 2 */
int (*xMremap)(sqlite3_file *fd,
sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap);
/* Methods above are valid for version 3 */
/* Additional methods may be added in future releases */
};
/*
** CAPI3REF: Standard File Control Opcodes
**
** These integer constants are opcodes for the xFileControl method
** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()]
** interface.
|
>
|
>
>
|
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
...
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
|
#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) #define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) #define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) #define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) ................................................................................ int (*xDeviceCharacteristics)(sqlite3_file*); /* Methods above are valid for version 1 */ int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ int (*xMremap)(sqlite3_file *fd, int flags, sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; #define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ /* ** CAPI3REF: Standard File Control Opcodes ** ** These integer constants are opcodes for the xFileControl method ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] ** interface. |