Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | When possible, use memory mapping when appending new pages to a database file. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | experimental-mmap |
Files: | files | file ages | folders |
SHA1: |
14135da3cdbafd699563a29608f32347 |
User & Date: | dan 2013-03-20 14:26:59.370 |
Context
2013-03-20
| ||
18:25 | Optimize the xMremap method in os_unix.c some. (check-in: 9529ed88a7 user: dan tags: experimental-mmap) | |
14:26 | When possible, use memory mapping when appending new pages to a database file. (check-in: 14135da3cd user: dan tags: experimental-mmap) | |
10:07 | Add test file mmap1.test. (check-in: aee1f53a74 user: dan tags: experimental-mmap) | |
Changes
Changes to src/os.c.
︙ | ︙ | |||
136 137 138 139 140 141 142 | int pgsz, int bExtend, /* True to extend file if necessary */ void volatile **pp /* OUT: Pointer to mapping */ ){ DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } | | > > > > > > > | | 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | int pgsz, int bExtend, /* True to extend file if necessary */ void volatile **pp /* OUT: Pointer to mapping */ ){ DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } int sqlite3OsMremap( sqlite3_file *id, /* Database file handle */ int flags, /* SQLITE_MREMAP_XXX flags */ i64 iOff, /* Offset at which mapping(s) start */ i64 nOld, /* Size of old mapping */ i64 nNew, /* Size of requested mapping */ void **pp /* IN/OUT: Pointer to mapped region */ ){ return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); } /* ** The next group of routines are convenience wrappers around the ** VFS methods. */ int sqlite3OsOpen( |
︙ | ︙ |
Changes to src/os.h.
︙ | ︙ | |||
255 256 257 258 259 260 261 | #define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 int sqlite3OsSectorSize(sqlite3_file *id); int sqlite3OsDeviceCharacteristics(sqlite3_file *id); int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); | | | 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 | #define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 int sqlite3OsSectorSize(sqlite3_file *id); int sqlite3OsDeviceCharacteristics(sqlite3_file *id); int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); /* ** Functions for accessing sqlite3_vfs methods */ int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); int sqlite3OsDelete(sqlite3_vfs *, const char *, int); |
︙ | ︙ |
Changes to src/os_unix.c.
︙ | ︙ | |||
4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 | #endif /* #ifndef SQLITE_OMIT_WAL */ /* ** Map, remap or unmap part of the database file. */ static int unixMremap( sqlite3_file *fd, /* Main database file */ sqlite3_int64 iOff, /* Offset to start mapping at */ sqlite3_int64 nOld, /* Size of old mapping, or zero */ sqlite3_int64 nNew, /* Size of new mapping, or zero */ void **ppMap /* IN/OUT: Old/new mappings */ ){ unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ assert( iOff==0 ); if( nOld!=0 ){ void *pOld = *ppMap; munmap(pOld, nOld); } if( nNew>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; | > > > > > > > > > > > > > > > > > > > > > > > > > > > > < | | | 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 | #endif /* #ifndef SQLITE_OMIT_WAL */ /* ** Map, remap or unmap part of the database file. */ static int unixMremap( sqlite3_file *fd, /* Main database file */ int flags, /* Mask of SQLITE_MREMAP_XXX flags */ sqlite3_int64 iOff, /* Offset to start mapping at */ sqlite3_int64 nOld, /* Size of old mapping, or zero */ sqlite3_int64 nNew, /* Size of new mapping, or zero */ void **ppMap /* IN/OUT: Old/new mappings */ ){ unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ i64 nRnd; /* nNew rounded up to 4096 */ assert( iOff==0 ); nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. ** If this is the case, extend the file on disk using ftruncate(). */ assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); if( flags & SQLITE_MREMAP_EXTEND ){ struct stat statbuf; /* Low-level file information */ rc = osFstat(p->h, &statbuf); if( rc==SQLITE_OK && nNew>statbuf.st_size ){ rc = robust_ftruncate(p->h, nNew); } if( rc!=SQLITE_OK ) return rc; } #if defined(_GNU_SOURCE) && defined(__linux__) if( nRnd && nOld ){ void *pOld = *ppMap; *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); if( pNew==MAP_FAILED ){ *ppMap = 0; return SQLITE_IOERR_MREMAP; } return SQLITE_OK; } #endif if( nOld!=0 ){ void *pOld = *ppMap; munmap(pOld, nOld); } if( nNew>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; rc = SQLITE_IOERR_MREMAP; } } *ppMap = pNew; return rc; } |
︙ | ︙ |
Changes to src/pager.c.
︙ | ︙ | |||
3850 3851 3852 3853 3854 3855 3856 | } /* ** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ | | | > | | 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 | } /* ** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } return SQLITE_OK; } /* ** Create, or recreate, the memory mapping of the database file. */ static int pagerMap(Pager *pPager, int bExtend){ int rc = SQLITE_OK; /* Return code */ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); assert( pPager->pMap==0 || pPager->nMap>0 ); assert( pPager->eState>=1 ); assert( pPager->nMmapOut==0 ); assert( pPager->nMapLimit>0 ); /* Figure out how large a mapping to request. Set variable sz to this ** value in bytes. */ nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; sz = (i64)nPg * pPager->pageSize; if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; if( sz!=pPager->nMapValid ){ int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ assert( pPager->pMap!=0 ); pPager->nMap = sz; }else{ assert( pPager->pMap==0 ); pPager->nMap = 0; } |
︙ | ︙ | |||
4242 4243 4244 4245 4246 4247 4248 | rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); } /* Before the first write, give the VFS a hint of what the final ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); | | > > > > > > > | 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 | rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); } /* Before the first write, give the VFS a hint of what the final ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); if( rc==SQLITE_OK && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ pPager->dbFileSize = pPager->dbSize; pagerMap(pPager, 1); } } while( rc==SQLITE_OK && pList ){ Pgno pgno = pList->pgno; /* If there are dirty pages in the page cache with page numbers greater ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to |
︙ | ︙ | |||
5269 5270 5271 5272 5273 5274 5275 | ** Otherwise, request the page from the PCache layer. */ if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ if( bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ | | | 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 | ** Otherwise, request the page from the PCache layer. */ if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ if( bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager, 0); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } if( pPg==0 ){ rc = pagerAcquireMapPage(pPager, pgno, &pPg); |
︙ | ︙ |
Changes to src/pager.h.
︙ | ︙ | |||
104 105 106 107 108 109 110 111 112 113 114 115 116 117 | int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); int sqlite3PagerSetJournalMode(Pager *, int); int sqlite3PagerGetJournalMode(Pager*); int sqlite3PagerOkToChangeJournalMode(Pager*); i64 sqlite3PagerJournalSizeLimit(Pager *, i64); | > | 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); void sqlite3PagerSetMmapsize(Pager *, int); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); int sqlite3PagerSetJournalMode(Pager *, int); int sqlite3PagerGetJournalMode(Pager*); int sqlite3PagerOkToChangeJournalMode(Pager*); i64 sqlite3PagerJournalSizeLimit(Pager *, i64); |
︙ | ︙ |
Changes to src/sqlite.h.in.
︙ | ︙ | |||
466 467 468 469 470 471 472 473 474 475 476 477 478 479 | #define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) #define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) #define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) | > | 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 | #define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) #define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) #define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) #define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) |
︙ | ︙ | |||
742 743 744 745 746 747 748 | int (*xDeviceCharacteristics)(sqlite3_file*); /* Methods above are valid for version 1 */ int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ | | > > | 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 | int (*xDeviceCharacteristics)(sqlite3_file*); /* Methods above are valid for version 1 */ int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ int (*xMremap)(sqlite3_file *fd, int flags, sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; #define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ /* ** CAPI3REF: Standard File Control Opcodes ** ** These integer constants are opcodes for the xFileControl method ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] ** interface. |
︙ | ︙ |