Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Defer opening and writing statement journals until the size reaches a threshold (currently 64KiB). |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
cb9302cca423de41305719a49208daa3 |
User & Date: | drh 2016-03-04 14:43:44.681 |
Context
2016-03-04
| ||
21:18 | Fix an assert() in sqlite3VarintLen(), even though it is impossible to hit in SQLite due to the way sqlite3VarintLen() is used. (check-in: 251424c586 user: drh tags: trunk) | |
18:45 | Merge changes from trunk. (check-in: 5294c977d9 user: drh tags: analyze-worst-case) | |
16:42 | Merge recent enhancements from trunk. Default page size is 4096. Writes to statement journals are avoided. (check-in: 456df3365e user: drh tags: sessions) | |
14:57 | Merge recent enhancements from trunk, and especially the changes that reduce the heap-memory footprint of schemas, and defer opening and writing to statement journals. (check-in: 2f0c195ccc user: drh tags: apple-osx) | |
14:43 | Defer opening and writing statement journals until the size reaches a threshold (currently 64KiB). (check-in: cb9302cca4 user: drh tags: trunk) | |
14:23 | Update test cases to taken deferred statement-journal opening into account. (Closed-Leaf check-in: 5b2fe5219a user: drh tags: memjournal-exp) | |
04:01 | Change the default cache_size to -2000 (which means 2000*1024 bytes independent of page_size). (check-in: 2682e8e413 user: drh tags: trunk) | |
Changes
Changes to src/journal.c.
︙ | ︙ | |||
20 21 22 23 24 25 26 27 28 29 30 31 32 33 | ** be used to service read() and write() requests. The actual file ** on disk is not created or populated until either: ** ** 1) The in-memory representation grows too large for the allocated ** buffer, or ** 2) The sqlite3JournalCreate() function is called. */ #ifdef SQLITE_ENABLE_ATOMIC_WRITE #include "sqliteInt.h" /* ** A JournalFile object is a subclass of sqlite3_file used by ** as an open file handle for journal files. | > | 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | ** be used to service read() and write() requests. The actual file ** on disk is not created or populated until either: ** ** 1) The in-memory representation grows too large for the allocated ** buffer, or ** 2) The sqlite3JournalCreate() function is called. */ #if 0 #ifdef SQLITE_ENABLE_ATOMIC_WRITE #include "sqliteInt.h" /* ** A JournalFile object is a subclass of sqlite3_file used by ** as an open file handle for journal files. |
︙ | ︙ | |||
249 250 251 252 253 254 255 256 | /* ** Return the number of bytes required to store a JournalFile that uses vfs ** pVfs to create the underlying on-disk files. */ int sqlite3JournalSize(sqlite3_vfs *pVfs){ return (pVfs->szOsFile+sizeof(JournalFile)); } #endif | > | 250 251 252 253 254 255 256 257 258 | /* ** Return the number of bytes required to store a JournalFile that uses vfs ** pVfs to create the underlying on-disk files. */ int sqlite3JournalSize(sqlite3_vfs *pVfs){ return (pVfs->szOsFile+sizeof(JournalFile)); } #endif #endif |
Changes to src/memjournal.c.
︙ | ︙ | |||
17 18 19 20 21 22 23 | #include "sqliteInt.h" /* Forward references to internal structures */ typedef struct MemJournal MemJournal; typedef struct FilePoint FilePoint; typedef struct FileChunk FileChunk; | < < < < < < < < < < > > > | > > > > > > > > > > > | | > > > > > > > > > > > > > > | | | | < < < | | | | | | | | | | | | | | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | < > > > | | | > > | | | | | | | | | | | | | | | | | | | | | | | | > > > > > > > > > > | | < | < | | < > < > | > < > | < < < | > > | > > > > | 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 | #include "sqliteInt.h" /* Forward references to internal structures */ typedef struct MemJournal MemJournal; typedef struct FilePoint FilePoint; typedef struct FileChunk FileChunk; /* ** The rollback journal is composed of a linked list of these structures. ** ** The zChunk array is always at least 8 bytes in size - usually much more. ** Its actual size is stored in the MemJournal.nChunkSize variable. */ struct FileChunk { FileChunk *pNext; /* Next chunk in the journal */ u8 zChunk[8]; /* Content of this chunk */ }; /* ** By default, allocate this many bytes of memory for each FileChunk object. */ #define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024 /* ** For chunk size nChunkSize, return the number of bytes that should ** be allocated for each FileChunk structure. */ #define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8)) /* ** An instance of this object serves as a cursor into the rollback journal. ** The cursor can be either for reading or writing. */ struct FilePoint { sqlite3_int64 iOffset; /* Offset from the beginning of the file */ FileChunk *pChunk; /* Specific chunk into which cursor points */ }; /* ** This structure is a subclass of sqlite3_file. Each open memory-journal ** is an instance of this class. */ struct MemJournal { const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ int nChunkSize; /* In-memory chunk-size */ int nBuf; /* Bytes of data before flushing */ int nSize; /* Bytes of data currently in memory */ FileChunk *pFirst; /* Head of in-memory chunk-list */ FilePoint endpoint; /* Pointer to the end of the file */ FilePoint readpoint; /* Pointer to the end of the last xRead() */ int flags; /* xOpen flags */ sqlite3_vfs *pVfs; /* The "real" underlying VFS */ const char *zJournal; /* Name of the journal file */ sqlite3_file *pReal; /* The "real" underlying file descriptor */ }; /* ** Read data from the in-memory journal file. This is the implementation ** of the sqlite3_vfs.xRead method. */ static int memjrnlRead( sqlite3_file *pJfd, /* The journal file from which to read */ void *zBuf, /* Put the results here */ int iAmt, /* Number of bytes to read */ sqlite_int64 iOfst /* Begin reading at this offset */ ){ MemJournal *p = (MemJournal *)pJfd; if( p->pReal ){ return sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); }else if( (iAmt+iOfst)>p->endpoint.iOffset ){ return SQLITE_IOERR_SHORT_READ; }else{ u8 *zOut = zBuf; int nRead = iAmt; int iChunkOffset; FileChunk *pChunk; if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; for(pChunk=p->pFirst; ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; pChunk=pChunk->pNext ){ iOff += p->nChunkSize; } }else{ pChunk = p->readpoint.pChunk; } iChunkOffset = (int)(iOfst%p->nChunkSize); do { int iSpace = p->nChunkSize - iChunkOffset; int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); zOut += nCopy; nRead -= iSpace; iChunkOffset = 0; } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); p->readpoint.iOffset = iOfst+iAmt; p->readpoint.pChunk = pChunk; } return SQLITE_OK; } /* ** Free the list of FileChunk structures headed at MemJournal.pFirst. */ static void memjrnlFreeChunks(MemJournal *p){ FileChunk *pIter; FileChunk *pNext; for(pIter=p->pFirst; pIter; pIter=pNext){ pNext = pIter->pNext; sqlite3_free(pIter); } p->pFirst = 0; } /* ** Flush the contents of memory to a real file on disk. */ static int createFile(MemJournal *p){ int rc = SQLITE_OK; if( !p->pReal ){ sqlite3_file *pReal = (sqlite3_file *)&p[1]; rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); if( rc==SQLITE_OK ){ int nChunk = p->nChunkSize; i64 iOff = 0; FileChunk *pIter; p->pReal = pReal; for(pIter=p->pFirst; pIter && rc==SQLITE_OK; pIter=pIter->pNext){ int nWrite = nChunk; if( pIter==p->endpoint.pChunk ){ nWrite = p->endpoint.iOffset % p->nChunkSize; if( nWrite==0 ) nWrite = p->nChunkSize; } rc = sqlite3OsWrite(pReal, pIter->zChunk, nWrite, iOff); iOff += nWrite; } if( rc!=SQLITE_OK ){ /* If an error occurred while writing to the file, close it before ** returning. This way, SQLite uses the in-memory journal data to ** roll back changes made to the internal page-cache before this ** function was called. */ sqlite3OsClose(pReal); p->pReal = 0; }else{ /* No error has occurred. Free the in-memory buffers. */ memjrnlFreeChunks(p); } } } return rc; } /* ** Write data to the file. */ static int memjrnlWrite( sqlite3_file *pJfd, /* The journal file into which to write */ const void *zBuf, /* Take data to be written from here */ int iAmt, /* Number of bytes to write */ sqlite_int64 iOfst /* Begin writing at this offset into the file */ ){ MemJournal *p = (MemJournal *)pJfd; int nWrite = iAmt; u8 *zWrite = (u8 *)zBuf; /* If the file has already been created on disk. */ if( p->pReal ){ return sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); } /* If the file should be created now. */ else if( p->nBuf>0 && (iAmt+iOfst)>p->nBuf ){ int rc = createFile(p); if( rc==SQLITE_OK ){ rc = memjrnlWrite(pJfd, zBuf, iAmt, iOfst); } return rc; } /* If the contents of this write should be stored in memory */ else{ /* An in-memory journal file should only ever be appended to. Random ** access writes are not required. The only exception to this is when ** the in-memory journal is being used by a connection using the ** atomic-write optimization. In this case the first 28 bytes of the ** journal file may be written as part of committing the transaction. */ assert( iOfst==p->endpoint.iOffset || iOfst==0 ); if( iOfst==0 && p->pFirst ){ assert( p->nChunkSize>iAmt ); memcpy(p->pFirst->zChunk, zBuf, iAmt); }else{ while( nWrite>0 ){ FileChunk *pChunk = p->endpoint.pChunk; int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); if( iChunkOffset==0 ){ /* New chunk is required to extend the file. */ FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); if( !pNew ){ return SQLITE_IOERR_NOMEM_BKPT; } pNew->pNext = 0; if( pChunk ){ assert( p->pFirst ); pChunk->pNext = pNew; }else{ assert( !p->pFirst ); p->pFirst = pNew; } p->endpoint.pChunk = pNew; } memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); zWrite += iSpace; nWrite -= iSpace; p->endpoint.iOffset += iSpace; } p->nSize = iAmt + iOfst; } } return SQLITE_OK; } /* ** Truncate the file. ** ** If the journal file is already on disk, truncate it there. Or, if it ** is still in main memory but is being truncated to zero bytes in size, ** ignore */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; if( p->pReal ){ return sqlite3OsTruncate(p->pReal, size); }else if( size==0 ){ memjrnlFreeChunks(p); p->nSize = 0; p->endpoint.pChunk = 0; p->endpoint.iOffset = 0; p->readpoint.pChunk = 0; p->readpoint.iOffset = 0; } return SQLITE_OK; } /* ** Close the file. */ static int memjrnlClose(sqlite3_file *pJfd){ MemJournal *p = (MemJournal *)pJfd; memjrnlFreeChunks(p); if( p->pReal ) sqlite3OsClose(p->pReal); return SQLITE_OK; } /* ** Sync the file. ** ** If the real file has been created, call its xSync method. Otherwise, ** syncing an in-memory journal is a no-op. */ static int memjrnlSync(sqlite3_file *pJfd, int flags){ MemJournal *p = (MemJournal *)pJfd; if( p->pReal ){ return sqlite3OsSync(p->pReal, flags); } return SQLITE_OK; } /* ** Query the size of the file in bytes. */ static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ MemJournal *p = (MemJournal *)pJfd; if( p->pReal ){ return sqlite3OsFileSize(p->pReal, pSize); } *pSize = (sqlite_int64) p->endpoint.iOffset; return SQLITE_OK; } /* ** Table of methods for MemJournal sqlite3_file object. */ |
︙ | ︙ | |||
226 227 228 229 230 231 232 | 0, /* xShmBarrier */ 0, /* xShmUnmap */ 0, /* xFetch */ 0 /* xUnfetch */ }; /* | | > > > > > > > > > | > > > > > > | | > > > > | > > > | > > > > > | > > > > > > > | > < | | > > > > > > > > > > > > | > | > | > > | > > > > > > > > > | > | | | 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 | 0, /* xShmBarrier */ 0, /* xShmUnmap */ 0, /* xFetch */ 0 /* xUnfetch */ }; /* ** Open a journal file. ** ** The behaviour of the journal file depends on the value of parameter ** nBuf. If nBuf is 0, then the journal file is always create and ** accessed using the underlying VFS. If nBuf is less than zero, then ** all content is always stored in main-memory. Finally, if nBuf is a ** positive value, then the journal file is initially created in-memory ** but may be flushed to disk later on. In this case the journal file is ** flushed to disk either when it grows larger than nBuf bytes in size, ** or when sqlite3JournalCreate() is called. */ int sqlite3JournalOpen( sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ const char *zName, /* Name of the journal file */ sqlite3_file *pJfd, /* Preallocated, blank file handle */ int flags, /* Opening flags */ int nBuf /* Bytes buffered before opening the file */ ){ MemJournal *p = (MemJournal*)pJfd; /* Zero the file-handle object. If nBuf was passed zero, initialize ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls ** made on the journal file-handle. */ memset(p, 0, sizeof(MemJournal) + (pVfs ? pVfs->szOsFile : 0)); if( nBuf==0 ){ return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); } if( nBuf>0 ){ p->nChunkSize = nBuf; }else{ p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); } p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods; p->nBuf = nBuf; p->flags = flags; p->zJournal = zName; p->pVfs = pVfs; return SQLITE_OK; } /* ** Open an in-memory journal file. */ void sqlite3MemJournalOpen(sqlite3_file *pJfd){ sqlite3JournalOpen(0, 0, pJfd, 0, -1); } #ifdef SQLITE_ENABLE_ATOMIC_WRITE /* ** If the argument p points to a MemJournal structure that is not an ** in-memory-only journal file (i.e. is one that was opened with a +ve ** nBuf parameter), and the underlying file has not yet been created, ** create it now. */ int sqlite3JournalCreate(sqlite3_file *p){ int rc = SQLITE_OK; if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nBuf>0 ){ rc = createFile((MemJournal*)p); } return rc; } #endif /* ** The file-handle passed as the only argument is open on a journal file. ** Return true if this "journal file" is currently stored in heap memory, ** or false otherwise. */ int sqlite3JournalIsInMemory(sqlite3_file *p){ return p->pMethods==&MemJournalMethods && ((MemJournal*)p)->pReal==0; } /* ** Return the number of bytes required to store a JournalFile that uses vfs ** pVfs to create the underlying on-disk files. */ int sqlite3JournalSize(sqlite3_vfs *pVfs){ return pVfs->szOsFile + sizeof(MemJournal); } |
Changes to src/os.c.
︙ | ︙ | |||
62 63 64 65 66 67 68 | ** sqlite3OsAccess() ** sqlite3OsFullPathname() ** */ #if defined(SQLITE_TEST) int sqlite3_memdebug_vfs_oom_test = 1; #define DO_OS_MALLOC_TEST(x) \ | | | 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | ** sqlite3OsAccess() ** sqlite3OsFullPathname() ** */ #if defined(SQLITE_TEST) int sqlite3_memdebug_vfs_oom_test = 1; #define DO_OS_MALLOC_TEST(x) \ if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3JournalIsInMemory(x))) { \ void *pTstAlloc = sqlite3Malloc(10); \ if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT; \ sqlite3_free(pTstAlloc); \ } #else #define DO_OS_MALLOC_TEST(x) #endif |
︙ | ︙ |
Changes to src/pager.c.
︙ | ︙ | |||
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 | ** ** If an IO error occurs, abandon processing and return the IO error code. ** Otherwise, return SQLITE_OK. */ static int zeroJournalHdr(Pager *pPager, int doTruncate){ int rc = SQLITE_OK; /* Return code */ assert( isOpen(pPager->jfd) ); if( pPager->journalOff ){ const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ IOTRACE(("JZEROHDR %p\n", pPager)) if( doTruncate || iLimit==0 ){ rc = sqlite3OsTruncate(pPager->jfd, 0); }else{ | > | 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 | ** ** If an IO error occurs, abandon processing and return the IO error code. ** Otherwise, return SQLITE_OK. */ static int zeroJournalHdr(Pager *pPager, int doTruncate){ int rc = SQLITE_OK; /* Return code */ assert( isOpen(pPager->jfd) ); assert( !sqlite3JournalIsInMemory(pPager->jfd) ); if( pPager->journalOff ){ const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ IOTRACE(("JZEROHDR %p\n", pPager)) if( doTruncate || iLimit==0 ){ rc = sqlite3OsTruncate(pPager->jfd, 0); }else{ |
︙ | ︙ | |||
1720 1721 1722 1723 1724 1725 1726 | ** if it is open and the pager is not in exclusive mode. */ static void releaseAllSavepoints(Pager *pPager){ int ii; /* Iterator for looping through Pager.aSavepoint */ for(ii=0; ii<pPager->nSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } | | | 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 | ** if it is open and the pager is not in exclusive mode. */ static void releaseAllSavepoints(Pager *pPager){ int ii; /* Iterator for looping through Pager.aSavepoint */ for(ii=0; ii<pPager->nSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ sqlite3OsClose(pPager->sjfd); } sqlite3_free(pPager->aSavepoint); pPager->aSavepoint = 0; pPager->nSavepoint = 0; pPager->nSubRec = 0; } |
︙ | ︙ | |||
1958 1959 1960 1961 1962 1963 1964 | releaseAllSavepoints(pPager); assert( isOpen(pPager->jfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ | | | | 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 | releaseAllSavepoints(pPager); assert( isOpen(pPager->jfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ if( sqlite3JournalIsInMemory(pPager->jfd) ){ /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ sqlite3OsClose(pPager->jfd); }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ if( pPager->journalOff==0 ){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); if( rc==SQLITE_OK && pPager->fullSync ){ |
︙ | ︙ | |||
1985 1986 1987 1988 1989 1990 1991 | ){ rc = zeroJournalHdr(pPager, hasMaster); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to | | | > | 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 | ){ rc = zeroJournalHdr(pPager, hasMaster); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to ** the database file, it will do so using an in-memory journal. */ int bDelete = !pPager->tempFile; assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); if( bDelete ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync); |
︙ | ︙ | |||
2725 2726 2727 2728 2729 2730 2731 | ** If a master journal file name is specified, but the file is not ** present on disk, then the journal is not hot and does not need to be ** played back. ** ** TODO: Technically the following is an error because it assumes that ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, | | | 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 | ** If a master journal file name is specified, but the file is not ** present on disk, then the journal is not hot and does not need to be ** played back. ** ** TODO: Technically the following is an error because it assumes that ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, ** mxPathname is 512, which is the same as the minimum allowable value ** for pageSize. */ zMaster = pPager->pTmpSpace; rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); if( rc==SQLITE_OK && zMaster[0] ){ rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); } |
︙ | ︙ | |||
4349 4350 4351 4352 4353 4354 4355 4356 | ** SQLITE_OK is returned if everything goes according to plan. An ** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() ** fails. */ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ | > > > > < < < > > | 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 | ** SQLITE_OK is returned if everything goes according to plan. An ** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() ** fails. */ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; int nBuf = 64*1024; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ nBuf = -1; } rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nBuf); } return rc; } /* ** Append a record of the current state of page pPg to the sub-journal. ** |
︙ | ︙ | |||
4574 4575 4576 4577 4578 4579 4580 | int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle | | < < < < < < < | < < < | 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 | int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; #ifndef SQLITE_OMIT_MEMORYDB if( flags & PAGER_MEMORY ){ memDb = 1; |
︙ | ︙ | |||
6663 6664 6665 6666 6667 6668 6669 | pPager->nSavepoint = nNew; /* If this is a release of the outermost savepoint, truncate ** the sub-journal to zero bytes in size. */ if( op==SAVEPOINT_RELEASE ){ if( nNew==0 && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ | | | 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 | pPager->nSavepoint = nNew; /* If this is a release of the outermost savepoint, truncate ** the sub-journal to zero bytes in size. */ if( op==SAVEPOINT_RELEASE ){ if( nNew==0 && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ if( sqlite3JournalIsInMemory(pPager->sjfd) ){ rc = sqlite3OsTruncate(pPager->sjfd, 0); assert( rc==SQLITE_OK ); } pPager->nSubRec = 0; } } /* Else this is a rollback operation, playback the specified savepoint. |
︙ | ︙ |
Changes to src/sqliteInt.h.
︙ | ︙ | |||
3994 3995 3996 3997 3998 3999 4000 | ** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). */ #define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ #define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ int sqlite3FindInIndex(Parse *, Expr *, u32, int*); | < | | > < < < < > < < | 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 | ** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). */ #define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ #define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ int sqlite3FindInIndex(Parse *, Expr *, u32, int*); int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); int sqlite3JournalSize(sqlite3_vfs *); #ifdef SQLITE_ENABLE_ATOMIC_WRITE int sqlite3JournalCreate(sqlite3_file *); #endif int sqlite3JournalIsInMemory(sqlite3_file *p); void sqlite3MemJournalOpen(sqlite3_file *); void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 int sqlite3SelectExprHeight(Select *); int sqlite3ExprCheckHeight(Parse*, int); #else #define sqlite3SelectExprHeight(x) 0 |
︙ | ︙ |
Changes to test/conflict.test.
︙ | ︙ | |||
281 282 283 284 285 286 287 | # cmd An UPDATE command to execute against table t1 # t0 True if there is an error from $cmd # t1 Content of "b" column of t1 assuming no error in $cmd # t2 Content of "x" column of t3 # t3 Number of temporary files for tables # t4 Number of temporary files for statement journals # | | | > > | | | | 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 | # cmd An UPDATE command to execute against table t1 # t0 True if there is an error from $cmd # t1 Content of "b" column of t1 assuming no error in $cmd # t2 Content of "x" column of t3 # t3 Number of temporary files for tables # t4 Number of temporary files for statement journals # # Update (2007-08-21): Since temporary table files are now opened lazily, # and none of the following tests use large quantities of data, t3 is always 0. # # Update (2016-03-04): Subjournals now also open lazily, so t4 is also always 0. # foreach {i conf1 cmd t0 t1 t2 t3 t4} { 1 {} UPDATE 1 {6 7 8 9} 1 0 0 2 REPLACE UPDATE 0 {7 6 9} 1 0 0 3 IGNORE UPDATE 0 {6 7 3 9} 1 0 0 4 FAIL UPDATE 1 {6 7 3 4} 1 0 0 5 ABORT UPDATE 1 {1 2 3 4} 1 0 0 6 ROLLBACK UPDATE 1 {1 2 3 4} 0 0 0 7 REPLACE {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 8 IGNORE {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 9 FAIL {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 10 ABORT {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 11 ROLLBACK {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 12 {} {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 13 {} {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 14 {} {UPDATE OR FAIL} 1 {6 7 3 4} 1 0 0 15 {} {UPDATE OR ABORT} 1 {1 2 3 4} 1 0 0 16 {} {UPDATE OR ROLLBACK} 1 {1 2 3 4} 0 0 0 } { if {$t0} {set t1 {UNIQUE constraint failed: t1.a}} if {[info exists TEMP_STORE] && $TEMP_STORE==3} { set t3 0 } else { set t3 [expr {$t3+$t4}] |
︙ | ︙ |
Changes to test/conflict2.test.
︙ | ︙ | |||
282 283 284 285 286 287 288 289 290 | # t1 Content of "b" column of t1 assuming no error in $cmd # t2 Content of "x" column of t3 # t3 Number of temporary files for tables # t4 Number of temporary files for statement journals # # Update: Since temporary table files are now opened lazily, and none # of the following tests use large quantities of data, t3 is always 0. # foreach {i conf1 cmd t0 t1 t2 t3 t4} { | > > > | | | | | | | | | | | 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 | # t1 Content of "b" column of t1 assuming no error in $cmd # t2 Content of "x" column of t3 # t3 Number of temporary files for tables # t4 Number of temporary files for statement journals # # Update: Since temporary table files are now opened lazily, and none # of the following tests use large quantities of data, t3 is always 0. # # Update (2016-03-04): Subjournals now only open when their size # exceeds 64KB. # foreach {i conf1 cmd t0 t1 t2 t3 t4} { 1 {} UPDATE 1 {6 7 8 9} 1 0 0 2 REPLACE UPDATE 0 {7 6 9} 1 0 0 3 IGNORE UPDATE 0 {6 7 3 9} 1 0 0 4 FAIL UPDATE 1 {6 7 3 4} 1 0 0 5 ABORT UPDATE 1 {1 2 3 4} 1 0 0 6 ROLLBACK UPDATE 1 {1 2 3 4} 0 0 0 7 REPLACE {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 8 IGNORE {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 9 FAIL {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 10 ABORT {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 11 ROLLBACK {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 12 {} {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 0 13 {} {UPDATE OR REPLACE} 0 {7 6 9} 1 0 0 14 {} {UPDATE OR FAIL} 1 {6 7 3 4} 1 0 0 15 {} {UPDATE OR ABORT} 1 {1 2 3 4} 1 0 0 16 {} {UPDATE OR ROLLBACK} 1 {1 2 3 4} 0 0 0 } { # When using in-memory journals, no temporary files are required for # statement journals. if {[permutation] == "inmemory_journal"} { set t4 0 } |
︙ | ︙ |
Changes to test/exclusive.test.
︙ | ︙ | |||
416 417 418 419 420 421 422 423 424 | BEGIN; INSERT INTO abc VALUES(1, 2, 3); INSERT INTO abc SELECT a+1, b+1, c+1 FROM abc; } } {} do_test exclusive-5.1 { # Three files are open: The db, journal and statement-journal. set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds | > | | 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 | BEGIN; INSERT INTO abc VALUES(1, 2, 3); INSERT INTO abc SELECT a+1, b+1, c+1 FROM abc; } } {} do_test exclusive-5.1 { # Three files are open: The db, journal and statement-journal. # (2016-03-04) The statement-journal is now opened lazily set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds } [expr 2 - ($TEMP_STORE>=2)] do_test exclusive-5.2 { execsql { COMMIT; } # One file open: the db. set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds |
︙ | ︙ | |||
442 443 444 445 446 447 448 449 450 | expr $sqlite_open_file_count-$extrafds } {2} do_test exclusive-5.4 { execsql { INSERT INTO abc SELECT a+10, b+10, c+10 FROM abc; } # Three files are open: The db, journal and statement-journal. set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds | > | > | | 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 | expr $sqlite_open_file_count-$extrafds } {2} do_test exclusive-5.4 { execsql { INSERT INTO abc SELECT a+10, b+10, c+10 FROM abc; } # Three files are open: The db, journal and statement-journal. # 2016-03-04: The statement-journal open is deferred set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds } [expr 2 - ($TEMP_STORE>=2)] do_test exclusive-5.5 { execsql { COMMIT; } # Three files are still open: The db, journal and statement-journal. # 2016-03-04: The statement-journal open is deferred set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds } [expr 2 - ($TEMP_STORE>=2)] do_test exclusive-5.6 { execsql { PRAGMA locking_mode = normal; SELECT * FROM abc; } } {normal 1 2 3 2 3 4 5 6 7 11 12 13 12 13 14 15 16 17} do_test exclusive-5.7 { |
︙ | ︙ |
Changes to test/stmt.test.
︙ | ︙ | |||
42 43 44 45 46 47 48 | set sqlite_open_file_count } {2} do_test stmt-1.4 { execsql { INSERT INTO t1 SELECT a+1, b+1 FROM t1; } set sqlite_open_file_count | > | > | | 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | set sqlite_open_file_count } {2} do_test stmt-1.4 { execsql { INSERT INTO t1 SELECT a+1, b+1 FROM t1; } set sqlite_open_file_count # 2016-03-04: statement-journal open deferred } {2} do_test stmt-1.5 { execsql COMMIT set sqlite_open_file_count } {1} do_test stmt-1.6.1 { execsql { BEGIN; INSERT INTO t1 SELECT a+2, b+2 FROM t1; } set sqlite_open_file_count } {2} do_test stmt-1.6.2 { execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 } set sqlite_open_file_count # 2016-03-04: statement-journal open deferred } {2} do_test stmt-1.7 { execsql COMMIT set sqlite_open_file_count } {1} proc filecount {testname sql expected} { |
︙ | ︙ | |||
80 81 82 83 84 85 86 | filecount stmt-2.1 { INSERT INTO t1 VALUES(9, 9) } 2 filecount stmt-2.2 { REPLACE INTO t1 VALUES(9, 9) } 2 filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9 } 2 filecount stmt-2.4 { INSERT INTO t1 SELECT 9, 9; INSERT INTO t1 SELECT 10, 10; | | | | 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | filecount stmt-2.1 { INSERT INTO t1 VALUES(9, 9) } 2 filecount stmt-2.2 { REPLACE INTO t1 VALUES(9, 9) } 2 filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9 } 2 filecount stmt-2.4 { INSERT INTO t1 SELECT 9, 9; INSERT INTO t1 SELECT 10, 10; } 2 do_test stmt-2.5 { execsql { CREATE INDEX i1 ON t1(b) } } {} filecount stmt-2.6 { REPLACE INTO t1 VALUES(5, 5); REPLACE INTO t1 VALUES(5, 5); } 2 finish_test |
Changes to test/tempdb.test.
︙ | ︙ | |||
72 73 74 75 76 77 78 | INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); INSERT INTO t2 VALUES(7, 8, 9); INSERT INTO t2 SELECT * FROM t1; } catchsql { INSERT INTO t1 SELECT * FROM t2 } set sqlite_open_file_count | | | 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); INSERT INTO t2 VALUES(7, 8, 9); INSERT INTO t2 SELECT * FROM t1; } catchsql { INSERT INTO t1 SELECT * FROM t2 } set sqlite_open_file_count } [expr 1 + (0==$jrnl_in_memory)] do_test tempdb-2.3 { execsql { PRAGMA temp_store = 'memory'; ROLLBACK; BEGIN; INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); |
︙ | ︙ |