Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Defer opening the file used for the temp database (where CREATE TEMP TABLE tables are stored) until the database is too large to reside entirely within the cache. There are likely still problems on this branch. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | tempfiles-lazy-open |
Files: | files | file ages | folders |
SHA1: |
be5a549eba6cf8e29cb6b9824fd6d0db |
User & Date: | dan 2016-04-05 21:07:58.179 |
Context
2016-04-06
| ||
15:39 | Add tests to this branch. Fix a problem with temporary databases in auto-vacuum mode. (check-in: afe9bd9b4b user: dan tags: tempfiles-lazy-open) | |
2016-04-05
| ||
21:07 | Defer opening the file used for the temp database (where CREATE TEMP TABLE tables are stored) until the database is too large to reside entirely within the cache. There are likely still problems on this branch. (check-in: be5a549eba user: dan tags: tempfiles-lazy-open) | |
20:59 | Carry table column types through into VIEW definitions, where possible. (check-in: fb555c3c2a user: drh tags: trunk) | |
Changes
Changes to src/pager.c.
︙ | ︙ | |||
2002 2003 2004 2005 2006 2007 2008 | } } #endif sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; | > | > > > | 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 | } } #endif sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; if( pPager->tempFile==0 || MEMDB ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); } sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ /* Drop the WAL write-lock, if any. Also, if the connection was in ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE ** lock held on the database file. */ |
︙ | ︙ | |||
2370 2371 2372 2373 2374 2375 2376 2377 2378 | ** already in the journal file (recorded in Pager.pInJournal) and ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to ** again within this transaction, it will be marked as dirty but ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially ** be written out into the database file before its journal file ** segment is synced. If a crash occurs during or following this, ** database corruption may ensue. */ assert( !pagerUseWal(pPager) ); | > > > > | | 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 | ** already in the journal file (recorded in Pager.pInJournal) and ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to ** again within this transaction, it will be marked as dirty but ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially ** be written out into the database file before its journal file ** segment is synced. If a crash occurs during or following this, ** database corruption may ensue. ** ** Update: Another exception is for temp files that are not ** in-memory databases. In this case the page may have been dirty ** at the start of the transaction. */ assert( !pagerUseWal(pPager) ); if( pPager->tempFile==0 ) sqlite3PcacheMakeClean(pPg); } pager_set_pagehash(pPg); /* If this was page 1, then restore the value of Pager.dbFileVers. ** Do this before any decoding. */ if( pgno==1 ){ memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); |
︙ | ︙ | |||
4254 4255 4256 4257 4258 4259 4260 | ** be obtained, SQLITE_BUSY is returned. */ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ int rc = SQLITE_OK; /* Return code */ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); | | > | 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 | ** be obtained, SQLITE_BUSY is returned. */ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ int rc = SQLITE_OK; /* Return code */ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); assert( pPager->tempFile==0 || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch ** is taken, as pager_wait_on_lock() is a no-op for temp-files. */ if( !isOpen(pPager->fd) ){ assert( pPager->tempFile && rc==SQLITE_OK ); |
︙ | ︙ | |||
5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 | assert( assert_pager_state(pPager) ); if( pPager->errCode ){ return pPager->errCode; }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); } } /* | > | 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 | assert( assert_pager_state(pPager) ); if( pPager->errCode ){ return pPager->errCode; }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ assert( pPager->tempFile==0 ); return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); } } /* |
︙ | ︙ | |||
6175 6176 6177 6178 6179 6180 6181 | PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eState<PAGER_WRITER_CACHEMOD ) return SQLITE_OK; | | > | < | 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 | PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eState<PAGER_WRITER_CACHEMOD ) return SQLITE_OK; assert( MEMDB==0 || pPager->tempFile ); if( pPager->tempFile ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); PgHdr *pPageOne = 0; if( pList==0 ){ /* Must have at least one page for the WAL commit flag. |
︙ | ︙ | |||
6841 6842 6843 6844 6845 6846 6847 | /* If the journal needs to be sync()ed before page pPg->pgno can ** be written to, store pPg->pgno in local variable needSyncPgno. ** ** If the isCommit flag is set, there is no need to remember that ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ | | | 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 | /* If the journal needs to be sync()ed before page pPg->pgno can ** be written to, store pPg->pgno in local variable needSyncPgno. ** ** If the isCommit flag is set, there is no need to remember that ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit && pPager->tempFile==0 ){ needSyncPgno = pPg->pgno; assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); } /* If the cache contains a page with page-number pgno, remove it |
︙ | ︙ |
Changes to src/pcache.c.
︙ | ︙ | |||
250 251 252 253 254 255 256 | assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); } /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new | | | 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 | assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); } /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new ** page because no clean pages are available for reuse and the cache ** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to ** allocate the new page and will only fail to allocate a new page on ** an OOM error. ** ** This routine should be invoked only after sqlite3PcacheFetch() fails. |
︙ | ︙ | |||
434 435 436 437 438 439 440 441 442 443 444 445 446 447 | */ void sqlite3PcacheCleanAll(PCache *pCache){ PgHdr *p; while( (p = pCache->pDirty)!=0 ){ sqlite3PcacheMakeClean(p); } } /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ void sqlite3PcacheClearSyncFlags(PCache *pCache){ PgHdr *p; for(p=pCache->pDirty; p; p=p->pDirtyNext){ | > > > > > > > > > > > | 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 | */ void sqlite3PcacheCleanAll(PCache *pCache){ PgHdr *p; while( (p = pCache->pDirty)!=0 ){ sqlite3PcacheMakeClean(p); } } /* ** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. */ void sqlite3PcacheClearWritable(PCache *pCache){ PgHdr *p; for(p=pCache->pDirty; p; p=p->pDirtyNext){ p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); } pCache->pSynced = pCache->pDirtyTail; } /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ void sqlite3PcacheClearSyncFlags(PCache *pCache){ PgHdr *p; for(p=pCache->pDirty; p; p=p->pDirtyNext){ |
︙ | ︙ | |||
480 481 482 483 484 485 486 | for(p=pCache->pDirty; p; p=pNext){ pNext = p->pDirtyNext; /* This routine never gets call with a positive pgno except right ** after sqlite3PcacheCleanAll(). So if there are dirty pages, ** it must be that pgno==0. */ assert( p->pgno>0 ); | | | 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 | for(p=pCache->pDirty; p; p=pNext){ pNext = p->pDirtyNext; /* This routine never gets call with a positive pgno except right ** after sqlite3PcacheCleanAll(). So if there are dirty pages, ** it must be that pgno==0. */ assert( p->pgno>0 ); if( p->pgno>pgno ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } } if( pgno==0 && pCache->nRefSum ){ sqlite3_pcache_page *pPage1; pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); |
︙ | ︙ |
Added test/temptable2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | # 2016 March 3 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix temptable2 do_execsql_test 1.1 { CREATE TEMP TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); } do_execsql_test 1.2 { WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100000 ) INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM X; } {} do_execsql_test 1.3 { PRAGMA temp.integrity_check; } {ok} #------------------------------------------------------------------------- # reset_db do_execsql_test 2.1 { CREATE TEMP TABLE t2(a, b); INSERT INTO t2 VALUES(1, 2); } {} do_execsql_test 2.2 { BEGIN; INSERT INTO t2 VALUES(3, 4); SELECT * FROM t2; } {1 2 3 4} do_execsql_test 2.3 { ROLLBACK; SELECT * FROM t2; } {1 2} #------------------------------------------------------------------------- # reset_db do_execsql_test 3.1.1 { PRAGMA main.cache_size = 10; PRAGMA temp.cache_size = 10; CREATE TEMP TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 ) INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; SELECT count(*) FROM t1; } {1000} do_execsql_test 3.1.2 { BEGIN; UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; ROLLBACK; } do_execsql_test 3.1.3 { SELECT count(*) FROM t1; } {1000} do_execsql_test 3.1.4 { PRAGMA temp.integrity_check } {ok} do_execsql_test 3.2.1 { BEGIN; UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0; SAVEPOINT abc; UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==1; ROLLBACK TO abc; UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==2; COMMIT; } do_execsql_test 3.2.2 { PRAGMA temp.integrity_check } {ok} #------------------------------------------------------------------------- # reset_db do_execsql_test 4.1.1 { PRAGMA main.cache_size = 10; PRAGMA temp.cache_size = 10; CREATE TEMP TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<10 ) INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x; SELECT count(*) FROM t1; PRAGMA temp.page_count; } {10 9} do_execsql_test 4.1.2 { BEGIN; UPDATE t1 SET b=randomblob(100); ROLLBACK; } do_execsql_test 4.1.3 { CREATE TEMP TABLE t2(a, b); CREATE INDEX i2 ON t2(a, b); WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; SELECT count(*) FROM t2; SELECT count(*) FROM t1; PRAGMA temp.page_count; } {500 10 292} do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok} #------------------------------------------------------------------------- # reset_db do_execsql_test 5.1.1 { PRAGMA main.cache_size = 10; PRAGMA temp.cache_size = 10; CREATE TEMP TABLE t2(a, b); CREATE INDEX i2 ON t2(a, b); WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 ) INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x; CREATE TEMP TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); INSERT INTO t1 VALUES(1, 2); PRAGMA temp.page_count; } {286} do_execsql_test 5.1.2 { BEGIN; UPDATE t1 SET a=2; UPDATE t2 SET a=randomblob(100); SELECT count(*) FROM t1; ROLLBACK; } {1} do_execsql_test 5.1.3 { UPDATE t2 SET a=randomblob(100); SELECT * FROM t1; } {1 2} do_execsql_test 5.1.4 { PRAGMA temp.integrity_check } {ok} finish_test |