Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix for 'truncate file' operations on in-memory databases. (CVS 6131) |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
83d1eafbde556f56969c6f285b6767d2 |
User & Date: | danielk1977 2009-01-07 15:18:21.000 |
Context
2009-01-07
| ||
15:33 | Conjecture: a journal header with nRec==0 must be the last header in the journal. Add asserts to make this conjecture explicit. (CVS 6132) (check-in: 15b5b5f90c user: drh tags: trunk) | |
15:18 | Fix for 'truncate file' operations on in-memory databases. (CVS 6131) (check-in: 83d1eafbde user: danielk1977 tags: trunk) | |
10:52 | Add a comment to the openSubjournal() function in pager.c. (CVS 6130) (check-in: 04387ae10a user: danielk1977 tags: trunk) | |
Changes
Changes to src/pager.c.
︙ | ︙ | |||
14 15 16 17 18 19 20 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** | | | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file ** locking to prevent two processes from writing the same database ** file simultaneously, or one process from reading the database while ** another is writing. ** ** @(#) $Id: pager.c,v 1.538 2009/01/07 15:18:21 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" /* ** Macros for troubleshooting. Normally turned off */ |
︙ | ︙ | |||
1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 | }else if( pPager->state==PAGER_SYNCED ){ pPager->state = PAGER_EXCLUSIVE; } pPager->dbOrigSize = 0; pPager->setMaster = 0; pPager->needSync = 0; /* lruListSetFirstSynced(pPager); */ if( !MEMDB ){ pPager->dbSizeValid = 0; } pPager->dbModified = 0; return (rc==SQLITE_OK?rc2:rc); } | > | 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 | }else if( pPager->state==PAGER_SYNCED ){ pPager->state = PAGER_EXCLUSIVE; } pPager->dbOrigSize = 0; pPager->setMaster = 0; pPager->needSync = 0; /* lruListSetFirstSynced(pPager); */ sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( !MEMDB ){ pPager->dbSizeValid = 0; } pPager->dbModified = 0; return (rc==SQLITE_OK?rc2:rc); } |
︙ | ︙ | |||
1458 1459 1460 1461 1462 1463 1464 | } sqlite3_free(pMaster); return rc; } /* | > | | | | 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 | } sqlite3_free(pMaster); return rc; } /* ** If the main database file is open and an exclusive lock is held, ** truncate the main file of the given pager to the specified number ** of pages. ** ** It might might be the case that the file on disk is smaller than nPage. ** This can happen, for example, if we are in the middle of a transaction ** which has extended the file size and the new pages are still all held ** in cache, then an INSERT or UPDATE does a statement rollback. Some ** operating system implementations can get confused if you try to ** truncate a file to some size that is larger than it currently is, ** so detect this case and write a single zero byte to the end of the new ** file instead. |
︙ | ︙ | |||
1487 1488 1489 1490 1491 1492 1493 | rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1); } if( rc==SQLITE_OK ){ pPager->dbFileSize = nPage; } } } | < < < < | 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 | rc = sqlite3OsWrite(pPager->fd, "", 1, newSize-1); } if( rc==SQLITE_OK ){ pPager->dbFileSize = nPage; } } } return rc; } /* ** Set the sectorSize for the given pager. ** ** The sector size is at least as big as the sector size reported |
︙ | ︙ | |||
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 | ** database file back to its original size. */ if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ rc = pager_truncate(pPager, mxPg); if( rc!=SQLITE_OK ){ goto end_playback; } } /* Copy original pages out of the journal and back into the database file. */ for(u=0; u<nRec; u++){ rc = pager_playback_one_page(pPager, 1, &pPager->journalOff, 0, 0); if( rc!=SQLITE_OK ){ | > | 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 | ** database file back to its original size. */ if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ rc = pager_truncate(pPager, mxPg); if( rc!=SQLITE_OK ){ goto end_playback; } pPager->dbSize = mxPg; } /* Copy original pages out of the journal and back into the database file. */ for(u=0; u<nRec; u++){ rc = pager_playback_one_page(pPager, 1, &pPager->journalOff, 0, 0); if( rc!=SQLITE_OK ){ |
︙ | ︙ | |||
2345 2346 2347 2348 2349 2350 2351 | pPager->state = (u8)locktype; IOTRACE(("LOCK %p %d\n", pPager, locktype)) } } return rc; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | | | < | 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 | pPager->state = (u8)locktype; IOTRACE(("LOCK %p %d\n", pPager, locktype)) } } return rc; } #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Truncate the in-memory database file image to nPage pages. This ** function does not actually modify the database file on disk. It ** just sets the internal state of the pager object so that the ** truncation will be done when the current transaction is committed. */ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assert( pPager->dbSizeValid ); assert( pPager->dbSize>=nPage ); pPager->dbSize = nPage; } |
︙ | ︙ | |||
2602 2603 2604 2605 2606 2607 2608 | if( !pPager->fd->pMethods ){ assert(pPager->tempFile); rc = sqlite3PagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); if( rc ) return rc; } /* If there are dirty pages in the page cache with page numbers greater | | | 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 | if( !pPager->fd->pMethods ){ assert(pPager->tempFile); rc = sqlite3PagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); if( rc ) return rc; } /* If there are dirty pages in the page cache with page numbers greater ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to ** make the file smaller (presumably by auto-vacuum code). Do not write ** any such pages to the file. */ if( pList->pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ i64 offset = (pList->pgno-1)*(i64)pPager->pageSize; char *pData = CODEC2(pPager, pList->pData, pList->pgno, 6); |
︙ | ︙ | |||
3959 3960 3961 3962 3963 3964 3965 | rc = writeMasterJournal(pPager, zMaster); if( rc!=SQLITE_OK ) goto sync_exit; rc = syncJournal(pPager); } } if( rc!=SQLITE_OK ) goto sync_exit; | < < < < < < < > > > > > > < < | 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 | rc = writeMasterJournal(pPager, zMaster); if( rc!=SQLITE_OK ) goto sync_exit; rc = syncJournal(pPager); } } if( rc!=SQLITE_OK ) goto sync_exit; /* Write all dirty pages to the database file */ pPg = sqlite3PcacheDirtyList(pPager->pPCache); rc = pager_write_pagelist(pPg); if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); /* The error might have left the dirty list all fouled up here, ** but that does not matter because if the if the dirty list did ** get corrupted, then the transaction will roll back and ** discard the dirty list. There is an assert in ** pager_get_all_dirty_pages() that verifies that no attempt ** is made to use an invalid dirty list. */ goto sync_exit; } sqlite3PcacheCleanAll(pPager->pPCache); if( pPager->dbSize<pPager->dbFileSize ){ assert( pPager->state>=PAGER_EXCLUSIVE ); rc = pager_truncate(pPager, pPager->dbSize); if( rc!=SQLITE_OK ) goto sync_exit; } /* Sync the database file. */ if( !pPager->noSync && !noSync ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } IOTRACE(("DBSYNC %p\n", pPager)) pPager->state = PAGER_SYNCED; } sync_exit: if( rc==SQLITE_IOERR_BLOCKED ){ /* pager_incr_changecounter() may attempt to obtain an exclusive * lock to spill the cache and return IOERR_BLOCKED. But since * there is no chance the cache is inconsistent, it is |
︙ | ︙ |
Changes to src/pager.h.
︙ | ︙ | |||
9 10 11 12 13 14 15 | ** May you share freely, never taking more than you give. ** ************************************************************************* ** This header file defines the interface that the sqlite page cache ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** | | | 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | ** May you share freely, never taking more than you give. ** ************************************************************************* ** This header file defines the interface that the sqlite page cache ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** ** @(#) $Id: pager.h,v 1.93 2009/01/07 15:18:21 danielk1977 Exp $ */ #ifndef _PAGER_H_ #define _PAGER_H_ /* ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise |
︙ | ︙ | |||
84 85 86 87 88 89 90 | #define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); int sqlite3PagerPageRefcount(DbPage*); int sqlite3PagerRef(DbPage*); int sqlite3PagerUnref(DbPage*); int sqlite3PagerWrite(DbPage*); int sqlite3PagerPagecount(Pager*, int*); | < | 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | #define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); int sqlite3PagerPageRefcount(DbPage*); int sqlite3PagerRef(DbPage*); int sqlite3PagerUnref(DbPage*); int sqlite3PagerWrite(DbPage*); int sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(DbPage*, int exFlag); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); u8 sqlite3PagerIsreadonly(Pager*); void sqlite3PagerDontRollback(DbPage*); int sqlite3PagerDontWrite(DbPage*); |
︙ | ︙ |
Changes to src/pcache1.c.
︙ | ︙ | |||
12 13 14 15 16 17 18 | ** ** This file implements the default page cache implementation (the ** sqlite3_pcache interface). It also contains part of the implementation ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. ** If the default page cache implementation is overriden, then neither of ** these two features are available. ** | | | 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | ** ** This file implements the default page cache implementation (the ** sqlite3_pcache interface). It also contains part of the implementation ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. ** If the default page cache implementation is overriden, then neither of ** these two features are available. ** ** @(#) $Id: pcache1.c,v 1.7 2009/01/07 15:18:21 danielk1977 Exp $ */ #include "sqliteInt.h" typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; |
︙ | ︙ | |||
43 44 45 46 47 48 49 50 51 52 53 54 55 56 | ** when the accessor is holding the global mutex (see pcache1EnterMutex() ** and pcache1LeaveMutex()). */ unsigned int nRecyclable; /* Number of pages in the LRU list */ unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ }; /* ** Each cache entry is represented by an instance of the following ** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated ** directly after the structure in memory (see the PGHDR1_TO_PAGE() ** macro below). | > > | 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | ** when the accessor is holding the global mutex (see pcache1EnterMutex() ** and pcache1LeaveMutex()). */ unsigned int nRecyclable; /* Number of pages in the LRU list */ unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ unsigned int iMaxKey; /* Largest key seen since xTruncate() */ }; /* ** Each cache entry is represented by an instance of the following ** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated ** directly after the structure in memory (see the PGHDR1_TO_PAGE() ** macro below). |
︙ | ︙ | |||
553 554 555 556 557 558 559 560 561 562 563 564 565 566 | pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; pCache->apHash[h] = pPage; } fetch_out: if( createFlag==1 ) sqlite3EndBenignMalloc(); pcache1LeaveMutex(); return (pPage ? PGHDR1_TO_PAGE(pPage) : 0); } /* | > > > | 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 | pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; pCache->apHash[h] = pPage; } fetch_out: if( pPage && iKey>pCache->iMaxKey ){ pCache->iMaxKey = iKey; } if( createFlag==1 ) sqlite3EndBenignMalloc(); pcache1LeaveMutex(); return (pPage ? PGHDR1_TO_PAGE(pPage) : 0); } /* |
︙ | ︙ | |||
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 | } *pp = pPage->pNext; h = iNew%pCache->nHash; pPage->iKey = iNew; pPage->pNext = pCache->apHash[h]; pCache->apHash[h] = pPage; pcache1LeaveMutex(); } /* ** Implementation of the sqlite3_pcache.xTruncate method. ** ** Discard all unpinned pages in the cache with a page number equal to ** or greater than parameter iLimit. Any pinned pages with a page number ** equal to or greater than iLimit are implicitly unpinned. */ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ PCache1 *pCache = (PCache1 *)p; pcache1EnterMutex(); | > > > > > | > > | 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 | } *pp = pPage->pNext; h = iNew%pCache->nHash; pPage->iKey = iNew; pPage->pNext = pCache->apHash[h]; pCache->apHash[h] = pPage; if( iNew>pCache->iMaxKey ){ pCache->iMaxKey = iNew; } pcache1LeaveMutex(); } /* ** Implementation of the sqlite3_pcache.xTruncate method. ** ** Discard all unpinned pages in the cache with a page number equal to ** or greater than parameter iLimit. Any pinned pages with a page number ** equal to or greater than iLimit are implicitly unpinned. */ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ PCache1 *pCache = (PCache1 *)p; pcache1EnterMutex(); if( iLimit<=pCache->iMaxKey ){ pcache1TruncateUnsafe(pCache, iLimit); pCache->iMaxKey = iLimit-1; } pcache1LeaveMutex(); } /* ** Implementation of the sqlite3_pcache.xDestroy method. ** ** Destroy a cache allocated using pcache1Create(). |
︙ | ︙ |
Changes to src/test2.c.
︙ | ︙ | |||
9 10 11 12 13 14 15 | ** May you share freely, never taking more than you give. ** ************************************************************************* ** Code for testing the pager.c module in SQLite. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** | | | 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | ** May you share freely, never taking more than you give. ** ************************************************************************* ** Code for testing the pager.c module in SQLite. This code ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** ** $Id: test2.c,v 1.65 2009/01/07 15:18:21 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" #include <stdlib.h> #include <string.h> #include <ctype.h> |
︙ | ︙ | |||
389 390 391 392 393 394 395 | static int pager_truncate( void *NotUsed, Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int argc, /* Number of arguments */ const char **argv /* Text of each argument */ ){ Pager *pPager; | < | < < < < | 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 | static int pager_truncate( void *NotUsed, Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int argc, /* Number of arguments */ const char **argv /* Text of each argument */ ){ Pager *pPager; int pgno; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " ID PGNO\"", 0); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); if( Tcl_GetInt(interp, argv[2], &pgno) ) return TCL_ERROR; sqlite3PagerTruncateImage(pPager, pgno); return TCL_OK; } /* ** Usage: page_unref PAGE ** |
︙ | ︙ |
Changes to test/memdb.test.
1 2 3 4 5 6 7 8 9 10 11 12 13 | # 2001 September 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is in-memory database backend. # | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | # 2001 September 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is in-memory database backend. # # $Id: memdb.test,v 1.16 2009/01/07 15:18:21 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable memorydb { |
︙ | ︙ | |||
406 407 408 409 410 411 412 413 414 415 416 417 | } 5 do_test memdb-8.2 { execsql { DELETE FROM t1; SELECT count(*) FROM t1; } } 0 } ;# ifcapable memorydb finish_test | > > > > > > > > > > > > > > > > > > > | 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 | } 5 do_test memdb-8.2 { execsql { DELETE FROM t1; SELECT count(*) FROM t1; } } 0 # Test that auto-vacuum works with in-memory databases. # do_test memdb-9.1 { db close sqlite3 db test.db db cache size 0 execsql { PRAGMA auto_vacuum = full; CREATE TABLE t1(a); INSERT INTO t1 VALUES(randstr(1000,1000)); INSERT INTO t1 VALUES(randstr(1000,1000)); INSERT INTO t1 VALUES(randstr(1000,1000)); } set memused [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] execsql { DELETE FROM t1 } set memused2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] expr {$memused2 + 2048 < $memused} } {1} } ;# ifcapable memorydb finish_test |
Changes to test/pager.test.
1 2 3 4 5 6 7 8 9 10 11 12 13 | # 2001 September 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | # 2001 September 15 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # # $Id: pager.test,v 1.34 2009/01/07 15:18:21 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl if {[info commands pager_open]!=""} { db close |
︙ | ︙ | |||
409 410 411 412 413 414 415 | pager_close [pager_open ptf2.db -15] } {} # Test truncate on an in-memory database is Ok. ifcapable memorydb { do_test pager-4.6.2 { set ::p2 [pager_open :memory: 10] | | | | 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 | pager_close [pager_open ptf2.db -15] } {} # Test truncate on an in-memory database is Ok. ifcapable memorydb { do_test pager-4.6.2 { set ::p2 [pager_open :memory: 10] pager_truncate $::p2 0 } {} do_test pager-4.6.3 { set page1 [page_get $::p2 1] for {set i 1} {$i<5} {incr i} { set p [page_get $::p2 $i] page_write $p "Page $i" pager_commit $::p2 page_unref $p } page_unref $page1 pager_truncate $::p2 3 } {} do_test pager-4.6.4 { pager_close $::p2 } {} } do_test pager-4.99 { |
︙ | ︙ |