/ Check-in [b844ac6f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Experimental changes that cause SQLite to use bytes 28..31 of the database header to determine the database size, rather than using the actual database size. This allows database space to be preallocated.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | experimental
Files: files | file ages | folders
SHA1:b844ac6fcb72595a71e5c5283ec461309a87ba79
User & Date: drh 2010-03-27 17:12:36
Context
2010-03-29
19:36
Fix issues with the io.test script. There is still a problem with a ROLLBACK TO not working on an initially empty database files. And much more testing is needed. check-in: 09786e2a user: drh tags: experimental
2010-03-27
17:12
Experimental changes that cause SQLite to use bytes 28..31 of the database header to determine the database size, rather than using the actual database size. This allows database space to be preallocated. check-in: b844ac6f user: drh tags: experimental
09:44
Increase the estimated cost of using a virtual table as the outer loop of a join when there exists an ORDER BY clause that is not satisfied by the virtual table. Fix for [775b39dd3c]. check-in: 9e075e70 user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/btree.c.

   512    512   ** optimization 2 above is omitted if the corresponding bit is already
   513    513   ** set in BtShared.pHasContent. The contents of the bitvec are cleared
   514    514   ** at the end of every transaction.
   515    515   */
   516    516   static int btreeSetHasContent(BtShared *pBt, Pgno pgno){
   517    517     int rc = SQLITE_OK;
   518    518     if( !pBt->pHasContent ){
   519         -    int nPage = 100;
   520         -    sqlite3PagerPagecount(pBt->pPager, &nPage);
   521         -    /* If sqlite3PagerPagecount() fails there is no harm because the
   522         -    ** nPage variable is unchanged from its default value of 100 */
   523         -    pBt->pHasContent = sqlite3BitvecCreate((u32)nPage);
          519  +    assert( pgno<=pBt->nPage );
          520  +    pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage);
   524    521       if( !pBt->pHasContent ){
   525    522         rc = SQLITE_NOMEM;
   526    523       }
   527    524     }
   528    525     if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){
   529    526       rc = sqlite3BitvecSet(pBt->pHasContent, pgno);
   530    527     }
................................................................................
  1560   1557   }
  1561   1558   
  1562   1559   /*
  1563   1560   ** Return the size of the database file in pages. If there is any kind of
  1564   1561   ** error, return ((unsigned int)-1).
  1565   1562   */
  1566   1563   static Pgno pagerPagecount(BtShared *pBt){
  1567         -  int nPage = -1;
  1568         -  int rc;
  1569         -  assert( pBt->pPage1 );
  1570         -  rc = sqlite3PagerPagecount(pBt->pPager, &nPage);
  1571         -  assert( rc==SQLITE_OK || nPage==-1 );
  1572         -  return (Pgno)nPage;
         1564  +  int nPage = (int)pBt->nPage;
         1565  +  return nPage;
  1573   1566   }
  1574   1567   
  1575   1568   /*
  1576   1569   ** Get a page from the pager and initialize it.  This routine is just a
  1577   1570   ** convenience wrapper around separate calls to btreeGetPage() and 
  1578   1571   ** btreeInitPage().
  1579   1572   **
................................................................................
  2250   2243     if( rc!=SQLITE_OK ) return rc;
  2251   2244     rc = btreeGetPage(pBt, 1, &pPage1, 0);
  2252   2245     if( rc!=SQLITE_OK ) return rc;
  2253   2246   
  2254   2247     /* Do some checking to help insure the file we opened really is
  2255   2248     ** a valid database file. 
  2256   2249     */
  2257         -  rc = sqlite3PagerPagecount(pBt->pPager, &nPage);
  2258         -  if( rc!=SQLITE_OK ){
         2250  +  nPage = get4byte(28+(u8*)pPage1->aData);
         2251  +  if( nPage==0 && (rc = sqlite3PagerPagecount(pBt->pPager, &nPage))!=0 ){
  2259   2252       goto page1_init_failed;
  2260   2253     }else if( nPage>0 ){
  2261   2254       int pageSize;
  2262   2255       int usableSize;
  2263   2256       u8 *page1 = pPage1->aData;
  2264   2257       rc = SQLITE_NOTADB;
  2265   2258       if( memcmp(page1, zMagicHeader, 16)!=0 ){
................................................................................
  2329   2322     */
  2330   2323     pBt->maxLocal = (pBt->usableSize-12)*64/255 - 23;
  2331   2324     pBt->minLocal = (pBt->usableSize-12)*32/255 - 23;
  2332   2325     pBt->maxLeaf = pBt->usableSize - 35;
  2333   2326     pBt->minLeaf = (pBt->usableSize-12)*32/255 - 23;
  2334   2327     assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) );
  2335   2328     pBt->pPage1 = pPage1;
         2329  +  pBt->nPage = nPage;
  2336   2330     return SQLITE_OK;
  2337   2331   
  2338   2332   page1_init_failed:
  2339   2333     releasePage(pPage1);
  2340   2334     pBt->pPage1 = 0;
  2341   2335     return rc;
  2342   2336   }
................................................................................
  2366   2360   ** into a new empty database by initializing the first page of
  2367   2361   ** the database.
  2368   2362   */
  2369   2363   static int newDatabase(BtShared *pBt){
  2370   2364     MemPage *pP1;
  2371   2365     unsigned char *data;
  2372   2366     int rc;
  2373         -  int nPage;
  2374   2367   
  2375   2368     assert( sqlite3_mutex_held(pBt->mutex) );
  2376         -  rc = sqlite3PagerPagecount(pBt->pPager, &nPage);
  2377         -  if( rc!=SQLITE_OK || nPage>0 ){
  2378         -    return rc;
         2369  +  if( pBt->nPage>0 ){
         2370  +    return SQLITE_OK;
  2379   2371     }
  2380   2372     pP1 = pBt->pPage1;
  2381   2373     assert( pP1!=0 );
  2382   2374     data = pP1->aData;
  2383   2375     rc = sqlite3PagerWrite(pP1->pDbPage);
  2384   2376     if( rc ) return rc;
  2385   2377     memcpy(data, zMagicHeader, sizeof(zMagicHeader));
................................................................................
  2397   2389     pBt->pageSizeFixed = 1;
  2398   2390   #ifndef SQLITE_OMIT_AUTOVACUUM
  2399   2391     assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 );
  2400   2392     assert( pBt->incrVacuum==1 || pBt->incrVacuum==0 );
  2401   2393     put4byte(&data[36 + 4*4], pBt->autoVacuum);
  2402   2394     put4byte(&data[36 + 7*4], pBt->incrVacuum);
  2403   2395   #endif
         2396  +  pBt->nPage = 1;
         2397  +  data[31] = 1;
  2404   2398     return SQLITE_OK;
  2405   2399   }
  2406   2400   
  2407   2401   /*
  2408   2402   ** Attempt to start a new transaction. A write-transaction
  2409   2403   ** is started if the second argument is nonzero, otherwise a read-
  2410   2404   ** transaction.  If the second argument is 2 or more and exclusive
................................................................................
  2765   2759   ** process is complete.  If nFin is zero, it is assumed that
  2766   2760   ** incrVacuumStep() will be called a finite amount of times
  2767   2761   ** which may or may not empty the freelist.  A full autovacuum
  2768   2762   ** has nFin>0.  A "PRAGMA incremental_vacuum" has nFin==0.
  2769   2763   */
  2770   2764   static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){
  2771   2765     Pgno nFreeList;           /* Number of pages still on the free-list */
         2766  +  int rc;
  2772   2767   
  2773   2768     assert( sqlite3_mutex_held(pBt->mutex) );
  2774   2769     assert( iLastPg>nFin );
  2775   2770   
  2776   2771     if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){
  2777         -    int rc;
  2778   2772       u8 eType;
  2779   2773       Pgno iPtrPage;
  2780   2774   
  2781   2775       nFreeList = get4byte(&pBt->pPage1->aData[36]);
  2782   2776       if( nFreeList==0 ){
  2783   2777         return SQLITE_DONE;
  2784   2778       }
................................................................................
  2846   2840     }
  2847   2841   
  2848   2842     if( nFin==0 ){
  2849   2843       iLastPg--;
  2850   2844       while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){
  2851   2845         if( PTRMAP_ISPAGE(pBt, iLastPg) ){
  2852   2846           MemPage *pPg;
  2853         -        int rc = btreeGetPage(pBt, iLastPg, &pPg, 0);
         2847  +        rc = btreeGetPage(pBt, iLastPg, &pPg, 0);
  2854   2848           if( rc!=SQLITE_OK ){
  2855   2849             return rc;
  2856   2850           }
  2857   2851           rc = sqlite3PagerWrite(pPg->pDbPage);
  2858   2852           releasePage(pPg);
  2859   2853           if( rc!=SQLITE_OK ){
  2860   2854             return rc;
  2861   2855           }
  2862   2856         }
  2863   2857         iLastPg--;
  2864   2858       }
  2865   2859       sqlite3PagerTruncateImage(pBt->pPager, iLastPg);
         2860  +    pBt->nPage = iLastPg;
  2866   2861     }
  2867   2862     return SQLITE_OK;
  2868   2863   }
  2869   2864   
  2870   2865   /*
  2871   2866   ** A write-transaction must be opened before calling this function.
  2872   2867   ** It performs a single unit of work towards an incremental vacuum.
................................................................................
  2882   2877     sqlite3BtreeEnter(p);
  2883   2878     assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE );
  2884   2879     if( !pBt->autoVacuum ){
  2885   2880       rc = SQLITE_DONE;
  2886   2881     }else{
  2887   2882       invalidateAllOverflowCache(pBt);
  2888   2883       rc = incrVacuumStep(pBt, 0, pagerPagecount(pBt));
         2884  +    if( rc==SQLITE_OK ){
         2885  +      rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
         2886  +      put4byte(&pBt->pPage1->aData[28], pBt->nPage);
         2887  +    }
  2889   2888     }
  2890   2889     sqlite3BtreeLeave(p);
  2891   2890     return rc;
  2892   2891   }
  2893   2892   
  2894   2893   /*
  2895   2894   ** This routine is called prior to sqlite3PagerCommit when a transaction
................................................................................
  2937   2936       }
  2938   2937       if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT;
  2939   2938   
  2940   2939       for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){
  2941   2940         rc = incrVacuumStep(pBt, nFin, iFree);
  2942   2941       }
  2943   2942       if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){
  2944         -      rc = SQLITE_OK;
  2945   2943         rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
  2946   2944         put4byte(&pBt->pPage1->aData[32], 0);
  2947   2945         put4byte(&pBt->pPage1->aData[36], 0);
         2946  +      put4byte(&pBt->pPage1->aData[28], nFin);
  2948   2947         sqlite3PagerTruncateImage(pBt->pPager, nFin);
         2948  +      pBt->nPage = nFin;
  2949   2949       }
  2950   2950       if( rc!=SQLITE_OK ){
  2951   2951         sqlite3PagerRollback(pPager);
  2952   2952       }
  2953   2953     }
  2954   2954   
  2955   2955     assert( nRef==sqlite3PagerRefcount(pPager) );
................................................................................
  3265   3265       BtShared *pBt = p->pBt;
  3266   3266       assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
  3267   3267       assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) );
  3268   3268       sqlite3BtreeEnter(p);
  3269   3269       rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint);
  3270   3270       if( rc==SQLITE_OK ){
  3271   3271         rc = newDatabase(pBt);
         3272  +      pBt->nPage = get4byte(28 + pBt->pPage1->aData);
  3272   3273       }
  3273   3274       sqlite3BtreeLeave(p);
  3274   3275     }
  3275   3276     return rc;
  3276   3277   }
  3277   3278   
  3278   3279   /*
................................................................................
  4829   4830         }
  4830   4831         releasePage(pPrevTrunk);
  4831   4832         pPrevTrunk = 0;
  4832   4833       }while( searchList );
  4833   4834     }else{
  4834   4835       /* There are no pages on the freelist, so create a new page at the
  4835   4836       ** end of the file */
  4836         -    int nPage = pagerPagecount(pBt);
  4837         -    *pPgno = nPage + 1;
  4838         -
  4839         -    if( *pPgno==PENDING_BYTE_PAGE(pBt) ){
  4840         -      (*pPgno)++;
  4841         -    }
         4837  +    rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
         4838  +    if( rc ) return rc;
         4839  +    pBt->nPage++;
         4840  +    if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++;
  4842   4841   
  4843   4842   #ifndef SQLITE_OMIT_AUTOVACUUM
  4844         -    if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, *pPgno) ){
         4843  +    if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, pBt->nPage) ){
  4845   4844         /* If *pPgno refers to a pointer-map page, allocate two new pages
  4846   4845         ** at the end of the file instead of one. The first allocated page
  4847   4846         ** becomes a new pointer-map page, the second is used by the caller.
  4848   4847         */
  4849   4848         MemPage *pPg = 0;
  4850         -      TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", *pPgno));
  4851         -      assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
  4852         -      rc = btreeGetPage(pBt, *pPgno, &pPg, 0);
         4849  +      TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage));
         4850  +      assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) );
         4851  +      rc = btreeGetPage(pBt, pBt->nPage, &pPg, 0);
  4853   4852         if( rc==SQLITE_OK ){
  4854   4853           rc = sqlite3PagerWrite(pPg->pDbPage);
  4855   4854           releasePage(pPg);
  4856   4855         }
  4857   4856         if( rc ) return rc;
  4858         -      (*pPgno)++;
  4859         -      if( *pPgno==PENDING_BYTE_PAGE(pBt) ){ (*pPgno)++; }
         4857  +      pBt->nPage++;
         4858  +      if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ){ pBt->nPage++; }
  4860   4859       }
  4861   4860   #endif
         4861  +    put4byte(28 + (u8*)pBt->pPage1->aData, pBt->nPage);
         4862  +    *pPgno = pBt->nPage;
  4862   4863   
  4863   4864       assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
  4864   4865       rc = btreeGetPage(pBt, *pPgno, ppPage, 0);
  4865   4866       if( rc ) return rc;
  4866   4867       rc = sqlite3PagerWrite((*ppPage)->pDbPage);
  4867   4868       if( rc!=SQLITE_OK ){
  4868   4869         releasePage(*ppPage);

Changes to src/btreeInt.h.

   416    416     u16 usableSize;       /* Number of usable bytes on each page */
   417    417     u16 maxLocal;         /* Maximum local payload in non-LEAFDATA tables */
   418    418     u16 minLocal;         /* Minimum local payload in non-LEAFDATA tables */
   419    419     u16 maxLeaf;          /* Maximum local payload in a LEAFDATA table */
   420    420     u16 minLeaf;          /* Minimum local payload in a LEAFDATA table */
   421    421     u8 inTransaction;     /* Transaction state */
   422    422     int nTransaction;     /* Number of open transactions (read + write) */
          423  +  u32 nPage;            /* Number of pages in the database */
   423    424     void *pSchema;        /* Pointer to space allocated by sqlite3BtreeSchema() */
   424    425     void (*xFreeSchema)(void*);  /* Destructor for BtShared.pSchema */
   425    426     sqlite3_mutex *mutex; /* Non-recursive mutex required to access this struct */
   426    427     Bitvec *pHasContent;  /* Set of pages moved to free-list this transaction */
   427    428   #ifndef SQLITE_OMIT_SHARED_CACHE
   428    429     int nRef;             /* Number of references to this structure */
   429    430     BtShared *pNext;      /* Next on a list of sharable BtShared structs */

Changes to src/pager.c.

  4152   4152     int rc = SQLITE_OK;
  4153   4153   
  4154   4154     /* This routine is not called unless a transaction has already been
  4155   4155     ** started.
  4156   4156     */
  4157   4157     assert( pPager->state>=PAGER_RESERVED );
  4158   4158   
  4159         -  /* If an error has been previously detected, we should not be
  4160         -  ** calling this routine.  Repeat the error for robustness.
         4159  +  /* If an error has been previously detected, report the same error
         4160  +  ** again.
  4161   4161     */
  4162         -  if( NEVER(pPager->errCode) )  return pPager->errCode;
         4162  +  if( pPager->errCode )  return pPager->errCode;
  4163   4163   
  4164   4164     /* Higher-level routines never call this function if database is not
  4165   4165     ** writable.  But check anyway, just for robustness. */
  4166   4166     if( NEVER(pPager->readOnly) ) return SQLITE_PERM;
  4167   4167   
  4168   4168     assert( !pPager->setMaster );
  4169   4169   
................................................................................
  4485   4485   
  4486   4486       if( rc==SQLITE_OK ){
  4487   4487         /* Increment the value just read and write it back to byte 24. */
  4488   4488         change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers);
  4489   4489         change_counter++;
  4490   4490         put32bits(((char*)pPgHdr->pData)+24, change_counter);
  4491   4491   
  4492         -      /* Also store the current database size in bytes 28..31 */
  4493         -      assert( pPager->dbSizeValid );
  4494         -      put32bits(((char*)pPgHdr->pData)+28, pPager->dbSize);
  4495         -
  4496   4492         /* Also store the SQLite version number in bytes 96..99 */
  4497   4493         assert( pPager->dbSizeValid );
  4498   4494         put32bits(((char*)pPgHdr->pData)+96, SQLITE_VERSION_NUMBER);
  4499   4495   
  4500   4496         /* If running in direct mode, write the contents of page 1 to the file. */
  4501   4497         if( DIRECT_MODE ){
  4502   4498           const void *zBuf = pPgHdr->pData;
................................................................................
  4566   4562     int noSync                      /* True to omit the xSync on the db file */
  4567   4563   ){
  4568   4564     int rc = SQLITE_OK;             /* Return code */
  4569   4565   
  4570   4566     /* The dbOrigSize is never set if journal_mode=OFF */
  4571   4567     assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF || pPager->dbOrigSize==0 );
  4572   4568   
  4573         -  /* If a prior error occurred, this routine should not be called.  ROLLBACK
  4574         -  ** is the appropriate response to an error, not COMMIT.  Guard against
  4575         -  ** coding errors by repeating the prior error. */
  4576         -  if( NEVER(pPager->errCode) ) return pPager->errCode;
         4569  +  /* If a prior error occurred, report that error again. */
         4570  +  if( pPager->errCode ) return pPager->errCode;
  4577   4571   
  4578   4572     PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", 
  4579   4573         pPager->zFilename, zMaster, pPager->dbSize));
  4580   4574   
  4581   4575     if( MEMDB && pPager->dbModified ){
  4582   4576       /* If this is an in-memory db, or no pages have been written to, or this
  4583   4577       ** function has already been called, it is mostly a no-op.  However, any
................................................................................
  4920   4914   int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){
  4921   4915     int rc = SQLITE_OK;                       /* Return code */
  4922   4916     int nCurrent = pPager->nSavepoint;        /* Current number of savepoints */
  4923   4917   
  4924   4918     if( nSavepoint>nCurrent && pPager->useJournal ){
  4925   4919       int ii;                                 /* Iterator variable */
  4926   4920       PagerSavepoint *aNew;                   /* New Pager.aSavepoint array */
         4921  +
         4922  +    rc = sqlite3PagerPagecount(pPager, 0);
         4923  +    if( rc ) return rc;
  4927   4924   
  4928   4925       /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM
  4929   4926       ** if the allocation fails. Otherwise, zero the new portion in case a 
  4930   4927       ** malloc failure occurs while populating it in the for(...) loop below.
  4931   4928       */
  4932   4929       aNew = (PagerSavepoint *)sqlite3Realloc(
  4933   4930           pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint

Changes to test/corrupt2.test.

   330    330       DELETE FROM t1 WHERE a = 1;
   331    331     } -corrupt {
   332    332       set nAppend [expr 1024*207 - [file size corrupt.db]]
   333    333       set fd [open corrupt.db r+]
   334    334       seek $fd 0 end
   335    335       puts -nonewline $fd [string repeat x $nAppend]
   336    336       close $fd
          337  +    hexio_write corrupt.db 28 00000000
   337    338     } -test {
   338    339       do_test corrupt2-6.4 {
   339    340         catchsql { 
   340    341           BEGIN EXCLUSIVE;
   341    342           COMMIT;
   342    343         }
   343    344       } {1 {database disk image is malformed}}
................................................................................
   518    519         db eval { INSERT INTO t1 SELECT NULL, randstr(50,50) FROM t1 }
   519    520       }
   520    521     } -corrupt {
   521    522       do_test corrupt2-13.1 {
   522    523         file size corrupt.db
   523    524       } $::sqlite_pending_byte
   524    525       hexio_write corrupt.db [expr $::sqlite_pending_byte+1023] 00
          526  +    hexio_write corrupt.db 28 00000000
   525    527     } -test {
   526    528       do_test corrupt2-13.2 {
   527    529         file size corrupt.db
   528    530       } [expr $::sqlite_pending_byte + 1024]
   529    531       do_test corrupt2-13.3 {
   530    532         catchsql { DELETE FROM t1 WHERE rowid < 30; }
   531    533       } {1 {database disk image is malformed}}
   532    534     }
   533    535   }
   534    536   
   535    537   finish_test

Changes to test/pragma.test.

   313    313         set in [open test.db r]
   314    314         fconfigure $in -translation binary
   315    315         puts -nonewline $out [read $in]
   316    316         seek $in 0
   317    317         puts -nonewline $out [read $in]
   318    318         close $in
   319    319         close $out
          320  +      hexio_write testerr.db 28 00000000
   320    321         execsql {REINDEX t2}
   321    322         execsql {PRAGMA integrity_check}
   322    323       } {ok}
   323    324       do_test pragma-3.8.1 {
   324    325         execsql {PRAGMA quick_check}
   325    326       } {ok}
   326    327       do_test pragma-3.9 {