Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Improved progress-handler and interrupt detection during PRAGMA integrity_check. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
6db42780a9e530bcc94490cc60805363 |
User & Date: | drh 2023-01-11 16:17:31.352 |
Context
2023-01-11
| ||
16:44 | Add an assert() to the byte-code engine that goes off if the OP_Halt opcode is invoked with SQLITE_INTERNAL. This causes the RIGHT JOIN error "Opcode jumps to ... which is outside the subroutine ..." to fail immediately, causing it to come more readily to tester's attention. There is at least one testcase in test/fuzzdata8.db that asserts due to this change. (check-in: b8f9944142 user: drh tags: code-generator-20230111) | |
16:25 | Fix problems with compressed LSM databases larger than 2GiB. (check-in: f884224578 user: dan tags: trunk) | |
16:17 | Improved progress-handler and interrupt detection during PRAGMA integrity_check. (check-in: 6db42780a9 user: drh tags: trunk) | |
00:27 | Add a new sqlite3_is_interrupted() interface that can be used by long-running app-defined functions and similar to see if they need to exit early due to an sqlite3_interrupt() call. (check-in: d030f34136 user: drh tags: trunk) | |
Changes
Changes to src/btree.c.
︙ | ︙ | |||
10177 10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204 10205 | ** testing and debugging only. */ Pager *sqlite3BtreePager(Btree *p){ return p->pBt->pPager; } #ifndef SQLITE_OMIT_INTEGRITY_CHECK /* ** Append a message to the error message string. */ static void checkAppendMsg( IntegrityCk *pCheck, const char *zFormat, ... ){ va_list ap; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; va_start(ap, zFormat); if( pCheck->errMsg.nChar ){ sqlite3_str_append(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); } sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); va_end(ap); if( pCheck->errMsg.accError==SQLITE_NOMEM ){ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | 10177 10178 10179 10180 10181 10182 10183 10184 10185 10186 10187 10188 10189 10190 10191 10192 10193 10194 10195 10196 10197 10198 10199 10200 10201 10202 10203 10204 10205 10206 10207 10208 10209 10210 10211 10212 10213 10214 10215 10216 10217 10218 10219 10220 10221 10222 10223 10224 10225 10226 10227 10228 10229 10230 10231 10232 10233 10234 10235 10236 10237 10238 10239 10240 10241 10242 10243 10244 10245 10246 10247 10248 10249 | ** testing and debugging only. */ Pager *sqlite3BtreePager(Btree *p){ return p->pBt->pPager; } #ifndef SQLITE_OMIT_INTEGRITY_CHECK /* ** Record an OOM error during integrity_check */ static void checkOom(IntegrityCk *pCheck){ pCheck->rc = SQLITE_NOMEM; pCheck->mxErr = 0; /* Causes integrity_check processing to stop */ if( pCheck->nErr==0 ) pCheck->nErr++; } /* ** Invoke the progress handler, if appropriate. Also check for an ** interrupt. */ static void checkProgress(IntegrityCk *pCheck){ sqlite3 *db = pCheck->db; if( AtomicLoad(&db->u1.isInterrupted) ){ pCheck->rc = SQLITE_INTERRUPT; pCheck->nErr++; pCheck->mxErr = 0; } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( db->xProgress ){ assert( db->nProgressOps>0 ); pCheck->nStep++; if( (pCheck->nStep % db->nProgressOps)==0 && db->xProgress(db->pProgressArg) ){ pCheck->rc = SQLITE_INTERRUPT; pCheck->nErr++; pCheck->mxErr = 0; } } #endif } /* ** Append a message to the error message string. */ static void checkAppendMsg( IntegrityCk *pCheck, const char *zFormat, ... ){ va_list ap; checkProgress(pCheck); if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; va_start(ap, zFormat); if( pCheck->errMsg.nChar ){ sqlite3_str_append(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); } sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); va_end(ap); if( pCheck->errMsg.accError==SQLITE_NOMEM ){ checkOom(pCheck); } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ #ifndef SQLITE_OMIT_INTEGRITY_CHECK /* |
︙ | ︙ | |||
10241 10242 10243 10244 10245 10246 10247 | checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ checkAppendMsg(pCheck, "2nd reference to page %d", iPage); return 1; } | < | 10277 10278 10279 10280 10281 10282 10283 10284 10285 10286 10287 10288 10289 10290 | checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ checkAppendMsg(pCheck, "2nd reference to page %d", iPage); return 1; } setPageReferenced(pCheck, iPage); return 0; } #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Check that the entry in the pointer-map for page iChild maps to |
︙ | ︙ | |||
10264 10265 10266 10267 10268 10269 10270 | ){ int rc; u8 ePtrmapType; Pgno iPtrmapParent; rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ | | | 10299 10300 10301 10302 10303 10304 10305 10306 10307 10308 10309 10310 10311 10312 10313 | ){ int rc; u8 ePtrmapType; Pgno iPtrmapParent; rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) checkOom(pCheck); checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ checkAppendMsg(pCheck, "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", |
︙ | ︙ | |||
10448 10449 10450 10451 10452 10453 10454 10455 10456 10457 10458 10459 10460 10461 | const char *saved_zPfx = pCheck->zPfx; int saved_v1 = pCheck->v1; int saved_v2 = pCheck->v2; u8 savedIsInit = 0; /* Check that the page exists */ pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; pCheck->zPfx = "Page %u: "; pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ | > > | 10483 10484 10485 10486 10487 10488 10489 10490 10491 10492 10493 10494 10495 10496 10497 10498 | const char *saved_zPfx = pCheck->zPfx; int saved_v1 = pCheck->v1; int saved_v2 = pCheck->v2; u8 savedIsInit = 0; /* Check that the page exists */ checkProgress(pCheck); if( pCheck->mxErr==0 ) goto end_of_check; pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; pCheck->zPfx = "Page %u: "; pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ |
︙ | ︙ | |||
10693 10694 10695 10696 10697 10698 10699 | ** happens when performing an integrity check on a single table. The ** zero is skipped, of course. But in addition, the freelist checks ** and the checks to make sure every page is referenced are also skipped, ** since obviously it is not possible to know which pages are covered by ** the unverified btrees. Except, if aRoot[1] is 1, then the freelist ** checks are still performed. */ | | | > | 10730 10731 10732 10733 10734 10735 10736 10737 10738 10739 10740 10741 10742 10743 10744 10745 10746 10747 10748 10749 10750 10751 | ** happens when performing an integrity check on a single table. The ** zero is skipped, of course. But in addition, the freelist checks ** and the checks to make sure every page is referenced are also skipped, ** since obviously it is not possible to know which pages are covered by ** the unverified btrees. Except, if aRoot[1] is 1, then the freelist ** checks are still performed. */ int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ char **pzOut /* OUT: Write the error message string here */ ){ Pgno i; IntegrityCk sCheck; BtShared *pBt = p->pBt; u64 savedDbFlags = pBt->db->flags; char zErr[100]; int bPartial = 0; /* True if not checking all btrees */ |
︙ | ︙ | |||
10722 10723 10724 10725 10726 10727 10728 10729 10730 10731 10732 10733 | if( aRoot[1]!=1 ) bCkFreelist = 0; } sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); assert( nRef>=0 ); sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; | > < < < < < < < | | | 10760 10761 10762 10763 10764 10765 10766 10767 10768 10769 10770 10771 10772 10773 10774 10775 10776 10777 10778 10779 10780 10781 10782 10783 10784 10785 10786 10787 10788 10789 10790 10791 10792 10793 | if( aRoot[1]!=1 ) bCkFreelist = 0; } sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); assert( nRef>=0 ); memset(&sCheck, 0, sizeof(sCheck)); sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; if( sCheck.nPage==0 ){ goto integrity_ck_cleanup; } sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); if( !sCheck.aPgRef ){ checkOom(&sCheck); goto integrity_ck_cleanup; } sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); if( sCheck.heap==0 ){ checkOom(&sCheck); goto integrity_ck_cleanup; } i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); /* Check the integrity of the freelist |
︙ | ︙ | |||
10828 10829 10830 10831 10832 10833 10834 | } /* Clean up and report errors. */ integrity_ck_cleanup: sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); | > | > > | < < | | 10860 10861 10862 10863 10864 10865 10866 10867 10868 10869 10870 10871 10872 10873 10874 10875 10876 10877 10878 10879 10880 10881 10882 10883 10884 | } /* Clean up and report errors. */ integrity_ck_cleanup: sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); *pnErr = sCheck.nErr; if( sCheck.nErr==0 ){ sqlite3_str_reset(&sCheck.errMsg); *pzOut = 0; }else{ *pzOut = sqlite3StrAccumFinish(&sCheck.errMsg); } /* Make sure this analysis did not leave any unref() pages. */ assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); sqlite3BtreeLeave(p); return sCheck.rc; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* ** Return the full pathname of the underlying database file. Return ** an empty string if the database is in-memory or a TEMP database. ** |
︙ | ︙ |
Changes to src/btree.h.
︙ | ︙ | |||
325 326 327 328 329 330 331 | i64 sqlite3BtreeOffset(BtCursor*); #endif int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); u32 sqlite3BtreePayloadSize(BtCursor*); sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); | | > > > > > > > > | 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 | i64 sqlite3BtreeOffset(BtCursor*); #endif int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); u32 sqlite3BtreePayloadSize(BtCursor*); sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ char **pzOut /* OUT: Write the error message string here */ ); struct Pager *sqlite3BtreePager(Btree*); i64 sqlite3BtreeRowCountEst(BtCursor*); #ifndef SQLITE_OMIT_INCRBLOB int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); void sqlite3BtreeIncrblobCursor(BtCursor *); |
︙ | ︙ |
Changes to src/btreeInt.h.
︙ | ︙ | |||
677 678 679 680 681 682 683 | #define ISAUTOVACUUM(pBt) (pBt->autoVacuum) #else #define ISAUTOVACUUM(pBt) 0 #endif /* | | | | > | 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 | #define ISAUTOVACUUM(pBt) (pBt->autoVacuum) #else #define ISAUTOVACUUM(pBt) 0 #endif /* ** This structure is passed around through all the PRAGMA integrity_check ** checking routines in order to keep track of some global state information. ** ** The aRef[] array is allocated so that there is 1 bit for each page in ** the database. As the integrity-check proceeds, for each page used in ** the database the corresponding bit is set. This allows integrity-check to ** detect pages that are used twice and orphaned pages (both of which ** indicate corruption). */ typedef struct IntegrityCk IntegrityCk; struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ u8 *aPgRef; /* 1 bit per page in the db (see above) */ Pgno nPage; /* Number of pages in the database */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int rc;; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ u32 nStep; /* Number of steps into the integrity_check process */ const char *zPfx; /* Error message prefix */ Pgno v1; /* Value for first %u substitution in zPfx */ int v2; /* Value for second %d substitution in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ }; |
︙ | ︙ |
Changes to src/vdbe.c.
︙ | ︙ | |||
6954 6955 6956 6957 6958 6959 6960 | assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); pIn1 = &aMem[pOp->p1]; assert( pOp->p5<db->nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); | | | | > | | 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 | assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); pIn1 = &aMem[pOp->p1]; assert( pOp->p5<db->nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); }else if( rc ){ sqlite3_free(z); goto abort_due_to_error; }else{ pnErr->u.i -= nErr-1; sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); } UPDATE_MAX_BLOBSIZE(pIn1); sqlite3VdbeChangeEncoding(pIn1, encoding); goto check_for_interrupt; |
︙ | ︙ |