Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix some problems with circular logs. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
4b36a0245fda855f497b4ba471c11088 |
User & Date: | dan 2013-10-28 10:47:26.918 |
Context
2013-10-28
| ||
10:56 | Fix a problem causing builds without LSM_NO_SYNC to fail. check-in: f8ce90c4a6 user: dan tags: trunk | |
10:47 | Fix some problems with circular logs. check-in: 4b36a0245f user: dan tags: trunk | |
2013-10-26
| ||
19:58 | Begin adding code to make the log file circular. check-in: ec6dda0ade user: dan tags: trunk | |
Changes
Changes to src/btInt.h.
︙ | ︙ | |||
192 193 194 195 196 197 198 199 200 201 202 203 204 205 | int sqlite4BtLogSnapshotOpen(BtLog*); int sqlite4BtLogSnapshotClose(BtLog*); int sqlite4BtLogSnapshotWritable(BtLog*); int sqlite4BtLogSize(BtLog*); int sqlite4BtLogCheckpoint(BtLog*); /* ** End of bt_log.c interface. *************************************************************************/ /************************************************************************* ** Interface to bt_lock.c functionality. | > > > > > > | 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | int sqlite4BtLogSnapshotOpen(BtLog*); int sqlite4BtLogSnapshotClose(BtLog*); int sqlite4BtLogSnapshotWritable(BtLog*); int sqlite4BtLogSize(BtLog*); int sqlite4BtLogCheckpoint(BtLog*); #ifndef NDEBUG void sqlite4BtDebugReadPage(u32 pgno, u8 *aData, int pgsz); #else # define sqlite4BtDebugReadPage(a,b,c) #endif /* ** End of bt_log.c interface. *************************************************************************/ /************************************************************************* ** Interface to bt_lock.c functionality. |
︙ | ︙ |
Changes to src/bt_log.c.
︙ | ︙ | |||
163 164 165 166 167 168 169 | ** of the input value as a little-endian integer. */ #define BYTESWAP32(x) ( \ (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ ) | < < < < < < < < | 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | ** of the input value as a little-endian integer. */ #define BYTESWAP32(x) ( \ (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ ) /* ** Generate or extend an 8 byte checksum based on the data in ** array aByte[] and the initial values of aIn[0] and aIn[1] (or ** initial values of 0 and 0 if aIn==NULL). ** ** The checksum is written back into aOut[] before returning. ** |
︙ | ︙ | |||
218 219 220 221 222 223 224 225 226 227 228 229 230 231 | }while( aData<aEnd ); } aOut[0] = s1; aOut[1] = s2; } /* ** Ensure that shared-memory chunk iChunk is mapped and available in ** the BtLog.apShm[] array. If an error occurs, return an SQLite4 error ** code. Otherwise, SQLITE4_OK. */ static int btLogMapShm(BtLog *pLog, int iChunk){ int rc = SQLITE4_OK; | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 | }while( aData<aEnd ); } aOut[0] = s1; aOut[1] = s2; } static void btDebugTopology(char *zStr, u32 *aLog){ fprintf(stderr, "%s: %d..%d %d..%d %d..%d\n", zStr, (int)aLog[0], (int)aLog[1], (int)aLog[2], (int)aLog[3], (int)aLog[4], (int)aLog[5] ); fflush(stderr); } static void btDebugCkptPage(u32 pgno, u8 *aData, int pgsz){ #if 0 static nCall = 0; u32 aCksum[2]; btLogChecksum(1, aData, pgsz, 0, aCksum); fprintf(stderr, "%d: Ckpt page %d (cksum=%08x%08x)\n", nCall++, (int)pgno, aCksum[0], aCksum[1] ); fflush(stderr); #endif } static void btDebugLogPage(u32 pgno, u32 iFrame, u8 *aData, int pgsz){ #if 0 static nCall = 0; u32 aCksum[2]; btLogChecksum(1, aData, pgsz, 0, aCksum); fprintf(stderr, "%d: Log page %d to frame %d (cksum=%08x%08x)\n", nCall++, (int)pgno, (int)iFrame, aCksum[0], aCksum[1] ); fflush(stderr); #endif } void sqlite4BtDebugReadPage(u32 pgno, u8 *aData, int pgsz){ #if 0 static nCall = 0; u32 aCksum[2]; btLogChecksum(1, aData, pgsz, 0, aCksum); fprintf(stderr, "%d: Read page %d (cksum=%08x%08x)\n", nCall++, (int)pgno, aCksum[0], aCksum[1] ); fflush(stderr); #endif } /* ** Ensure that shared-memory chunk iChunk is mapped and available in ** the BtLog.apShm[] array. If an error occurs, return an SQLite4 error ** code. Otherwise, SQLITE4_OK. */ static int btLogMapShm(BtLog *pLog, int iChunk){ int rc = SQLITE4_OK; |
︙ | ︙ | |||
743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 | return rc; } static int btLogHashSearch( BtLog *pLog, /* Log module handle */ int iSide, /* 0 or 1 - the side of hash table to read */ int iHash, /* Index of hash to query */ u32 pgno, /* query for this page number */ u32 *piFrame /* OUT: Frame number for matching entry */ ){ ht_slot *aHash; u32 *aPgno; u32 iZero; int rc; rc = btLogFindHash(pLog, iSide, iHash, &aHash, &aPgno, &iZero); if( rc==SQLITE4_OK ){ int nCollide = HASHTABLE_NSLOT*2; int iSlot; u32 iFrame = 0; iSlot=btLogHashKey(pLog, pgno); for( ; aHash[iSlot]; iSlot=btLogHashNext(pLog, iSlot)){ if( aPgno[aHash[iSlot]-1]==pgno ){ | > | > | 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 | return rc; } static int btLogHashSearch( BtLog *pLog, /* Log module handle */ int iSide, /* 0 or 1 - the side of hash table to read */ int iHash, /* Index of hash to query */ u32 iHi, /* Consider no frames after this one */ u32 pgno, /* query for this page number */ u32 *piFrame /* OUT: Frame number for matching entry */ ){ ht_slot *aHash; u32 *aPgno; u32 iZero; int rc; rc = btLogFindHash(pLog, iSide, iHash, &aHash, &aPgno, &iZero); if( rc==SQLITE4_OK ){ int nCollide = HASHTABLE_NSLOT*2; int iSlot; u32 iFrame = 0; iSlot=btLogHashKey(pLog, pgno); for( ; aHash[iSlot]; iSlot=btLogHashNext(pLog, iSlot)){ if( aPgno[aHash[iSlot]-1]==pgno ){ u32 iCandidate = iZero + aHash[iSlot] - 1; if( iCandidate<=iHi ) iFrame = iCandidate; } if( (nCollide--)==0 ) return btErrorBkpt(SQLITE4_CORRUPT); } *piFrame = iFrame; if( iFrame==0 ){ rc = SQLITE4_NOTFOUND; |
︙ | ︙ | |||
792 793 794 795 796 797 798 | int sqlite4BtLogRead(BtLog *pLog, u32 pgno, u8 *aData){ const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock)); int rc = SQLITE4_NOTFOUND; u32 iFrame = 0; int i; /* Loop through regions (c), (b) and (a) of the log file. In that order. */ | | | | | | 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 | int sqlite4BtLogRead(BtLog *pLog, u32 pgno, u8 *aData){ const int pgsz = sqlite4BtPagerPagesize((BtPager*)(pLog->pLock)); int rc = SQLITE4_NOTFOUND; u32 iFrame = 0; int i; /* Loop through regions (c), (b) and (a) of the log file. In that order. */ for(i=2; i>=0 && rc==SQLITE4_NOTFOUND; i--){ u32 iLo = pLog->snapshot.aLog[i*2+0]; u32 iHi = pLog->snapshot.aLog[i*2+1]; int iSide; int iHash; int iHashLast; iHash = btLogFrameHash(pLog, iHi); iHashLast = btLogFrameHash(pLog, iLo); iSide = (pLog->snapshot.iHashSide + (i==0)) % 2; for( ; rc==SQLITE4_NOTFOUND && iHash>=iHashLast; iHash--){ rc = btLogHashSearch(pLog, iSide, iHash, iHi, pgno, &iFrame); if( rc==SQLITE4_OK && (iFrame<iLo || iFrame>iHi) ){ rc = SQLITE4_NOTFOUND; } } } if( rc==SQLITE4_OK ){ bt_env *pVfs = pLog->pLock->pVfs; i64 iOff; assert( rc==SQLITE4_OK ); iOff = btLogFrameOffset(pLog, pgsz, iFrame); rc = pVfs->xRead(pLog->pFd, iOff + sizeof(BtFrameHdr), aData, pgsz); #if 0 |
︙ | ︙ | |||
899 900 901 902 903 904 905 | memset(&frame, 0, sizeof(frame)); frame.pgno = pgno; frame.ctrl = (bCommit ? BT_FRAME_COMMIT : 0) + iNextFrame; a = pLog->snapshot.aFrameCksum; btLogChecksum(1, (u8*)&frame, offsetof(BtFrameHdr,aCksum), a, frame.aCksum); btLogChecksum(1, aData, pgsz, frame.aCksum, frame.aCksum); | | < < < | 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 | memset(&frame, 0, sizeof(frame)); frame.pgno = pgno; frame.ctrl = (bCommit ? BT_FRAME_COMMIT : 0) + iNextFrame; a = pLog->snapshot.aFrameCksum; btLogChecksum(1, (u8*)&frame, offsetof(BtFrameHdr,aCksum), a, frame.aCksum); btLogChecksum(1, aData, pgsz, frame.aCksum, frame.aCksum); btDebugLogPage(pgno, iFrame, aData, pgsz); /* Write the frame header to the log file. */ rc = btLogWriteData(pLog, iOff, (u8*)&frame, sizeof(frame)); } pLog->snapshot.iNextFrame = iNextFrame; /* Write the frame contents to the log file. */ |
︙ | ︙ | |||
924 925 926 927 928 929 930 | pLog->snapshot.iHashSide = (pLog->snapshot.iHashSide+1) %2; } rc = btLogHashInsert(pLog, pgno, iFrame); } /* Update the private copy of the shm-header */ if( rc==SQLITE4_OK ){ | < < | 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 | pLog->snapshot.iHashSide = (pLog->snapshot.iHashSide+1) %2; } rc = btLogHashInsert(pLog, pgno, iFrame); } /* Update the private copy of the shm-header */ if( rc==SQLITE4_OK ){ if( btLogIsEmpty(pLog) ){ assert( iFrame==1 ); aLog[4] = iFrame; }else if( iFrame==1 ){ assert( aLog[0]==0 && aLog[1]==0 && aLog[2]==0 && aLog[3]==0 ); aLog[0] = aLog[4]; aLog[1] = aLog[5]; aLog[4] = iFrame; }else if( iFrame!=aLog[5]+1 ){ assert( iFrame>aLog[5] ); assert( aLog[2]==0 && aLog[3]==0 ); aLog[2] = aLog[4]; aLog[3] = aLog[5]; aLog[4] = iFrame; } aLog[5] = iFrame; memcpy(pLog->snapshot.aFrameCksum, frame.aCksum, sizeof(frame.aCksum)); } /* If this is a COMMIT, also update the shared shm-header. */ if( bCommit ){ rc = btLogUpdateSharedHdr(pLog); } |
︙ | ︙ | |||
1016 1017 1018 1019 1020 1021 1022 | if( rc==SQLITE4_OK ){ int iRegion; u32 *aLog = pLog->snapshot.aLog; BtShm *pShm = btLogShm(pLog); u32 iCkpt = pShm->ckpt.iFirstRecover; | < < < < | 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 | if( rc==SQLITE4_OK ){ int iRegion; u32 *aLog = pLog->snapshot.aLog; BtShm *pShm = btLogShm(pLog); u32 iCkpt = pShm->ckpt.iFirstRecover; for(iRegion=0; iRegion<3; iRegion++){ if( aLog[iRegion*2] ){ if( iCkpt>=aLog[iRegion*2] && iCkpt<=aLog[iRegion*2+1] ){ aLog[iRegion*2] = iCkpt; break; }else{ aLog[iRegion*2] = 0; aLog[iRegion*2+1] = 0; } } } } return rc; } int sqlite4BtLogSnapshotClose(BtLog *pLog){ sqlite4BtLockReaderUnlock(pLog->pLock); |
︙ | ︙ | |||
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 | u32 **paPgno, /* OUT: s4_malloc'd array of sorted pgno */ int *pnPgno, /* OUT: Number of entries in *paPgno */ u32 *piLastFrame ){ u32 *aLog = pLog->snapshot.aLog;/* Log file topology */ u32 i; u32 *aPgno; /* Returned array */ u32 *aSpace; /* Temporary space used by merge-sort */ int nMax; int rc = SQLITE4_OK; int iRegion; /* Determine an upper limit on the number of distinct page numbers. This ** limit is used to allocate space for the returned array. */ nMax = 0; for(iRegion=0; iRegion<3; iRegion++){ if( aLog[iRegion*2] ){ nMax += 1 + aLog[iRegion*2+1] - aLog[iRegion*2+0]; } } /* Allocate space to collect all page numbers. */ aPgno = (u32*)sqlite4_malloc(pLog->pLock->pEnv, sizeof(u32)*nMax*2); if( aPgno==0 ) rc = btErrorBkpt(SQLITE4_NOMEM); aSpace = &aPgno[nMax]; /* Copy the required page numbers into the allocated array */ for(iRegion=0; iRegion<3; iRegion++){ u32 iFirst = aLog[iRegion*2]; u32 iLast = aLog[iRegion*2+1]; if( iFirst ){ for(i=iFirst; rc==SQLITE4_OK && i<=iLast; i++){ int iHash = btLogFrameHash(pLog, i); u32 *aPage; ht_slot *aHash; u32 iZero; /* It doesn't matter which 'side' of the hash table is requested here, ** as only the page-number array, not the aHash[] table, will be used. ** And it is the same for both sides. Hence the constant 0 passed as ** the second argument to btLogFindHash(). */ rc = btLogFindHash(pLog, 0, iHash, &aHash, &aPage, &iZero); if( rc==SQLITE4_OK ){ | > > | | | | 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 | u32 **paPgno, /* OUT: s4_malloc'd array of sorted pgno */ int *pnPgno, /* OUT: Number of entries in *paPgno */ u32 *piLastFrame ){ u32 *aLog = pLog->snapshot.aLog;/* Log file topology */ u32 i; u32 *aPgno; /* Returned array */ int nPgno; /* Elements in aPgno[] */ u32 *aSpace; /* Temporary space used by merge-sort */ int nMax; int rc = SQLITE4_OK; int iRegion; /* Determine an upper limit on the number of distinct page numbers. This ** limit is used to allocate space for the returned array. */ nMax = 0; for(iRegion=0; iRegion<3; iRegion++){ if( aLog[iRegion*2] ){ nMax += 1 + aLog[iRegion*2+1] - aLog[iRegion*2+0]; } } /* Allocate space to collect all page numbers. */ aPgno = (u32*)sqlite4_malloc(pLog->pLock->pEnv, sizeof(u32)*nMax*2); if( aPgno==0 ) rc = btErrorBkpt(SQLITE4_NOMEM); aSpace = &aPgno[nMax]; nPgno = 0; /* Copy the required page numbers into the allocated array */ for(iRegion=0; iRegion<3; iRegion++){ u32 iFirst = aLog[iRegion*2]; u32 iLast = aLog[iRegion*2+1]; if( iFirst ){ for(i=iFirst; rc==SQLITE4_OK && i<=iLast; i++){ int iHash = btLogFrameHash(pLog, i); u32 *aPage; ht_slot *aHash; u32 iZero; /* It doesn't matter which 'side' of the hash table is requested here, ** as only the page-number array, not the aHash[] table, will be used. ** And it is the same for both sides. Hence the constant 0 passed as ** the second argument to btLogFindHash(). */ rc = btLogFindHash(pLog, 0, iHash, &aHash, &aPage, &iZero); if( rc==SQLITE4_OK ){ aPgno[nPgno++] = aPage[i-iZero]; } } *piLastFrame = iLast; } } /* Sort the contents of the array in ascending order. This step also ** eliminates any duplicate page numbers. */ if( rc==SQLITE4_OK ){ btLogMergeSort(aPgno, &nPgno, aSpace); *pnPgno = nPgno; *paPgno = aPgno; }else{ sqlite4_free(pLog->pLock->pEnv, aPgno); *paPgno = 0; *pnPgno = 0; } |
︙ | ︙ | |||
1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 | } /* Copy data from the log file to the database file. */ for(i=0; rc==SQLITE4_OK && i<nPgno; i++){ u32 pgno = aPgno[i]; rc = sqlite4BtLogRead(pLog, pgno, aBuf); if( rc==SQLITE4_OK ){ i64 iOff = (i64)pgsz * (pgno-1); rc = pVfs->xWrite(pFd, iOff, aBuf, pgsz); } } /* Update the first field of the checkpoint-header. This tells readers ** that they need not consider anything that in the log before this | > | 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 | } /* Copy data from the log file to the database file. */ for(i=0; rc==SQLITE4_OK && i<nPgno; i++){ u32 pgno = aPgno[i]; rc = sqlite4BtLogRead(pLog, pgno, aBuf); if( rc==SQLITE4_OK ){ btDebugCkptPage(pgno, aBuf, pgsz); i64 iOff = (i64)pgsz * (pgno-1); rc = pVfs->xWrite(pFd, iOff, aBuf, pgsz); } } /* Update the first field of the checkpoint-header. This tells readers ** that they need not consider anything that in the log before this |
︙ | ︙ |
Changes to src/bt_pager.c.
|
| | | 1 2 3 4 5 6 7 8 | /*; ** 2013 September 14 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. |
︙ | ︙ | |||
249 250 251 252 253 254 255 | /* ** Close a pager database handle. */ int sqlite4BtPagerClose(BtPager *p){ int i; int rc; | | | 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 | /* ** Close a pager database handle. */ int sqlite4BtPagerClose(BtPager *p){ int i; int rc; rc = sqlite4BtLockDisconnect((BtLock*)p, btCheckpoint, btCleanup); if( p->btl.pFd ){ p->btl.pVfs->xClose(p->btl.pFd); } btPurgeCache(p); sqlite4BtLogClose(p->pLog, 0); |
︙ | ︙ | |||
591 592 593 594 595 596 597 598 599 600 601 602 603 604 | rc = btHashAdd(p, pRet); } if( rc!=SQLITE4_OK ){ btFreePage(p, pRet); pRet = 0; } } } assert( (pRet!=0)==(rc==SQLITE4_OK) ); if( rc==SQLITE4_OK ){ p->nTotalRef++; pRet->nRef++; } | > | 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 | rc = btHashAdd(p, pRet); } if( rc!=SQLITE4_OK ){ btFreePage(p, pRet); pRet = 0; } } sqlite4BtDebugReadPage(pgno, pRet->aData, p->dbhdr.pgsz); } assert( (pRet!=0)==(rc==SQLITE4_OK) ); if( rc==SQLITE4_OK ){ p->nTotalRef++; pRet->nRef++; } |
︙ | ︙ |