Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix problems with delete markers and range scans.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: ccf5a6bb6a0b1e6dfbef72fab9a561909bc86a16
User & Date: dan 2013-12-06 20:11:58.904
Context
2013-12-07
20:38
Add notes describing schedule pages to bt.wiki. check-in: 1df60437f3 user: dan tags: trunk
2013-12-06
20:11
Fix problems with delete markers and range scans. check-in: ccf5a6bb6a user: dan tags: trunk
2013-12-05
20:08
Support scan queries on fast-insert data. Still some problems. check-in: 0cd2ab7e9e user: dan tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to lsm-test/lsmtest1.c.
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
){
  Datatest2 aTest[] = {
      /* defn,                                 nRange, nWrite, nIter */
    { {DATA_RANDOM,     20,25,     100,200},   10000,  10,     50   },
    { {DATA_RANDOM,     20,25,     100,200},   10000,  200,    50   },
    { {DATA_RANDOM,     20,25,     100,200},   100,    10,     1000 },
    { {DATA_RANDOM,     20,25,     100,200},   100,    200,    50   },

#if 0
    { {DATA_RANDOM,     20,25,     100,200},       200, 50 }
#endif
  };

  int i;

  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
    char *zName = getName2(zSystem, &aTest[i]);
    if( testCaseBegin(pRc, zPattern, "%s", zName) ){







<
<
<
<







465
466
467
468
469
470
471




472
473
474
475
476
477
478
){
  Datatest2 aTest[] = {
      /* defn,                                 nRange, nWrite, nIter */
    { {DATA_RANDOM,     20,25,     100,200},   10000,  10,     50   },
    { {DATA_RANDOM,     20,25,     100,200},   10000,  200,    50   },
    { {DATA_RANDOM,     20,25,     100,200},   100,    10,     1000 },
    { {DATA_RANDOM,     20,25,     100,200},   100,    200,    50   },




  };

  int i;

  for(i=0; *pRc==LSM_OK && i<ArraySize(aTest); i++){
    char *zName = getName2(zSystem, &aTest[i]);
    if( testCaseBegin(pRc, zPattern, "%s", zName) ){
Changes to lsm-test/lsmtest_main.c.
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
  int *pRc                        /* IN/OUT: Error code */
){
  int nVal = (zVal ? strlen(zVal) : 0);
  testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
}

void testFetchCompare(
  TestDb *pDb1, 
  TestDb *pDb2, 
  void *pKey, int nKey, 
  int *pRc
){
  int rc;
  void *pDbVal1;
  void *pDbVal2;
  int nDbVal1;
  int nDbVal2;

  static int nCall = 0;
  nCall++;

  rc = tdb_fetch(pDb1, pKey, nKey, &pDbVal1, &nDbVal1);
  testSetError(rc);

  rc = tdb_fetch(pDb2, pKey, nKey, &pDbVal2, &nDbVal2);
  testSetError(rc);

  if( *pRc==0 
   && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1)))
  ){
    testSetError(1);
  }







|
|












|


|







141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
  int *pRc                        /* IN/OUT: Error code */
){
  int nVal = (zVal ? strlen(zVal) : 0);
  testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
}

void testFetchCompare(
  TestDb *pControl, 
  TestDb *pDb, 
  void *pKey, int nKey, 
  int *pRc
){
  int rc;
  void *pDbVal1;
  void *pDbVal2;
  int nDbVal1;
  int nDbVal2;

  static int nCall = 0;
  nCall++;

  rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1);
  testSetError(rc);

  rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2);
  testSetError(rc);

  if( *pRc==0 
   && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1)))
  ){
    testSetError(1);
  }
Changes to src/bt.h.
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

int sqlite4BtReplace(bt_db*, const void *pK, int nK, const void *pV, int nV);
int sqlite4BtDelete(bt_cursor*);

int sqlite4BtSetCookie(bt_db*, unsigned int iVal);
int sqlite4BtGetCookie(bt_db*, unsigned int *piVal);

/*
** Append a human readable representation of the b-tree page passed via
** arguments a+n to the buffer passed as the first argument. If argument
** pgno is non-zero it is included in the output.
*/
int sqlite4BtDebugPage(sqlite4_buffer*, unsigned int pgno, char *a, int n);

/*
** kvstore xControl() method.
**
** BT_CONTROL_INFO:
**   If the second argument to sqlite4BtControl() is BT_CONTROL_INFO, then
**   the third is expected to be a pointer to an instance of type bt_info.







<
<
<
<
<
<







111
112
113
114
115
116
117






118
119
120
121
122
123
124

int sqlite4BtReplace(bt_db*, const void *pK, int nK, const void *pV, int nV);
int sqlite4BtDelete(bt_cursor*);

int sqlite4BtSetCookie(bt_db*, unsigned int iVal);
int sqlite4BtGetCookie(bt_db*, unsigned int *piVal);








/*
** kvstore xControl() method.
**
** BT_CONTROL_INFO:
**   If the second argument to sqlite4BtControl() is BT_CONTROL_INFO, then
**   the third is expected to be a pointer to an instance of type bt_info.
201
202
203
204
205
206
207
208
209
210

211
212
213
214
215
216
217
typedef struct bt_info bt_info;
struct bt_info {
  int eType;
  unsigned int pgno;
  sqlite4_buffer output;
};

#define BT_INFO_PAGEDUMP   1
#define BT_INFO_FILENAME   2
#define BT_INFO_HDRDUMP    3


typedef struct bt_logsizecb bt_logsizecb;
struct bt_logsizecb {
  void *pCtx;                     /* A copy of this is passed to xLogsize() */
  void (*xLogsize)(void*, int);   /* Callback function */
};








|
|
|
>







195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
typedef struct bt_info bt_info;
struct bt_info {
  int eType;
  unsigned int pgno;
  sqlite4_buffer output;
};

#define BT_INFO_PAGEDUMP       1
#define BT_INFO_FILENAME       2
#define BT_INFO_HDRDUMP        3
#define BT_INFO_PAGEDUMP_ASCII 4

typedef struct bt_logsizecb bt_logsizecb;
struct bt_logsizecb {
  void *pCtx;                     /* A copy of this is passed to xLogsize() */
  void (*xLogsize)(void*, int);   /* Callback function */
};

Changes to src/bt_main.c.
461
462
463
464
465
466
467

















468
469
470
471
472
473






474
475
476
477
478
479
480
  va_start(ap, zFormat);
  zAppend = sqlite4_vmprintf(0, zFormat, ap);
  va_end(ap);

  sqlite4_buffer_append(pBuf, zAppend, strlen(zAppend));
  sqlite4_free(0, zAppend);
}


















/*
** Append a human-readable interpretation of the b-tree page in aData/nData
** to buffer pBuf.
*/
static void btPageToAscii(u32 pgno, u8 *aData, int nData, sqlite4_buffer *pBuf){






  int i;
  int nCell = (int)btCellCount(aData, nData);

  sqlite4BtBufAppendf(pBuf, "Page %d: ", pgno);
  sqlite4BtBufAppendf(pBuf, "nCell=%d ", nCell);
  sqlite4BtBufAppendf(pBuf, "iFree=%d ", (int)btFreeOffset(aData, nData));
  sqlite4BtBufAppendf(pBuf, "flags=%d ", (int)btFlags(aData));







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





|
>
>
>
>
>
>







461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
  va_start(ap, zFormat);
  zAppend = sqlite4_vmprintf(0, zFormat, ap);
  va_end(ap);

  sqlite4_buffer_append(pBuf, zAppend, strlen(zAppend));
  sqlite4_free(0, zAppend);
}

#include <ctype.h>

void btBufferAppendBlob(
  sqlite4_buffer *pBuf, 
  int bAscii, 
  u8 *pBlob, int nBlob
){
  int j;
  for(j=0; j<nBlob; j++){
    if( bAscii ){
      sqlite4BtBufAppendf(pBuf, "%c", isalnum((int)pBlob[j]) ? pBlob[j] : '.');
    }else{
      sqlite4BtBufAppendf(pBuf, "%02X", (int)pBlob[j]);
    }
  }
}

/*
** Append a human-readable interpretation of the b-tree page in aData/nData
** to buffer pBuf.
*/
static void btPageToAscii(
  u32 pgno,                       /* Page number */
  int bAscii,                     /* True to print keys and values as ASCII */
  BtPager *pPager,                /* Pager object (or NULL) */
  u8 *aData, int nData,           /* Buffer containing page data */
  sqlite4_buffer *pBuf            /* Output buffer */
){
  int i;
  int nCell = (int)btCellCount(aData, nData);

  sqlite4BtBufAppendf(pBuf, "Page %d: ", pgno);
  sqlite4BtBufAppendf(pBuf, "nCell=%d ", nCell);
  sqlite4BtBufAppendf(pBuf, "iFree=%d ", (int)btFreeOffset(aData, nData));
  sqlite4BtBufAppendf(pBuf, "flags=%d ", (int)btFlags(aData));
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519





520


521
522
523




524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
  for(i=0; i<nCell; i++){
    int nKey;
    int j;
    u8 *pCell = btCellFind(aData, nData, i);
    sqlite4BtBufAppendf(pBuf, "  Cell %d: ", i);

    pCell += sqlite4BtVarintGet32(pCell, &nKey);
    for(j=0; j<nKey; j++){
      sqlite4BtBufAppendf(pBuf, "%02X", (int)pCell[j]);
    }

    if( btFlags(aData) & BT_PGFLAGS_INTERNAL ){
      sqlite4BtBufAppendf(pBuf, "  child=%d ", (int)btGetU32(&pCell[j]));
    }else{
      int nVal;
      pCell += nKey;
      sqlite4BtBufAppendf(pBuf, "  ");
      pCell += sqlite4BtVarintGet32(pCell, &nVal);
      if( nVal>=2 ){
        for(j=0; j<(nVal-2); j++){
          sqlite4BtBufAppendf(pBuf, "%02X", (int)pCell[j]);
        }
      }else{
        sqlite4BtBufAppendf(pBuf, "delete-key");
      }
    }
    sqlite4BtBufAppendf(pBuf, "\n");
  }
}








int sqlite4BtDebugPage(sqlite4_buffer *pBuf, u32 pgno, char *aData, int nData){
  btPageToAscii(pgno, (u8*)aData, nData, pBuf);
  return SQLITE4_OK;




}

#ifndef NDEBUG
#include <stdio.h>

static void printPage(FILE *f, u32 pgno, u8 *aData, int nData){
  sqlite4_buffer buf;

  sqlite4_buffer_init(&buf, 0);
  btPageToAscii(pgno, aData, nData, &buf);
  sqlite4_buffer_append(&buf, "", 1);

  fprintf(f, "%s", (char*)buf.p);
  sqlite4_buffer_clear(&buf);
}

int printPgdataToStderr(u32 pgno, u8 *aData, int nData){







|
|
|
<

|


<



|
<
<






|
>
>
>
>
>

>
>
|
|
<
>
>
>
>









|







514
515
516
517
518
519
520
521
522
523

524
525
526
527

528
529
530
531


532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548

549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
  for(i=0; i<nCell; i++){
    int nKey;
    int j;
    u8 *pCell = btCellFind(aData, nData, i);
    sqlite4BtBufAppendf(pBuf, "  Cell %d: ", i);

    pCell += sqlite4BtVarintGet32(pCell, &nKey);
    btBufferAppendBlob(pBuf, bAscii, pCell, nKey);
    pCell += nKey;


    if( btFlags(aData) & BT_PGFLAGS_INTERNAL ){
      sqlite4BtBufAppendf(pBuf, "  child=%d ", (int)btGetU32(pCell));
    }else{
      int nVal;

      sqlite4BtBufAppendf(pBuf, "  ");
      pCell += sqlite4BtVarintGet32(pCell, &nVal);
      if( nVal>=2 ){
        btBufferAppendBlob(pBuf, bAscii, pCell, nVal-2);


      }else{
        sqlite4BtBufAppendf(pBuf, "delete-key");
      }
    }
    sqlite4BtBufAppendf(pBuf, "\n");
  }

  if( pPager && btFlags(aData) & BT_PGFLAGS_INTERNAL ){
    for(i=0; i<btCellCount(aData, nData); i++){
      BtPage *pChild;
      u8 *aChild;
      u32 child;

      child = btChildPgno(aData, nData, i);
      sqlite4BtPageGet(pPager, child, &pChild);
      aChild = sqlite4BtPageData(pChild);
      btPageToAscii(child, bAscii, pPager, aChild, nData, pBuf);

      sqlite4BtPageRelease(pChild);
    }
  }
  sqlite4BtBufAppendf(pBuf, "\n");
}

#ifndef NDEBUG
#include <stdio.h>

static void printPage(FILE *f, u32 pgno, u8 *aData, int nData){
  sqlite4_buffer buf;

  sqlite4_buffer_init(&buf, 0);
  btPageToAscii(pgno, 0, 0, aData, nData, &buf);
  sqlite4_buffer_append(&buf, "", 1);

  fprintf(f, "%s", (char*)buf.p);
  sqlite4_buffer_clear(&buf);
}

int printPgdataToStderr(u32 pgno, u8 *aData, int nData){
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354

  return rc;
}

static int fiCsrEnd(FiCursor *pCsr, int bLast){
  bt_db *db = pCsr->base.pDb;
  BtDbHdr *pHdr = sqlite4BtPagerDbhdr(db->pPager);
  int bHit = 0;             /* Found at least one entry */
  int iSub = 0;             /* Index of sub-cursor to seek */
  BtCursor mcsr;            /* Used to iterate through the meta-tree */
  int rc;                   /* Return code */

  fiCsrReset(pCsr);

  /* Loop through the set of f-tree levels. Open a cursor on each. */







<







1369
1370
1371
1372
1373
1374
1375

1376
1377
1378
1379
1380
1381
1382

  return rc;
}

static int fiCsrEnd(FiCursor *pCsr, int bLast){
  bt_db *db = pCsr->base.pDb;
  BtDbHdr *pHdr = sqlite4BtPagerDbhdr(db->pPager);

  int iSub = 0;             /* Index of sub-cursor to seek */
  BtCursor mcsr;            /* Used to iterate through the meta-tree */
  int rc;                   /* Return code */

  fiCsrReset(pCsr);

  /* Loop through the set of f-tree levels. Open a cursor on each. */
3211
3212
3213
3214
3215
3216
3217















3218
3219
3220
3221
3222
3223
3224
  }

  btCheckPageRefs(db);
  db->bFastInsertOp = 0;
  return rc;
}

















/*
** Delete the entry that the cursor currently points to.
*/
int sqlite4BtDelete(bt_cursor *pBase){
  bt_db *db = pBase->pDb;
  int rc;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
  }

  btCheckPageRefs(db);
  db->bFastInsertOp = 0;
  return rc;
}

void sqlite4BtDebugTree(bt_db *db, int iCall, u32 iRoot){
  BtPage *pPg;
  sqlite4_buffer buf;
  int pgsz;

  pgsz = sqlite4BtPagerPagesize(db->pPager);
  sqlite4_buffer_init(&buf, 0);
  sqlite4BtPageGet(db->pPager, iRoot, &pPg);
  btPageToAscii(iRoot, 1, db->pPager, sqlite4BtPageData(pPg), pgsz, &buf);
  fprintf(stderr, "%d TREE at %d:\n", iCall, (int)iRoot);
  fprintf(stderr, "%.*s", buf.n, (char*)buf.p);
  sqlite4_buffer_clear(&buf);
  sqlite4BtPageRelease(pPg);
}


/*
** Delete the entry that the cursor currently points to.
*/
int sqlite4BtDelete(bt_cursor *pBase){
  bt_db *db = pBase->pDb;
  int rc;
3238
3239
3240
3241
3242
3243
3244



3245
3246
3247



3248
3249
3250
3251
3252
3253




3254

3255
3256
3257
3258
3259
3260
3261
    }
    if( rc==SQLITE4_OK ){
      rc = btBalanceIfUnderfull(pCsr);
    }

    btCsrReleaseAll(pCsr);
  }else{



    void *pKey;
    int nKey;




    rc = sqlite4BtCsrKey(pBase, &pKey, &nKey);
    if( rc==SQLITE4_OK ){
      int bFastInsertOp = db->bFastInsertOp;
      db->bFastInsertOp = 1;
      rc = sqlite4BtReplace(db, pKey, nKey, 0, -1);
      db->bFastInsertOp = bFastInsertOp;




    }

  }
  return rc;
}

int sqlite4BtSetCookie(bt_db *db, unsigned int iVal){
  return sqlite4BtPagerSetCookie(db->pPager, iVal);
}







>
>
>



>
>
>
|





>
>
>
>

>







3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
    }
    if( rc==SQLITE4_OK ){
      rc = btBalanceIfUnderfull(pCsr);
    }

    btCsrReleaseAll(pCsr);
  }else{
    FiCursor *pCsr = (FiCursor*)pBase;
    BtCursor *pSub = &pCsr->aBt[pCsr->iBt];

    void *pKey;
    int nKey;

    rc = btCsrBuffer(pSub, 0);
    pKey = pSub->ovfl.buf.p;
    nKey = pSub->ovfl.nKey;

    if( rc==SQLITE4_OK ){
      int bFastInsertOp = db->bFastInsertOp;
      db->bFastInsertOp = 1;
      rc = sqlite4BtReplace(db, pKey, nKey, 0, -1);
      db->bFastInsertOp = bFastInsertOp;
#if 0
static int iCall = 0;
sqlite4BtDebugTree(db, ++iCall, 513); 
#endif
    }

  }
  return rc;
}

int sqlite4BtSetCookie(bt_db *db, unsigned int iVal){
  return sqlite4BtPagerSetCookie(db->pPager, iVal);
}
3279
3280
3281
3282
3283
3284
3285

3286
3287
3288
3289
3290
3291
3292

3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
  if( iCtx==0 ) sqlite4BtCommit(db, 0);
}

static int btControlInfo(bt_db *db, bt_info *pInfo){
  int rc = SQLITE4_OK;

  switch( pInfo->eType ){

    case BT_INFO_PAGEDUMP: {
      int iCtx;                   /* ControlTransaction() context */
      rc = btControlTransaction(db, &iCtx);
      if( rc==SQLITE4_OK ){
        BtPage *pPg = 0;
        rc = sqlite4BtPageGet(db->pPager, pInfo->pgno, &pPg);
        if( rc==SQLITE4_OK ){

          u8 *aData;
          int nData;
          aData = sqlite4BtPageData(pPg);
          nData = sqlite4BtPagerPagesize(db->pPager);
          btPageToAscii(pInfo->pgno, aData, nData, &pInfo->output);
          sqlite4_buffer_append(&pInfo->output, "", 1);
          sqlite4BtPageRelease(pPg);
        }
        btControlTransactionDone(db, iCtx);
      }
      break;
    }







>







>




|







3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
  if( iCtx==0 ) sqlite4BtCommit(db, 0);
}

static int btControlInfo(bt_db *db, bt_info *pInfo){
  int rc = SQLITE4_OK;

  switch( pInfo->eType ){
    case BT_INFO_PAGEDUMP_ASCII:
    case BT_INFO_PAGEDUMP: {
      int iCtx;                   /* ControlTransaction() context */
      rc = btControlTransaction(db, &iCtx);
      if( rc==SQLITE4_OK ){
        BtPage *pPg = 0;
        rc = sqlite4BtPageGet(db->pPager, pInfo->pgno, &pPg);
        if( rc==SQLITE4_OK ){
          int bAscii = (pInfo->eType==BT_INFO_PAGEDUMP_ASCII);
          u8 *aData;
          int nData;
          aData = sqlite4BtPageData(pPg);
          nData = sqlite4BtPagerPagesize(db->pPager);
          btPageToAscii(pInfo->pgno, bAscii, 0, aData, nData, &pInfo->output);
          sqlite4_buffer_append(&pInfo->output, "", 1);
          sqlite4BtPageRelease(pPg);
        }
        btControlTransactionDone(db, iCtx);
      }
      break;
    }