SQLite

Check-in [a9a2aeab29]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts4-experimental
Files: files | file ages | folders
SHA1: a9a2aeab298ba2ac0b8835e61406e2d83bf7e39b
User & Date: dan 2014-05-16 10:30:44.076
Context
2014-05-16
15:48
Add extra test to backcompat.test to ensure that old and new versions of FTS may work together on the same incremental merge operation. (Closed-Leaf check-in: 3997d47bb9 user: dan tags: fts4-experimental)
10:30
Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation. (check-in: a9a2aeab29 user: dan tags: fts4-experimental)
2014-05-15
19:05
Merge latest trunk changes with this branch. (check-in: 5809986ffc user: dan tags: fts4-experimental)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_write.c.
3632
3633
3634
3635
3636
3637
3638

3639
3640
3641
3642
3643
3644
3645
  int nLeafEst;                   /* Space allocated for leaf blocks */
  int nWork;                      /* Number of leaf pages flushed */
  sqlite3_int64 iAbsLevel;        /* Absolute level of input segments */
  int iIdx;                       /* Index of *output* segment in iAbsLevel+1 */
  sqlite3_int64 iStart;           /* Block number of first allocated block */
  sqlite3_int64 iEnd;             /* Block number of last allocated block */
  sqlite3_int64 nLeafData;        /* Bytes of leaf page data so far */

  NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
};

/*
** An object of the following type is used to read data from a single
** FTS segment node. See the following functions:
**







>







3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
  int nLeafEst;                   /* Space allocated for leaf blocks */
  int nWork;                      /* Number of leaf pages flushed */
  sqlite3_int64 iAbsLevel;        /* Absolute level of input segments */
  int iIdx;                       /* Index of *output* segment in iAbsLevel+1 */
  sqlite3_int64 iStart;           /* Block number of first allocated block */
  sqlite3_int64 iEnd;             /* Block number of last allocated block */
  sqlite3_int64 nLeafData;        /* Bytes of leaf page data so far */
  u8 bNoLeafData;                 /* If true, store 0 for segment size */
  NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
};

/*
** An object of the following type is used to read data from a single
** FTS segment node. See the following functions:
**
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
  if( rc==SQLITE_OK ){
    rc = fts3WriteSegdir(p, 
        pWriter->iAbsLevel+1,               /* level */
        pWriter->iIdx,                      /* idx */
        pWriter->iStart,                    /* start_block */
        pWriter->aNodeWriter[0].iBlock,     /* leaves_end_block */
        pWriter->iEnd,                      /* end_block */
        pWriter->nLeafData,                 /* end_block */
        pRoot->block.a, pRoot->block.n      /* root */
    );
  }
  sqlite3_free(pRoot->block.a);
  sqlite3_free(pRoot->key.a);

  *pRc = rc;







|







4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
  if( rc==SQLITE_OK ){
    rc = fts3WriteSegdir(p, 
        pWriter->iAbsLevel+1,               /* level */
        pWriter->iIdx,                      /* idx */
        pWriter->iStart,                    /* start_block */
        pWriter->aNodeWriter[0].iBlock,     /* leaves_end_block */
        pWriter->iEnd,                      /* end_block */
        (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0),   /* end_block */
        pRoot->block.a, pRoot->block.n      /* root */
    );
  }
  sqlite3_free(pRoot->block.a);
  sqlite3_free(pRoot->key.a);

  *pRc = rc;
4176
4177
4178
4179
4180
4181
4182

4183
4184
4185
4186
4187
4188
4189
    if( sqlite3_step(pSelect)==SQLITE_ROW ){
      iStart = sqlite3_column_int64(pSelect, 1);
      iLeafEnd = sqlite3_column_int64(pSelect, 2);
      fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
      if( pWriter->nLeafData<0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }

      nRoot = sqlite3_column_bytes(pSelect, 4);
      aRoot = sqlite3_column_blob(pSelect, 4);
    }else{
      return sqlite3_reset(pSelect);
    }

    /* Check for the zero-length marker in the %_segments table */







>







4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
    if( sqlite3_step(pSelect)==SQLITE_ROW ){
      iStart = sqlite3_column_int64(pSelect, 1);
      iLeafEnd = sqlite3_column_int64(pSelect, 2);
      fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData);
      if( pWriter->nLeafData<0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }
      pWriter->bNoLeafData = (pWriter->nLeafData==0);
      nRoot = sqlite3_column_bytes(pSelect, 4);
      aRoot = sqlite3_column_blob(pSelect, 4);
    }else{
      return sqlite3_reset(pSelect);
    }

    /* Check for the zero-length marker in the %_segments table */
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
        }
      }

      if( nSeg!=0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }
      fts3IncrmergeRelease(p, pWriter, &rc);
      if( nSeg==0 ){
        fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
      }
    }

    sqlite3Fts3SegReaderFinish(pCsr);
  }








|







4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
        }
      }

      if( nSeg!=0 ){
        pWriter->nLeafData = pWriter->nLeafData * -1;
      }
      fts3IncrmergeRelease(p, pWriter, &rc);
      if( nSeg==0 && pWriter->bNoLeafData==0 ){
        fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData);
      }
    }

    sqlite3Fts3SegReaderFinish(pCsr);
  }

Changes to test/fts4growth.test.
362
363
364
365
366
367
368





369












370


371
372



373
374








375








376



377


378


379
380


381




382
383
384
385
386
do_execsql_test 6.5 {
  INSERT INTO x5 SELECT words FROM t1;
  SELECT level, idx, end_block FROM x5_segdir;
} {
  0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
}






#do_test 3.2 {












  #t1_to_x2


  #execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level}
#} {0 13 1 15 2 5}




#proc second {x} { lindex $x 1 }








#db func second second








#for {set i 0} {$i <1000} {incr i} {



#  t1_to_x2


#  db eval {


#    SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
#  } {


#    puts "$i.$level: $c"




#  }
#}

finish_test








>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
|
|
>
>
>
|
|
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
|
>
>
>
|
>
>
|
>
>
|
|
>
>
|
>
>
>
>
|
|



362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
do_execsql_test 6.5 {
  INSERT INTO x5 SELECT words FROM t1;
  SELECT level, idx, end_block FROM x5_segdir;
} {
  0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006}
}

#--------------------------------------------------------------------------
# Ensure that if part of an incremental merge is performed by an old
# version that does not support storing segment sizes in the end_block
# field, no size is stored in the final segment (as it would be incorrect).
#
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE x6 USING fts4;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  INSERT INTO x6 SELECT words FROM t1;
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
}

do_execsql_test 7.2 {
  INSERT INTO x6(x6) VALUES('merge=25,4');
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
  1 0 {16014 -51226}
}

do_execsql_test 7.3 {
  UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1;
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
  1 0 16014
}

do_execsql_test 7.4 {
  INSERT INTO x6(x6) VALUES('merge=25,4');
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
  0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
  1 0 16014
}

do_execsql_test 7.5 {
  INSERT INTO x6(x6) VALUES('merge=2500,4');
  SELECT level, idx, end_block FROM x6_segdir;
} {
  0 0 {598 118006} 0 1 {718 118006} 1 0 16014
}

do_execsql_test 7.6 {
  INSERT INTO x6(x6) VALUES('merge=2500,2');
  SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir;
} {
  2 0 23695 24147 {41262 633507}
}

do_execsql_test 7.7 {
  SELECT sum(length(block)) FROM x6_segments 
  WHERE blockid BETWEEN 23695 AND 24147
} {633507}



finish_test