SQLite

Check-in [bff2168370]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix some bugs in the incremental merge code. Some remain.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts4-incr-merge
Files: files | file ages | folders
SHA1: bff21683705a61b8b8672e0b44c287d1dc7c32a9
User & Date: dan 2012-03-13 19:56:34.883
Context
2012-03-14
11:51
Fix another bug in the incremental merge code. (check-in: f97b12e095 user: dan tags: fts4-incr-merge)
2012-03-13
19:56
Fix some bugs in the incremental merge code. Some remain. (check-in: bff2168370 user: dan tags: fts4-incr-merge)
2012-03-09
12:52
Minor commenting and stylistic changes only. (check-in: a1747086c5 user: drh tags: fts4-incr-merge)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_write.c.
3716
3717
3718
3719
3720
3721
3722
3723




3724
3725

3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
      LayerWriter *pLayer;

      pWriter->nLeafEst = ((iEnd - iStart) + 1) / FTS_MAX_APPENDABLE_HEIGHT;
      pWriter->iStart = iStart;
      pWriter->iEnd = iEnd;
      pWriter->iAbsLevel = iAbsLevel;
      pWriter->iIdx = iIdx;





      pLayer = &pWriter->aLayer[nHeight];
      pLayer->iBlock = pWriter->iStart + nHeight*FTS_MAX_APPENDABLE_HEIGHT;

      blobGrowBuffer(&pLayer->block, MAX(nRoot, p->nNodeSize), &rc);
      if( rc==SQLITE_OK ){
        memcpy(pLayer->block.a, aRoot, nRoot);
        pLayer->block.n = nRoot;
      }

      for(i=(int)aRoot[0]; i>=0 && rc==SQLITE_OK; i--){
        pLayer = &pWriter->aLayer[i];
        NodeReader reader;

        rc = nodeReaderInit(&reader, pLayer->block.a, pLayer->block.n);
        while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
        blobGrowBuffer(&pLayer->key, reader.term.n, &rc);
        if( rc==SQLITE_OK ){








>
>
>
>

|
>






|







3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
      LayerWriter *pLayer;

      pWriter->nLeafEst = ((iEnd - iStart) + 1) / FTS_MAX_APPENDABLE_HEIGHT;
      pWriter->iStart = iStart;
      pWriter->iEnd = iEnd;
      pWriter->iAbsLevel = iAbsLevel;
      pWriter->iIdx = iIdx;

      for(i=nHeight+1; i<FTS_MAX_APPENDABLE_HEIGHT; i++){
        pWriter->aLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst;
      }

      pLayer = &pWriter->aLayer[nHeight];
      pLayer->iBlock = pWriter->iStart;
      pLayer->iBlock += pWriter->nLeafEst*FTS_MAX_APPENDABLE_HEIGHT;
      blobGrowBuffer(&pLayer->block, MAX(nRoot, p->nNodeSize), &rc);
      if( rc==SQLITE_OK ){
        memcpy(pLayer->block.a, aRoot, nRoot);
        pLayer->block.n = nRoot;
      }

      for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
        pLayer = &pWriter->aLayer[i];
        NodeReader reader;

        rc = nodeReaderInit(&reader, pLayer->block.a, pLayer->block.n);
        while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
        blobGrowBuffer(&pLayer->key, reader.term.n, &rc);
        if( rc==SQLITE_OK ){
3861
3862
3863
3864
3865
3866
3867




3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883

  /* Insert the marker in the %_segments table to make sure nobody tries
  ** to steal the space just allocated. This is also used to identify 
  ** appendable segments.  */
  if( rc==SQLITE_OK ){
    rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0);
  }





  /* Set up the array of LayerWriter objects */
  for(i=0; i<FTS_MAX_APPENDABLE_HEIGHT; i++){
    pWriter->aLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst;
  }

  pWriter->iAbsLevel = iAbsLevel;
  pWriter->nLeafEst = nLeafEst;
  pWriter->iIdx = iIdx;
  return SQLITE_OK;
}

/*
** Remove an entry from the %_segdir table. This involves running the 
** following two statements:
**







>
>
>
>





<
<
<
<







3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881




3882
3883
3884
3885
3886
3887
3888

  /* Insert the marker in the %_segments table to make sure nobody tries
  ** to steal the space just allocated. This is also used to identify 
  ** appendable segments.  */
  if( rc==SQLITE_OK ){
    rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0);
  }

  pWriter->iAbsLevel = iAbsLevel;
  pWriter->nLeafEst = nLeafEst;
  pWriter->iIdx = iIdx;

  /* Set up the array of LayerWriter objects */
  for(i=0; i<FTS_MAX_APPENDABLE_HEIGHT; i++){
    pWriter->aLayer[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst;
  }




  return SQLITE_OK;
}

/*
** Remove an entry from the %_segdir table. This involves running the 
** following two statements:
**
3958
3959
3960
3961
3962
3963
3964








3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976

3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989

3990
3991
3992
3993
3994
3995
3996
    pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild);
  }else{
    assert( pNode->nAlloc>=1 );
    pNode->n = 1;
  }
}









static int fts3TruncateNode(
  const char *aNode,              /* Current node image */
  int nNode,                      /* Size of aNode in bytes */
  Blob *pNew,                     /* OUT: Write new node image here */
  const char *zTerm,              /* Omit all terms smaller than this */
  int nTerm,                      /* Size of zTerm in bytes */
  sqlite3_int64 *piBlock          /* OUT: Block number in next layer down */
){
  NodeReader reader;              /* Reader object */
  Blob prev = {0, 0, 0};
  int rc = SQLITE_OK;
  int bStarted = 0;


  /* Allocate required output space */
  blobGrowBuffer(pNew, nNode, &rc);
  if( rc!=SQLITE_OK ) return rc;
  pNew->n = 0;

  /* Populate new node buffer */
  for(rc = nodeReaderInit(&reader, aNode, nNode); 
      rc==SQLITE_OK && reader.aNode; 
      rc = nodeReaderNext(&reader)
  ){
    if( bStarted==0 ){
      if( fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm)<0 ) continue;

      pNew->a[0] = aNode[0];
      fts3StartNode(pNew, (int)aNode[0], reader.iChild);
      *piBlock = reader.iChild;
      bStarted = 1;
    }
    rc = fts3AppendToNode(
        pNew, &prev, reader.term.a, reader.term.n,







>
>
>
>
>
>
>
>












>












|
>







3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
    pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild);
  }else{
    assert( pNode->nAlloc>=1 );
    pNode->n = 1;
  }
}

/*
** The first two arguments are a pointer to and the size of a segment b-tree
** node. The node may be a leaf or an internal node.
**
** This function creates a new node image in blob object *pNew by copying
** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes)
** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode.
*/
static int fts3TruncateNode(
  const char *aNode,              /* Current node image */
  int nNode,                      /* Size of aNode in bytes */
  Blob *pNew,                     /* OUT: Write new node image here */
  const char *zTerm,              /* Omit all terms smaller than this */
  int nTerm,                      /* Size of zTerm in bytes */
  sqlite3_int64 *piBlock          /* OUT: Block number in next layer down */
){
  NodeReader reader;              /* Reader object */
  Blob prev = {0, 0, 0};
  int rc = SQLITE_OK;
  int bStarted = 0;
  int bLeaf = aNode[0]=='\0';     /* True for a leaf node */

  /* Allocate required output space */
  blobGrowBuffer(pNew, nNode, &rc);
  if( rc!=SQLITE_OK ) return rc;
  pNew->n = 0;

  /* Populate new node buffer */
  for(rc = nodeReaderInit(&reader, aNode, nNode); 
      rc==SQLITE_OK && reader.aNode; 
      rc = nodeReaderNext(&reader)
  ){
    if( bStarted==0 ){
      int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm);
      if( res<0 || (bLeaf==0 && res==0) ) continue;
      pNew->a[0] = aNode[0];
      fts3StartNode(pNew, (int)aNode[0], reader.iChild);
      *piBlock = reader.iChild;
      bStarted = 1;
    }
    rc = fts3AppendToNode(
        pNew, &prev, reader.term.a, reader.term.n,
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
    memset(&filter, 0, sizeof(Fts3SegFilter));
    filter.flags = FTS3_SEGMENT_REQUIRE_POS;

    rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
    assert( rc!=SQLITE_ABORT );
    if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){
      rc = fts3IncrmergeWriter(p, iAbsLevel, csr.zTerm, csr.nTerm, &writer);
      assert( rc!=SQLITE_ABORT );
      if( rc==SQLITE_OK ){
        do {
          rc = fts3IncrmergeAppend(p, &writer, &csr);
          assert( rc!=SQLITE_ABORT );
          if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, &csr);
          assert( rc!=SQLITE_ABORT );
          if( writer.nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
        }while( rc==SQLITE_ROW );
      }
    }
    assert( rc!=SQLITE_ABORT );
    fts3IncrmergeRelease(p, &writer, &rc);
    nRem -= (1 + writer.nWork);







<



<

<







4191
4192
4193
4194
4195
4196
4197

4198
4199
4200

4201

4202
4203
4204
4205
4206
4207
4208
    memset(&filter, 0, sizeof(Fts3SegFilter));
    filter.flags = FTS3_SEGMENT_REQUIRE_POS;

    rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
    assert( rc!=SQLITE_ABORT );
    if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){
      rc = fts3IncrmergeWriter(p, iAbsLevel, csr.zTerm, csr.nTerm, &writer);

      if( rc==SQLITE_OK ){
        do {
          rc = fts3IncrmergeAppend(p, &writer, &csr);

          if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, &csr);

          if( writer.nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
        }while( rc==SQLITE_ROW );
      }
    }
    assert( rc!=SQLITE_ABORT );
    fts3IncrmergeRelease(p, &writer, &rc);
    nRem -= (1 + writer.nWork);