/ Check-in [208e3cb6]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an fts5 bug in large incremental merges.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 208e3cb6b6dc8c7d824b64dec2034004c9fcbba5
User & Date: dan 2015-02-26 14:54:03
Context
2015-02-26
20:49
Optimize copying data from fts5 in-memory hash tables to top level segments. check-in: 8e3ca632 user: dan tags: fts5
14:54
Fix an fts5 bug in large incremental merges. check-in: 208e3cb6 user: dan tags: fts5
2015-02-25
19:24
Instead of the 4-byte fields, use regular varints for the poslist-size field in fts5_hash.c. check-in: 7eb022d7 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

1179
1180
1181
1182
1183
1184
1185

1186
1187
1188
1189
1190
1191
1192
....
1197
1198
1199
1200
1201
1202
1203
1204
1205

1206
1207
1208
1209
1210
1211
1212
1213
1214
1215

1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226

1227
1228
1229
1230
1231
1232
1233
....
3339
3340
3341
3342
3343
3344
3345

3346
3347
3348
3349
3350
3351
3352
....
3459
3460
3461
3462
3463
3464
3465

3466
3467
3468
3469
3470
3471
3472

    fts5DataWrite(p, FTS5_STRUCTURE_ROWID(iIdx), buf.p, buf.n);
    fts5BufferFree(&buf);
  }
}

#if 0

static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
  int rc = SQLITE_OK;
  Fts5Buffer buf;
  memset(&buf, 0, sizeof(buf));
  fts5DebugStructure(&rc, &buf, pStruct);
  fprintf(stdout, "%s: %s\n", zCaption, buf.p);
  fflush(stdout);
................................................................................
#endif

static int fts5SegmentSize(Fts5StructureSegment *pSeg){
  return 1 + pSeg->pgnoLast - pSeg->pgnoFirst;
}

/*
** Return a copy of index structure pStruct. Except, promote as many segments
** as possible to level iPromote. If an OOM occurs, NULL is returned.

*/
static void fts5StructurePromoteTo(
  Fts5Index *p,
  int iPromote,
  int szPromote,
  Fts5Structure *pStruct
){
  int il, is;
  Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote];


  for(il=iPromote+1; il<pStruct->nLevel; il++){
    Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
    if( pLvl->nMerge ) return;
    for(is=pLvl->nSeg-1; is>=0; is--){
      int sz = fts5SegmentSize(&pLvl->aSeg[is]);
      if( sz>szPromote ) return;
      fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1);
      if( p->rc ) return;
      memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment));
      pOut->nSeg++;
      pLvl->nSeg--;

    }
  }
}

/*
** A new segment has just been written to level iLvl of index structure
** pStruct. This function determines if any segments should be promoted
................................................................................
      fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
      assert( nRem==0 || p->rc==SQLITE_OK );
      if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
        fts5StructurePromote(p, iBestLvl+1, pStruct);
      }
      *ppStruct = pStruct;
    }

  }
}

static void fts5IndexCrisisMerge(
  Fts5Index *p,                   /* FTS5 backend object */
  int iIdx,                       /* Index to work on */
  Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
................................................................................
      pSeg->iSegid = iSegid;
      pSeg->nHeight = nHeight;
      pSeg->pgnoFirst = 1;
      pSeg->pgnoLast = pgnoLast;
    }
    fts5StructurePromote(p, 0, pStruct);
  }


  if( p->pConfig->nAutomerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast);
  fts5IndexCrisisMerge(p, iHash, &pStruct);
  fts5StructureWrite(p, iHash, pStruct);
  fts5StructureRelease(pStruct);
}








>







 







|
|
>










>
|
|
|
|
|
|
|
|
|
|
|
>







 







>







 







>







1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
....
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
....
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
....
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478

    fts5DataWrite(p, FTS5_STRUCTURE_ROWID(iIdx), buf.p, buf.n);
    fts5BufferFree(&buf);
  }
}

#if 0
static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*);
static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
  int rc = SQLITE_OK;
  Fts5Buffer buf;
  memset(&buf, 0, sizeof(buf));
  fts5DebugStructure(&rc, &buf, pStruct);
  fprintf(stdout, "%s: %s\n", zCaption, buf.p);
  fflush(stdout);
................................................................................
#endif

static int fts5SegmentSize(Fts5StructureSegment *pSeg){
  return 1 + pSeg->pgnoLast - pSeg->pgnoFirst;
}

/*
** Return a copy of index structure pStruct. Except, promote as many 
** segments as possible to level iPromote. If an OOM occurs, NULL is 
** returned.
*/
static void fts5StructurePromoteTo(
  Fts5Index *p,
  int iPromote,
  int szPromote,
  Fts5Structure *pStruct
){
  int il, is;
  Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote];

  if( pOut->nMerge==0 ){
    for(il=iPromote+1; il<pStruct->nLevel; il++){
      Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
      if( pLvl->nMerge ) return;
      for(is=pLvl->nSeg-1; is>=0; is--){
        int sz = fts5SegmentSize(&pLvl->aSeg[is]);
        if( sz>szPromote ) return;
        fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1);
        if( p->rc ) return;
        memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment));
        pOut->nSeg++;
        pLvl->nSeg--;
      }
    }
  }
}

/*
** A new segment has just been written to level iLvl of index structure
** pStruct. This function determines if any segments should be promoted
................................................................................
      fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
      assert( nRem==0 || p->rc==SQLITE_OK );
      if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
        fts5StructurePromote(p, iBestLvl+1, pStruct);
      }
      *ppStruct = pStruct;
    }

  }
}

static void fts5IndexCrisisMerge(
  Fts5Index *p,                   /* FTS5 backend object */
  int iIdx,                       /* Index to work on */
  Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
................................................................................
      pSeg->iSegid = iSegid;
      pSeg->nHeight = nHeight;
      pSeg->pgnoFirst = 1;
      pSeg->pgnoLast = pgnoLast;
    }
    fts5StructurePromote(p, 0, pStruct);
  }


  if( p->pConfig->nAutomerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast);
  fts5IndexCrisisMerge(p, iHash, &pStruct);
  fts5StructureWrite(p, iHash, pStruct);
  fts5StructureRelease(pStruct);
}