/ Check-in [ab0a4f44]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Following an incr-merge operation that does not completely consume its input segments, store context in the rowid==1 row of the %_stat table that allows the next incr-merge to pick up where the previous left off.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-incr-merge
Files: files | file ages | folders
SHA1:ab0a4f44fb67e9f0cb82297b80e728ca58cdb0fb
User & Date: dan 2012-03-22 16:48:12
Context
2012-03-22
17:48
Add test cases to fts4merge.test. check-in: ecab2083 user: dan tags: fts4-incr-merge
16:48
Following an incr-merge operation that does not completely consume its input segments, store context in the rowid==1 row of the %_stat table that allows the next incr-merge to pick up where the previous left off. check-in: ab0a4f44 user: dan tags: fts4-incr-merge
2012-03-21
14:34
Add fts4merge3.test, for testing that older versions of FTS4 may interoperate with incr-merge capable versions. check-in: 903ec512 user: dan tags: fts4-incr-merge
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_write.c.

62
63
64
65
66
67
68






69
70
71
72
73
74
75
...
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
...
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
...
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
...
420
421
422
423
424
425
426
427














428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
....
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102

3103
3104
3105
3106
3107
3108
3109
....
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131

3132
3133
3134
3135
3136
3137
3138
3139
....
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286

3287
3288
3289
3290
3291
3292
3293
3294
3295

3296
3297
3298
3299
3300
3301
3302
....
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977


























3978
3979
3980
3981
3982
3983
3984
....
3997
3998
3999
4000
4001
4002
4003

4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
....
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204

4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216

4217
4218
4219
4220
4221
4222
4223
....
4269
4270
4271
4272
4273
4274
4275
4276

4277
4278

4279
4280
4281
4282
4283
4284
4285
....
4292
4293
4294
4295
4296
4297
4298

4299
4300
4301
4302
4303
4304
4305

4306
4307


































































4308
4309
4310
4311
4312
4313
4314
....
4315
4316
4317
4318
4319
4320
4321
4322
4323





4324
4325
4326



4327
4328
4329
4330
4331
4332
4333
4334






4335
4336
4337
4338
4339
4340
4341
4342
4343



4344
4345

4346

4347
4348
4349
4350
4351
4352
4353
4354
4355
4356




4357
4358
4359
4360

4361
4362




4363
4364





4365



4366

4367
4368
4369
4370
4371
4372




4373
4374
4375
4376
4377
4378
4379
4380

4381
4382
4383
4384






4385
4386
4387
4388
4389
4390
4391
# define FTS3_NODE_CHUNKSIZE       test_fts3_node_chunksize
# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
#else
# define FTS3_NODE_CHUNKSIZE (4*1024) 
# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
#endif








/*
** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic
** and incremental merge operation that takes place. This is used for 
** debugging FTS only, it should not usually be turned on in production
** systems.
*/
................................................................................
#define SQL_SELECT_SEGDIR_MAX_LEVEL   15
#define SQL_DELETE_SEGDIR_LEVEL       16
#define SQL_DELETE_SEGMENTS_RANGE     17
#define SQL_CONTENT_INSERT            18
#define SQL_DELETE_DOCSIZE            19
#define SQL_REPLACE_DOCSIZE           20
#define SQL_SELECT_DOCSIZE            21
#define SQL_SELECT_DOCTOTAL           22
#define SQL_REPLACE_DOCTOTAL          23

#define SQL_SELECT_ALL_PREFIX_LEVEL   24
#define SQL_DELETE_ALL_TERMS_SEGDIR   25
#define SQL_DELETE_SEGDIR_RANGE       26
#define SQL_SELECT_ALL_LANGID         27
#define SQL_FIND_MERGE_LEVEL          28
#define SQL_MAX_LEAF_NODE_ESTIMATE    29
................................................................................

/* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
/* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
/* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
/* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=0",
/* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
/* 24 */  "",
/* 25 */  "",

/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",

/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
** of the oldest level in the db that contains at least ? segments. Or,
** if no level in the FTS index contains more than ? segments, the statement
** returns zero rows.  */
/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>?"
         "  ORDER BY (level %% 1024) DESC LIMIT 1",

/* Estimate the upper limit on the number of leaf nodes in a new segment
** created by merging the oldest :2 segments from absolute level :1. See 
** function fts3Incrmerge() for details.  */
/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
         "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
................................................................................
  *pp = pStmt;
  return rc;
}


static int fts3SelectDocsize(
  Fts3Table *pTab,                /* FTS3 table handle */
  int eStmt,                      /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
  sqlite3_int64 iDocid,           /* Docid to bind for SQL_SELECT_DOCSIZE */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  sqlite3_stmt *pStmt = 0;        /* Statement requested from fts3SqlStmt() */
  int rc;                         /* Return code */

  assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );

  rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
  if( rc==SQLITE_OK ){
    if( eStmt==SQL_SELECT_DOCSIZE ){
      sqlite3_bind_int64(pStmt, 1, iDocid);
    }
    rc = sqlite3_step(pStmt);
    if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
      rc = sqlite3_reset(pStmt);
      if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
      pStmt = 0;
    }else{
      rc = SQLITE_OK;
................................................................................
  return rc;
}

int sqlite3Fts3SelectDoctotal(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);














}

int sqlite3Fts3SelectDocsize(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_int64 iDocid,           /* Docid to read size data for */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
}

/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
** is executed.
**
................................................................................
  if( *pRC ) return;
  a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
  if( a==0 ){
    *pRC = SQLITE_NOMEM;
    return;
  }
  pBlob = (char*)&a[nStat];
  rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
  if( rc ){
    sqlite3_free(a);
    *pRC = rc;
    return;
  }

  if( sqlite3_step(pStmt)==SQLITE_ROW ){
    fts3DecodeIntArray(nStat, a,
         sqlite3_column_blob(pStmt, 0),
         sqlite3_column_bytes(pStmt, 0));
  }else{
    memset(a, 0, sizeof(u32)*(nStat) );
  }
................................................................................
      x = 0;
    }else{
      x = x + aSzIns[i] - aSzDel[i];
    }
    a[i+1] = x;
  }
  fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
  rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
  if( rc ){
    sqlite3_free(a);
    *pRC = rc;
    return;
  }

  sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
  sqlite3_step(pStmt);
  *pRC = sqlite3_reset(pStmt);
  sqlite3_free(a);
}

/*
** Merge the entire database so that there is one segment for each 
................................................................................
  nByte = sizeof(Fts3SegReader *) * nSeg;
  pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);

  if( pCsr->apSegment==0 ){
    rc = SQLITE_NOMEM;
  }else{
    memset(pCsr->apSegment, 0, nByte);
    pCsr->nSegment = nSeg;
    rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
  }
  if( rc==SQLITE_OK ){
    int i;
    int rc2;
    sqlite3_bind_int64(pStmt, 1, iAbsLevel);

    for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<nSeg; i++){
      rc = sqlite3Fts3SegReaderNew(i, 0,
          sqlite3_column_int64(pStmt, 1),        /* segdir.start_block */
          sqlite3_column_int64(pStmt, 2),        /* segdir.leaves_end_block */
          sqlite3_column_int64(pStmt, 3),        /* segdir.end_block */
          sqlite3_column_blob(pStmt, 4),         /* segdir.root */
          sqlite3_column_bytes(pStmt, 4),        /* segdir.root */
          &pCsr->apSegment[i]
      );

    }
    rc2 = sqlite3_reset(pStmt);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}
................................................................................

    rc2 = sqlite3_reset(pSelect);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}

/* 
** Either allocate an output segment or locate an existing appendable 
** output segment to append to. An "appendable" output segment is
** slightly different to a normal one, as the required range of keys in
** the %_segments table must be allocated up front.


























**
** In the %_segdir table, a segment is defined by the values in three
** columns:
**
**     start_block
**     leaves_end_block
**     end_block
................................................................................
**
** In the actual code below, the value "16" is replaced with the 
** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT.
*/
static int fts3IncrmergeWriter( 
  Fts3Table *p,                   /* Fts3 table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level of input segments */

  Fts3MultiSegReader *pCsr,       /* Cursor that data will be read from */
  IncrmergeWriter *pWriter        /* Populate this object */
){
  int rc;                         /* Return Code */
  int i;                          /* Iterator variable */
  int nLeafEst;                   /* Blocks allocated for leaf nodes */
  int iIdx;                       /* Index of output segment */
  sqlite3_stmt *pLeafEst = 0;     /* SQL used to determine nLeafEst */
  sqlite3_stmt *pFirstBlock = 0;  /* SQL used to determine first block */
  sqlite3_stmt *pOutputIdx = 0;   /* SQL used to find output index */
  const char *zKey = pCsr->zTerm; /* First key to be appended to output */
  int nKey = pCsr->nTerm;         /* Size of zKey in bytes */

  rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1);
    sqlite3_step(pOutputIdx);
    iIdx = sqlite3_column_int(pOutputIdx, 0) - 1;
    rc = sqlite3_reset(pOutputIdx);
  }
  if( rc!=SQLITE_OK ) return rc;

  assert( zKey );
  assert( pWriter->nLeafEst==0 );
  rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx, zKey, nKey, pWriter);
  if( rc!=SQLITE_OK || pWriter->nLeafEst ) return rc;
  iIdx++;

  /* Calculate nLeafEst. */
  rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pLeafEst, 1, iAbsLevel);
    sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment);
    if( SQLITE_ROW==sqlite3_step(pLeafEst) ){
................................................................................
** SQLite error code otherwise.
*/
static int fts3TruncateSegment(
  Fts3Table *p,                   /* FTS3 table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level of segment to modify */
  int iIdx,                       /* Index within level of segment to modify */
  const char *zTerm,              /* Remove terms smaller than this */
  int nTerm                       /* Number of bytes in buffer zTerm */
){
  int rc = SQLITE_OK;             /* Return code */
  Blob root = {0,0,0};            /* New root page image */
  Blob block = {0,0,0};           /* Buffer used for any other block */
  sqlite3_int64 iBlock = 0;       /* Block id */
  sqlite3_int64 iNewStart = 0;    /* New value for iStartBlock */
  sqlite3_int64 iOldStart = 0;    /* Old value for iStartBlock */
  int rc2;                        /* sqlite3_reset() return code */
  sqlite3_stmt *pFetch = 0;       /* Statement used to fetch segdir */

  assert( p->aStmt[SQL_SELECT_SEGDIR] );

  pFetch = p->aStmt[SQL_SELECT_SEGDIR];

  sqlite3_bind_int64(pFetch, 1, iAbsLevel);
  sqlite3_bind_int(pFetch, 2, iIdx);
  if( SQLITE_ROW==sqlite3_step(pFetch) ){
    const char *aRoot = sqlite3_column_blob(pFetch, 4);
    int nRoot = sqlite3_column_bytes(pFetch, 4);
    iOldStart = sqlite3_column_int64(pFetch, 1);
    rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock);
  }
  rc2 = sqlite3_reset(pFetch);
  if( rc==SQLITE_OK ) rc = rc2;


  while( rc==SQLITE_OK && iBlock ){
    char *aBlock = 0;
    int nBlock = 0;
    iNewStart = iBlock;

    rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0);
................................................................................
** its data was copied to the output segment by the incrmerge operation)
** or modified in place so that it no longer contains those entries that
** have been duplicated in the output segment.
*/
static int fts3IncrmergeChomp(
  Fts3Table *p,                   /* FTS table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level containing segments */
  Fts3MultiSegReader *pCsr        /* Chomp all segments opened by this cursor */

){
  int i;

  int rc = SQLITE_OK;

  for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){
    Fts3SegReader *pSeg = 0;
    int j;

    /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding
................................................................................

    if( pSeg->aNode==0 ){
      /* Seg-reader is at EOF. Remove the entire input segment. */
      rc = fts3DeleteSegment(p, pSeg);
      if( rc==SQLITE_OK ){
        rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx);
      }

    }else{
      /* The incremental merge did not copy all the data from this 
      ** segment to the upper level. The segment is modified in place
      ** so that it contains no keys smaller than zTerm/nTerm. */ 
      const char *zTerm = pSeg->zTerm;
      int nTerm = pSeg->nTerm;
      rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm);

    }
  }



































































  return rc;
}

/*
** Attempt an incremental merge that writes nMerge leaf blocks.
**
................................................................................
** Incremental merges happen nMin segments at a time. The two
** segments to be merged are the nMin oldest segments (the ones with
** the smallest indexes) in the highest level that contains at least
** nMin segments. Multiple merges might occur in an attempt to write the 
** quota of nMerge leaf blocks.
*/
static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
  int rc = SQLITE_OK;             /* Return code */
  int nRem = nMerge;              /* Number of leaf pages yet to  be written */






  assert( nMin>=2 );




  while( rc==SQLITE_OK && nRem>0 ){
    sqlite3_int64 iAbsLevel;        /* Absolute level number to work on */
    sqlite3_stmt *pFindLevel = 0;   /* SQL used to determine iAbsLevel */
    Fts3MultiSegReader *pCsr;       /* Cursor used to read input data */
    Fts3SegFilter *pFilter;         /* Filter used with cursor pCsr */
    IncrmergeWriter *pWriter;       /* Writer object */
    const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);







    /* Determine which level to merge segments from. Any level, from any
    ** prefix or language index may be selected. Stack variable iAbsLevel 
    ** is set to the absolute level number of the level to merge from.  */
    rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
    sqlite3_bind_int(pFindLevel, 1, nMin);
    if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){
      /* There are no levels with nMin or more segments. Or an error has
      ** occurred. Either way, exit early.  */
      return sqlite3_reset(pFindLevel);



    }
    iAbsLevel = sqlite3_column_int64(pFindLevel, 0);

    sqlite3_reset(pFindLevel);


    /* Allocate space for the cursor, filter and writer objects */
    pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
    if( !pWriter ) return SQLITE_NOMEM;
    memset(pWriter, 0, nAlloc);
    pFilter = (Fts3SegFilter *)&pWriter[1];
    pCsr = (Fts3MultiSegReader *)&pFilter[1];

    /* Open a cursor to iterate through the contents of indexes 0 and 1 of
    ** the selected absolute level. */




    pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
    rc = fts3IncrmergeCsr(p, iAbsLevel, nMin, pCsr);
    fts3LogMerge(nMin, iAbsLevel);
    if( rc==SQLITE_OK ){

      rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter);
    }




    if( rc==SQLITE_OK ){
      if( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){





        rc = fts3IncrmergeWriter(p, iAbsLevel, pCsr, pWriter);



        if( rc==SQLITE_OK ){

          do {
            rc = fts3IncrmergeAppend(p, pWriter, pCsr);
            if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
            if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
          }while( rc==SQLITE_ROW );
        }




      }
    }
    fts3IncrmergeRelease(p, pWriter, &rc);
    nRem -= (1 + pWriter->nWork);

    /* Update or delete the input segments */
    if( rc==SQLITE_OK ){
      rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr);

    }

    sqlite3Fts3SegReaderFinish(pCsr);
    sqlite3_free(pWriter);






  }

  return rc;
}

/*
** Process statements of the form:







>
>
>
>
>
>







 







|
|







 







|
|











|







 







<






<
<
|

<
|
<







 







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>







|







 







|





>







 







|





>
|







 







<






>









>







 








|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>






<


<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







|







<


|
>
|
<
|
|
|
|
|
|
|
|
|
|
>







 







|
>


>







 







>







>


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|

>
>
>
>
>



>
>
>

<
<





>
>
>
>
>
>
|
|
|
|
|
|
|
|
|
>
>
>
|
|
>
|
>








|
|
>
>
>
>

|
<
<
>
|
<
>
>
>
>
|
<
>
>
>
>
>
|
>
>
>
|
>
|
|
|
|
|
|
>
>
>
>
|
|
<
<

<
<
<
>




>
>
>
>
>
>







62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
...
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
...
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
...
394
395
396
397
398
399
400

401
402
403
404
405
406


407
408

409

410
411
412
413
414
415
416
...
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
....
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
....
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
....
3290
3291
3292
3293
3294
3295
3296

3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
....
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
....
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054

4055
4056


















4057
4058
4059
4060
4061
4062
4063
....
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226

4227
4228
4229
4230
4231

4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
....
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
....
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
....
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431


4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475


4476
4477

4478
4479
4480
4481
4482

4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505


4506



4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
# define FTS3_NODE_CHUNKSIZE       test_fts3_node_chunksize
# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
#else
# define FTS3_NODE_CHUNKSIZE (4*1024) 
# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
#endif

/*
** The two values that may be meaningfully bound to the :1 parameter in
** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
*/
#define FTS_STAT_DOCTOTAL      0
#define FTS_STAT_INCRMERGEHINT 1

/*
** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic
** and incremental merge operation that takes place. This is used for 
** debugging FTS only, it should not usually be turned on in production
** systems.
*/
................................................................................
#define SQL_SELECT_SEGDIR_MAX_LEVEL   15
#define SQL_DELETE_SEGDIR_LEVEL       16
#define SQL_DELETE_SEGMENTS_RANGE     17
#define SQL_CONTENT_INSERT            18
#define SQL_DELETE_DOCSIZE            19
#define SQL_REPLACE_DOCSIZE           20
#define SQL_SELECT_DOCSIZE            21
#define SQL_SELECT_STAT               22
#define SQL_REPLACE_STAT              23

#define SQL_SELECT_ALL_PREFIX_LEVEL   24
#define SQL_DELETE_ALL_TERMS_SEGDIR   25
#define SQL_DELETE_SEGDIR_RANGE       26
#define SQL_SELECT_ALL_LANGID         27
#define SQL_FIND_MERGE_LEVEL          28
#define SQL_MAX_LEAF_NODE_ESTIMATE    29
................................................................................

/* 16 */  "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
/* 17 */  "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
/* 18 */  "INSERT INTO %Q.'%q_content' VALUES(%s)",
/* 19 */  "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
/* 20 */  "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
/* 21 */  "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
/* 22 */  "SELECT value FROM %Q.'%q_stat' WHERE id=?",
/* 23 */  "REPLACE INTO %Q.'%q_stat' VALUES(?,?)",
/* 24 */  "",
/* 25 */  "",

/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",

/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
** of the oldest level in the db that contains at least ? segments. Or,
** if no level in the FTS index contains more than ? segments, the statement
** returns zero rows.  */
/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
         "  ORDER BY (level %% 1024) DESC LIMIT 1",

/* Estimate the upper limit on the number of leaf nodes in a new segment
** created by merging the oldest :2 segments from absolute level :1. See 
** function fts3Incrmerge() for details.  */
/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
         "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
................................................................................
  *pp = pStmt;
  return rc;
}


static int fts3SelectDocsize(
  Fts3Table *pTab,                /* FTS3 table handle */

  sqlite3_int64 iDocid,           /* Docid to bind for SQL_SELECT_DOCSIZE */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  sqlite3_stmt *pStmt = 0;        /* Statement requested from fts3SqlStmt() */
  int rc;                         /* Return code */



  rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0);
  if( rc==SQLITE_OK ){

    sqlite3_bind_int64(pStmt, 1, iDocid);

    rc = sqlite3_step(pStmt);
    if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
      rc = sqlite3_reset(pStmt);
      if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
      pStmt = 0;
    }else{
      rc = SQLITE_OK;
................................................................................
  return rc;
}

int sqlite3Fts3SelectDoctotal(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  sqlite3_stmt *pStmt = 0;
  int rc;
  rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
    if( sqlite3_step(pStmt)!=SQLITE_ROW
     || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB
    ){
      rc = sqlite3_reset(pStmt);
      if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
      pStmt = 0;
    }
  }
  *ppStmt = pStmt;
  return rc;
}

int sqlite3Fts3SelectDocsize(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_int64 iDocid,           /* Docid to read size data for */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  return fts3SelectDocsize(pTab, iDocid, ppStmt);
}

/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
** is executed.
**
................................................................................
  if( *pRC ) return;
  a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
  if( a==0 ){
    *pRC = SQLITE_NOMEM;
    return;
  }
  pBlob = (char*)&a[nStat];
  rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
  if( rc ){
    sqlite3_free(a);
    *pRC = rc;
    return;
  }
  sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
  if( sqlite3_step(pStmt)==SQLITE_ROW ){
    fts3DecodeIntArray(nStat, a,
         sqlite3_column_blob(pStmt, 0),
         sqlite3_column_bytes(pStmt, 0));
  }else{
    memset(a, 0, sizeof(u32)*(nStat) );
  }
................................................................................
      x = 0;
    }else{
      x = x + aSzIns[i] - aSzDel[i];
    }
    a[i+1] = x;
  }
  fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
  if( rc ){
    sqlite3_free(a);
    *pRC = rc;
    return;
  }
  sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
  sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
  sqlite3_step(pStmt);
  *pRC = sqlite3_reset(pStmt);
  sqlite3_free(a);
}

/*
** Merge the entire database so that there is one segment for each 
................................................................................
  nByte = sizeof(Fts3SegReader *) * nSeg;
  pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);

  if( pCsr->apSegment==0 ){
    rc = SQLITE_NOMEM;
  }else{
    memset(pCsr->apSegment, 0, nByte);

    rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
  }
  if( rc==SQLITE_OK ){
    int i;
    int rc2;
    sqlite3_bind_int64(pStmt, 1, iAbsLevel);
    assert( pCsr->nSegment==0 );
    for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<nSeg; i++){
      rc = sqlite3Fts3SegReaderNew(i, 0,
          sqlite3_column_int64(pStmt, 1),        /* segdir.start_block */
          sqlite3_column_int64(pStmt, 2),        /* segdir.leaves_end_block */
          sqlite3_column_int64(pStmt, 3),        /* segdir.end_block */
          sqlite3_column_blob(pStmt, 4),         /* segdir.root */
          sqlite3_column_bytes(pStmt, 4),        /* segdir.root */
          &pCsr->apSegment[i]
      );
      pCsr->nSegment++;
    }
    rc2 = sqlite3_reset(pStmt);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}
................................................................................

    rc2 = sqlite3_reset(pSelect);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}

/*
** Determine the largest segment index value that exists within absolute
** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus
** one before returning SQLITE_OK. Or, if there are no segments at all 
** within level iAbsLevel, set *piIdx to zero.
**
** If an error occurs, return an SQLite error code. The final value of
** *piIdx is undefined in this case.
*/
static int fts3IncrmergeOutputIdx( 
  Fts3Table *p,                   /* FTS Table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute index of input segments */
  int *piIdx                      /* OUT: Next free index at iAbsLevel+1 */
){
  int rc;
  sqlite3_stmt *pOutputIdx = 0;   /* SQL used to find output index */

  rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1);
    sqlite3_step(pOutputIdx);
    *piIdx = sqlite3_column_int(pOutputIdx, 0);
    rc = sqlite3_reset(pOutputIdx);
  }

  return rc;
}

/* 
** Allocate an appendable output segment on absolute level iAbsLevel+1
** with idx value iIdx.
**
** In the %_segdir table, a segment is defined by the values in three
** columns:
**
**     start_block
**     leaves_end_block
**     end_block
................................................................................
**
** In the actual code below, the value "16" is replaced with the 
** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT.
*/
static int fts3IncrmergeWriter( 
  Fts3Table *p,                   /* Fts3 table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level of input segments */
  int iIdx,                       /* Index of new output segment */
  Fts3MultiSegReader *pCsr,       /* Cursor that data will be read from */
  IncrmergeWriter *pWriter        /* Populate this object */
){
  int rc;                         /* Return Code */
  int i;                          /* Iterator variable */
  int nLeafEst;                   /* Blocks allocated for leaf nodes */

  sqlite3_stmt *pLeafEst = 0;     /* SQL used to determine nLeafEst */
  sqlite3_stmt *pFirstBlock = 0;  /* SQL used to determine first block */



















  /* Calculate nLeafEst. */
  rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pLeafEst, 1, iAbsLevel);
    sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment);
    if( SQLITE_ROW==sqlite3_step(pLeafEst) ){
................................................................................
** SQLite error code otherwise.
*/
static int fts3TruncateSegment(
  Fts3Table *p,                   /* FTS3 table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level of segment to modify */
  int iIdx,                       /* Index within level of segment to modify */
  const char *zTerm,              /* Remove terms smaller than this */
  int nTerm                      /* Number of bytes in buffer zTerm */
){
  int rc = SQLITE_OK;             /* Return code */
  Blob root = {0,0,0};            /* New root page image */
  Blob block = {0,0,0};           /* Buffer used for any other block */
  sqlite3_int64 iBlock = 0;       /* Block id */
  sqlite3_int64 iNewStart = 0;    /* New value for iStartBlock */
  sqlite3_int64 iOldStart = 0;    /* Old value for iStartBlock */

  sqlite3_stmt *pFetch = 0;       /* Statement used to fetch segdir */

  rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0);
  if( rc==SQLITE_OK ){
    int rc2;                      /* sqlite3_reset() return code */

    sqlite3_bind_int64(pFetch, 1, iAbsLevel);
    sqlite3_bind_int(pFetch, 2, iIdx);
    if( SQLITE_ROW==sqlite3_step(pFetch) ){
      const char *aRoot = sqlite3_column_blob(pFetch, 4);
      int nRoot = sqlite3_column_bytes(pFetch, 4);
      iOldStart = sqlite3_column_int64(pFetch, 1);
      rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock);
    }
    rc2 = sqlite3_reset(pFetch);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  while( rc==SQLITE_OK && iBlock ){
    char *aBlock = 0;
    int nBlock = 0;
    iNewStart = iBlock;

    rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0);
................................................................................
** its data was copied to the output segment by the incrmerge operation)
** or modified in place so that it no longer contains those entries that
** have been duplicated in the output segment.
*/
static int fts3IncrmergeChomp(
  Fts3Table *p,                   /* FTS table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level containing segments */
  Fts3MultiSegReader *pCsr,       /* Chomp all segments opened by this cursor */
  int *pnRem                      /* Number of segments not deleted */
){
  int i;
  int nRem = 0;
  int rc = SQLITE_OK;

  for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){
    Fts3SegReader *pSeg = 0;
    int j;

    /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding
................................................................................

    if( pSeg->aNode==0 ){
      /* Seg-reader is at EOF. Remove the entire input segment. */
      rc = fts3DeleteSegment(p, pSeg);
      if( rc==SQLITE_OK ){
        rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx);
      }
      *pnRem = 0;
    }else{
      /* The incremental merge did not copy all the data from this 
      ** segment to the upper level. The segment is modified in place
      ** so that it contains no keys smaller than zTerm/nTerm. */ 
      const char *zTerm = pSeg->zTerm;
      int nTerm = pSeg->nTerm;
      rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm);
      nRem++;
    }
  }

  *pnRem = nRem;
  return rc;
}

/*
** Store an incr-merge hint in the database.
*/
static int fts3IncrmergeHintStore(
  Fts3Table *p,                   /* FTS3 table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level to read input data from */
  int nMerge                      /* Number of segments to merge */
){
  char aBlob[FTS3_VARINT_MAX * 2];
  int nBlob = 0;
  int rc;
  sqlite3_stmt *pReplace = 0;

  assert( p->bHasStat );
  nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], iAbsLevel);
  nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], nMerge);

  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
    sqlite3_bind_blob(pReplace, 2, aBlob, nBlob, SQLITE_TRANSIENT);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }

  return rc;
}

/*
** Load an incr-merge hint from the database.
**
** The incr-merge hint, if one exists, is stored in the rowid==1 row of
** the %_stat table.
*/
static int fts3IncrmergeHintLoad(
  Fts3Table *p,                   /* FTS3 table handle */
  sqlite3_int64 *piAbsLevel,      /* Absolute level to read input data from */
  int *pnMerge                    /* Number of segments to merge */
){
  sqlite3_stmt *pSelect = 0;
  int rc;

  *pnMerge = 0;
  *piAbsLevel = 0;

  rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT);
    if( SQLITE_ROW==sqlite3_step(pSelect) ){
      char *aHint = sqlite3_column_blob(pSelect, 0);
      int nHint = sqlite3_column_bytes(pSelect, 0);
      if( aHint ){
        int i;
        char aBlob[FTS3_VARINT_MAX * 2];
        memcpy(aBlob, aHint, MAX(sizeof(aBlob), nHint));
        i = sqlite3Fts3GetVarint(aBlob, piAbsLevel);
        sqlite3Fts3GetVarint32(&aBlob[i], pnMerge);
      }
    }
    rc = sqlite3_reset(pSelect);
  }

  return rc;
}

/*
** Attempt an incremental merge that writes nMerge leaf blocks.
**
................................................................................
** Incremental merges happen nMin segments at a time. The two
** segments to be merged are the nMin oldest segments (the ones with
** the smallest indexes) in the highest level that contains at least
** nMin segments. Multiple merges might occur in an attempt to write the 
** quota of nMerge leaf blocks.
*/
static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
  int rc;                         /* Return code */
  int nRem = nMerge;              /* Number of leaf pages yet to  be written */
  int bUseHint = 1;               /* True if hint has not yet been attempted */
  sqlite3_int64 iHintAbsLevel = 0;/* Hint level */
  int nHintSeg = 0;               /* Hint number of segments */
  int nSeg = 0;                   /* Number of input segments */
  sqlite3_int64 iAbsLevel = 0;    /* Absolute level number to work on */

  assert( nMin>=2 );

  rc = fts3IncrmergeHintLoad(p, &iHintAbsLevel, &nHintSeg);
  if( nHintSeg==0 ) bUseHint = 0;

  while( rc==SQLITE_OK && nRem>0 ){


    Fts3MultiSegReader *pCsr;       /* Cursor used to read input data */
    Fts3SegFilter *pFilter;         /* Filter used with cursor pCsr */
    IncrmergeWriter *pWriter;       /* Writer object */
    const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);

    if( bUseHint ){
      iAbsLevel = iHintAbsLevel;
      nSeg = nHintSeg;
    }else{
      sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */

      /* Determine which level to merge segments from. Any level, from any
      ** prefix or language index may be selected. Stack variable iAbsLevel 
      ** is set to the absolute level number of the level to merge from.  */
      rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
      sqlite3_bind_int(pFindLevel, 1, nMin);
      if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){
        /* There are no levels with nMin or more segments. Or an error has
         ** occurred. Either way, exit early.  */
        rc = sqlite3_reset(pFindLevel);
        iAbsLevel = 0;
        nSeg = 0;
        break;
      }
      iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
      nSeg = nMin;
      sqlite3_reset(pFindLevel);
    }

    /* Allocate space for the cursor, filter and writer objects */
    pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
    if( !pWriter ) return SQLITE_NOMEM;
    memset(pWriter, 0, nAlloc);
    pFilter = (Fts3SegFilter *)&pWriter[1];
    pCsr = (Fts3MultiSegReader *)&pFilter[1];

    /* Open a cursor to iterate through the contents of the oldest nSeg 
    ** indexes of absolute level iAbsLevel. If this cursor is opened using 
    ** the 'hint' parameters, it is possible that there are less than nSeg
    ** segments available in level iAbsLevel. In this case, no work is
    ** done on iAbsLevel - fall through to the next iteration of the loop 
    ** to start work on some other level.  */
    pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
    rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);


    if( pCsr->nSegment==nSeg && SQLITE_OK==rc
     && SQLITE_OK ==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))

     && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
    ){
      int iIdx = 0;               /* Largest idx in level (iAbsLevel+1) */
      rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
      if( rc==SQLITE_OK ){

        if( bUseHint && iIdx>0 ){
          const char *zKey = pCsr->zTerm;
          int nKey = pCsr->nTerm;
          rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
        }else{
          rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
        }
      }

      if( rc==SQLITE_OK && pWriter->nLeafEst ){
        fts3LogMerge(nSeg, iAbsLevel);
        do {
          rc = fts3IncrmergeAppend(p, pWriter, pCsr);
          if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
          if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
        }while( rc==SQLITE_ROW );

        /* Update or delete the input segments */
        if( rc==SQLITE_OK ){
          nRem -= (1 + pWriter->nWork);
          rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg);
        }
      }






      fts3IncrmergeRelease(p, pWriter, &rc);
    }

    sqlite3Fts3SegReaderFinish(pCsr);
    sqlite3_free(pWriter);
    bUseHint = 0;
  }

  /* Write the hint values into the %_stat table for the next incr-merger */
  if( rc==SQLITE_OK && (iAbsLevel!=iHintAbsLevel || nHintSeg!=nSeg) ){
    rc = fts3IncrmergeHintStore(p, iAbsLevel, nSeg);
  }

  return rc;
}

/*
** Process statements of the form:

Changes to test/fts4merge.test.

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
...
110
111
112
113
114
115
116
117
118
119
120
121
122
123










124


125
































    SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three'
  } {123 132 213 231 312 321}
}

do_execsql_test 1.5 { 
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
} {
  0 {0 1 2 3} 
  1 {0 1 2} 
  2 0
  3 0
}

#-------------------------------------------------------------------------
# Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are
# handled correctly.
#
................................................................................
}

do_execsql_test 3.3 { 
  INSERT INTO t2(t2) VALUES('merge=1000000,2');
  SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
} {
  0 0 
  1 {0 1} 
  2 0 
  3 {0 1} 
  4 {0 1} 
  5 0
}











finish_test










































<
|
<







 







<
|
|
|
|


>
>
>
>
>
>
>
>
>
>
|
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
59
60
61
62
63
64
65

66

67
68
69
70
71
72
73
...
108
109
110
111
112
113
114

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
    SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three'
  } {123 132 213 231 312 321}
}

do_execsql_test 1.5 { 
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
} {

  2 {0 1}

  3 0
}

#-------------------------------------------------------------------------
# Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are
# handled correctly.
#
................................................................................
}

do_execsql_test 3.3 { 
  INSERT INTO t2(t2) VALUES('merge=1000000,2');
  SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
} {
  0 0 

  2 0
  3 0 
  4 0
  6 0
}

#-------------------------------------------------------------------------
# Test cases 4.*
#
reset_db
do_execsql_test 4.1 {
  PRAGMA page_size = 512;
  CREATE VIRTUAL TABLE t4 USING fts4;
  PRAGMA main.page_size;
} {512}

do_test 4.2 {
  foreach x {a c b d e f g h i j k l m n o p} {
    execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')"
  }
  execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level}
} {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}}

foreach {tn expect} {
  1  "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0"
  2  "0 {0 1 2 3 4 5 6 7 8 9 10 11 12}    1 0"
  3  "0 {0 1 2 3 4 5 6 7 8 9 10 11}       1 0"
  4  "0 {0 1 2 3 4 5 6 7 8 9 10}          1 0"
  5  "0 {0 1 2 3 4 5 6 7 8 9}             1 0"
  6  "0 {0 1 2 3 4 5 6 7 8}               1 0"
  7  "0 {0 1 2 3 4 5 6 7}                 1 0"
  8  "0 {0 1 2 3 4 5 6}                   1 0"
  9  "0 {0 1 2 3 4 5}                     1 0"
} {
  do_execsql_test 4.3.$tn {
    INSERT INTO t4(t4) VALUES('merge=1,16');
    SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
  } $expect
}

do_execsql_test 4.4.1 {
  SELECT quote(value) FROM t4_stat WHERE rowid=1
} {X'0006'}

do_execsql_test 4.4.2 {
  DELETE FROM t4_stat WHERE rowid=1;
  INSERT INTO t4(t4) VALUES('merge=1,12');
  SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
} "0 {0 1 2 3 4 5}                     1 0"


finish_test