/ Check-in [7ed9d2f2]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Allow multiple incremental merges to proceed concurrently. This is required to prevent a large crisis-merge from occuring while an even larger incremental-merge is underway.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts4-incr-merge-exp
Files: files | file ages | folders
SHA1: 7ed9d2f24a650b424b97dfc19b8042c4cf09c82c
User & Date: dan 2012-03-27 11:48:02
Context
2012-03-27
13:44
Merge the fts4-incr-merge-exp branch with fts4-incr-merge. check-in: eb00b958 user: dan tags: fts4-incr-merge
11:48
Allow multiple incremental merges to proceed concurrently. This is required to prevent a large crisis-merge from occuring while an even larger incremental-merge is underway. Closed-Leaf check-in: 7ed9d2f2 user: dan tags: fts4-incr-merge-exp
00:38
Minor correct errors in the file format description for FTS3/4 contained in the fts3.c header comment. check-in: fb8aacdd user: drh tags: fts4-incr-merge
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_write.c.

324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
....
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463

4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477

4478
4479
4480


4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494

4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506

4507

4508
4509
4510
4511















































4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529


4530
4531






4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546

4547
4548
4549


4550



4551
4552
4553
4554
4555




4556













4557
4558
4559






4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571




4572
4573
4574
4575
4576
4577
4578

4579

4580

4581
4582
4583
4584
4585
4586
4587
4588
4589
....
4602
4603
4604
4605
4606
4607
4608




4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624


4625
4626
4627
4628
4629
4630
4631

/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
** of the oldest level in the db that contains at least ? segments. Or,
** if no level in the FTS index contains more than ? segments, the statement
** returns zero rows.  */
/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
         "  ORDER BY (level %% 1024) DESC LIMIT 1",

/* Estimate the upper limit on the number of leaf nodes in a new segment
** created by merging the oldest :2 segments from absolute level :1. See 
** function sqlite3Fts3Incrmerge() for details.  */
/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
         "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",

................................................................................
  *pnRem = nRem;
  return rc;
}

/*
** Store an incr-merge hint in the database.
*/
static int fts3IncrmergeHintStore(
  Fts3Table *p,                   /* FTS3 table handle */
  sqlite3_int64 iAbsLevel,        /* Absolute level to read input data from */
  int nMerge                      /* Number of segments to merge */
){
  char aBlob[FTS3_VARINT_MAX * 2];
  int nBlob = 0;
  int rc;
  sqlite3_stmt *pReplace = 0;

  assert( p->bHasStat );
  nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], iAbsLevel);
  nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], nMerge);


  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
    sqlite3_bind_blob(pReplace, 2, aBlob, nBlob, SQLITE_TRANSIENT);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }

  return rc;
}

/*
** Load an incr-merge hint from the database.

**
** The incr-merge hint, if one exists, is stored in the rowid==1 row of
** the %_stat table.


*/
static int fts3IncrmergeHintLoad(
  Fts3Table *p,                   /* FTS3 table handle */
  sqlite3_int64 *piAbsLevel,      /* Absolute level to read input data from */
  int *pnMerge                    /* Number of segments to merge */
){
  sqlite3_stmt *pSelect = 0;
  int rc;

  *pnMerge = 0;
  *piAbsLevel = 0;

  rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0);
  if( rc==SQLITE_OK ){

    sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT);
    if( SQLITE_ROW==sqlite3_step(pSelect) ){
      const char *aHint = sqlite3_column_blob(pSelect, 0);
      int nHint = sqlite3_column_bytes(pSelect, 0);
      if( aHint ){
        int i;
        char aBlob[FTS3_VARINT_MAX * 2];
        memcpy(aBlob, aHint, MAX(sizeof(aBlob), nHint));
        i = sqlite3Fts3GetVarint(aBlob, piAbsLevel);
        sqlite3Fts3GetVarint32(&aBlob[i], pnMerge);
      }
    }

    rc = sqlite3_reset(pSelect);

  }

  return rc;
}
















































/*
** Attempt an incremental merge that writes nMerge leaf blocks.
**
** Incremental merges happen nMin segments at a time. The two
** segments to be merged are the nMin oldest segments (the ones with
** the smallest indexes) in the highest level that contains at least
** nMin segments. Multiple merges might occur in an attempt to write the 
** quota of nMerge leaf blocks.
*/
int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
  int rc;                         /* Return code */
  int nRem = nMerge;              /* Number of leaf pages yet to  be written */
  int bUseHint = 1;               /* True if hint has not yet been attempted */
  sqlite3_int64 iHintAbsLevel = 0;/* Hint level */
  int nHintSeg = 0;               /* Hint number of segments */
  int nSeg = 0;                   /* Number of input segments */
  sqlite3_int64 iAbsLevel = 0;    /* Absolute level number to work on */



  assert( nMin>=2 );







  rc = fts3IncrmergeHintLoad(p, &iHintAbsLevel, &nHintSeg);
  if( nHintSeg==0 ) bUseHint = 0;

  while( rc==SQLITE_OK && nRem>0 ){
    Fts3MultiSegReader *pCsr;       /* Cursor used to read input data */
    Fts3SegFilter *pFilter;         /* Filter used with cursor pCsr */
    IncrmergeWriter *pWriter;       /* Writer object */
    const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);

    if( bUseHint ){
      iAbsLevel = iHintAbsLevel;
      nSeg = nHintSeg;
    }else{
      sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */


      /* Determine which level to merge segments from. Any level, from any
      ** prefix or language index may be selected. Stack variable iAbsLevel 


      ** is set to the absolute level number of the level to merge from.  */



      rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
      sqlite3_bind_int(pFindLevel, 1, nMin);
      if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){
        /* There are no levels with nMin or more segments. Or an error has
         ** occurred. Either way, exit early.  */




        rc = sqlite3_reset(pFindLevel);













        iAbsLevel = 0;
        nSeg = 0;
        break;






      }
      iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
      nSeg = nMin;
      sqlite3_reset(pFindLevel);
    }

    /* Allocate space for the cursor, filter and writer objects */
    pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
    if( !pWriter ) return SQLITE_NOMEM;
    memset(pWriter, 0, nAlloc);
    pFilter = (Fts3SegFilter *)&pWriter[1];
    pCsr = (Fts3MultiSegReader *)&pFilter[1];





    /* Open a cursor to iterate through the contents of the oldest nSeg 
    ** indexes of absolute level iAbsLevel. If this cursor is opened using 
    ** the 'hint' parameters, it is possible that there are less than nSeg
    ** segments available in level iAbsLevel. In this case, no work is
    ** done on iAbsLevel - fall through to the next iteration of the loop 
    ** to start work on some other level.  */

    pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;

    rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);

    if( pCsr->nSegment==nSeg && SQLITE_OK==rc
     && SQLITE_OK ==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
     && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
    ){
      int iIdx = 0;               /* Largest idx in level (iAbsLevel+1) */
      rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
      if( rc==SQLITE_OK ){
        if( bUseHint && iIdx>0 ){
          const char *zKey = pCsr->zTerm;
................................................................................
          if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
        }while( rc==SQLITE_ROW );

        /* Update or delete the input segments */
        if( rc==SQLITE_OK ){
          nRem -= (1 + pWriter->nWork);
          rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg);




        }
      }

      fts3IncrmergeRelease(p, pWriter, &rc);
    }

    sqlite3Fts3SegReaderFinish(pCsr);
    sqlite3_free(pWriter);
    bUseHint = 0;
  }

  /* Write the hint values into the %_stat table for the next incr-merger */
  if( rc==SQLITE_OK && (iAbsLevel!=iHintAbsLevel || nHintSeg!=nSeg) ){
    rc = fts3IncrmergeHintStore(p, iAbsLevel, nSeg);
  }



  return rc;
}

/*
** Convert the text beginning at *pz into an integer and return
** its value.  Advance *pz to point to the first character past
** the integer.







|







 







|
<
<
<
<
<
<
<

<
<
<
<
>




|








|
>

<
|
>
>

|
<
<
<
<



|
<
<


>





|
|
|
|
<
|
|
>
|
>




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>













|
|
|


>
>

<
>
>
>
>
>
>

|
<
<

<
<
<
<
<
|
<
<
<
|
>

<
<
>
>
|
>
>
>
|
|
|
<
|
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
<
>
>
>
>
>
>

<
<
<


<
<
<
<
<
<
>
>
>
>







>

>
|
>
|
|







 







>
>
>
>







<
<



|
|


>
>







324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
....
4444
4445
4446
4447
4448
4449
4450
4451







4452




4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469

4470
4471
4472
4473
4474




4475
4476
4477
4478


4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490

4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567

4568
4569
4570
4571
4572
4573
4574
4575


4576





4577



4578
4579
4580


4581
4582
4583
4584
4585
4586
4587
4588
4589

4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610

4611
4612
4613
4614
4615
4616
4617



4618
4619






4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
....
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674


4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690

/* This statement is used to determine which level to read the input from
** when performing an incremental merge. It returns the absolute level number
** of the oldest level in the db that contains at least ? segments. Or,
** if no level in the FTS index contains more than ? segments, the statement
** returns zero rows.  */
/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
         "  ORDER BY (level %% 1024) ASC LIMIT 1",

/* Estimate the upper limit on the number of leaf nodes in a new segment
** created by merging the oldest :2 segments from absolute level :1. See 
** function sqlite3Fts3Incrmerge() for details.  */
/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
         "  FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",

................................................................................
  *pnRem = nRem;
  return rc;
}

/*
** Store an incr-merge hint in the database.
*/
static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){







  sqlite3_stmt *pReplace = 0;




  int rc;                         /* Return code */

  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
    sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }

  return rc;
}

/*
** Load an incr-merge hint from the database. The incr-merge hint, if one 
** exists, is stored in the rowid==1 row of the %_stat table.
**

** If successful, populate blob *pHint with the value read from the %_stat
** table and return SQLITE_OK. Otherwise, if an error occurs, return an
** SQLite error code.
*/
static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){




  sqlite3_stmt *pSelect = 0;
  int rc;

  pHint->n = 0;


  rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0);
  if( rc==SQLITE_OK ){
    int rc2;
    sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT);
    if( SQLITE_ROW==sqlite3_step(pSelect) ){
      const char *aHint = sqlite3_column_blob(pSelect, 0);
      int nHint = sqlite3_column_bytes(pSelect, 0);
      if( aHint ){
        blobGrowBuffer(pHint, nHint, &rc);
        if( rc==SQLITE_OK ){
          memcpy(pHint->a, aHint, nHint);
          pHint->n = nHint;

        }
      }
    }
    rc2 = sqlite3_reset(pSelect);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}

/*
** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
** Otherwise, append an entry to the hint stored in blob *pHint. Each entry
** consists of two varints, the absolute level number of the input segments 
** and the number of input segments.
**
** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs,
** set *pRc to an SQLite error code before returning.
*/
static void fts3IncrmergeHintPush(
  Blob *pHint,                    /* Hint blob to append to */
  i64 iAbsLevel,                  /* First varint to store in hint */
  int nInput,                     /* Second varint to store in hint */
  int *pRc                        /* IN/OUT: Error code */
){
  blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc);
  if( *pRc==SQLITE_OK ){
    pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel);
    pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput);
  }
}

/*
** Read the last entry (most recently pushed) from the hint blob *pHint
** and then remove the entry. Write the two values read to *piAbsLevel and 
** *pnInput before returning.
**
** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does
** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB.
*/
static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
  const int nHint = pHint->n;
  int i;

  i = pHint->n-2;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
  while( i>0 && (pHint->a[i-1] & 0x80) ) i--;

  pHint->n = i;
  i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
  i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput);
  if( i!=nHint ) return SQLITE_CORRUPT_VTAB;

  return SQLITE_OK;
}


/*
** Attempt an incremental merge that writes nMerge leaf blocks.
**
** Incremental merges happen nMin segments at a time. The two
** segments to be merged are the nMin oldest segments (the ones with
** the smallest indexes) in the highest level that contains at least
** nMin segments. Multiple merges might occur in an attempt to write the 
** quota of nMerge leaf blocks.
*/
int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
  int rc;                         /* Return code */
  int nRem = nMerge;              /* Number of leaf pages yet to  be written */
  Fts3MultiSegReader *pCsr;       /* Cursor used to read input data */
  Fts3SegFilter *pFilter;         /* Filter used with cursor pCsr */
  IncrmergeWriter *pWriter;       /* Writer object */
  int nSeg = 0;                   /* Number of input segments */
  sqlite3_int64 iAbsLevel = 0;    /* Absolute level number to work on */
  Blob hint = {0, 0, 0};          /* Hint read from %_stat table */
  int bDirtyHint = 0;             /* True if blob 'hint' has been modified */


  /* Allocate space for the cursor, filter and writer objects */
  const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);
  pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
  if( !pWriter ) return SQLITE_NOMEM;
  pFilter = (Fts3SegFilter *)&pWriter[1];
  pCsr = (Fts3MultiSegReader *)&pFilter[1];

  rc = fts3IncrmergeHintLoad(p, &hint);


  while( rc==SQLITE_OK && nRem>0 ){





    const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex;



    sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */
    int bUseHint = 0;             /* True if attempting to append */



    /* Search the %_segdir table for the absolute level with the smallest
    ** relative level number that contains at least nMin segments, if any.
    ** If one is found, set iAbsLevel to the absolute level number and
    ** nSeg to nMin. If no level with at least nMin segments can be found, 
    ** set nSeg to -1.
    */
    rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
    sqlite3_bind_int(pFindLevel, 1, nMin);
    if( sqlite3_step(pFindLevel)==SQLITE_ROW ){

      iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
      nSeg = nMin;
    }else{
      nSeg = -1;
    }
    rc = sqlite3_reset(pFindLevel);

    /* If the hint read from the %_stat table is not empty, check if the
    ** last entry in it specifies a relative level smaller than or equal
    ** to the level identified by the block above (if any). If so, this 
    ** iteration of the loop will work on merging at the hinted level.
    */
    if( rc==SQLITE_OK && hint.n ){
      int nHint = hint.n;
      sqlite3_int64 iHintAbsLevel = 0;      /* Hint level */
      int nHintSeg = 0;                     /* Hint number of segments */

      rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg);
      if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){
        iAbsLevel = iHintAbsLevel;
        nSeg = nHintSeg;

        bUseHint = 1;
        bDirtyHint = 1;
      }else{
        /* This undoes the effect of the HintPop() above - so that no entry
        ** is removed from the hint blob.  */
        hint.n = nHint;
      }



    }







    /* If nSeg is less that zero, then there is no level with at least
    ** nMin segments and no hint in the %_stat table. No work to do.
    ** Exit early in this case.  */
    if( nSeg<0 ) break;

    /* Open a cursor to iterate through the contents of the oldest nSeg 
    ** indexes of absolute level iAbsLevel. If this cursor is opened using 
    ** the 'hint' parameters, it is possible that there are less than nSeg
    ** segments available in level iAbsLevel. In this case, no work is
    ** done on iAbsLevel - fall through to the next iteration of the loop 
    ** to start work on some other level.  */
    memset(pWriter, 0, nAlloc);
    pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
    if( rc==SQLITE_OK ){
      rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
    }
    if( SQLITE_OK==rc && pCsr->nSegment==nSeg
     && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
     && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
    ){
      int iIdx = 0;               /* Largest idx in level (iAbsLevel+1) */
      rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
      if( rc==SQLITE_OK ){
        if( bUseHint && iIdx>0 ){
          const char *zKey = pCsr->zTerm;
................................................................................
          if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
        }while( rc==SQLITE_ROW );

        /* Update or delete the input segments */
        if( rc==SQLITE_OK ){
          nRem -= (1 + pWriter->nWork);
          rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg);
          if( nSeg!=0 ){
            bDirtyHint = 1;
            fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc);
          }
        }
      }

      fts3IncrmergeRelease(p, pWriter, &rc);
    }

    sqlite3Fts3SegReaderFinish(pCsr);


  }

  /* Write the hint values into the %_stat table for the next incr-merger */
  if( bDirtyHint && rc==SQLITE_OK ){
    rc = fts3IncrmergeHintStore(p, &hint);
  }

  sqlite3_free(pWriter);
  sqlite3_free(hint.a);
  return rc;
}

/*
** Convert the text beginning at *pz into an integer and return
** its value.  Advance *pz to point to the first character past
** the integer.

Changes to test/fts4merge.test.

192
193
194
195
196
197
198
199

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227

228
229
230
231
232
233
234
235
236
237


238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
} {
  0 {0 1 2 3 4 5 6 7} 
  1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
  2 {0 1 2}
}

do_execsql_test 5.3 {
  INSERT INTO t1(t1) VALUES('merge=1,4');

  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
} {
  0 {0 1 2 3 4 5 6 7} 
  1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
  2 {0 1 2 3}
}

do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'}
do_test 5.5 {
  foreach docid [execsql {SELECT docid FROM t1}] {
    execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
  }
} {}

do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0104'}

do_execsql_test 5.7 {
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} 
  1 {0 1 2 3 4 5 6 7 8 9 10 11} 
  2 {0 1 2 3 4 5 6 7}
  X'0104'
}

do_execsql_test 5.8 {
  INSERT INTO t1(t1) VALUES('merge=1,4');

  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15} 
  1 {0 1 2 3 4 5 6 7 8 9 10 11} 
  2 {0 1 2 3 4 5 6 7}
  3 {0}
  X'0204'
}



do_test 5.9 {
  set L [expr 16*16*8 + 16*4 + 1]
  foreach docid [execsql {
      SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L
  }] {
    execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
  }
} {}

do_execsql_test 5.10 {
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 0   1 0   2 0   3 {0 1}
  X'0204'
}

do_execsql_test 5.11 {
  INSERT INTO t1(t1) VALUES('merge=10,4');
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 0   1 0   2 0   3 {0 1}
  X'0000'
}

#-------------------------------------------------------------------------
# Test cases 6.*
#
# At one point the following test caused an assert() to fail (because the
# second 'merge=1,2' operation below actually "merges" a single input







|
>


|
|



|






|





|
|

|



|
>



|
|
|
<
<


>
>

|











|
<



|



|
<







192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235


236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253

254
255
256
257
258
259
260
261

262
263
264
265
266
267
268
} {
  0 {0 1 2 3 4 5 6 7} 
  1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
  2 {0 1 2}
}

do_execsql_test 5.3 {
  INSERT INTO t1(t1) VALUES('merge=1,5');
  INSERT INTO t1(t1) VALUES('merge=1,5');
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
} {
  0 {0 1 2}
  1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 
  2 {0 1 2 3}
}

do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}
do_test 5.5 {
  foreach docid [execsql {SELECT docid FROM t1}] {
    execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
  }
} {}

do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}

do_execsql_test 5.7 {
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 {0 1 2 3 4 5 6 7 8 9 10} 
  1 {0 1 2 3 4 5 6 7 8 9 10 11 12} 
  2 {0 1 2 3 4 5 6 7}
  X'0105'
}

do_execsql_test 5.8 {
  INSERT INTO t1(t1) VALUES('merge=1,6');
  INSERT INTO t1(t1) VALUES('merge=1,6');
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 {0 1 2 3 4} 
  1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
  2 {0 1 2 3 4 5 6 7 8} X'0106'


}

do_test 5.8.1 { fts3_integrity_check t1 } ok

do_test 5.9 {
  set L [expr 16*16*7 + 16*3 + 12]
  foreach docid [execsql {
      SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L
  }] {
    execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
  }
} {}

do_execsql_test 5.10 {
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 0 1 {0 1} 2 0 3 0 X'0106'

}

do_execsql_test 5.11 {
  INSERT INTO t1(t1) VALUES('merge=1,6');
  SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  SELECT quote(value) from t1_stat WHERE rowid=1;
} {
  0 0 1 {0 1} 2 0 3 0 X''

}

#-------------------------------------------------------------------------
# Test cases 6.*
#
# At one point the following test caused an assert() to fail (because the
# second 'merge=1,2' operation below actually "merges" a single input