SQLite

Check-in [d6e2637df1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the way fts5 internally allocates segment ids in order to eliminated non-determinism from the module.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: d6e2637df16764aa9723a30ea2eb8a631d28cb2b
User & Date: dan 2016-03-21 09:56:19.361
Context
2016-03-21
14:46
Add the sqlite3_system_errno() interface. (check-in: 4bd12b57ea user: drh tags: trunk)
10:49
Merge updates from trunk. (check-in: 86ab864396 user: drh tags: sqlite_system_errno)
09:56
Change the way fts5 internally allocates segment ids in order to eliminated non-determinism from the module. (check-in: d6e2637df1 user: dan tags: trunk)
00:38
Change the Vdbe.aMem array so that it is zero-based instead of one-based. (check-in: c39081e878 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_index.c.
︙︙
3449
3450
3451
3452
3453
3454
3455
3456


3457


3458
3459
3460
3461
3462
3463


3464
3465
3466
3467













3468
3469
3470
3471
3472
3473
3474

static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
  int iSegid = 0;

  if( p->rc==SQLITE_OK ){
    if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
      p->rc = SQLITE_FULL;
    }else{
      while( iSegid==0 ){


        int iLvl, iSeg;


        sqlite3_randomness(sizeof(u32), (void*)&iSegid);
        iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
            if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
              iSegid = 0;


            }
          }
        }
      }













    }
  }

  return iSegid;
}

/*








|
>
>
|
>
>
|
<
|
|
|
<
>
>
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>








3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462

3463
3464
3465

3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491

static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
  int iSegid = 0;

  if( p->rc==SQLITE_OK ){
    if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
      p->rc = SQLITE_FULL;
    }else{
      /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following
      ** array is 63 elements, or 252 bytes, in size.  */
      u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32];
      int iLvl, iSeg;
      int i;
      u32 mask;
      memset(aUsed, 0, sizeof(aUsed));

      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
          int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;

          if( iId<=FTS5_MAX_SEGMENT ){
            aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
          }
        }
      }

      for(i=0; aUsed[i]==0xFFFFFFFF; i++);
      mask = aUsed[i];
      for(iSegid=0; mask & (1 << iSegid); iSegid++);
      iSegid += 1 + i*32;

#ifdef SQLITE_DEBUG
      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
          assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid );
        }
      }
      assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT );
#endif
    }
  }

  return iSegid;
}

/*
︙︙
3905
3906
3907
3908
3909
3910
3911

3912

3913
3914
3915
3916
3917
3918
3919

  Fts5PageWriter *pLeaf = &pWriter->writer;
  if( p->rc==SQLITE_OK ){
    assert( pLeaf->pgno>=1 );
    if( pLeaf->buf.n>4 ){
      fts5WriteFlushLeaf(p, pWriter);
    }
    *pnLeaf = pLeaf->pgno-1;

    fts5WriteFlushBtree(p, pWriter);

  }
  fts5BufferFree(&pLeaf->term);
  fts5BufferFree(&pLeaf->buf);
  fts5BufferFree(&pLeaf->pgidx);
  fts5BufferFree(&pWriter->btterm);

  for(i=0; i<pWriter->nDlidx; i++){








>
|
>








3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938

  Fts5PageWriter *pLeaf = &pWriter->writer;
  if( p->rc==SQLITE_OK ){
    assert( pLeaf->pgno>=1 );
    if( pLeaf->buf.n>4 ){
      fts5WriteFlushLeaf(p, pWriter);
    }
    *pnLeaf = pLeaf->pgno-1;
    if( pLeaf->pgno>1 ){
      fts5WriteFlushBtree(p, pWriter);
    }
  }
  fts5BufferFree(&pLeaf->term);
  fts5BufferFree(&pLeaf->buf);
  fts5BufferFree(&pLeaf->pgidx);
  fts5BufferFree(&pWriter->btterm);

  for(i=0; i<pWriter->nDlidx; i++){
︙︙
Changes to ext/fts5/tool/fts5txt2db.tcl.
︙︙
13
14
15
16
17
18
19

20
21
22
23
24
25
26

    {fts5                 "use fts5 (this is the default)"}
    {fts4                 "use fts4"}
    {colsize   "10 10 10" "list of column sizes"}
    {tblname   "t1"       "table name to create"}
    {detail    "full"     "Fts5 detail mode to use"}
    {repeat    1          "Load each file this many times"}
    {prefix    ""         "Fts prefix= option"}

    database
    file...
  } {
  This script is designed to create fts4/5 tables with more than one column.
  The -colsize option should be set to a Tcl list of integer values, one for
  each column in the table. Each value is the number of tokens that will be
  inserted into the column value for each row. For example, setting the -colsize








>








13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

    {fts5                 "use fts5 (this is the default)"}
    {fts4                 "use fts4"}
    {colsize   "10 10 10" "list of column sizes"}
    {tblname   "t1"       "table name to create"}
    {detail    "full"     "Fts5 detail mode to use"}
    {repeat    1          "Load each file this many times"}
    {prefix    ""         "Fts prefix= option"}
    {trans     1          "True to use a transaction"}
    database
    file...
  } {
  This script is designed to create fts4/5 tables with more than one column.
  The -colsize option should be set to a Tcl list of integer values, one for
  each column in the table. Each value is the number of tokens that will be
  inserted into the column value for each row. For example, setting the -colsize
︙︙
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228

set cols [create_table]
set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])"
foreach c [lrange $cols 1 end] {
  append sql ", \$R($c)"
}
append sql ")"

db eval BEGIN
  while {$i < $N} {
    foreach c $cols s $A(colsize) {
      set R($c) [lrange $tokens $i [expr $i+$s-1]]
      incr i $s
    }
    db eval $sql
  }
db eval COMMIT











|







|




211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229

set cols [create_table]
set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])"
foreach c [lrange $cols 1 end] {
  append sql ", \$R($c)"
}
append sql ")"

if {$A(trans)} { db eval BEGIN }
  while {$i < $N} {
    foreach c $cols s $A(colsize) {
      set R($c) [lrange $tokens $i [expr $i+$s-1]]
      incr i $s
    }
    db eval $sql
  }
if {$A(trans)} { db eval COMMIT }