/ Check-in [9f8d678a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Simplify the way position lists are copied when merging data.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 9f8d678a0ea75e169daf8b3f00bd05f52a050ea6
User & Date: dan 2014-08-11 20:26:34
Context
2014-08-12
08:36
Automatically resize the hash table used by fts5. check-in: f1cb48f4 user: dan tags: fts5
2014-08-11
20:26
Simplify the way position lists are copied when merging data. check-in: 9f8d678a user: dan tags: fts5
19:44
Replace the hash table borrowed from fts3. check-in: 617e2fac user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

2808
2809
2810
2811
2812
2813
2814




























2815
2816
2817
2818
2819
2820
2821
....
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
....
3176
3177
3178
3179
3180
3181
3182

3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
  Fts5PageWriter *pPage = &pWriter->aWriter[0];
  fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
  if( pPage->buf.n>=p->pgsz ){
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
}





























static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){
  fts5BufferAppendVarint(&p->rc, &pWriter->aWriter[0].buf, 0);
}

/*
** Flush any data cached by the writer object to the database. Free any
................................................................................

      /* Append the rowid to the output */
      fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));

      /* Copy the position list from input to output */
      fts5WriteAppendPoslistInt(p, &writer, sPos.nRem);
      for(/* noop */; !fts5ChunkIterEof(p, &sPos); fts5ChunkIterNext(p, &sPos)){
        int iOff = 0;
        while( iOff<sPos.n ){
          int iVal;
          iOff += getVarint32(&sPos.p[iOff], iVal);
          fts5WriteAppendPoslistInt(p, &writer, iVal);
        }
      }
    }

    fts5ChunkIterRelease(&sPos);
  }

  /* Flush the last leaf page to disk. Set the output segment b-tree height
................................................................................

static int fts5FlushNewEntry(
  void *pCtx, 
  i64 iRowid, 
  const u8 *aPoslist, 
  int nPoslist
){

  Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
  int rc = SQLITE_OK;
  int i = 0;

  /* Append the rowid itself */
  fts5WriteAppendRowid(p->pIdx, &p->writer, iRowid);

  /* Append the size of the position list in bytes */
  fts5WriteAppendPoslistInt(p->pIdx, &p->writer, nPoslist);

  /* Copy the position list to the output segment */
  while( i<nPoslist ){
    int iVal;
    i += getVarint32(&aPoslist[i], iVal);
    fts5WriteAppendPoslistInt(p->pIdx, &p->writer, iVal);
  }

  return rc;
}

/*
** Flush the contents of in-memory hash table iHash to a new level-0 
** segment on disk. Also update the corresponding structure record.
**







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







<
<
<
<
|
<







 







>


<







|
<
<
<
|
<
<







2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
....
3065
3066
3067
3068
3069
3070
3071




3072

3073
3074
3075
3076
3077
3078
3079
....
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208

3209
3210
3211
3212
3213
3214
3215
3216



3217


3218
3219
3220
3221
3222
3223
3224
  Fts5PageWriter *pPage = &pWriter->aWriter[0];
  fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
  if( pPage->buf.n>=p->pgsz ){
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
}

static void fts5WriteAppendPoslistData(
  Fts5Index *p, 
  Fts5SegWriter *pWriter, 
  const u8 *aData, 
  int nData
){
  Fts5PageWriter *pPage = &pWriter->aWriter[0];
  const u8 *a = aData;
  int n = nData;
  
  while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pgsz ){
    int nReq = p->pgsz - pPage->buf.n;
    int nCopy = 0;
    while( nCopy<nReq ){
      i64 dummy;
      nCopy += getVarint(&a[nCopy], (u64*)&dummy);
    }
    fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a);
    a += nCopy;
    n -= nCopy;
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
  if( n>0 ){
    fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a);
  }
}

static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){
  fts5BufferAppendVarint(&p->rc, &pWriter->aWriter[0].buf, 0);
}

/*
** Flush any data cached by the writer object to the database. Free any
................................................................................

      /* Append the rowid to the output */
      fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));

      /* Copy the position list from input to output */
      fts5WriteAppendPoslistInt(p, &writer, sPos.nRem);
      for(/* noop */; !fts5ChunkIterEof(p, &sPos); fts5ChunkIterNext(p, &sPos)){




        fts5WriteAppendPoslistData(p, &writer, sPos.p, sPos.n);

      }
    }

    fts5ChunkIterRelease(&sPos);
  }

  /* Flush the last leaf page to disk. Set the output segment b-tree height
................................................................................

static int fts5FlushNewEntry(
  void *pCtx, 
  i64 iRowid, 
  const u8 *aPoslist, 
  int nPoslist
){
  Fts5Buffer *pBuf;
  Fts5FlushCtx *p = (Fts5FlushCtx*)pCtx;
  int rc = SQLITE_OK;


  /* Append the rowid itself */
  fts5WriteAppendRowid(p->pIdx, &p->writer, iRowid);

  /* Append the size of the position list in bytes */
  fts5WriteAppendPoslistInt(p->pIdx, &p->writer, nPoslist);

  /* And the poslist data */



  fts5WriteAppendPoslistData(p->pIdx, &p->writer, aPoslist, nPoslist);


  return rc;
}

/*
** Flush the contents of in-memory hash table iHash to a new level-0 
** segment on disk. Also update the corresponding structure record.
**