/ Check-in [b4ac61ae]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further minor performance improvements and code-size reductions related to fts5 column filters on detail=col tables.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-perf
Files: files | file ages | folders
SHA1: b4ac61aeee976296e7719949cd4fb496147a29e8
User & Date: dan 2016-01-26 20:08:50
Context
2016-01-26
20:19
Performance improvements for fts5, particularly detail=col mode. check-in: a3d7b8ac user: dan tags: trunk
20:08
Further minor performance improvements and code-size reductions related to fts5 column filters on detail=col tables. Leaf check-in: b4ac61ae user: dan tags: fts5-perf
19:30
Improve the performance of fts5 column filters on detail=col tables. check-in: 249a2d07 user: dan tags: fts5-perf
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048

5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096

5097
5098
5099
5100
5101
5102
5103
....
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
    pIter->base.pData = pIter->poslist.p;
  }
}

/*
** xSetOutputs callback used when: detail=col when there is a column filter.
**
**   * detail=col,
**   * there is a column filter, and
**   * the table contains 32 or fewer columns.
*/
static void fts5IterSetOutputs_Col32(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;
  pIter->base.iRowid = pSeg->iRowid;

  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  assert( pColset );

  fts5BufferZero(&pIter->poslist);
  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
    int i;
    int iPrev = 0;
    u32 m = 0;
    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
    u8 *pEnd = (u8*)&a[pSeg->nPos]; 

    while( a<pEnd ){
      iPrev += (int)a[0] - 2;
      m |= (1 << iPrev);
      a++;
    }

    iPrev = 0;
    a = pIter->poslist.p;
    for(i=0; i<pColset->nCol; i++){
      int iCol = pColset->aiCol[i];
      if( m & (1 << iCol) ){
        *a++ = (iCol - iPrev) + 2;
        iPrev = iCol;
      }
    }
    pIter->poslist.n = a - pIter->poslist.p;

  }else{
    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
  }


  pIter->base.pData = pIter->poslist.p;
  pIter->base.nData = pIter->poslist.n;
}

/*
** xSetOutputs callback used by detail=col when there is a column filter.
*/
static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;
  pIter->base.iRowid = pSeg->iRowid;

  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  assert( pColset );

  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
    /* All data is stored on the current page. Populate the output 
    ** variables to point into the body of the page object. */
    Fts5PoslistWriter writer = {0};
    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
    int n = pSeg->nPos;
    int iCol = 0;
    int iCVal = pColset->aiCol[0];
    i64 iPos = 0;
    int iOff = 0;

    fts5BufferZero(&pIter->poslist);
    while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){
      while( iPos>=iCVal ){
        if( iPos==iCVal ){
          sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos);
        }
        if( ++iCol>=pColset->nCol ) goto setoutputs_col_out;
        assert( pColset->aiCol[iCol]>iCVal );
        iCVal = pColset->aiCol[iCol];
      }
    }

  }else{
    /* The data is distributed over two or more pages. Copy it into the
    ** Fts5Iter.poslist buffer and then set the output pointer to point
    ** to this buffer.  */
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
  }

setoutputs_col_out:
  pIter->base.pData = pIter->poslist.p;
  pIter->base.nData = pIter->poslist.n;

}

/*
** xSetOutputs callback used by detail=full when there is a column filter.
*/
static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;
................................................................................

  else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
    pIter->xSetOutputs = fts5IterSetOutputs_Full;
  }

  else{
    assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
    if( pConfig->nCol<=32 ){
      pIter->xSetOutputs = fts5IterSetOutputs_Col32;
      sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
    }else{
      pIter->xSetOutputs = fts5IterSetOutputs_Col;
    }
  }
}








|
|
|
<
<

|
<
<
<
<
<
<

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
>





|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>







 







|
|







5000
5001
5002
5003
5004
5005
5006
5007
5008
5009


5010
5011






5012

























5013


5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
....
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
    pIter->base.pData = pIter->poslist.p;
  }
}

/*
** xSetOutputs callback used by detail=col when there is a column filter
** and there are 100 or more columns. Also called as a fallback from
** fts5IterSetOutputs_Col100 if the column-list spans more than one page.


*/
static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){






  fts5BufferZero(&pIter->poslist);

























  fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);


  pIter->base.iRowid = pSeg->iRowid;
  pIter->base.pData = pIter->poslist.p;
  pIter->base.nData = pIter->poslist.n;
}

/*
** xSetOutputs callback used when: 
**
**   * detail=col,
**   * there is a column filter, and
**   * the table contains 100 or fewer columns. 
**
** The last point is to ensure all column numbers are stored as 
** single-byte varints.
*/
static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){

  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  assert( pIter->pColset );

  if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){
    fts5IterSetOutputs_Col(pIter, pSeg);
  }else{
    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
    int iPrev = 0;
    int *aiCol = pIter->pColset->aiCol;
    int *aiColEnd = &aiCol[pIter->pColset->nCol];

    u8 *aOut = pIter->poslist.p;
    int iPrevOut = 0;

    pIter->base.iRowid = pSeg->iRowid;

    while( a<pEnd ){
      iPrev += (int)a++[0] - 2;
      while( *aiCol<iPrev ){
        aiCol++;
        if( aiCol==aiColEnd ) goto setoutputs_col_out;
      }
      if( *aiCol==iPrev ){
        *aOut++ = (iPrev - iPrevOut) + 2;
        iPrevOut = iPrev;
      }
    }

setoutputs_col_out:
    pIter->base.pData = pIter->poslist.p;
    pIter->base.nData = aOut - pIter->poslist.p;
  }
}

/*
** xSetOutputs callback used by detail=full when there is a column filter.
*/
static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;
................................................................................

  else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
    pIter->xSetOutputs = fts5IterSetOutputs_Full;
  }

  else{
    assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
    if( pConfig->nCol<=100 ){
      pIter->xSetOutputs = fts5IterSetOutputs_Col100;
      sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
    }else{
      pIter->xSetOutputs = fts5IterSetOutputs_Col;
    }
  }
}