SQLite

Check-in [b4ac61aeee]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further minor performance improvements and code-size reductions related to fts5 column filters on detail=col tables.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5-perf
Files: files | file ages | folders
SHA1: b4ac61aeee976296e7719949cd4fb496147a29e8
User & Date: dan 2016-01-26 20:08:50.348
Context
2016-01-26
20:19
Performance improvements for fts5, particularly detail=col mode. (check-in: a3d7b8ac53 user: dan tags: trunk)
20:08
Further minor performance improvements and code-size reductions related to fts5 column filters on detail=col tables. (Leaf check-in: b4ac61aeee user: dan tags: fts5-perf)
19:30
Improve the performance of fts5 column filters on detail=col tables. (check-in: 249a2d070c user: dan tags: fts5-perf)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_index.c.
︙︙
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054







5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086





5087
5088

5089
5090
5091





5092



5093


5094
5095
5096

5097
5098
5099
5100
5101
5102
5103

    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
    pIter->base.pData = pIter->poslist.p;
  }
}

/*
** xSetOutputs callback used when: detail=col when there is a column filter.
**
**   * detail=col,
**   * there is a column filter, and
**   * the table contains 32 or fewer columns.
*/
static void fts5IterSetOutputs_Col32(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;
  pIter->base.iRowid = pSeg->iRowid;

  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  assert( pColset );

  fts5BufferZero(&pIter->poslist);
  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
    int i;
    int iPrev = 0;
    u32 m = 0;
    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
    u8 *pEnd = (u8*)&a[pSeg->nPos]; 

    while( a<pEnd ){
      iPrev += (int)a[0] - 2;
      m |= (1 << iPrev);
      a++;
    }

    iPrev = 0;
    a = pIter->poslist.p;
    for(i=0; i<pColset->nCol; i++){
      int iCol = pColset->aiCol[i];
      if( m & (1 << iCol) ){
        *a++ = (iCol - iPrev) + 2;
        iPrev = iCol;
      }
    }
    pIter->poslist.n = a - pIter->poslist.p;

  }else{
    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
  }

  pIter->base.pData = pIter->poslist.p;
  pIter->base.nData = pIter->poslist.n;
}

/*
** xSetOutputs callback used by detail=col when there is a column filter.







*/
static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;
  pIter->base.iRowid = pSeg->iRowid;

  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  assert( pColset );

  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
    /* All data is stored on the current page. Populate the output 
    ** variables to point into the body of the page object. */
    Fts5PoslistWriter writer = {0};
    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
    int n = pSeg->nPos;
    int iCol = 0;
    int iCVal = pColset->aiCol[0];
    i64 iPos = 0;
    int iOff = 0;

    fts5BufferZero(&pIter->poslist);
    while( 0==sqlite3Fts5PoslistNext64(a, n, &iOff, &iPos) ){
      while( iPos>=iCVal ){
        if( iPos==iCVal ){
          sqlite3Fts5PoslistWriterAppend(&pIter->poslist, &writer, iPos);
        }
        if( ++iCol>=pColset->nCol ) goto setoutputs_col_out;
        assert( pColset->aiCol[iCol]>iCVal );
        iCVal = pColset->aiCol[iCol];
      }
    }

  }else{





    /* The data is distributed over two or more pages. Copy it into the
    ** Fts5Iter.poslist buffer and then set the output pointer to point

    ** to this buffer.  */
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);





  }






setoutputs_col_out:
  pIter->base.pData = pIter->poslist.p;
  pIter->base.nData = pIter->poslist.n;

}

/*
** xSetOutputs callback used by detail=full when there is a column filter.
*/
static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;








|
<
<
|
|

|
<
<
<
<
<
<

<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
|





|
>
>
>
>
>
>
>

|
<
<


|

|
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<

>
>
>
>
>
|
|
>
|
|
|
>
>
>
>
>
|
>
>
>
|
>
>

|
|
>








5000
5001
5002
5003
5004
5005
5006
5007


5008
5009
5010
5011






5012

























5013

5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029


5030
5031
5032
5033
5034









5035












5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070

    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
    pIter->base.pData = pIter->poslist.p;
  }
}

/*
** xSetOutputs callback used by detail=col when there is a column filter


** and there are 100 or more columns. Also called as a fallback from
** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
*/
static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){






  fts5BufferZero(&pIter->poslist);

























  fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);

  pIter->base.iRowid = pSeg->iRowid;
  pIter->base.pData = pIter->poslist.p;
  pIter->base.nData = pIter->poslist.n;
}

/*
** xSetOutputs callback used when: 
**
**   * detail=col,
**   * there is a column filter, and
**   * the table contains 100 or fewer columns. 
**
** The last point is to ensure all column numbers are stored as 
** single-byte varints.
*/
static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){



  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  assert( pIter->pColset );

  if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){









    fts5IterSetOutputs_Col(pIter, pSeg);












  }else{
    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
    int iPrev = 0;
    int *aiCol = pIter->pColset->aiCol;
    int *aiColEnd = &aiCol[pIter->pColset->nCol];

    u8 *aOut = pIter->poslist.p;
    int iPrevOut = 0;

    pIter->base.iRowid = pSeg->iRowid;

    while( a<pEnd ){
      iPrev += (int)a++[0] - 2;
      while( *aiCol<iPrev ){
        aiCol++;
        if( aiCol==aiColEnd ) goto setoutputs_col_out;
      }
      if( *aiCol==iPrev ){
        *aOut++ = (iPrev - iPrevOut) + 2;
        iPrevOut = iPrev;
      }
    }

setoutputs_col_out:
    pIter->base.pData = pIter->poslist.p;
    pIter->base.nData = aOut - pIter->poslist.p;
  }
}

/*
** xSetOutputs callback used by detail=full when there is a column filter.
*/
static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
  Fts5Colset *pColset = pIter->pColset;
︙︙
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157


  else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
    pIter->xSetOutputs = fts5IterSetOutputs_Full;
  }

  else{
    assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
    if( pConfig->nCol<=32 ){
      pIter->xSetOutputs = fts5IterSetOutputs_Col32;
      sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
    }else{
      pIter->xSetOutputs = fts5IterSetOutputs_Col;
    }
  }
}









|
|








5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124


  else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
    pIter->xSetOutputs = fts5IterSetOutputs_Full;
  }

  else{
    assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
    if( pConfig->nCol<=100 ){
      pIter->xSetOutputs = fts5IterSetOutputs_Col100;
      sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
    }else{
      pIter->xSetOutputs = fts5IterSetOutputs_Col;
    }
  }
}
︙︙