SQLite: Check-in [a3b50e4f]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview

Comment:	Fix a bug triggered by optimizing an FTS3 table when there are no segments on disk but pending terms in the hash table.
Downloads:	Tarball \| ZIP archive \| SQL archive
Timelines:	family \| ancestors \| descendants \| both \| trunk
Files:	files \| file ages \| folders
SHA1:	a3b50e4f80ca2dacc1f72435b8c72d55ffdb64e7
User & Date:	dan 2009-12-29 10:32:37

Context

2009-12-29
23:39		Within the special new.* and old.* tables of a trigger, recognize all the original table names even if those names overload the "rowid", "oid", or "_rowid_" special names. Ticket [34d2ae1c6d0]. (check-in: 1a0e5fa9 user: drh tags: trunk)
10:32		Fix a bug triggered by optimizing an FTS3 table when there are no segments on disk but pending terms in the hash table. (check-in: a3b50e4f user: dan tags: trunk)
2009-12-24
16:00		Immediately purge entries from the column cache when the associated register undergoes an affinity change. Ticket [eb5548a849]. Enhance the SQLITE_TESTCTRL_OPTIMIZATIONS setting of sqlite3_test_control so that it can disable the column cache for testing purposes, in an effort to prevent future problems of a similar nature to this one. (check-in: ea4e57e1 user: drh tags: trunk)

Changes

Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

**
** The first integer in each data is the number of hits that the simple
** query has in the current column.
**
** If the GLOBALCOUNT flag is set, then this is followed by the total
** number of hits the simple query has in the current column of *all*
** selected rows.



**
** If the POSITIONLIST flag is set, then this is followed by <local-count>
** integers - the positions of each of the hits for the current column/query.
*/
#define FTS3_MATCHINFO_GLOBALCOUNT  0x00000001
#define FTS3_MATCHINFO_POSITIONLIST 0x00000002


typedef struct MatchInfo MatchInfo;
struct MatchInfo {
  int rc;                         /* Return code. SQLITE_OK if no error */
  sqlite3_int64 iDocid;           /* Docid of entry to return data for */
  Fts3Table *pTab;                /* FTS3 Virtual table */
  int flags;                      /* Output flags (see above) */
................................................................................
  sqlite3_context *pCtx,
  Fts3Expr *pExpr,
  MatchInfo *pInfo
){
  int eType = pExpr->eType;
  if( eType==FTSQUERY_NOT || pInfo->rc ){
    return;
  }else if( eType!=FTSQUERY_PHRASE && eType!=FTSQUERY_NEAR ){
    assert( pExpr->pLeft && pExpr->pRight );
    fts3ExprMatchInfo(pCtx, pExpr->pLeft, pInfo);
    if( pInfo->rc==SQLITE_OK ){
      fts3ExprMatchInfo(pCtx, pExpr->pRight, pInfo);
    }
  }else{

    Fts3Table *pTab = pInfo->pTab;

    /* If it is not loaded already, load the doclist for this simple query
    ** from the FTS3 full-text index. 
    */
    if( pExpr->isLoaded==0 ){
      pInfo->rc = evalFts3Expr(pTab,pExpr,&pExpr->aDoclist,&pExpr->nDoclist,1);
................................................................................
              memset(pExpr->aHist, 0, nByte);

              /* Scan the entire doclist to populate Fts3Expr.aHist[]. */ 
              while( pCsr<pEnd ){
                while( *pCsr++ & 0x80 );
                while( *pCsr ){
                  sqlite3_int64 iCol = 0;
                  if( *pCsr==0x01 ) pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
                  pExpr->aHist[iCol] += fts3ColumnlistCount(&pCsr);
                }
                pCsr++;
              }
            }

            fts3MatchInfoAppend(pInfo, pExpr->aHist[i]);
          }





          if( i==0 ){
            if( *pExpr->pCurrent==0x01 ) continue;
          }else{
            sqlite3_int64 iCol;
            char *pList = pExpr->pCurrent;
            if( *pList==0x00 ) continue;
................................................................................

          if( pInfo->flags&FTS3_MATCHINFO_POSITIONLIST ){
            int nLocal = 0;
            sqlite3_int64 iOffset = 0;
            char *pList = pExpr->pCurrent;
            while( *pList&0xFE ){
              fts3GetDeltaVarint(&pList, &iOffset); iOffset -= 2;
              fts3MatchInfoAppend(pInfo, iOffset);
              nLocal++;
            }
            pExpr->pCurrent = pList;
            pInfo->aOut[iLocalOff] = nLocal;
          }else{
            pInfo->aOut[iLocalOff] = fts3ColumnlistCount(&pExpr->pCurrent);
          }
................................................................................
  if( nVal==2 ){
    int i;
    const unsigned char *zFlags = sqlite3_value_text(apVal[1]);
    for(i=0; zFlags[i]; i++){
      switch( zFlags[i] ){
        case 'g': flags |= FTS3_MATCHINFO_GLOBALCOUNT; break;
        case 'p': flags |= FTS3_MATCHINFO_POSITIONLIST; break;

        default: {
          char zErr[18];
          memcpy(zErr, "Unknown flag: \"%c\"", 18);
          zErr[16] = (char)zFlags[i];
          sqlite3_result_error(pContext, zErr, -1);
          return;
        }

**
** The first integer in each data is the number of hits that the simple
** query has in the current column.
**
** If the GLOBALCOUNT flag is set, then this is followed by the total
** number of hits the simple query has in the current column of *all*
** selected rows.
**
** If the PHRASELENGTH flag is set, this is followed by the number of
** tokens in the phrase.
**
** If the POSITIONLIST flag is set, then this is followed by <local-count>
** integers - the positions of each of the hits for the current column/query.
*/
#define FTS3_MATCHINFO_GLOBALCOUNT  0x00000001
#define FTS3_MATCHINFO_POSITIONLIST 0x00000002
#define FTS3_MATCHINFO_PHRASELENGTH 0x00000004

typedef struct MatchInfo MatchInfo;
struct MatchInfo {
  int rc;                         /* Return code. SQLITE_OK if no error */
  sqlite3_int64 iDocid;           /* Docid of entry to return data for */
  Fts3Table *pTab;                /* FTS3 Virtual table */
  int flags;                      /* Output flags (see above) */
................................................................................
  sqlite3_context *pCtx,
  Fts3Expr *pExpr,
  MatchInfo *pInfo
){
  int eType = pExpr->eType;
  if( eType==FTSQUERY_NOT || pInfo->rc ){
    return;
  }else if( eType!=FTSQUERY_PHRASE ){
    assert( pExpr->pLeft && pExpr->pRight );
    fts3ExprMatchInfo(pCtx, pExpr->pLeft, pInfo);
    if( pInfo->rc==SQLITE_OK ){
      fts3ExprMatchInfo(pCtx, pExpr->pRight, pInfo);
    }
  }else{
    int nPhrase = pExpr->pPhrase->nToken;
    Fts3Table *pTab = pInfo->pTab;

    /* If it is not loaded already, load the doclist for this simple query
    ** from the FTS3 full-text index. 
    */
    if( pExpr->isLoaded==0 ){
      pInfo->rc = evalFts3Expr(pTab,pExpr,&pExpr->aDoclist,&pExpr->nDoclist,1);
................................................................................
              memset(pExpr->aHist, 0, nByte);

              /* Scan the entire doclist to populate Fts3Expr.aHist[]. */ 
              while( pCsr<pEnd ){
                while( *pCsr++ & 0x80 );
                while( *pCsr ){
                  sqlite3_int64 iCol = 0;
                  if( *pCsr==0x01 ) pCsr += sqlite3Fts3GetVarint(++pCsr, &iCol);
                  pExpr->aHist[iCol] += fts3ColumnlistCount(&pCsr);
                }
                pCsr++;
              }
            }

            fts3MatchInfoAppend(pInfo, pExpr->aHist[i]);
          }

          if( pInfo->flags&FTS3_MATCHINFO_PHRASELENGTH ){
            fts3MatchInfoAppend(pInfo, nPhrase);
          }

          if( i==0 ){
            if( *pExpr->pCurrent==0x01 ) continue;
          }else{
            sqlite3_int64 iCol;
            char *pList = pExpr->pCurrent;
            if( *pList==0x00 ) continue;
................................................................................

          if( pInfo->flags&FTS3_MATCHINFO_POSITIONLIST ){
            int nLocal = 0;
            sqlite3_int64 iOffset = 0;
            char *pList = pExpr->pCurrent;
            while( *pList&0xFE ){
              fts3GetDeltaVarint(&pList, &iOffset); iOffset -= 2;
              fts3MatchInfoAppend(pInfo, iOffset+1-nPhrase);
              nLocal++;
            }
            pExpr->pCurrent = pList;
            pInfo->aOut[iLocalOff] = nLocal;
          }else{
            pInfo->aOut[iLocalOff] = fts3ColumnlistCount(&pExpr->pCurrent);
          }
................................................................................
  if( nVal==2 ){
    int i;
    const unsigned char *zFlags = sqlite3_value_text(apVal[1]);
    for(i=0; zFlags[i]; i++){
      switch( zFlags[i] ){
        case 'g': flags |= FTS3_MATCHINFO_GLOBALCOUNT; break;
        case 'p': flags |= FTS3_MATCHINFO_POSITIONLIST; break;
        case 'n': flags |= FTS3_MATCHINFO_PHRASELENGTH; break;
        default: {
          char zErr[18];
          memcpy(zErr, "Unknown flag: \"%c\"", 18);
          zErr[16] = (char)zFlags[i];
          sqlite3_result_error(pContext, zErr, -1);
          return;
        }

Changes to ext/fts3/fts3_write.c.

  const char *zVal = (const char *)sqlite3_value_text(pVal);
  int nVal = sqlite3_value_bytes(pVal);

  if( !zVal ){
    return SQLITE_NOMEM;
  }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
    rc = fts3SegmentMerge(p, -1);
    if( rc==SQLITE_DONE || rc==SQLITE_OK ){
      rc = SQLITE_OK;

      sqlite3Fts3PendingTermsClear(p);
    }
#ifdef SQLITE_TEST
  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
    p->nNodeSize = atoi(&zVal[9]);
    rc = SQLITE_OK;
  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){








|

>

  const char *zVal = (const char *)sqlite3_value_text(pVal);
  int nVal = sqlite3_value_bytes(pVal);

  if( !zVal ){
    return SQLITE_NOMEM;
  }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
    rc = fts3SegmentMerge(p, -1);
    if( rc==SQLITE_DONE ){
      rc = SQLITE_OK;
    }else{
      sqlite3Fts3PendingTermsClear(p);
    }
#ifdef SQLITE_TEST
  }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
    p->nNodeSize = atoi(&zVal[9]);
    rc = SQLITE_OK;
  }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){