/ Check-in [4cc048c3]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add hidden column "rank". Currently this always returns the same value as the bm25() function.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 4cc048c3651e830a6aeded924c7f3a60b634e133
User & Date: dan 2014-07-30 19:41:58
Context
2014-07-30
20:26
Fix things so that the fts5 extension API works with "ORDER BY rank" queries. check-in: f1b4e1a9 user: dan tags: fts5
19:41
Add hidden column "rank". Currently this always returns the same value as the bm25() function. check-in: 4cc048c3 user: dan tags: fts5
2014-07-28
20:14
Add the "loadfts" program, for performance testing the loading of data into fts3/fts4/fts5 tables. check-in: 770b9540 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

    52     52   */
    53     53   struct Fts5Table {
    54     54     sqlite3_vtab base;              /* Base class used by SQLite core */
    55     55     Fts5Config *pConfig;            /* Virtual table configuration */
    56     56     Fts5Index *pIndex;              /* Full-text index */
    57     57     Fts5Storage *pStorage;          /* Document store */
    58     58     Fts5Global *pGlobal;            /* Global (connection wide) data */
           59  +  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
    59     60   };
    60     61   
    61     62   struct Fts5MatchPhrase {
    62     63     Fts5Buffer *pPoslist;           /* Pointer to current poslist */
    63     64     int nTerm;                      /* Size of phrase in terms */
    64     65   };
           66  +
           67  +/*
           68  +** Variable pStmt is set to a compiled SQL statement of the form:
           69  +**
           70  +**   SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
           71  +**
           72  +*/
           73  +struct Fts5Sorter {
           74  +  sqlite3_stmt *pStmt;
           75  +  i64 iRowid;                     /* Current rowid */
           76  +  u8 *aPoslist;                   /* Position lists for current row */
           77  +  int nIdx;                       /* Number of entries in aIdx[] */
           78  +  int aIdx[0];                    /* Offsets into aPoslist for current row */
           79  +};
    65     80   
    66     81   /*
    67     82   ** Virtual-table cursor object.
    68     83   */
    69     84   struct Fts5Cursor {
    70     85     sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
    71     86     int idxNum;                     /* idxNum passed to xFilter() */
    72     87     sqlite3_stmt *pStmt;            /* Statement used to read %_content */
    73     88     Fts5Expr *pExpr;                /* Expression for MATCH queries */
           89  +  Fts5Sorter *pSorter;            /* Sorter for "ORDER BY rank" queries */
    74     90     int csrflags;                   /* Mask of cursor flags (see below) */
    75     91     Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
           92  +  Fts5Auxiliary *pRank;           /* Rank callback (or NULL) */
    76     93   
    77     94     /* Variables used by auxiliary functions */
    78     95     i64 iCsrId;                     /* Cursor id */
    79     96     Fts5Auxiliary *pAux;            /* Currently executing extension function */
    80         -  Fts5Auxdata *pAuxdata;          /* First in linked list of aux-data */
           97  +  Fts5Auxdata *pAuxdata;          /* First in linked list of saved aux-data */
    81     98     int *aColumnSize;               /* Values for xColumnSize() */
    82     99   };
    83    100   
    84    101   /*
    85    102   ** Values for Fts5Cursor.csrflags
    86    103   */
    87    104   #define FTS5CSR_REQUIRE_CONTENT   0x01
................................................................................
   223    240   ){
   224    241     return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
   225    242   }
   226    243   
   227    244   /*
   228    245   ** The three query plans xBestIndex may choose between.
   229    246   */
   230         -#define FTS5_PLAN_SCAN    1       /* No usable constraint */
   231         -#define FTS5_PLAN_MATCH   2       /* (<tbl> MATCH ?) */
   232         -#define FTS5_PLAN_ROWID   3       /* (rowid = ?) */
          247  +#define FTS5_PLAN_SCAN           1       /* No usable constraint */
          248  +#define FTS5_PLAN_MATCH          2       /* (<tbl> MATCH ?) */
          249  +#define FTS5_PLAN_SORTED_MATCH   3       /* (<tbl> MATCH ? ORDER BY rank) */
          250  +#define FTS5_PLAN_ROWID          4       /* (rowid = ?) */
          251  +#define FTS5_PLAN_SOURCE         5       /* A source cursor for SORTED_MATCH */
   233    252   
   234    253   #define FTS5_PLAN(idxNum) ((idxNum) & 0x7)
   235    254   
   236    255   #define FTS5_ORDER_DESC   8       /* ORDER BY rowid DESC */
   237    256   #define FTS5_ORDER_ASC   16       /* ORDER BY rowid ASC */
   238    257   
   239    258   /*
................................................................................
   280    299     if( iCons>=0 ){
   281    300       pInfo->aConstraintUsage[iCons].argvIndex = 1;
   282    301       pInfo->aConstraintUsage[iCons].omit = 1;
   283    302     }else{
   284    303       pInfo->estimatedCost = 10000000.0;
   285    304     }
   286    305   
   287         -  if( pInfo->nOrderBy==1 && pInfo->aOrderBy[0].iColumn<0 ){
   288         -    pInfo->orderByConsumed = 1;
   289         -    ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC;
          306  +  if( pInfo->nOrderBy==1 ){
          307  +    int iSort = pInfo->aOrderBy[0].iColumn;
          308  +    if( iSort<0 ){
          309  +      /* ORDER BY rowid [ASC|DESC] */
          310  +      pInfo->orderByConsumed = 1;
          311  +    }else if( iSort==(pConfig->nCol+1) && ePlan==FTS5_PLAN_MATCH ){
          312  +      /* ORDER BY rank [ASC|DESC] */
          313  +      pInfo->orderByConsumed = 1;
          314  +      ePlan = FTS5_PLAN_SORTED_MATCH;
          315  +    }
          316  +
          317  +    if( pInfo->orderByConsumed ){
          318  +      ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC;
          319  +    }
   290    320     }
   291    321      
   292    322     pInfo->idxNum = ePlan;
   293    323     return SQLITE_OK;
   294    324   }
   295    325   
   296    326   /*
................................................................................
   337    367     Fts5Auxdata *pData;
   338    368     Fts5Auxdata *pNext;
   339    369   
   340    370     if( pCsr->pStmt ){
   341    371       int eStmt = fts5StmtType(pCsr->idxNum);
   342    372       sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
   343    373     }
   344         -  sqlite3Fts5ExprFree(pCsr->pExpr);
          374  +  if( pCsr->pSorter ){
          375  +    Fts5Sorter *pSorter = pCsr->pSorter;
          376  +    sqlite3_finalize(pSorter->pStmt);
          377  +    sqlite3_free(pSorter);
          378  +  }
          379  +  
          380  +  if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
          381  +    sqlite3Fts5ExprFree(pCsr->pExpr);
          382  +  }
   345    383   
   346    384     for(pData=pCsr->pAuxdata; pData; pData=pNext){
   347    385       pNext = pData->pNext;
   348    386       if( pData->xDelete ) pData->xDelete(pData->pPtr);
   349    387       sqlite3_free(pData);
   350    388     }
   351    389   
................................................................................
   353    391     for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
   354    392     *pp = pCsr->pNext;
   355    393   
   356    394     sqlite3_free(pCsr);
   357    395     return SQLITE_OK;
   358    396   }
   359    397   
          398  +static int fts5SorterNext(Fts5Cursor *pCsr){
          399  +  Fts5Sorter *pSorter = pCsr->pSorter;
          400  +  int rc;
          401  +
          402  +  rc = sqlite3_step(pSorter->pStmt);
          403  +  if( rc==SQLITE_DONE ){
          404  +    rc = SQLITE_OK;
          405  +    CsrFlagSet(pCsr, FTS5CSR_EOF);
          406  +  }else if( rc==SQLITE_ROW ){
          407  +    rc = SQLITE_OK;
          408  +    pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
          409  +  }
          410  +
          411  +  return rc;
          412  +}
   360    413   
   361    414   /*
   362    415   ** Advance the cursor to the next row in the table that matches the 
   363    416   ** search criteria.
   364    417   **
   365    418   ** Return SQLITE_OK if nothing goes wrong.  SQLITE_OK is returned
   366    419   ** even if we reach end-of-file.  The fts5EofMethod() will be called
................................................................................
   367    420   ** subsequently to determine whether or not an EOF was hit.
   368    421   */
   369    422   static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   370    423     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   371    424     int ePlan = FTS5_PLAN(pCsr->idxNum);
   372    425     int rc = SQLITE_OK;
   373    426   
   374         -  if( ePlan!=FTS5_PLAN_MATCH ){
   375         -    rc = sqlite3_step(pCsr->pStmt);
   376         -    if( rc!=SQLITE_ROW ){
   377         -      CsrFlagSet(pCsr, FTS5CSR_EOF);
   378         -      rc = sqlite3_reset(pCsr->pStmt);
   379         -    }else{
   380         -      rc = SQLITE_OK;
   381         -    }
   382         -  }else{
   383         -    rc = sqlite3Fts5ExprNext(pCsr->pExpr);
   384         -    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
   385         -      CsrFlagSet(pCsr, FTS5CSR_EOF);
   386         -    }
   387         -    CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
   388         -  }
          427  +  switch( ePlan ){
          428  +    case FTS5_PLAN_MATCH:
          429  +    case FTS5_PLAN_SOURCE:
          430  +      rc = sqlite3Fts5ExprNext(pCsr->pExpr);
          431  +      if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
          432  +        CsrFlagSet(pCsr, FTS5CSR_EOF);
          433  +      }
          434  +      CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
          435  +      break;
          436  +
          437  +    case FTS5_PLAN_SORTED_MATCH: {
          438  +      rc = fts5SorterNext(pCsr);
          439  +      break;
          440  +    }
          441  +
          442  +    default:
          443  +      rc = sqlite3_step(pCsr->pStmt);
          444  +      if( rc!=SQLITE_ROW ){
          445  +        CsrFlagSet(pCsr, FTS5CSR_EOF);
          446  +        rc = sqlite3_reset(pCsr->pStmt);
          447  +      }else{
          448  +        rc = SQLITE_OK;
          449  +      }
          450  +      break;
          451  +  }
          452  +  
          453  +  return rc;
          454  +}
          455  +
          456  +static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){
          457  +  Fts5Config *pConfig = pTab->pConfig;
          458  +  Fts5Sorter *pSorter;
          459  +  int nPhrase;
          460  +  int nByte;
          461  +  int rc = SQLITE_OK;
          462  +  char *zSql;
   389    463     
          464  +  nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
          465  +  nByte = sizeof(Fts5Sorter) + sizeof(int) * nPhrase;
          466  +  pSorter = (Fts5Sorter*)sqlite3_malloc(nByte);
          467  +  if( pSorter==0 ) return SQLITE_NOMEM;
          468  +  memset(pSorter, 0, nByte);
          469  +
          470  +  zSql = sqlite3_mprintf("SELECT rowid, %Q FROM %Q.%Q ORDER BY +%s %s",
          471  +      pConfig->zName, pConfig->zDb, pConfig->zName, FTS5_RANK_NAME,
          472  +      bAsc ? "ASC" : "DESC"
          473  +  );
          474  +  if( zSql==0 ){
          475  +    rc = SQLITE_NOMEM;
          476  +  }else{
          477  +    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
          478  +    sqlite3_free(zSql);
          479  +  }
          480  +
          481  +  pCsr->pSorter = pSorter;
          482  +  if( rc==SQLITE_OK ){
          483  +    assert( pTab->pSortCsr==0 );
          484  +    pTab->pSortCsr = pCsr;
          485  +    rc = fts5SorterNext(pCsr);
          486  +    pTab->pSortCsr = 0;
          487  +  }
          488  +
          489  +  if( rc!=SQLITE_OK ){
          490  +    sqlite3_finalize(pSorter->pStmt);
          491  +    sqlite3_free(pSorter);
          492  +    pCsr->pSorter = 0;
          493  +  }
          494  +
   390    495     return rc;
   391    496   }
   392    497   
   393    498   static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){
   394    499     int rc;
   395    500     rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
   396    501     if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
................................................................................
   410    515     int idxNum,                     /* Strategy index */
   411    516     const char *idxStr,             /* Unused */
   412    517     int nVal,                       /* Number of elements in apVal */
   413    518     sqlite3_value **apVal           /* Arguments for the indexing scheme */
   414    519   ){
   415    520     Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
   416    521     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   417         -  int rc = SQLITE_OK;
   418         -  int ePlan = FTS5_PLAN(idxNum);
   419         -  int eStmt = fts5StmtType(idxNum);
   420    522     int bAsc = ((idxNum & FTS5_ORDER_ASC) ? 1 : 0);
          523  +  int rc = SQLITE_OK;
   421    524   
   422         -  pCsr->idxNum = idxNum;
   423    525     assert( pCsr->pStmt==0 );
   424    526     assert( pCsr->pExpr==0 );
   425    527     assert( pCsr->csrflags==0 );
   426         -
   427         -  rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
   428         -  if( rc==SQLITE_OK ){
   429         -    if( ePlan==FTS5_PLAN_MATCH ){
   430         -      char **pzErr = &pTab->base.zErrMsg;
   431         -      const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
   432         -      rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
   433         -      if( rc==SQLITE_OK ){
   434         -        rc = fts5CursorFirst(pTab, pCsr, bAsc);
   435         -      }
   436         -    }else{
   437         -      if( ePlan==FTS5_PLAN_ROWID ){
   438         -        sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
   439         -      }
   440         -      rc = fts5NextMethod(pCursor);
          528  +  assert( pCsr->pRank==0 );
          529  +
          530  +  if( pTab->pSortCsr ){
          531  +    pCsr->idxNum = FTS5_PLAN_SOURCE;
          532  +    pCsr->pRank = pTab->pSortCsr->pRank;
          533  +    pCsr->pExpr = pTab->pSortCsr->pExpr;
          534  +    rc = fts5CursorFirst(pTab, pCsr, bAsc);
          535  +  }else{
          536  +    int ePlan = FTS5_PLAN(idxNum);
          537  +    int eStmt = fts5StmtType(idxNum);
          538  +    pCsr->idxNum = idxNum;
          539  +    rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
          540  +    if( rc==SQLITE_OK ){
          541  +      if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){
          542  +        char **pzErr = &pTab->base.zErrMsg;
          543  +        const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
          544  +        pCsr->pRank = pTab->pGlobal->pAux;
          545  +        rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
          546  +        if( rc==SQLITE_OK ){
          547  +          if( ePlan==FTS5_PLAN_MATCH ){
          548  +            rc = fts5CursorFirst(pTab, pCsr, bAsc);
          549  +          }else{
          550  +            rc = fts5CursorFirstSorted(pTab, pCsr, bAsc);
          551  +          }
          552  +        }
          553  +      }else{
          554  +        if( ePlan==FTS5_PLAN_ROWID ){
          555  +          sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
          556  +        }
          557  +        rc = fts5NextMethod(pCursor);
          558  +      }
   441    559       }
   442    560     }
   443    561   
   444    562     return rc;
   445    563   }
   446    564   
   447    565   /* 
................................................................................
   460    578   ** rowid should be written to *pRowid.
   461    579   */
   462    580   static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
   463    581     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   464    582     int ePlan = FTS5_PLAN(pCsr->idxNum);
   465    583     
   466    584     assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
   467         -  if( ePlan!=FTS5_PLAN_MATCH ){
   468         -    *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
   469         -  }else{
   470         -    *pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
          585  +  switch( ePlan ){
          586  +    case FTS5_PLAN_SOURCE:
          587  +    case FTS5_PLAN_MATCH:
          588  +      *pRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
          589  +      break;
          590  +
          591  +    case FTS5_PLAN_SORTED_MATCH:
          592  +      *pRowid = pCsr->pSorter->iRowid;
          593  +      break;
          594  +
          595  +    default:
          596  +      *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
          597  +      break;
   471    598     }
   472    599   
   473    600     return SQLITE_OK;
   474    601   }
   475    602   
   476    603   /*
   477    604   ** If the cursor requires seeking (bSeekRequired flag is set), seek it.
................................................................................
   490    617       }else{
   491    618         rc = sqlite3_reset(pCsr->pStmt);
   492    619         if( rc==SQLITE_OK ){
   493    620           rc = SQLITE_CORRUPT_VTAB;
   494    621         }
   495    622       }
   496    623     }
   497         -  return rc;
   498         -}
   499         -
   500         -/* 
   501         -** This is the xColumn method, called by SQLite to request a value from
   502         -** the row that the supplied cursor currently points to.
   503         -*/
   504         -static int fts5ColumnMethod(
   505         -  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
   506         -  sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
   507         -  int iCol                        /* Index of column to read value from */
   508         -){
   509         -  Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig;
   510         -  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
   511         -  int rc = SQLITE_OK;
   512         -  
   513         -  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
   514         -
   515         -  if( iCol==pConfig->nCol ){
   516         -    /* User is requesting the value of the special column with the same name
   517         -    ** as the table. Return the cursor integer id number. This value is only
   518         -    ** useful in that it may be passed as the first argument to an FTS5
   519         -    ** auxiliary function.  */
   520         -    sqlite3_result_int64(pCtx, pCsr->iCsrId);
   521         -  }else{
   522         -    rc = fts5SeekCursor(pCsr);
   523         -    if( rc==SQLITE_OK ){
   524         -      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
   525         -    }
   526         -  }
   527    624     return rc;
   528    625   }
   529    626   
   530    627   /*
   531    628   ** This function is called to handle an FTS INSERT command. In other words,
   532    629   ** an INSERT statement of the form:
   533    630   **
................................................................................
   569    666   ){
   570    667     Fts5Table *pTab = (Fts5Table*)pVtab;
   571    668     Fts5Config *pConfig = pTab->pConfig;
   572    669     int eType0;                     /* value_type() of apVal[0] */
   573    670     int eConflict;                  /* ON CONFLICT for this DML */
   574    671     int rc = SQLITE_OK;             /* Return code */
   575    672   
   576         -  assert( nArg==1 || nArg==(2 + pConfig->nCol + 1) );
          673  +  /* A delete specifies a single argument - the rowid of the row to remove.
          674  +  ** Update and insert operations pass:
          675  +  **
          676  +  **   1. The "old" rowid, or NULL.
          677  +  **   2. The "new" rowid.
          678  +  **   3. Values for each of the nCol matchable columns.
          679  +  **   4. Values for the two hidden columns (<tablename> and "rank").
          680  +  */
          681  +  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
   577    682   
   578    683     if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
   579    684       return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]);
   580    685     }
   581    686   
   582    687     eType0 = sqlite3_value_type(apVal[0]);
   583    688     eConflict = sqlite3_vtab_on_conflict(pConfig->db);
................................................................................
   835    940         }
   836    941       }
   837    942     }
   838    943   
   839    944     fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
   840    945     return rc;
   841    946   }
          947  +
          948  +static void fts5ApiInvoke(
          949  +  Fts5Auxiliary *pAux,
          950  +  Fts5Cursor *pCsr,
          951  +  sqlite3_context *context,
          952  +  int argc,
          953  +  sqlite3_value **argv
          954  +){
          955  +  assert( pCsr->pAux==0 );
          956  +  pCsr->pAux = pAux;
          957  +  pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv);
          958  +  pCsr->pAux = 0;
          959  +}
   842    960   
   843    961   static void fts5ApiCallback(
   844    962     sqlite3_context *context,
   845    963     int argc,
   846    964     sqlite3_value **argv
   847    965   ){
   848    966   
................................................................................
   857    975     for(pCsr=pAux->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
   858    976       if( pCsr->iCsrId==iCsrId ) break;
   859    977     }
   860    978     if( pCsr==0 ){
   861    979       char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
   862    980       sqlite3_result_error(context, zErr, -1);
   863    981     }else{
   864         -    assert( pCsr->pAux==0 );
   865         -    pCsr->pAux = pAux;
   866         -    pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc-1, &argv[1]);
   867         -    pCsr->pAux = 0;
          982  +    fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
          983  +  }
          984  +}
          985  +
          986  +/* 
          987  +** This is the xColumn method, called by SQLite to request a value from
          988  +** the row that the supplied cursor currently points to.
          989  +*/
          990  +static int fts5ColumnMethod(
          991  +  sqlite3_vtab_cursor *pCursor,   /* Cursor to retrieve value from */
          992  +  sqlite3_context *pCtx,          /* Context for sqlite3_result_xxx() calls */
          993  +  int iCol                        /* Index of column to read value from */
          994  +){
          995  +  Fts5Config *pConfig = ((Fts5Table*)(pCursor->pVtab))->pConfig;
          996  +  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
          997  +  int rc = SQLITE_OK;
          998  +  
          999  +  assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
         1000  +
         1001  +  if( iCol==pConfig->nCol ){
         1002  +    if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){
         1003  +      /* todo */
         1004  +    }else{
         1005  +      /* User is requesting the value of the special column with the same name
         1006  +      ** as the table. Return the cursor integer id number. This value is only
         1007  +      ** useful in that it may be passed as the first argument to an FTS5
         1008  +      ** auxiliary function.  */
         1009  +      sqlite3_result_int64(pCtx, pCsr->iCsrId);
         1010  +    }
         1011  +  }else if( iCol==pConfig->nCol+1 ){
         1012  +    /* The value of the "rank" column. */
         1013  +    if( pCsr->pRank ){
         1014  +      fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, 0, 0);
         1015  +    }
         1016  +  }else{
         1017  +    rc = fts5SeekCursor(pCsr);
         1018  +    if( rc==SQLITE_OK ){
         1019  +      sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
         1020  +    }
   868   1021     }
         1022  +  return rc;
   869   1023   }
   870   1024   
   871   1025   
   872   1026   /*
   873   1027   ** This routine implements the xFindFunction method for the FTS3
   874   1028   ** virtual table.
   875   1029   */

Changes to ext/fts5/fts5Int.h.

    24     24   ** less than 32. If it is set to anything large than that, an #error
    25     25   ** directive in fts5_index.c will cause the build to fail.
    26     26   */
    27     27   #define FTS5_MAX_PREFIX_INDEXES 31
    28     28   
    29     29   #define FTS5_DEFAULT_NEARDIST 10
    30     30   
           31  +/* Name of rank column */
           32  +#define FTS5_RANK_NAME "rank"
           33  +
    31     34   /**************************************************************************
    32     35   ** Interface to code in fts5_config.c. fts5_config.c contains contains code
    33     36   ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
    34     37   */
    35     38   
    36     39   typedef struct Fts5Config Fts5Config;
    37     40   
................................................................................
   390    393   void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
   391    394   
   392    395   void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
   393    396   void sqlite3Fts5ParseSetColumn(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
   394    397   void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
   395    398   void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
   396    399   
   397         -
   398    400   /*
   399    401   ** End of interface to code in fts5_expr.c.
   400    402   **************************************************************************/
   401    403   
   402    404   
   403    405   /**************************************************************************
   404    406   ** Interface to code in fts5.c. 
................................................................................
   419    421   
   420    422   /**************************************************************************
   421    423   ** Interface to code in fts5_aux.c. 
   422    424   */
   423    425   
   424    426   int sqlite3Fts5AuxInit(Fts5Global*);
   425    427   /*
   426         -** End of interface to code in fts5_expr.c.
          428  +** End of interface to code in fts5_aux.c.
          429  +**************************************************************************/
          430  +
          431  +/**************************************************************************
          432  +** Interface to code in fts5_sorter.c. 
          433  +*/
          434  +typedef struct Fts5Sorter Fts5Sorter;
          435  +
          436  +int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp);
          437  +
          438  +/*
          439  +** End of interface to code in fts5_sorter.c.
   427    440   **************************************************************************/
   428    441   
   429    442   #endif

Changes to ext/fts5/fts5_aux.c.

   952    952   int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
   953    953     struct Builtin {
   954    954       const char *zFunc;            /* Function name (nul-terminated) */
   955    955       void *pUserData;              /* User-data pointer */
   956    956       fts5_extension_function xFunc;/* Callback function */
   957    957       void (*xDestroy)(void*);      /* Destructor function */
   958    958     } aBuiltin [] = {
   959         -    { "bm25",      0, fts5Bm25Function,    0 },
   960    959       { "bm25debug", (void*)1, fts5Bm25Function,    0 },
   961    960       { "snippet",   0, fts5SnippetFunction, 0 },
   962    961       { "fts5_test", 0, fts5TestFunction,    0 },
          962  +    { "bm25",      0, fts5Bm25Function,    0 },
   963    963     };
   964    964   
   965    965     int rc = SQLITE_OK;             /* Return code */
   966    966     int i;                          /* To iterate through builtin functions */
   967    967   
   968    968     for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
   969    969       rc = sqlite3Fts5CreateAux(pGlobal, 

Changes to ext/fts5/fts5_config.c.

   161    161     if( pRet==0 ) return SQLITE_NOMEM;
   162    162     memset(pRet, 0, sizeof(Fts5Config));
   163    163     pRet->db = db;
   164    164   
   165    165     pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg);
   166    166     pRet->zDb = fts5Strdup(azArg[1]);
   167    167     pRet->zName = fts5Strdup(azArg[2]);
   168         -  if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
          168  +  if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
          169  +    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
          170  +    rc = SQLITE_ERROR;
          171  +  }else if( pRet->azCol==0 || pRet->zDb==0 || pRet->zName==0 ){
   169    172       rc = SQLITE_NOMEM;
   170    173     }else{
   171    174       int i;
   172    175       for(i=3; rc==SQLITE_OK && i<nArg; i++){
   173    176         char *zDup = fts5Strdup(azArg[i]);
   174    177         if( zDup==0 ){
   175    178           rc = SQLITE_NOMEM;
................................................................................
   185    188               sqlite3Fts5Dequote(zArg);
   186    189               rc = fts5ConfigParseSpecial(pRet, zDup, zArg, pzErr);
   187    190               sqlite3_free(zDup);
   188    191               zDup = 0;
   189    192             }
   190    193           }
   191    194   
   192         -        /* If it is not a special directive, it must be a column name */
          195  +        /* If it is not a special directive, it must be a column name. In
          196  +        ** this case, check that it is not the reserved column name "rank". */
   193    197           if( zDup ){
   194    198             sqlite3Fts5Dequote(zDup);
   195    199             pRet->azCol[pRet->nCol++] = zDup;
          200  +          if( sqlite3_stricmp(zDup, FTS5_RANK_NAME)==0 ){
          201  +            *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zDup);
          202  +            rc = SQLITE_ERROR;
          203  +          }
   196    204           }
   197    205         }
   198    206       }
   199    207     }
   200    208   
   201    209     if( rc==SQLITE_OK && pRet->pTokenizer==0 ){
   202    210       rc = fts5ConfigDefaultTokenizer(pRet);
................................................................................
   245    253       zOld = zSql;
   246    254       zSql = sqlite3_mprintf("%s%s%Q", zOld, (i==0?"":", "), pConfig->azCol[i]);
   247    255       sqlite3_free(zOld);
   248    256     }
   249    257   
   250    258     if( zSql ){
   251    259       zOld = zSql;
   252         -    zSql = sqlite3_mprintf("%s, %Q HIDDEN)", zOld, pConfig->zName);
          260  +    zSql = sqlite3_mprintf("%s, %Q HIDDEN, %s HIDDEN)", 
          261  +        zOld, pConfig->zName, FTS5_RANK_NAME
          262  +    );
   253    263       sqlite3_free(zOld);
   254    264     }
   255    265   
   256    266     if( zSql==0 ){
   257    267       rc = SQLITE_NOMEM;
   258    268     }else{
   259    269       rc = sqlite3_declare_vtab(pConfig->db, zSql);

Changes to ext/fts5/fts5_expr.c.

    30     30   void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
    31     31   
    32     32   struct Fts5Expr {
    33     33     Fts5Index *pIndex;
    34     34     Fts5ExprNode *pRoot;
    35     35     int bAsc;
    36     36     int nPhrase;                    /* Number of phrases in expression */
    37         -  Fts5ExprPhrase **apPhrase;      /* Pointers to phrase objects */
           37  +  Fts5ExprPhrase **apExprPhrase;  /* Pointers to phrase objects */
    38     38   };
    39     39   
    40     40   /*
    41     41   ** eType:
    42     42   **   Expression node type. Always one of:
    43     43   **
    44     44   **       FTS5_AND                 (pLeft, pRight valid)
................................................................................
   212    212     if( sParse.rc==SQLITE_OK ){
   213    213       *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr));
   214    214       if( pNew==0 ){
   215    215         sParse.rc = SQLITE_NOMEM;
   216    216       }else{
   217    217         pNew->pRoot = sParse.pExpr;
   218    218         pNew->pIndex = 0;
   219         -      pNew->apPhrase = sParse.apPhrase;
          219  +      pNew->apExprPhrase = sParse.apPhrase;
   220    220         pNew->nPhrase = sParse.nPhrase;
   221    221         sParse.apPhrase = 0;
   222    222       }
   223    223     }
   224    224   
   225    225     sqlite3_free(sParse.apPhrase);
   226    226     *pzErr = sParse.zErr;
................................................................................
   271    271     /* Components of the new expression object */
   272    272     Fts5Expr *pNew;
   273    273     Fts5ExprPhrase **apPhrase;
   274    274     Fts5ExprNode *pNode;
   275    275     Fts5ExprNearset *pNear;
   276    276     Fts5ExprPhrase *pCopy;
   277    277   
   278         -  pOrig = pExpr->apPhrase[iPhrase];
          278  +  pOrig = pExpr->apExprPhrase[iPhrase];
   279    279     pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
   280    280     apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
   281    281     pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
   282    282     pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
   283    283         sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
   284    284     );
   285    285     pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
................................................................................
   292    292     }
   293    293   
   294    294     if( rc==SQLITE_OK ){
   295    295       /* All the allocations succeeded. Put the expression object together. */
   296    296       pNew->pIndex = pExpr->pIndex;
   297    297       pNew->pRoot = pNode;
   298    298       pNew->nPhrase = 1;
   299         -    pNew->apPhrase = apPhrase;
   300         -    pNew->apPhrase[0] = pCopy;
          299  +    pNew->apExprPhrase = apPhrase;
          300  +    pNew->apExprPhrase[0] = pCopy;
   301    301   
   302    302       pNode->eType = FTS5_STRING;
   303    303       pNode->pNear = pNear;
   304    304   
   305    305       pNear->iCol = -1;
   306    306       pNear->nPhrase = 1;
   307    307       pNear->apPhrase[0] = pCopy;
................................................................................
   341    341   
   342    342   /*
   343    343   ** Free the expression object passed as the only argument.
   344    344   */
   345    345   void sqlite3Fts5ExprFree(Fts5Expr *p){
   346    346     if( p ){
   347    347       sqlite3Fts5ParseNodeFree(p->pRoot);
   348         -    sqlite3_free(p->apPhrase);
          348  +    sqlite3_free(p->apExprPhrase);
   349    349       sqlite3_free(p);
   350    350     }
   351    351   }
   352    352   
   353    353   /*
   354    354   ** All individual term iterators in pPhrase are guaranteed to be valid and
   355    355   ** pointing to the same rowid when this function is called. This function 
................................................................................
  1584   1584   }
  1585   1585   
  1586   1586   /*
  1587   1587   ** Return the number of terms in the iPhrase'th phrase in pExpr.
  1588   1588   */
  1589   1589   int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
  1590   1590     if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;
  1591         -  return pExpr->apPhrase[iPhrase]->nTerm;
         1591  +  return pExpr->apExprPhrase[iPhrase]->nTerm;
  1592   1592   }
  1593   1593   
  1594   1594   /*
  1595   1595   ** This function is used to access the current position list for phrase
  1596   1596   ** iPhrase.
  1597   1597   */
  1598   1598   int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
  1599   1599     if( iPhrase>=0 && iPhrase<pExpr->nPhrase ){
  1600         -    Fts5ExprPhrase *pPhrase = pExpr->apPhrase[iPhrase];
         1600  +    Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
  1601   1601       Fts5ExprNode *pNode = pPhrase->pNode;
  1602   1602       if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
  1603   1603         *pa = pPhrase->poslist.p;
  1604   1604         return pPhrase->poslist.n;
  1605   1605       }
  1606   1606     }
  1607   1607     *pa = 0;
  1608   1608     return 0;
  1609   1609   }
  1610   1610   

Changes to test/fts5aa.test.

   271    271   foreach {rowid x y} $d10 {
   272    272     do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
   273    273     do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
   274    274   }
   275    275   
   276    276   do_execsql_test 10.4.1 { DELETE FROM t1 }
   277    277   do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
          278  +
          279  +#-------------------------------------------------------------------------
          280  +#
          281  +do_catchsql_test 11.1 {
          282  +  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
          283  +} {1 {reserved fts5 column name: rank}}
          284  +do_catchsql_test 11.2 {
          285  +  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
          286  +} {1 {reserved fts5 table name: rank}}
   278    287   
   279    288   finish_test
   280    289   

Changes to test/fts5ae.test.

   257    257     1 {a} {0 2}
   258    258     2 {b} {3 1}
   259    259     3 {c} {1 0}
   260    260     4 {d} {2 3}
   261    261     5 {g AND (e OR f)} {5 4}
   262    262     6 {j AND (h OR i)} {5 6}
   263    263   } {
   264         -  do_execsql_test 8.2.$tn {
          264  +  do_execsql_test 8.2.$tn.1 {
   265    265       SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8) DESC;
   266    266     } $res
          267  +
          268  +  do_execsql_test 8.2.$tn.2 {
          269  +    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank DESC;
          270  +  } $res
          271  +
          272  +  do_execsql_test 8.3.$tn.3 {
          273  +    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank DESC;
          274  +  } $res
   267    275   }
   268    276   
   269    277   
   270    278   finish_test
   271    279