SQLite

TESTSRC += \
  $(TOP)/ext/misc/amatch.c \
  $(TOP)/ext/misc/closure.c \
  $(TOP)/ext/misc/eval.c \
  $(TOP)/ext/misc/fileio.c \
  $(TOP)/ext/misc/fuzzer.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  $(TOP)/ext/misc/ieee754.c \
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/wholenumber.c

  $(TOP)\ext\misc\amatch.c \
  $(TOP)\ext\misc\closure.c \
  $(TOP)\ext\misc\eval.c \
  $(TOP)\ext\misc\fileio.c \
  $(TOP)\ext\misc\fuzzer.c \
  fts5.c \
  $(TOP)\ext\fts5\fts5_tcl.c \
  $(TOP)\ext\fts5\fts5_test_mi.c \
  $(TOP)\ext\misc\ieee754.c \
  $(TOP)\ext\misc\nextchar.c \
  $(TOP)\ext\misc\percentile.c \
  $(TOP)\ext\misc\regexp.c \
  $(TOP)\ext\misc\spellfix.c \
  $(TOP)\ext\misc\totype.c \
  $(TOP)\ext\misc\wholenumber.c

**   the FTS5 table.
**
**   If parameter iCol is greater than or equal to the number of columns
**   in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
**   an OOM condition or IO error), an appropriate SQLite error code is 
**   returned.
**
** xColumnCount(pFts):
**   Return the number of columns in the table.
**
** xColumnSize(pFts, iCol, pnToken):
**   If parameter iCol is less than zero, set output variable *pnToken
**   to the total number of tokens in the current row. Or, if iCol is
**   non-negative but less than the number of columns in the table, set
**   *pnToken to the number of tokens in column iCol of the current row.
**
**   If parameter iCol is greater than or equal to the number of columns
**   in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
**   an OOM condition or IO error), an appropriate SQLite error code is 
**   returned.
**



** xColumnText:
**   This function attempts to retrieve the text of column iCol of the
**   current document. If successful, (*pz) is set to point to a buffer
**   containing the text in utf-8 encoding, (*pn) is set to the size in bytes
**   (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
**   if an error occurs, an SQLite error code is returned and the final values
**   of (*pz) and (*pn) are undefined.

  }
  z = Tcl_GetStringFromObj(objv[2], &n);

  iVal = f5t_fts5HashKey(nSlot, z, n);
  Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
  return TCL_OK;
}

static int f5tRegisterMatchinfo(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  int rc;
  sqlite3 *db = 0;

  if( objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB");
    return TCL_ERROR;
  }
  if( f5tDbPointer(interp, objv[1], &db) ){
    return TCL_ERROR;
  }

  rc = sqlite3Fts5TestRegisterMatchinfo(db);
  if( rc!=SQLITE_OK ){
    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
    return TCL_ERROR;
  }
  return TCL_OK;
}

/*
** Entry point.
*/
int Fts5tcl_Init(Tcl_Interp *interp){
  static struct Cmd {
    char *zName;
    Tcl_ObjCmdProc *xProc;
    int bTokenizeCtx;
  } aCmd[] = {
    { "sqlite3_fts5_create_tokenizer",   f5tCreateTokenizer, 1 },
    { "sqlite3_fts5_token",              f5tTokenizerReturn, 1 },
    { "sqlite3_fts5_tokenize",           f5tTokenize, 0 },
    { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
    { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
    { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }
  };
  int i;
  F5tTokenizerContext *pContext;

  pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext));
  memset(pContext, 0, sizeof(*pContext));

/*
** 2015 Aug 04
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains test code only, it is not included in release 
** versions of FTS5. It contains the implementation of an FTS5 auxiliary
** function very similar to the FTS4 function matchinfo():
**
**     https://www.sqlite.org/fts3.html#matchinfo
**
** Known differences are that:
**
**  1) this function uses the FTS5 definition of "matchable phrase", which
**     excludes any phrases that are part of an expression sub-tree that
**     does not match the current row. This comes up for MATCH queries 
**     such as:
**
**         "a OR (b AND c)"
**
**     In FTS4, if a single row contains instances of tokens "a" and "c", 
**     but not "b", all instances of "c" are considered matches. In FTS5,
**     they are not (as the "b AND c" sub-tree does not match the current
**     row.
**
**  2) ...
**     
** This file exports a single function that may be called to register the
** matchinfo() implementation with a database handle:
**
**   int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db);
*/


#ifdef SQLITE_TEST
#ifdef SQLITE_ENABLE_FTS5

#include "fts5.h"
#include <tcl.h>
#include <assert.h>
#include <string.h>

typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
typedef unsigned int u32;

struct Fts5MatchinfoCtx {
  int nCol;                       /* Number of cols in FTS5 table */
  int nPhrase;                    /* Number of phrases in FTS5 query */
  char *zArg;                     /* nul-term'd copy of 2nd arg */
  int nRet;                       /* Number of elements in aRet[] */
  u32 *aRet;                      /* Array of 32-bit unsigned ints to return */
};



/*
** Return a pointer to the fts5_api pointer for database connection db.
** If an error occurs, return NULL and leave an error in the database 
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
static fts5_api *fts5_api_from_db(sqlite3 *db){
  fts5_api *pRet = 0;
  sqlite3_stmt *pStmt = 0;

  if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
   && SQLITE_ROW==sqlite3_step(pStmt) 
   && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
  ){
    memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
  }
  sqlite3_finalize(pStmt);
  return pRet;
}


/*
** Argument f should be a flag accepted by matchinfo() (a valid character
** in the string passed as the second argument). If it is not, 0 is 
** returned. Otherwise, if f is a valid matchinfo flag, the value returned
** is the number of 32-bit integers added to the output array if the
** table has nCol columns and the query nPhrase phrases.
*/
static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
  int ret = 0;
  switch( f ){
    case 'p': ret = 1; break;
    case 'c': ret = 1; break;
    case 'x': ret = 3 * nCol * nPhrase; break;
    case 'y': ret = nCol * nPhrase; break;
    case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
    case 'n': ret = 1; break;
    case 'a': ret = nCol; break;
    case 'l': ret = nCol; break;
    case 's': ret = nCol; break;
  }
  return ret;
}

static int fts5MatchinfoIter(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  Fts5MatchinfoCtx *p,
  int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*)
){
  int i;
  int n = 0;
  int rc = SQLITE_OK;
  char f;
  for(i=0; (f = p->zArg[i]); i++){
    rc = x(pApi, pFts, p, f, &p->aRet[n]);
    if( rc!=SQLITE_OK ) break;
    n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f);
  }
  return rc;
}

static int fts5MatchinfoXCb(
  const Fts5ExtensionApi *pApi,
  Fts5Context *pFts,
  void *pUserData
){
  u32 *aOut = (u32*)pUserData;
  int nCol = pApi->xColumnCount(pFts);
  int nInst;
  int iPrev = -1;
  int rc;
  int i;

  rc = pApi->xInstCount(pFts, &nInst);
  for(i=0; rc==SQLITE_OK && i<nInst; i++){
    int iPhrase, iCol, iOff;
    rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
    aOut[iCol*3 + 1]++;
    if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
    iPrev = iCol;
  }

  return rc;
}

static int fts5MatchinfoGlobalCb(
  const Fts5ExtensionApi *pApi,
  Fts5Context *pFts,
  Fts5MatchinfoCtx *p,
  char f,
  u32 *aOut
){
  int rc = SQLITE_OK;
  switch( f ){
    case 'p':
      aOut[0] = p->nPhrase; 
      break;

    case 'c':
      aOut[0] = p->nCol; 
      break;

    case 'x': {
      int i;
      for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){
        void *pPtr = (void*)&aOut[i * p->nCol * 3];
        rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb);
      }
      break;
    }

    case 'n': {
      sqlite3_int64 nRow;
      rc = pApi->xRowCount(pFts, &nRow);
      aOut[0] = (u32)nRow;
      break;
    }

    case 'a': {
      sqlite3_int64 nRow = 0;
      rc = pApi->xRowCount(pFts, &nRow);
      if( nRow==0 ){
        memset(aOut, 0, sizeof(u32) * p->nCol);
      }else{
        int i;
        for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
          sqlite3_int64 nToken;
          rc = pApi->xColumnTotalSize(pFts, i, &nToken);
          if( rc==SQLITE_OK){
            aOut[i] = (u32)((2*nToken + nRow) / (2*nRow));
          }
        }
      }
      break;
    }

  }
  return rc;
}

static int fts5MatchinfoLocalCb(
  const Fts5ExtensionApi *pApi,
  Fts5Context *pFts,
  Fts5MatchinfoCtx *p,
  char f,
  u32 *aOut
){
  int i;
  int rc = SQLITE_OK;

  switch( f ){
    case 'b': 
    case 'x':
    case 'y': {
      int nInst;
      int nMul = (f=='x' ? 3 : 1);

      if( f=='b' ){
        int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
        for(i=0; i<nInt; i++) aOut[i] = 0;
      }else{
        for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
      }

      rc = pApi->xInstCount(pFts, &nInst);
      for(i=0; rc==SQLITE_OK && i<nInst; i++){
        int iPhrase, iOff, iCol = 0;
        rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
        if( f=='b' ){
          aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << (iCol%32));
        }else{
          aOut[nMul * (iCol + iPhrase * p->nCol)]++;
        }
      }

      break;
    }

    case 'l': {
      for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
        int nToken;
        rc = pApi->xColumnSize(pFts, i, &nToken);
        aOut[i] = (u32)nToken;
      }
      break;
    }

    case 's':
      memset(aOut, 0, sizeof(u32) * p->nCol);
      break;
  }
  return rc;
}
 
static Fts5MatchinfoCtx *fts5MatchinfoNew(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning error message */
  const char *zArg                /* Matchinfo flag string */
){
  Fts5MatchinfoCtx *p;
  int nCol;
  int nPhrase;
  int i;
  int nInt;
  int nByte;
  int rc;

  nCol = pApi->xColumnCount(pFts);
  nPhrase = pApi->xPhraseCount(pFts);

  nInt = 0;
  for(i=0; zArg[i]; i++){
    int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
    if( n==0 ){
      char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
      sqlite3_result_error(pCtx, zErr, -1);
      sqlite3_free(zErr);
      return 0;
    }
    nInt += n;
  }

  nByte = sizeof(Fts5MatchinfoCtx)          /* The struct itself */
         + sizeof(u32) * nInt               /* The p->aRet[] array */
         + (i+1);                           /* The p->zArg string */
  p = (Fts5MatchinfoCtx*)sqlite3_malloc(nByte);
  if( p==0 ){
    sqlite3_result_error_nomem(pCtx);
    return 0;
  }
  memset(p, 0, nByte);

  p->nCol = nCol;
  p->nPhrase = nPhrase;
  p->aRet = (u32*)&p[1];
  p->nRet = nInt;
  p->zArg = (char*)&p->aRet[nInt];
  memcpy(p->zArg, zArg, i);

  rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb);
  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
    sqlite3_free(p);
    p = 0;
  }

  return p;
}

static void fts5MatchinfoFunc(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  const char *zArg;
  Fts5MatchinfoCtx *p;
  int rc;

  if( nVal>0 ){
    zArg = (const char*)sqlite3_value_text(apVal[0]);
  }else{
    zArg = "pcx";
  }

  p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
  if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
    p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
    pApi->xSetAuxdata(pFts, p, sqlite3_free);
    if( p==0 ) return;
  }

  rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }else{
    /* No errors has occured, so return a copy of the array of integers. */
    int nByte = p->nRet * sizeof(u32);
    sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT);
  }
}

int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
  int rc;                         /* Return code */
  fts5_api *pApi;                 /* FTS5 API functions */

  /* Extract the FTS5 API pointer from the database handle. The 
  ** fts5_api_from_db() function above is copied verbatim from the 
  ** FTS5 documentation. Refer there for details. */
  pApi = fts5_api_from_db(db);

  /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
  ** with this database handle, or an error (OOM perhaps?) has occurred.
  **
  ** Also check that the fts5_api object is version 1 or newer (there 
  ** is no actual version of FTS5 that would return an API object of version
  ** 0, but FTS5 extensions should check the API version before using it). */
  if( pApi==0 || pApi->iVersion<1 ){
    return SQLITE_ERROR;
  }

  /* Register the implementation of matchinfo() */
  rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);

  return rc;
}

#endif /* SQLITE_ENABLE_FTS5 */
#endif /* SQLITE_TEST */

  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/wholenumber.c \
  $(TOP)/ext/misc/vfslog.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  $(TOP)/ext/fts5/fts5_test_mi.c \
  fts5.c


#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c

TESTSRC2 = \