SQLite

static int fts5ApiTokenize(
  Fts5Context *pCtx, 
  const char *pText, int nText, 
  void *pUserData,
  int (*xToken)(void*, const char*, int, int, int, int)
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken);
}

static int fts5ApiPhraseCount(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
}

**   are numbered starting from zero.
**
** xRowid:
**   Returns the rowid of the current row.
**
** xPoslist:
**   Iterate through instances of phrase iPhrase in the current row. 
**
** xTokenize:
**   Tokenize text using the tokenizer belonging to the FTS5 table.
*/
struct Fts5ExtensionApi {
  int iVersion;                   /* Currently always set to 1 */

  void *(*xUserData)(Fts5Context*);

  int (*xColumnCount)(Fts5Context*);

  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  assert( 0 );
}

static int fts5TestCallback(
  void *pContext,                 /* Pointer to Fts5Buffer object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd,                       /* End offset of token */
  int iPos                        /* Position offset of token */
){
  int rc = SQLITE_OK;
  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
  if( pBuf->n!=0 ){
    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
  }
  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
  return rc;
}


static void fts5TestFunction(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
    iRowid = pApi->xRowid(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " tokenize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
    Fts5Buffer buf;
    memset(&buf, 0, sizeof(buf));
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
      rc = pApi->xColumnText(pFts, i, &z, &n);
      if( rc==SQLITE_OK ){
        Fts5Buffer buf1;
        memset(&buf1, 0, sizeof(Fts5Buffer));
        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
        sqlite3_free(buf1.p);
      }
    }
    if( zReq==0 ){
      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
    }else{
      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
    }
    sqlite3_free(buf.p);
  }

  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  sqlite3Fts5BufferFree(&s);

  ORDER BY rowid DESC;
} {
  3 {2 3}
  2 {2 3}
  1 {2 3}
}

#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}

do_execsql_test 6.2 {
  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
} {
  2 {{horatio than are} {dreamt of in your philosophy}}
  1 {{there are more} {things in heaven and earth}}
}

finish_test