SQLite

  }else{
    *pnToken = 0;
    rc = SQLITE_RANGE;
  }
  return rc;
}













static int fts5ApiSetAuxdata(
  Fts5Context *pCtx,              /* Fts5 context */
  void *pPtr,                     /* Pointer to save as auxdata */
  void(*xDelete)(void*)           /* Destructor for pPtr (or NULL) */
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Auxdata *pData;

  fts5ApiPhraseCount,
  fts5ApiPhraseSize,
  fts5ApiInstCount,
  fts5ApiInst,
  fts5ApiRowid,
  fts5ApiColumnText,
  fts5ApiColumnSize,

  fts5ApiQueryPhrase,
  fts5ApiSetAuxdata,
  fts5ApiGetAuxdata,
};


/*

/*
** Find a tokenizer. This is the implementation of the 
** fts5_api.xFindTokenizer() method.
*/
static int fts5FindTokenizer(
  fts5_api *pApi,                 /* Global context (one per db handle) */
  const char *zName,              /* Name of new function */
  void **ppUserData,
  fts5_tokenizer *pTokenizer      /* Populate this object */
){
  Fts5Global *pGlobal = (Fts5Global*)pApi;
  int rc = SQLITE_OK;
  Fts5TokenizerModule *pTok;

  for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
    if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
  }

  if( pTok ){
    *pTokenizer = pTok->x;
    *ppUserData = pTok->pUserData;
  }else{
    memset(pTokenizer, 0, sizeof(fts5_tokenizer));
    rc = SQLITE_ERROR;
  }

  return rc;
}

  int (*xInstCount)(Fts5Context*, int *pnInst);
  int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);

  sqlite3_int64 (*xRowid)(Fts5Context*);
  int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
  int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);


  int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
    int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
  );
  int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
  void *(*xGetAuxdata)(Fts5Context*, int bClear);
};




/* 
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/

/*************************************************************************
** CUSTOM TOKENIZERS
**

    void (*xDestroy)(void*)
  );

  /* Find an existing tokenizer */
  int (*xFindTokenizer)(
    fts5_api *pApi,
    const char *zName,
    void **ppContext,
    fts5_tokenizer *pTokenizer
  );

  /* Create a new auxiliary function */
  int (*xCreateFunction)(
    fts5_api *pApi,
    const char *zName,

** directive in fts5_index.c will cause the build to fail.
*/
#define FTS5_MAX_PREFIX_INDEXES 31

#define FTS5_DEFAULT_NEARDIST 10
#define FTS5_DEFAULT_RANK     "bm25"

/* Name of rank and rowid columns */
#define FTS5_RANK_NAME "rank"
#define FTS5_ROWID_NAME "rowid"

/**************************************************************************
** Interface to code in fts5.c. 
*/
typedef struct Fts5Global Fts5Global;

int sqlite3Fts5GetTokenizer(

#define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
#define fts5BufferAppend32(a,b,c)     sqlite3Fts5BufferAppend32(a,b,c)

/* Write and decode big-endian 32-bit integer values */
void sqlite3Fts5Put32(u8*, int);
int sqlite3Fts5Get32(const u8*);

#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)

typedef struct Fts5PoslistReader Fts5PoslistReader;
struct Fts5PoslistReader {
  /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
  int iCol;                       /* If (iCol>=0), this column only */
  const u8 *a;                    /* Position list to iterate through */
  int n;                          /* Size of buffer at a[] in bytes */
  int i;                          /* Current offset in a[] */

#include "fts5Int.h"

#define FTS5_DEFAULT_PAGE_SIZE   1000
#define FTS5_DEFAULT_AUTOMERGE      4

/* Maximum allowed page size */
#define FTS5_MAX_PAGE_SIZE (128*1024)

static int fts5_iswhitespace(char x){
  return (x==' ');
}

static int fts5_isopenquote(char x){
  return (x=='"' || x=='\'' || x=='[' || x=='`');
}

/*
** Argument pIn points to a character that is part of a nul-terminated 
** string. Return a pointer to the first character following *pIn in 
** the string that is not a white-space character.
*/
static const char *fts5ConfigSkipWhitespace(const char *pIn){
  const char *p = pIn;
  if( p ){
    while( fts5_iswhitespace(*p) ){ p++; }
  }
  return p;
}

/*
** Argument pIn points to a character that is part of a nul-terminated 
** string. Return a pointer to the first character following *pIn in 
** the string that is not a "bareword" character.
*/
static const char *fts5ConfigSkipBareword(const char *pIn){
  const char *p = pIn;
  while( *p      && *p!=' ' && *p!=':' && *p!='!' && *p!='@' 
      && *p!='#' && *p!='$' && *p!='%' && *p!='^' && *p!='&' 
      && *p!='*' && *p!='(' && *p!=')' && *p!='='
  ){
    p++;
  }
  if( p==pIn ) p = 0;
  return p;
}

static int fts5_isdigit(char a){
  return (a>='0' && a<='9');
}



static const char *fts5ConfigSkipLiteral(const char *pIn){
  const char *p = pIn;
  if( p ){
    switch( *p ){
      case 'n': case 'N':
        if( sqlite3_strnicmp("null", p, 4)==0 ){
          p = &p[4];
        }else{
          p = 0;
        }
        break;
        
      case 'x': case 'X':
        p++;
        if( *p=='\'' ){
          p++;
          while( (*p>='a' && *p<='f') 
              || (*p>='A' && *p<='F') 
              || (*p>='0' && *p<='9') 
          ){
            p++;
          }
          if( *p=='\'' && 0==((p-pIn)%2) ){
            p++;
          }else{
            p = 0;
          }
        }else{
          p = 0;
        }
        break;

      case '\'':
        p++;
        while( p ){
          if( *p=='\'' ){
            p++;
            if( *p!='\'' ) break;
          }
          p++;
          if( *p==0 ) p = 0;
        }
        break;

      default:
        /* maybe a number */
        if( *p=='+' || *p=='-' ) p++;
        while( fts5_isdigit(*p) ) p++;

        /* At this point, if the literal was an integer, the parse is 
        ** finished. Or, if it is a floating point value, it may continue
        ** with either a decimal point or an 'E' character. */
        if( *p=='.' && fts5_isdigit(p[1]) ){
          p += 2;
          while( fts5_isdigit(*p) ) p++;
        }
        if( p==pIn ) p = 0;

        break;
    }
  }

  return p;
}

static int fts5Dequote(char *z){
  char q;
  int iIn = 1;
  int iOut = 0;
  int bRet = 1;
  q = z[0];

  assert( q=='[' || q=='\'' || q=='"' || q=='`' );
  if( q=='[' ) q = ']';  

  while( z[iIn] ){
    if( z[iIn]==q ){
      if( z[iIn+1]!=q ){
        if( z[iIn+1]=='\0' ) bRet = 0;
        break;
      }
      z[iOut++] = q;
      iIn += 2;
    }else{
      z[iOut++] = z[iIn++];
    }
  }
  z[iOut] = '\0';

  return bRet;
}

/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters.  The conversion is done in-place.  If the
** input does not begin with a quote character, then this routine
** is a no-op.
**
** Examples:
**
**     "abc"   becomes   abc
**     'xyz'   becomes   xyz
**     [pqr]   becomes   pqr
**     `mno`   becomes   mno
*/
void sqlite3Fts5Dequote(char *z){
  char quote;                     /* Quote character (if any ) */

  assert( 0==fts5_iswhitespace(z[0]) );
  quote = z[0];
  if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){


    fts5Dequote(z);


  }







}

/*
** Trim any white-space from the right of nul-terminated string z.
*/
static char *fts5TrimString(char *z){
  int n = strlen(z);
  while( n>0 && fts5_iswhitespace(z[n-1]) ){
    z[--n] = '\0';
  }
  while( fts5_iswhitespace(*z) ) z++;
  return z;
}

/*
** Parse the "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message
** may be left in *pzErr. It is the responsibility of the caller to
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
  Fts5Global *pGlobal,
  Fts5Config *pConfig,            /* Configuration object to update */
  const char *zCmd,               /* Special command to parse */
  int nCmd,                       /* Size of zCmd in bytes */
  const char *zArg,               /* Argument to parse */
  char **pzErr                    /* OUT: Error message */
){
  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    int rc = SQLITE_OK;
    const char *p;
    if( pConfig->aPrefix ){
      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
      rc = SQLITE_ERROR;
    }else{
      pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
    }
    p = zArg;

        rc = SQLITE_ERROR;
      }
      pConfig->aPrefix[pConfig->nPrefix] = nPre;
      pConfig->nPrefix++;
    }
    return rc;
  }

  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    int rc = SQLITE_OK;
    const char *p = (const char*)zArg;
    int nArg = strlen(zArg) + 1;
    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
    char *pSpace = pDel;

    if( azArg && pSpace ){
      if( pConfig->pTok ){
        *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
        rc = SQLITE_ERROR;
      }else{
        for(nArg=0; p && *p; nArg++){
          const char *p2 = fts5ConfigSkipWhitespace(p);
          if( p2 && *p2=='\'' ){
            p = fts5ConfigSkipLiteral(p2);
          }else{
            p = fts5ConfigSkipBareword(p2);
          }
          if( p ){
            memcpy(pSpace, p2, p-p2);
            azArg[nArg] = pSpace;
            sqlite3Fts5Dequote(pSpace);
            pSpace += (p - p2) + 1;
            p = fts5ConfigSkipWhitespace(p);
          }
        }
        if( p==0 ){
          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
          rc = SQLITE_ERROR;
        }else{
          rc = sqlite3Fts5GetTokenizer(pGlobal, 
              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi
          );
          if( rc!=SQLITE_OK ){
            *pzErr = sqlite3_mprintf("error in tokenizer constructor");
          }
        }
      }
    }

    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;
  }

  *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
  return SQLITE_ERROR;
}

/*
** Duplicate the string passed as the only argument into a buffer allocated

/*
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
  return sqlite3Fts5GetTokenizer(
      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
  );
}

/*
** Arguments nArg/azArg contain the string arguments passed to the xCreate

  int nArg,                       /* Number of arguments */
  const char **azArg,             /* Array of nArg CREATE VIRTUAL TABLE args */
  Fts5Config **ppOut,             /* OUT: Results of parse */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE_OK;             /* Return code */
  Fts5Config *pRet;               /* New object to return */
  int i;

  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
  pRet->db = db;
  pRet->iCookie = -1;

  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
  pRet->zDb = fts5Strdup(&rc, azArg[1]);
  pRet->zName = fts5Strdup(&rc, azArg[2]);

  if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
    rc = SQLITE_ERROR;
  }


  for(i=3; rc==SQLITE_OK && i<nArg; i++){
    char *zDup = fts5Strdup(&rc, azArg[i]);
    if( zDup ){
      char *zCol = 0;
      int bParseError = 0;

      /* Check if this is a quoted column name */
      if( fts5_isopenquote(zDup[0]) ){
        bParseError = fts5Dequote(zDup);
        zCol = zDup;

      }else{
        char *z = (char*)fts5ConfigSkipBareword(zDup);
        if( *z=='\0' ){
          zCol = zDup;
        }else{
          int nCmd = z - zDup;

          z = (char*)fts5ConfigSkipWhitespace(z);

          if( *z!='=' ){
            bParseError = 1;
          }else{
            z++;
            z = fts5TrimString(z);
            if( fts5_isopenquote(*z) ){
              if( fts5Dequote(z) ) bParseError = 1;
            }else{
              char *z2 = (char*)fts5ConfigSkipBareword(z);
              if( *z2 ) bParseError = 1;
            }
            if( bParseError==0 ){
              rc = fts5ConfigParseSpecial(pGlobal, pRet, zDup, nCmd, z, pzErr);
            }
          }
        }
      }


      if( bParseError ){
        assert( *pzErr==0 );
        *pzErr = sqlite3_mprintf("parse error in \"%s\"", zDup);
        rc = SQLITE_ERROR;
      }else if( zCol ){
        if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) 
         || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) 
        ){
          *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
          rc = SQLITE_ERROR;
        }else{
          pRet->azCol[pRet->nCol++] = zCol;
          zDup = 0;
        }
      }

      sqlite3_free(zDup);
    }
  }

  /* If a tokenizer= option was successfully parsed, the tokenizer has
  ** already been allocated. Otherwise, allocate an instance of the default
  ** tokenizer (simple) now.  */
  if( rc==SQLITE_OK && pRet->pTok==0 ){
    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
  }

  if( rc!=SQLITE_OK ){
    sqlite3Fts5ConfigFree(pRet);
    *ppOut = 0;

  const char *pText, int nText,   /* Text to tokenize */
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
){
  return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
}





































































































/*
** Argument pIn points to the first character in what is expected to be
** a comma-separated list of SQL literals followed by a ')' character.
** If it actually is this, return a pointer to the ')'. Otherwise, return
** NULL to indicate a parse error.
*/
static const char *fts5ConfigSkipArgs(const char *pIn){

    if( *p!='(' ) rc = SQLITE_ERROR;
    p++;
  }
  if( rc==SQLITE_OK ){
    const char *pArgs; 
    p = fts5ConfigSkipWhitespace(p);
    pArgs = p;
    if( *p!=')' ){
      p = fts5ConfigSkipArgs(p);
      if( p==0 ){
        rc = SQLITE_ERROR;
      }else if( p!=pArgs ){
        zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
        if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
      }
    }
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(zRank);
    assert( zRankArgs==0 );
  }else{

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
**
*/

#ifdef SQLITE_TEST

#include "fts5.h"
#include <tcl.h>
#include <string.h>
#include <assert.h>

/*************************************************************************

  if( Tcl_GetCommandInfo(interp, z, &cmdInfo) ){
    p = (struct SqliteDb*)cmdInfo.objClientData;
    *ppDb = p->db;
    return TCL_OK;
  }
  return TCL_ERROR;
}

/* End of code that accesses the SqliteDb struct.
**************************************************************************/

static int f5tDbAndApi(
  Tcl_Interp *interp, 
  Tcl_Obj *pObj, 
  sqlite3 **ppDb, 
  fts5_api **ppApi
){
  sqlite3 *db = 0;
  int rc = f5tDbPointer(interp, pObj, &db);
  if( rc!=TCL_OK ){
    return TCL_ERROR;
  }else{
    sqlite3_stmt *pStmt = 0;
    fts5_api *pApi = 0;

    rc = sqlite3_prepare_v2(db, "SELECT fts5()", -1, &pStmt, 0);
    if( rc!=SQLITE_OK ){
      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
      return TCL_ERROR;
    }

    if( SQLITE_ROW==sqlite3_step(pStmt) ){
      const void *pPtr = sqlite3_column_blob(pStmt, 0);
      memcpy((void*)&pApi, pPtr, sizeof(pApi));
    }

    if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
      return TCL_ERROR;
    }

    *ppDb = db;
    *ppApi = pApi;
  }

  return TCL_OK;
}

typedef struct F5tFunction F5tFunction;
struct F5tFunction {
  Tcl_Interp *interp;
  Tcl_Obj *pScript;
};

  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  char *zName;
  Tcl_Obj *pScript;
  sqlite3 *db = 0;

  fts5_api *pApi = 0;
  F5tFunction *pCtx = 0;
  int rc;

  if( objc!=4 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
    return TCL_ERROR;
  }

  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;

  zName = Tcl_GetString(objv[2]);
  pScript = objv[3];
















  pCtx = (F5tFunction*)ckalloc(sizeof(F5tFunction));
  pCtx->interp = interp;
  pCtx->pScript = pScript;
  Tcl_IncrRefCount(pScript);

  rc = pApi->xCreateFunction(
      pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy
  );
  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
    return TCL_ERROR;
  }

  return TCL_OK;
}

static int xTokenizeCb2(
  void *pCtx, 
  const char *zToken, int nToken, 
  int iStart, int iEnd, int iPos
){
  Tcl_Obj *pRet = (Tcl_Obj*)pCtx;
  Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
  Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iStart));
  Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iEnd));
  Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
  return SQLITE_OK;
}


/*
**      sqlite3_fts5_tokenize DB TOKENIZER TEXT
**
** Description...
*/
static int f5tTokenize(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  char *zName;
  char *zText;
  int nText;
  sqlite3 *db = 0;
  fts5_api *pApi = 0;
  Fts5Tokenizer *pTok = 0;
  fts5_tokenizer tokenizer;
  Tcl_Obj *pRet = 0;
  void *pUserdata;
  int rc;

  if( objc!=4 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB NAME TEXT");
    return TCL_ERROR;
  }
  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
  zName = Tcl_GetString(objv[2]);
  zText = Tcl_GetStringFromObj(objv[3], &nText);

  rc = pApi->xFindTokenizer(pApi, zName, &pUserdata, &tokenizer);
  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "no such tokenizer: ", zName, 0);
    return TCL_ERROR;
  }

  rc = tokenizer.xCreate(pUserdata, 0, 0, &pTok);
  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
    return TCL_ERROR;
  }

  pRet = Tcl_NewObj();
  Tcl_IncrRefCount(pRet);
  rc = tokenizer.xTokenize(pTok, pRet, zText, nText, xTokenizeCb2);
  tokenizer.xDelete(pTok);
  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
    Tcl_DecrRefCount(pRet);
    return TCL_ERROR;
  }


  Tcl_SetObjResult(interp, pRet);
  Tcl_DecrRefCount(pRet);
  return TCL_OK;
}

/*************************************************************************
** Start of tokenizer wrapper.
*/

typedef struct F5tTokenizerContext F5tTokenizerContext;
typedef struct F5tTokenizerCb F5tTokenizerCb;
typedef struct F5tTokenizerModule F5tTokenizerModule;
typedef struct F5tTokenizerModule F5tTokenizerInstance;

struct F5tTokenizerContext {
  void *pCtx;
  int (*xToken)(void*, const char*, int, int, int, int);
};

struct F5tTokenizerModule {
  Tcl_Interp *interp;
  Tcl_Obj *pScript;
  F5tTokenizerContext *pContext;
};

static int f5tTokenizerCreate(
  void *pCtx, 
  const char **azArg, 
  int nArg, 
  Fts5Tokenizer **ppOut
){
  F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
  Tcl_Obj *pEval;
  int rc = TCL_OK;
  int i;

  pEval = Tcl_DuplicateObj(pMod->pScript);
  Tcl_IncrRefCount(pEval);
  for(i=0; rc==TCL_OK && i<nArg; i++){
    Tcl_Obj *pObj = Tcl_NewStringObj(azArg[i], -1);
    rc = Tcl_ListObjAppendElement(pMod->interp, pEval, pObj);
  }

  if( rc==TCL_OK ){
    rc = Tcl_EvalObjEx(pMod->interp, pEval, TCL_GLOBAL_ONLY);
  }
  Tcl_DecrRefCount(pEval);

  if( rc==TCL_OK ){
    F5tTokenizerInstance *pInst = ckalloc(sizeof(F5tTokenizerInstance));
    memset(pInst, 0, sizeof(F5tTokenizerInstance));
    pInst->interp = pMod->interp;
    pInst->pScript = Tcl_GetObjResult(pMod->interp);
    pInst->pContext = pMod->pContext;
    Tcl_IncrRefCount(pInst->pScript);
    *ppOut = (Fts5Tokenizer*)pInst;
  }

  return rc;
}


static void f5tTokenizerDelete(Fts5Tokenizer *p){
  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
  Tcl_DecrRefCount(pInst->pScript);
  ckfree(pInst);
}

static int f5tTokenizerTokenize(
  Fts5Tokenizer *p, 
  void *pCtx,
  const char *pText, int nText, 
  int (*xToken)(void*, const char*, int, int, int, int)
){
  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
  void *pOldCtx;
  int (*xOldToken)(void*, const char*, int, int, int, int);
  Tcl_Obj *pEval;
  int rc;

  pOldCtx = pInst->pContext->pCtx;
  xOldToken = pInst->pContext->xToken;

  pEval = Tcl_DuplicateObj(pInst->pScript);
  Tcl_IncrRefCount(pEval);
  rc = Tcl_ListObjAppendElement(
      pInst->interp, pEval, Tcl_NewStringObj(pText, nText)
  );
  if( rc==TCL_OK ){
    rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY);
  }
  Tcl_DecrRefCount(pEval);

  pInst->pContext->pCtx = pOldCtx;
  pInst->pContext->xToken = xOldToken;
  return rc;
}

extern const char *sqlite3ErrName(int);

/*
** sqlite3_fts5_token TEXT START END POS
*/
static int f5tTokenizerReturn(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
  int iStart;
  int iEnd;
  int iPos;
  int nToken;
  char *zToken;
  int rc;

  assert( p );
  if( objc!=5 ){
    Tcl_WrongNumArgs(interp, 1, objv, "TEXT START END POS");
    return TCL_ERROR;
  }
  if( p->xToken==0 ){
    Tcl_AppendResult(interp, 
        "sqlite3_fts5_token may only be used by tokenizer callback", 0
    );
    return TCL_ERROR;
  }

  zToken = Tcl_GetStringFromObj(objv[1], &nToken);
  if( Tcl_GetIntFromObj(interp, objv[2], &iStart) 
   || Tcl_GetIntFromObj(interp, objv[3], &iEnd) 
   || Tcl_GetIntFromObj(interp, objv[4], &iPos) 
  ){
    return TCL_ERROR;
  }

  rc = p->xToken(p->pCtx, zToken, nToken, iStart, iEnd, iPos);
  Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
  return TCL_OK;
}

static void f5tDelTokenizer(void *pCtx){
  F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
  Tcl_DecrRefCount(pMod->pScript);
  ckfree(pMod);
}

/*
**      sqlite3_fts5_create_tokenizer DB NAME SCRIPT
**
** Register a tokenizer named NAME implemented by script SCRIPT. When
** a tokenizer instance is created (fts5_tokenizer.xCreate), any tokenizer
** arguments are appended to SCRIPT and the result executed.
**
** The value returned by (SCRIPT + args) is itself a tcl script. This 
** script - call it SCRIPT2 - is executed to tokenize text using the
** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize
** text SCRIPT2 is invoked with a single argument appended to it - the
** text to tokenize.
**
** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
** token within the tokenized text.
*/
static int f5tCreateTokenizer(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData;
  sqlite3 *db;
  fts5_api *pApi;
  char *zName;
  Tcl_Obj *pScript;
  fts5_tokenizer t;
  F5tTokenizerModule *pMod;
  int rc;

  if( objc!=4 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
    return TCL_ERROR;
  }
  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
    return TCL_ERROR;
  }
  zName = Tcl_GetString(objv[2]);
  pScript = objv[3];

  t.xCreate = f5tTokenizerCreate;
  t.xTokenize = f5tTokenizerTokenize;
  t.xDelete = f5tTokenizerDelete;

  pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
  pMod->interp = interp;
  pMod->pScript = pScript;
  pMod->pContext = pContext;
  Tcl_IncrRefCount(pScript);
  rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
    return TCL_ERROR;
  }

  return TCL_OK;
}

static void xF5tFree(ClientData clientData){
  ckfree(clientData);
}

/*
** Entry point.
*/
int Fts5tcl_Init(Tcl_Interp *interp){
  static struct Cmd {
    char *zName;
    Tcl_ObjCmdProc *xProc;
    int bTokenizeCtx;
  } aCmd[] = {
    { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 },
    { "sqlite3_fts5_token",            f5tTokenizerReturn, 1 },
    { "sqlite3_fts5_tokenize",         f5tTokenize, 0 },
    { "sqlite3_fts5_create_function",  f5tCreateFunction, 0 }
  };
  int i;
  F5tTokenizerContext *pContext;

  pContext = ckalloc(sizeof(F5tTokenizerContext));
  memset(pContext, 0, sizeof(*pContext));

  for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
    struct Cmd *p = &aCmd[i];
    void *pCtx = 0;
    if( p->bTokenizeCtx ) pCtx = (void*)pContext;
    Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree));
  }

  return TCL_OK;
}

#endif

**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#include "fts5.h"
#include <string.h>
#include <assert.h>


/*
** Create a "simple" tokenizer.
*/
static int fts5SimpleCreate(
  void *pCtx, 

    is = ie+1;
  }while( is<nText && rc==SQLITE_OK );
  
  if( pFold!=aFold ) sqlite3_free(pFold);
  if( rc==SQLITE_DONE ) rc = SQLITE_OK;
  return rc;
}

/**************************************************************************
** Start of porter2 stemmer implementation.
*/

/* Any tokens larger than this (in bytes) are passed through without
** stemming. */
#define FTS5_PORTER_MAX_TOKEN 64

typedef struct PorterTokenizer PorterTokenizer;
struct PorterTokenizer {
  fts5_tokenizer tokenizer;       /* Parent tokenizer module */
  Fts5Tokenizer *pTokenizer;      /* Parent tokenizer instance */
  char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
};

/*
** Delete a "porter" tokenizer.
*/
static void fts5PorterDelete(Fts5Tokenizer *pTok){
  if( pTok ){
    PorterTokenizer *p = (PorterTokenizer*)pTok;
    if( p->pTokenizer ){
      p->tokenizer.xDelete(p->pTokenizer);
    }
    sqlite3_free(p);
  }
}

/*
** Create a "porter" tokenizer.
*/
static int fts5PorterCreate(
  void *pCtx, 
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  fts5_api *pApi = (fts5_api*)pCtx;
  int rc = SQLITE_OK;
  PorterTokenizer *pRet;
  void *pUserdata = 0;

  pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
    rc = pApi->xFindTokenizer(pApi, "simple", &pUserdata, &pRet->tokenizer);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    rc = pRet->tokenizer.xCreate(pUserdata, 0, 0, &pRet->pTokenizer);
  }

  if( rc!=SQLITE_OK ){
    fts5PorterDelete((Fts5Tokenizer*)pRet);
    pRet = 0;
  }
  *ppOut = (Fts5Tokenizer*)pRet;
  return rc;
}

typedef struct PorterContext PorterContext;
struct PorterContext {
  void *pCtx;
  int (*xToken)(void*, const char*, int, int, int, int);
  char *aBuf;
};

typedef struct PorterRule PorterRule;
struct PorterRule {
  const char *zSuffix;
  int nSuffix;
  int (*xCond)(char *zStem, int nStem);
  const char *zOutput;
  int nOutput;
};

static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
  int ret = -1;
  int nBuf = *pnBuf;
  PorterRule *p;


  for(p=aRule; p->zSuffix; p++){
    assert( strlen(p->zSuffix)==p->nSuffix );
    assert( strlen(p->zOutput)==p->nOutput );
    if( nBuf<p->nSuffix ) continue;
    if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
  }

  if( p->zSuffix ){
    int nStem = nBuf - p->nSuffix;
    if( p->xCond==0 || p->xCond(aBuf, nStem) ){
      memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
      *pnBuf = nStem + p->nOutput;
      ret = p - aRule;
    }
  }

  return ret;
}

static int fts5PorterIsVowel(char c, int bYIsVowel){
  return (
      c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y')
  );
}

static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){
  int i;
  int bCons = bPrevCons;

  /* Scan for a vowel */
  for(i=0; i<nStem; i++){
    if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break;
  }

  /* Scan for a consonent */
  for(i++; i<nStem; i++){
    if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1;
  }
  return 0;
}

/* porter rule condition: (m > 0) */
static int fts5Porter_MGt0(char *zStem, int nStem){
  return !!fts5PorterGobbleVC(zStem, nStem, 0);
}

/* porter rule condition: (m > 1) */
static int fts5Porter_MGt1(char *zStem, int nStem){
  int n;
  n = fts5PorterGobbleVC(zStem, nStem, 0);
  if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
    return 1;
  }
  return 0;
}

/* porter rule condition: (m = 1) */
static int fts5Porter_MEq1(char *zStem, int nStem){
  int n;
  n = fts5PorterGobbleVC(zStem, nStem, 0);
  if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
    return 1;
  }
  return 0;
}

/* porter rule condition: (*o) */
static int fts5Porter_Ostar(char *zStem, int nStem){
  if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){
    return 0;
  }else{
    int i;
    int mask = 0;
    int bCons = 0;
    for(i=0; i<nStem; i++){
      bCons = !fts5PorterIsVowel(zStem[i], bCons);
      assert( bCons==0 || bCons==1 );
      mask = (mask << 1) + bCons;
    }
    return ((mask & 0x0007)==0x0005);
  }
}

/* porter rule condition: (m > 1 and (*S or *T)) */
static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){
  return nStem>0
      && (zStem[nStem-1]=='s' || zStem[nStem-1]=='t')
      && fts5Porter_MGt1(zStem, nStem);
}

/* porter rule condition: (*v*) */
static int fts5Porter_Vowel(char *zStem, int nStem){
  int i;
  for(i=0; i<nStem; i++){
    if( fts5PorterIsVowel(zStem[i], i>0) ){
      return 1;
    }
  }
  return 0;
}

static int fts5PorterCb(
  void *pCtx, 
  const char *pToken, 
  int nToken, 
  int iStart, 
  int iEnd, 
  int iPos
){
  PorterContext *p = (PorterContext*)pCtx;

  PorterRule aStep1A[] = {
    { "sses", 4,  0, "ss", 2 },
    { "ies",  3,  0, "i",  1  },
    { "ss",   2,  0, "ss", 2 },
    { "s",    1,  0, "",   0 },
    { 0, 0, 0, 0 }
  };

  PorterRule aStep1B[] = {
    { "eed", 3,  fts5Porter_MGt0,  "ee", 2 },
    { "ed",  2,  fts5Porter_Vowel, "",   0 },
    { "ing", 3,  fts5Porter_Vowel, "",   0 },
    { 0, 0, 0, 0 }
  };

  PorterRule aStep1B2[] = {
    { "at", 2,  0, "ate", 3 },
    { "bl", 2,  0, "ble", 3 },
    { "iz", 2,  0, "ize", 3 },
    { 0, 0, 0, 0 }
  };

  PorterRule aStep1C[] = {
    { "y",  1,  fts5Porter_Vowel, "i", 1 },
    { 0, 0, 0, 0 }
  };

  PorterRule aStep2[] = {
    { "ational", 7, fts5Porter_MGt0, "ate", 3}, 
    { "tional", 6, fts5Porter_MGt0, "tion", 4}, 
    { "enci", 4, fts5Porter_MGt0, "ence", 4}, 
    { "anci", 4, fts5Porter_MGt0, "ance", 4}, 
    { "izer", 4, fts5Porter_MGt0, "ize", 3}, 
    { "logi", 4, fts5Porter_MGt0, "log", 3},     /* added post 1979 */
    { "bli", 3, fts5Porter_MGt0, "ble", 3},      /* modified post 1979 */
    { "alli", 4, fts5Porter_MGt0, "al", 2}, 
    { "entli", 5, fts5Porter_MGt0, "ent", 3}, 
    { "eli", 3, fts5Porter_MGt0, "e", 1}, 
    { "ousli", 5, fts5Porter_MGt0, "ous", 3}, 
    { "ization", 7, fts5Porter_MGt0, "ize", 3}, 
    { "ation", 5, fts5Porter_MGt0, "ate", 3}, 
    { "ator", 4, fts5Porter_MGt0, "ate", 3}, 
    { "alism", 5, fts5Porter_MGt0, "al", 2}, 
    { "iveness", 7, fts5Porter_MGt0, "ive", 3}, 
    { "fulness", 7, fts5Porter_MGt0, "ful", 3}, 
    { "ousness", 7, fts5Porter_MGt0, "ous", 3}, 
    { "aliti", 5, fts5Porter_MGt0, "al", 2}, 
    { "iviti", 5, fts5Porter_MGt0, "ive", 3}, 
    { "biliti", 6, fts5Porter_MGt0, "ble", 3}, 
    { 0, 0, 0, 0 }
  };

  PorterRule aStep3[] = {
    { "icate", 5, fts5Porter_MGt0, "ic", 2}, 
    { "ative", 5, fts5Porter_MGt0, "", 0}, 
    { "alize", 5, fts5Porter_MGt0, "al", 2}, 
    { "iciti", 5, fts5Porter_MGt0, "ic", 2}, 
    { "ical", 4, fts5Porter_MGt0, "ic", 2}, 
    { "ful", 3, fts5Porter_MGt0, "", 0}, 
    { "ness", 4, fts5Porter_MGt0, "", 0}, 
    { 0, 0, 0, 0 }
  };

  PorterRule aStep4[] = {
    { "al", 2, fts5Porter_MGt1, "", 0}, 
    { "ance", 4, fts5Porter_MGt1, "", 0}, 
    { "ence", 4, fts5Porter_MGt1, "", 0}, 
    { "er", 2, fts5Porter_MGt1, "", 0}, 
    { "ic", 2, fts5Porter_MGt1, "", 0}, 
    { "able", 4, fts5Porter_MGt1, "", 0}, 
    { "ible", 4, fts5Porter_MGt1, "", 0}, 
    { "ant", 3, fts5Porter_MGt1, "", 0}, 
    { "ement", 5, fts5Porter_MGt1, "", 0}, 
    { "ment", 4, fts5Porter_MGt1, "", 0}, 
    { "ent", 3, fts5Porter_MGt1, "", 0}, 
    { "ion", 3, fts5Porter_MGt1_and_S_or_T, "", 0}, 
    { "ou", 2, fts5Porter_MGt1, "", 0}, 
    { "ism", 3, fts5Porter_MGt1, "", 0}, 
    { "ate", 3, fts5Porter_MGt1, "", 0}, 
    { "iti", 3, fts5Porter_MGt1, "", 0}, 
    { "ous", 3, fts5Porter_MGt1, "", 0}, 
    { "ive", 3, fts5Porter_MGt1, "", 0}, 
    { "ize", 3, fts5Porter_MGt1, "", 0}, 
    { 0, 0, 0, 0 }
  };


  char *aBuf;
  int nBuf;
  int n;

  if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through;
  aBuf = p->aBuf;
  nBuf = nToken;
  memcpy(aBuf, pToken, nBuf);

  /* Step 1. */
  fts5PorterApply(aBuf, &nBuf, aStep1A);
  n = fts5PorterApply(aBuf, &nBuf, aStep1B);
  if( n==1 || n==2 ){
    if( fts5PorterApply(aBuf, &nBuf, aStep1B2)<0 ){
      char c = aBuf[nBuf-1];
      if( fts5PorterIsVowel(c, 0)==0 
       && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2] 
      ){
        nBuf--;
      }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){
        aBuf[nBuf++] = 'e';
      }
    }
  }
  fts5PorterApply(aBuf, &nBuf, aStep1C);

  /* Steps 2 through 4. */
  fts5PorterApply(aBuf, &nBuf, aStep2);
  fts5PorterApply(aBuf, &nBuf, aStep3);
  fts5PorterApply(aBuf, &nBuf, aStep4);

  /* Step 5a. */
  if( nBuf>0 && aBuf[nBuf-1]=='e' ){
    if( fts5Porter_MGt1(aBuf, nBuf-1) 
     || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
    ){
      nBuf--;
    }
  }

  /* Step 5b. */
  if( nBuf>1 && aBuf[nBuf-1]=='l' 
   && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1) 
  ){
    nBuf--;
  }

  return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd, iPos);

 pass_through:
  return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd, iPos);
}

/*
** Tokenize using the porter tokenizer.
*/
static int fts5PorterTokenize(
  Fts5Tokenizer *pTokenizer,
  void *pCtx,
  const char *pText, int nText,
  int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos)
){
  PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
  PorterContext sCtx;
  sCtx.xToken = xToken;
  sCtx.pCtx = pCtx;
  sCtx.aBuf = p->aBuf;
  return p->tokenizer.xTokenize(
      p->pTokenizer, (void*)&sCtx, pText, nText, fts5PorterCb
  );
}

/*
** Register all built-in tokenizers with FTS5.
*/
int sqlite3Fts5TokenizerInit(fts5_api *pApi){
  struct BuiltinTokenizer {
    const char *zName;

    fts5_tokenizer x;
  } aBuiltin[] = {
    { "porter",  { fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize } },
    { "simple",  { fts5SimpleCreate, fts5SimpleDelete, fts5SimpleTokenize } }
  };
  
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
    rc = pApi->xCreateTokenizer(pApi,
        aBuiltin[i].zName,
        (void*)pApi,
        &aBuiltin[i].x,
        0
    );
  }

  return SQLITE_OK;
}

# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the fts5 tokenizers
#

if {![info exists testdir]} {
  set testdir [file join [file dirname [info script]] .. .. test]
}
source $testdir/tester.tcl
set testprefix fts5tokenizer



do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
  DROP TABLE ft1;
}
do_execsql_test 1.1 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize='porter');
  DROP TABLE ft1;
}
do_execsql_test 1.2 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = porter);
  DROP TABLE ft1;
}
do_execsql_test 1.3 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter');
  DROP TABLE ft1;
}
do_execsql_test 1.4 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter simple');
  DROP TABLE ft1;
}

do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
  INSERT INTO ft1 VALUES('embedded databases');
}
do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1
do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1
do_execsql_test 2.3 { 
  SELECT rowid FROM ft1 WHERE ft1 MATCH 'database embedding' 
} 1


proc tcl_create {args} { 
  set ::targs $args
  error "failed" 
}
sqlite3_fts5_create_tokenizer db tcl tcl_create

foreach {tn directive expected} {
  1 {tokenize='tcl a b c'}             {a b c}
  2 {tokenize='tcl ''d'' ''e'' ''f'''} {d e f}
  3 {tokenize="tcl 'g' 'h' 'i'"}       {g h i}
  4 {tokenize = tcl}                   {}
} {
  do_catchsql_test 3.$tn.1 "
    CREATE VIRTUAL TABLE ft2 USING fts5(x, $directive)
  " {1 {error in tokenizer constructor}}
  do_test 3.$tn.2 { set ::targs } $expected
}


do_catchsql_test 4.1 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize = tcl abc);
} {1 {parse error in "tokenize = tcl abc"}}
do_catchsql_test 4.2 {
  CREATE VIRTUAL TABLE ft2 USING fts5(x y)
} {1 {parse error in "x y"}}

finish_test