SQLite

  u8 *abUnindexed;                /* True for unindexed columns */
  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  int eContent;                   /* An FTS5_CONTENT value */
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
  int bOffsets;                   /* "offsets=" option value (dflt==1) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;

  /* Values loaded from the %_config table */
  int iCookie;                    /* Incremented when %_config is modified */
  int pgsz;                       /* Approximate page size used in %_data */

/* Current expected value of %_config table 'version' field */
#define FTS5_CURRENT_VERSION 4

#define FTS5_CONTENT_NORMAL   0
#define FTS5_CONTENT_NONE     1
#define FTS5_CONTENT_EXTERNAL 2

#define FTS5_DETAIL_FULL    0
#define FTS5_DETAIL_NONE    1
#define FTS5_DETAIL_COLUMNS 2



int sqlite3Fts5ConfigParse(
    Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
);
void sqlite3Fts5ConfigFree(Fts5Config*);

/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);

int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);

Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
int sqlite3Fts5ExprPopulatePoslists(
    Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int
);

int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);

/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y.  */

  assert( 0==fts5_iswhitespace(z[0]) );
  quote = z[0];
  if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
    fts5Dequote(z);
  }
}


struct Fts5Enum {
  const char *zName;
  int eVal;
};
typedef struct Fts5Enum Fts5Enum;

static int fts5ConfigSetEnum(
  const Fts5Enum *aEnum, 
  const char *zEnum, 
  int *peVal
){
  int nEnum = strlen(zEnum);
  int i;
  int iVal = -1;

  for(i=0; aEnum[i].zName; i++){
    if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
      if( iVal>=0 ) return SQLITE_ERROR;
      iVal = aEnum[i].eVal;
    }
  }

  *peVal = iVal;
  return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
}

/*
** Parse a "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message
** may be left in *pzErr. It is the responsibility of the caller to

      rc = SQLITE_ERROR;
    }else{
      pConfig->bColumnsize = (zArg[0]=='1');
    }
    return rc;
  }

  if( sqlite3_strnicmp("offsets", zCmd, nCmd)==0 ){
    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
      *pzErr = sqlite3_mprintf("malformed offsets=... directive");
  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
    const Fts5Enum aDetail[] = {
      { "none", FTS5_DETAIL_NONE },
      { "full", FTS5_DETAIL_FULL },
      { "columns", FTS5_DETAIL_COLUMNS },
      { 0, 0 }
    };

    if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){
      *pzErr = sqlite3_mprintf("malformed detail=... directive");
      rc = SQLITE_ERROR;
    }else{
      pConfig->bOffsets = (zArg[0]=='1');
    }
    return rc;
  }

  *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
  return SQLITE_ERROR;
}

  nByte = nArg * (sizeof(char*) + sizeof(u8));
  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
  pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
  pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
  pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
  pRet->bColumnsize = 1;
  pRet->bOffsets = 1;
  pRet->eDetail = FTS5_DETAIL_FULL;
#ifdef SQLITE_DEBUG
  pRet->bPrefixIndex = 1;
#endif
  if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
    rc = SQLITE_ERROR;
  }

}

void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNearset *pNear, 
  Fts5Colset *pColset 
){
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
    pParse->zErr = sqlite3_mprintf(
      "fts5: column queries are not supported (detail=none)"
    );
    sqlite3_free(pColset);
    return;
  }

  if( pNear ){
    pNear->pColset = pColset;
  }else{
    sqlite3_free(pColset);
  }
}

        for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
          pNear->apPhrase[iPhrase]->pNode = pRet;
        }
        if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
          if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){
            pRet->eType = FTS5_TERM;
          }
        }else if( pParse->pConfig->bOffsets==0 ){
        }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
          assert( pParse->rc==SQLITE_OK );
          pParse->rc = SQLITE_ERROR;
          assert( pParse->zErr==0 );
          pParse->zErr = sqlite3_mprintf(
              "fts5: %s queries are not supported (offsets=0)", 
              "fts5: %s queries are not supported (detail!=full)", 
              pNear->nPhrase==1 ? "phrase": "NEAR"
          );
          sqlite3_free(pRet);
          pRet = 0;
        }
      }else{
        fts5ExprAddChildren(pRet, pLeft);

    for(i=0; i<pNear->nPhrase; i++){
      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];

      zRet = fts5PrintfAppend(zRet, " {");
      for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
        char *zTerm = pPhrase->aTerm[iTerm].zTerm;
        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
        if( pPhrase->aTerm[iTerm].bPrefix ){
          zRet = fts5PrintfAppend(zRet, "*");
        }
      }

      if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
      if( zRet==0 ) return 0;
    }

  }else{

    nRet = pPhrase->poslist.n;
  }else{
    *pa = 0;
    nRet = 0;
  }
  return nRet;
}

Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
  int i;
  Fts5PoslistWriter *pRet;
  for(i=0; i<pExpr->nPhrase; i++){
    Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
    assert( pExpr->apExprPhrase[i]->nTerm==1 );
    pBuf->n = 0;
  }
  pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
  if( pRet ){
    memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
  }
  return pRet;
}

struct Fts5ExprCtx {
  Fts5Expr *pExpr;
  Fts5PoslistWriter *aWriter;
  i64 iOff;
};
typedef struct Fts5ExprCtx Fts5ExprCtx;

static int fts5ExprPopulatePoslistsCb(
  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
  int tflags,                /* Mask of FTS5_TOKEN_* flags */
  const char *pToken,        /* Pointer to buffer containing token */
  int nToken,                /* Size of token in bytes */
  int iStart,                /* Byte offset of token within input text */
  int iEnd                   /* Byte offset of end of token within input text */
){
  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
  Fts5Expr *pExpr = p->pExpr;
  int i;

  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
  for(i=0; i<pExpr->nPhrase; i++){
    Fts5ExprTerm *pTerm;
    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
      int nTerm = strlen(pTerm->zTerm);
      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
       && memcmp(pTerm->zTerm, pToken, nTerm)==0
      ){
        int rc = sqlite3Fts5PoslistWriterAppend(
            &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff
        );
        if( rc ) return rc;
        break;
      }
    }
  }
  return SQLITE_OK;
}

int sqlite3Fts5ExprPopulatePoslists(
  Fts5Config *pConfig,
  Fts5Expr *pExpr, 
  Fts5PoslistWriter *aWriter,
  int iCol, 
  const char *z, int n
){
  Fts5ExprCtx sCtx;
  sCtx.pExpr = pExpr;
  sCtx.aWriter = aWriter;
  sCtx.iOff = (((i64)iCol) << 32) - 1;

  return sqlite3Fts5Tokenize(pConfig, 
      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
  );
}

** This file contains the implementation of an in-memory hash table used
** to accumuluate "term -> doclist" content before it is flused to a level-0
** segment.
*/


struct Fts5Hash {
  int bOffsets;                   /* Copy of Fts5Config.bOffsets */
  int eDetail;                    /* Copy of Fts5Config.eDetail */
  int *pnByte;                    /* Pointer to bytes counter */
  int nEntry;                     /* Number of entries currently in hash */
  int nSlot;                      /* Size of aSlot[] array */
  Fts5HashEntry *pScan;           /* Current ordered scan item */
  Fts5HashEntry **aSlot;          /* Array of hash slots */
};

  *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash));
  if( pNew==0 ){
    rc = SQLITE_NOMEM;
  }else{
    int nByte;
    memset(pNew, 0, sizeof(Fts5Hash));
    pNew->pnByte = pnByte;
    pNew->bOffsets = pConfig->bOffsets;
    pNew->eDetail = pConfig->eDetail;

    pNew->nSlot = 1024;
    nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
    pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
    if( pNew->aSlot==0 ){
      sqlite3_free(pNew);
      *ppNew = 0;

  char bByte,                     /* First byte of token */
  const char *pToken, int nToken  /* Token to add or remove to or from index */
){
  unsigned int iHash;
  Fts5HashEntry *p;
  u8 *pPtr;
  int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
  int bNew = pHash->bOffsets;     /* If non-delete entry should be written */
  int bNew;                       /* If non-delete entry should be written */
  
  bNew = (pHash->eDetail==FTS5_DETAIL_FULL);

  /* Attempt to locate an existing hash entry */
  iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
    if( p->zKey[0]==bByte 
     && memcmp(&p->zKey[1], pToken, nToken)==0 
     && p->zKey[nToken+1]==0 

    assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
    p->zKey[nToken+1] = '\0';
    p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
    p->iSzPoslist = p->nData;
    p->nData += 1;
    p->iRowid = iRowid;
    p->iCol = (pHash->bOffsets-1);
    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
    p->pHashNext = pHash->aSlot[iHash];
    pHash->aSlot[iHash] = p;
    pHash->nEntry++;
    nIncr += p->nData;
  }

  /* Check there is enough space to append a new entry. Worst case scenario

  /* If this is a new rowid, append the 4-byte size field for the previous
  ** entry, and the new rowid for this entry.  */
  if( iRowid!=p->iRowid ){
    fts5HashAddPoslistSize(p);
    p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
    p->iSzPoslist = p->nData;
    p->nData += 1;
    p->iCol = (pHash->bOffsets-1);
    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
    p->iPos = 0;
    p->iRowid = iRowid;
    bNew = 1;
  }

  if( iCol>=0 ){
    /* Append a new column value, if necessary */
    assert( iCol>=p->iCol );
    if( iCol!=p->iCol ){
      if( pHash->bOffsets==0 ){
      if( pHash->eDetail==FTS5_DETAIL_FULL ){
        bNew = 1;
        p->iCol = iPos = iCol;
      }else{
        pPtr[p->nData++] = 0x01;
        p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
        p->iCol = iCol;
        p->iPos = 0;
      }else{
        bNew = 1;
        p->iCol = iPos = iCol;
      }
    }

    /* Append the new position offset, if necessary */
    if( bNew ){
      p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
      p->iPos = iPos;

  Fts5Colset *pColset,
  Fts5Buffer *pBuf
){
  if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
    if( pColset==0 ){
      fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
    }else{
      if( p->pConfig->bOffsets==0 ){
      if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){
        PoslistOffsetsCtx sCtx;
        memset(&sCtx, 0, sizeof(sCtx));
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
      }else{
        PoslistCallbackCtx sCtx;
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        sCtx.eState = fts5IndexColsetTest(pColset, 0);
        assert( sCtx.eState==0 || sCtx.eState==1 );
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
      }else{
        PoslistOffsetsCtx sCtx;
        memset(&sCtx, 0, sizeof(sCtx));
        sCtx.pBuf = pBuf;
        sCtx.pColset = pColset;
        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
      }
    }
  }
}

/*
** IN/OUT parameter (*pa) points to a position list n bytes in size. If

  Fts5Buffer *pBuf
){
  if( p->rc==SQLITE_OK ){
    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
    assert( fts5MultiIterEof(p, pMulti)==0 );
    assert( pSeg->nPos>0 );
    if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
      if( p->pConfig->bOffsets
      if( p->pConfig->eDetail==FTS5_DETAIL_FULL
       && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
       && (pColset==0 || pColset->nCol==1)
      ){
        const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
        int nPos;
        if( pColset ){
          nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);

  Fts5IndexIter *pIter, 
  Fts5Colset *pColset,            /* Column filter (or NULL) */
  const u8 **pp,                  /* OUT: Pointer to position-list data */
  int *pn,                        /* OUT: Size of position-list in bytes */
  i64 *piRowid                    /* OUT: Current rowid */
){
  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  int eDetail = pIter->pIndex->pConfig->eDetail;

  assert( pIter->pIndex->rc==SQLITE_OK );
  *piRowid = pSeg->iRowid;
  if( pIter->pIndex->pConfig->bOffsets 
  if( eDetail==FTS5_DETAIL_FULL 
   && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  ){
    u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
    if( pColset==0 || pIter->bFiltered ){
      *pn = pSeg->nPos;
      *pp = pPos;
    }else if( pColset->nCol==1 ){
      *pp = pPos;
      *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]);
    }else{
      fts5BufferZero(&pIter->poslist);
      fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist);
      *pp = pIter->poslist.p;
      *pn = pIter->poslist.n;
    }
  }else{
    fts5BufferZero(&pIter->poslist);
    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
    if( eDetail==FTS5_DETAIL_FULL ){
    *pp = pIter->poslist.p;
      *pp = pIter->poslist.p;
    }
    *pn = pIter->poslist.n;
  }
  return fts5IndexReturn(pIter->pIndex);
}

/*
** This function is similar to sqlite3Fts5IterPoslist(), except that it

  const char *z,                  /* Index key to query for */
  int n,                          /* Size of index key in bytes */
  int flags,                      /* Flags for Fts5IndexQuery */
  u64 *pCksum                     /* IN/OUT: Checksum value */
){
  u64 cksum = *pCksum;
  Fts5IndexIter *pIdxIter = 0;
  Fts5Buffer buf = {0, 0, 0};
  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);

  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
    i64 dummy;
    const u8 *pPos;
    int nPos;
    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
    rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy);
    rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
    if( rc==SQLITE_OK ){
      Fts5PoslistReader sReader;
      for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader);
      for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
          sReader.bEof==0;
          sqlite3Fts5PoslistReaderNext(&sReader)
      ){
        int iCol = FTS5_POS2COLUMN(sReader.iPos);
        int iOff = FTS5_POS2OFFSET(sReader.iPos);
        cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
      }
      rc = sqlite3Fts5IterNext(pIdxIter);
    }
  }
  sqlite3Fts5IterClose(pIdxIter);
  fts5BufferFree(&buf);

  *pCksum = cksum;
  return rc;
}


/*

*/
#define FTS5CSR_REQUIRE_CONTENT   0x01
#define FTS5CSR_REQUIRE_DOCSIZE   0x02
#define FTS5CSR_REQUIRE_INST      0x04
#define FTS5CSR_EOF               0x08
#define FTS5CSR_FREE_ZRANK        0x10
#define FTS5CSR_REQUIRE_RESEEK    0x20
#define FTS5CSR_REQUIRE_POSLIST   0x40

#define BitFlagAllTest(x,y) (((x) & (y))==(y))
#define BitFlagTest(x,y)    (((x) & (y))!=0)


/*
** Macros to Set(), Clear() and Test() cursor flags.

  return pTab->pConfig->eContent==FTS5_CONTENT_NONE;
}

/*
** Return true if pTab is an offsetless table.
*/
static int fts5IsOffsetless(Fts5Table *pTab){
  return pTab->pConfig->bOffsets==0;
  return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL;
}

/*
** Delete a virtual table handle allocated by fts5InitVtab(). 
*/
static void fts5FreeVtab(Fts5Table *pTab){
  if( pTab ){

** specific to the previous row stored by the cursor object.
*/
static void fts5CsrNewrow(Fts5Cursor *pCsr){
  CsrFlagSet(pCsr, 
      FTS5CSR_REQUIRE_CONTENT 
    | FTS5CSR_REQUIRE_DOCSIZE 
    | FTS5CSR_REQUIRE_INST 
    | FTS5CSR_REQUIRE_POSLIST 
  );
}

static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  Fts5Auxdata *pData;
  Fts5Auxdata *pNext;

  int rc;
  Fts5Table *pTab = (Fts5Table*)pVtab;
  fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
  rc = sqlite3Fts5StorageRollback(pTab->pStorage);
  return rc;
}

static int fts5CsrPoslist(Fts5Cursor*, int, const u8**);

static void *fts5ApiUserData(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return pCsr->pAux->pUserData;
}

static int fts5ApiColumnCount(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;

  return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
}

static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}

static int fts5ApiColumnText(
  Fts5Context *pCtx, 
  int iCol, 
  const char **pz, 
  int *pn
){
  int rc = SQLITE_OK;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
    *pz = 0;
    *pn = 0;
  }else{
    rc = fts5SeekCursor(pCsr, 0);
    if( rc==SQLITE_OK ){
      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
    }
  }
  return rc;
}

static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
  int n;
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
    Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
    if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
      Fts5PoslistWriter *aWriter;
      int i;
assert( pCsr->pSorter==0 );  /* fixme */
      aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr);
      if( aWriter==0 ) rc = SQLITE_NOMEM;
      for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
        int n; const char *z;
        rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
        if( rc==SQLITE_OK ){
          rc = sqlite3Fts5ExprPopulatePoslists(
              pConfig, pCsr->pExpr, aWriter, i, z, n
          );
        }
      }
      sqlite3_free(aWriter);
    }
    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
  }

  if( pCsr->pSorter ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
    n = pSorter->aIdx[iPhrase] - i1;
    *pa = &pSorter->aPoslist[i1];
  }else{
    n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);

  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  int rc = SQLITE_OK;
  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
   || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
  ){
    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
      rc = SQLITE_RANGE;
#if 0
    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 2];
      *piOff = -1;
#endif
    }else{
      *piPhrase = pCsr->aInst[iIdx*3];
      *piCol = pCsr->aInst[iIdx*3 + 1];
      *piOff = pCsr->aInst[iIdx*3 + 2];
    }
  }
  return rc;
}

static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  return fts5CursorRowid((Fts5Cursor*)pCtx);
}

static int fts5ApiColumnText(
  Fts5Context *pCtx, 
  int iCol, 
  const char **pz, 
  int *pn
){
  int rc = SQLITE_OK;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
    *pz = 0;
    *pn = 0;
  }else{
    rc = fts5SeekCursor(pCsr, 0);
    if( rc==SQLITE_OK ){
      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
    }
  }
  return rc;
}

static int fts5ColumnSizeCb(
  void *pContext,                 /* Pointer to int */
  int tflags,
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd                        /* End offset of token */

  Fts5Context *pCtx, 
  Fts5PhraseIter *pIter, 
  int *piCol, int *piOff
){
  if( pIter->a>=pIter->b ){
    *piCol = -1;
    *piOff = -1;
#if 0
  }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){
    int iVal;
    pIter->a += fts5GetVarint32(pIter->a, iVal);
    *piCol += (iVal-2);
    *piOff = -1;
#endif
  }else{
    int iVal;
    pIter->a += fts5GetVarint32(pIter->a, iVal);
    if( iVal==1 ){
      pIter->a += fts5GetVarint32(pIter->a, iVal);
      *piCol = iVal;
      *piOff = 0;

  fts5ApiColumnSize,
  fts5ApiQueryPhrase,
  fts5ApiSetAuxdata,
  fts5ApiGetAuxdata,
  fts5ApiPhraseFirst,
  fts5ApiPhraseNext,
};


/*
** Implementation of API function xQueryPhrase().
*/
static int fts5ApiQueryPhrase(
  Fts5Context *pCtx, 
  int iPhrase, 

      if( pConfig->bColumnsize ){
        rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
      }
      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
        if( pConfig->abUnindexed[i] ) continue;
        ctx.iCol = i;
        ctx.szCol = 0;
        if( pConfig->bOffsets==0 ){
        if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
        }
        if( rc==SQLITE_OK ){
          rc = sqlite3Fts5Tokenize(pConfig, 
              FTS5_TOKENIZE_DOCUMENT,
              (const char*)sqlite3_column_text(pScan, i+1),
              sqlite3_column_bytes(pScan, i+1),

      
      for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
        set p [lindex $phraselist $iPhrase]
        set nPm1 [expr {[llength $p] - 1}]
        set iFirst [expr $iFL - $O(-near) - [llength $p]]

        for {set i $iFirst} {$i <= $iFL} {incr i} {
          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
          set lCand [lrange $col $i [expr $i+$nPm1]]

          set bMatch 1
          foreach tok $p term $lCand {
            if {[string match $tok $term]==0} {
              #puts "$tok $term failed"
              set bMatch 0
            }
          }
          if {$bMatch} { 
            #puts "match at $i"
            lappend B($iPhrase) $i 
          }

          #if {$lCand == $p} { lappend B($iPhrase) $i }
        }
        if {[llength $B($iPhrase)] == 0} break
      }

      if {$iPhrase==$nPhrase} {
        for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
          set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]

      foreach a $A($iCol,$iPhrase) {
        lappend res "$counter.$iCol.$a"
      }
    }
    incr counter
  }

  #puts $res
  #puts "$aCol -> $res"
  sort_poslist $res
}

#-------------------------------------------------------------------------
# Usage:
#
#   sort_poslist LIST

}

# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored int the global 
# Assuming the table contains the same records as stored in the global 
# $::data array (see above), this function returns a list containing one
# element for each match in the dataset. The elements are themselves lists
# formatted as follows:
#
#   <rowid> {<phrase 0 matches> <phrase 1 matches>...}
#
# where each <phrase X matches> element is a list of phrase matches in the

  return [concat {*}$res]
}

#
# End of test code
#-------------------------------------------------------------------------

proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    lappend res [string map {{ } .} [$cmd xInst $i]]
  }
  set res
}


foreach {tn2 sql} {
  1  {}
  2  {BEGIN}
} {
  reset_db
  sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
  fts5_aux_test_functions db

  do_execsql_test 1.0 {
  do_execsql_test 1.$tn2.0 {
    CREATE VIRTUAL TABLE xx USING fts5(x,y);
    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
  }

  execsql $sql

  do_test $tn2.1.1 {
  do_test 1.$tn2.1.1 {
    foreach {id x y} $data {
      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
    }
    execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
  } {}

    8 "c"
    9 "no"
    10 "L O O L V V K"
  } {
    set expr "\"$phrase\""
    set res [matchdata 1 $expr]

    do_execsql_test $tn2.1.2.$tn.[llength $res] { 
    do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  #-------------------------------------------------------------------------
  # Test some AND and OR queries.
  #

    2.2 "a+b OR c"
    2.3 "d+c OR u"
    2.4 "d+c OR u+d"

    3.1 { a AND b AND c }
  } {
    set res [matchdata 1 $expr]
    do_execsql_test $tn2.2.$tn.[llength $res] { 
    do_execsql_test 1.$tn2.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  #-------------------------------------------------------------------------
  # Queries on a specific column.
  #

    4.1 {{"x" "y"}:a}
    4.2 {{"y" x}:a}
    4.3 {{x "x"}:b}
    4.4 {{"y" y}:b}
  } {
    set res [matchdata 1 $expr]
    do_execsql_test $tn2.3.$tn.[llength $res] { 
    do_execsql_test 1.$tn2.3.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  #-------------------------------------------------------------------------
  # Some NEAR queries.
  #
  foreach {tn expr} {
    1 "NEAR(a b)"
    2 "NEAR(r c)"
    2 { NEAR(r c, 5) }
    3 { NEAR(r c, 3) }
    4 { NEAR(r c, 2) }
    5 { NEAR(r c, 0) }
    6 { NEAR(a b c) }
    7 { NEAR(a b c, 8) }
    8  { x : NEAR(r c) }
    9  { y : NEAR(r c) }
  } {
    set res [matchdata 1 $expr]
    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
    do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}

  foreach {tn expr tclexpr} {
    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
  } {
    do_execsql_test $tn2.5.$tn {
    do_execsql_test 1.$tn2.5.$tn {
      SELECT fts5_expr_tcl($expr, 'N $x')
    } [list $tclexpr]
  }

  #-------------------------------------------------------------------------
  #
  do_execsql_test $tn2.6.integrity {
  do_execsql_test 1.$tn2.6.integrity {
    INSERT INTO xx(xx) VALUES('integrity-check');
  }
  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
  foreach {bAsc sql} {
    1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
    0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
  } {

      15 { a OR b AND c }
      16 { c AND b OR a }
      17 { c AND (b OR a) }
      18 { c NOT (b OR a) }
      19 { c NOT b OR a AND d }
    } {
      set res [matchdata 0 $expr $bAsc]
      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
      do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res
    }
  }
}

do_execsql_test 3.1 {
do_execsql_test 2.1 {
  SELECT fts5_expr_tcl('a AND b');
} {{AND [nearset -- {a}] [nearset -- {b}]}}

# Some tests for detail=col tables and detail=none.
#
foreach {tn2 sql} {
  1  {
    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
  }
  2  {
    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
    BEGIN;
  }
  3  {
    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none);
    BEGIN;
  }
  4  {
    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none);
  }
} {
  reset_db
  fts5_aux_test_functions db

  execsql $sql

  do_execsql_test 3.$tn2.0 {
    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
  }


  do_test 3.$tn2.1.1 {
    foreach {id x y} $data {
      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
    }
    execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
  } {}

  foreach {tn q} {
    1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L"
    11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d"
  } {
    set res [matchdata 1 $q]

    do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q
    } $res
  }

}

finish_test

  5 {
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    BEGIN;
  }

  6 {
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0);
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }

  7 {
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5");
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5");
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }

  8 {
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5");
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5");
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    BEGIN;
  }

} {

  do_test $T.1 { 

#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS5 module.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5offsets
set testprefix fts5detail

# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

fts5_aux_test_functions db

#--------------------------------------------------------------------------
# Simple tests.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0);
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
  INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
  INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
  INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
  INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
  INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
  INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
  INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7

  do_execsql_test 1.2.$tn.2 {
    SELECT rowid FROM t1($match || '*');
  } $res
}

do_catchsql_test 1.3.1 {
  SELECT rowid FROM t1('h + d');
} {1 {fts5: phrase queries are not supported (offsets=0)}}
} {1 {fts5: phrase queries are not supported (detail!=full)}}

do_catchsql_test 1.3.2 {
  SELECT rowid FROM t1('NEAR(h d)');
} {1 {fts5: NEAR queries are not supported (offsets=0)}}
} {1 {fts5: NEAR queries are not supported (detail!=full)}}


#-------------------------------------------------------------------------
# integrity-check with both offsets= and prefix= options.
# integrity-check with both detail= and prefix= options.
#
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, offsets=0, prefix="1");
  CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1");
  INSERT INTO t2(a) VALUES('aa ab');
}

#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r}

do_execsql_test 2.1 {
  INSERT INTO t2(t2) VALUES('integrity-check');
}

do_execsql_test 2.2 {
  SELECT fts5_test_poslist(t2) FROM t2('aa');
} {0.0.0}

set ::pc 0
#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
#exit

#-------------------------------------------------------------------------
# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs
# work with offsets=0 tables.
# work with detail=col tables.
#
set data {
  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
  2  {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc}
  3  {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca}
  4  {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca}
  5  {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb}

  35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb}
  36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa}
  37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb}
  38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac}
  39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba}
  40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca}
}

set data {
  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
}

proc matchdata {expr {bAsc 1}} {

  set tclexpr [db one {
    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z')
  }]
  set res [list]

  #puts "$expr -> $tclexpr"
  foreach {id x y z} $::data {
    set cols [list $x $y $z]
    set ::pc 0
    #set hits [lsort -command instcompare [eval $tclexpr]]
    set hits [eval $tclexpr]
    if {[llength $hits]>0} {
      lappend res [list $id $hits]
    }
  }

  if {$bAsc} {
    set res [lsort -integer -increasing -index 0 $res]
  } else {
    set res [lsort -integer -decreasing -index 0 $res]
  }

  return [concat {*}$res]
}

foreach {tn tbl} {
  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, offsets=0) }
  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) }
} {
#break
  reset_db
  fts5_aux_test_functions db
  execsql $tbl
  foreach {id x y z} $data {
    execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) }
  }
  foreach {tn2 expr} {
    1 aaa    2 ccc    3 bab    4 aac
    5 aa*    6 cc*    7 ba*    8 aa*
    9 a*     10 b*   11 c*
  } {

    set res [list]
    set res [matchdata $expr]
    foreach {id x y z} $data {
      if {[lsearch [concat $x $y $z] $expr]>=0} {
        lappend res $id
        set inst [list]
        if {[lsearch $x $expr]>=0} { lappend inst 0.0.-1 }
        if {[lsearch $y $expr]>=0} { lappend inst 0.1.-1 }
        if {[lsearch $z $expr]>=0} { lappend inst 0.2.-1 }
        lappend res $inst
      }
    }

    do_execsql_test 3.$tn.$tn2.1 {
      SELECT rowid, fts5_test_poslist(t3) FROM t3($expr)
    } $res

    do_execsql_test 3.$tn.$tn2.2 {
      SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr)
    } $res
  }

}

}

#-------------------------------------------------------------------------
# Simple tests for detail=none tables.
#
breakpoint
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none);
  INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g');
  INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9');
}

do_catchsql_test 4.1 {
  SELECT * FROM t4('a:a')
} {1 {fts5: column queries are not supported (detail=none)}}

finish_test