SQLite

static int fts5ApiColumnCount(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
}

static int fts5ApiColumnAvgSize(Fts5Context *pCtx, int iCol, int *pnToken){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  return sqlite3Fts5StorageAvgsize(pTab->pStorage, iCol, pnToken);
}

static int fts5ApiTokenize(
  Fts5Context *pCtx, 
  const char *pText, int nText, 
  void *pUserData,
  int (*xToken)(void*, const char*, int, int, int, int)

int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);

/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5IndexInit(sqlite3*);

void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);

int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf);
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);

/*
** End of interface to code in fts5_index.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_storage.c. fts5_storage.c contains contains 
** code to access the data stored in the %_content and %_docsize tables.

int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg);


/*
** End of interface to code in fts5_storage.c.
**************************************************************************/

  int i;

  if( nVal>=1 ){
    zReq = (const char*)sqlite3_value_text(apVal[0]);
  }

  memset(&s, 0, sizeof(Fts5Buffer));
  nCol = pApi->xColumnCount(pFts);

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnavgsize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnavgsize") ){
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      int colsz = 0;
      rc = pApi->xColumnAvgSize(pFts, i, &colsz);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
    }
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount ");
  }

  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnsize ");
  }

static void fts5DataReference(Fts5Data *pData){
  pData->nRef++;
}

/*
** INSERT OR REPLACE a record into the %_data table.
*/
static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
  if( p->rc!=SQLITE_OK ) return;

  if( p->pWriter==0 ){
    int rc;
    Fts5Config *pConfig = p->pConfig;
    char *zSql = sqlite3_mprintf(
        "REPLACE INTO '%q'.%Q(id, block) VALUES(?,?)", pConfig->zDb, p->zDataTbl

    if( rc==SQLITE_OK ){
      memset(&s, 0, sizeof(Fts5Structure));
      for(i=0; i<pConfig->nPrefix+1; i++){
        fts5StructureWrite(p, i, &s);
      }
      rc = p->rc;
    }
    sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
  }

  if( rc ){
    sqlite3Fts5IndexClose(p, 0);
    *pp = 0;
  }
  return rc;

      fts5StructureRelease(pIter->pStruct);
      fts5BufferFree(&pIter->poslist);
    }
    fts5CloseReader(pIter->pIndex);
    sqlite3_free(pIter);
  }
}

/*
** Read the "averages" record into the buffer supplied as the second 
** argument. Return SQLITE_OK if successful, or an SQLite error code
** if an error occurs.
*/
int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){
  fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID);
  return p->rc;
}

/*
** Replace the current "averages" record with the contents of the buffer 
** supplied as the second argument.
*/
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
  fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
  return p->rc;
}

*/

#include "fts5Int.h"

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
  i64 nTotalRow;                  /* Total number of rows in FTS table */
  i64 *aTotalSize;                /* Total sizes of each column */ 
  sqlite3_stmt *aStmt[9];
};


#if FTS5_STMT_SCAN_ASC!=0 
# error "FTS5_STMT_SCAN_ASC mismatch" 
#endif

  Fts5Index *pIndex, 
  int bCreate, 
  Fts5Storage **pp,
  char **pzErr                    /* OUT: Error message */
){
  int rc;
  Fts5Storage *p;                 /* New object */
  int nByte;                      /* Bytes of space to allocate */

  nByte = sizeof(Fts5Storage)               /* Fts5Storage object */
        + pConfig->nCol * sizeof(i64);      /* Fts5Storage.aTotalSize[] */
  *pp = p = (Fts5Storage*)sqlite3_malloc(nByte);
  if( !p ) return SQLITE_NOMEM;

  memset(p, 0, nByte);
  p->aTotalSize = (i64*)&p[1];
  p->pConfig = pConfig;
  p->pIndex = pIndex;

  if( bCreate ){
    int i;
    char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
    if( zDefn==0 ){

      for(iCol=1; iCol<=pConfig->nCol; iCol++){
        rc = sqlite3Fts5Tokenize(pConfig, 
            (const char*)sqlite3_column_text(pSeek, iCol),
            sqlite3_column_bytes(pSeek, iCol),
            (void*)&ctx,
            fts5StorageInsertCallback
        );
        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
      }
      p->nTotalRow--;
    }
    rc2 = sqlite3_reset(pSeek);
    if( rc==SQLITE_OK ) rc = rc2;
  }

  return rc;
}

    sqlite3_bind_int64(pReplace, 1, iRowid);
    sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  return rc;
}

/*
** Load the contents of the "averages" record from disk into the
** p->nTotalRow and p->aTotalSize[] variables.
**
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/
static int fts5StorageLoadTotals(Fts5Storage *p){
  int nCol = p->pConfig->nCol;
  Fts5Buffer buf;
  int rc;
  memset(&buf, 0, sizeof(buf));

  memset(p->aTotalSize, 0, sizeof(i64) * nCol);
  p->nTotalRow = 0;
  rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf);
  if( rc==SQLITE_OK && buf.n ){
    int i = 0;
    int iCol;
    i += getVarint(&buf.p[i], (u64*)&p->nTotalRow);
    for(iCol=0; i<buf.n && iCol<nCol; iCol++){
      i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
    }
  }
  sqlite3_free(buf.p);

  return rc;
}

/*
** Store the current contents of the p->nTotalRow and p->aTotalSize[] 
** variables in the "averages" record on disk.
**
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/
static int fts5StorageSaveTotals(Fts5Storage *p){
  int nCol = p->pConfig->nCol;
  int i;
  Fts5Buffer buf;
  int rc = SQLITE_OK;
  memset(&buf, 0, sizeof(buf));

  sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
  for(i=0; i<nCol; i++){
    sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
  }
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
  }
  sqlite3_free(buf.p);

  return rc;
}


/*
** Insert a new row into the FTS table.
*/
int sqlite3Fts5StorageInsert(
  Fts5Storage *p,                 /* Storage module to write to */
  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
  int eConflict,                  /* on conflict clause */
  i64 *piRowid                    /* OUT: rowid of new record */
){
  Fts5Config *pConfig = p->pConfig;
  int rc = SQLITE_OK;             /* Return code */
  sqlite3_stmt *pInsert;          /* Statement used to write %_content table */
  int eStmt;                      /* Type of statement used on %_content */
  int i;                          /* Counter variable */
  Fts5InsertCtx ctx;              /* Tokenization callback context object */
  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */

  memset(&buf, 0, sizeof(Fts5Buffer));
  rc = fts5StorageLoadTotals(p);

  /* Insert the new row into the %_content table. */
  if( rc==SQLITE_OK ){
    if( eConflict==SQLITE_REPLACE ){
      eStmt = FTS5_STMT_REPLACE_CONTENT;
      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
      }
    }else{
      eStmt = FTS5_STMT_INSERT_CONTENT;
    }
  }
  if( rc==SQLITE_OK ){
    rc = fts5StorageGetStmt(p, eStmt, &pInsert);
  }
  for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
    rc = sqlite3_bind_value(pInsert, i, apVal[i]);
  }

    rc = sqlite3Fts5Tokenize(pConfig, 
        (const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
        sqlite3_value_bytes(apVal[ctx.iCol+2]),
        (void*)&ctx,
        fts5StorageInsertCallback
    );
    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
  }
  p->nTotalRow++;

  /* Write the %_docsize record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageInsertDocsize(p, *piRowid, &buf);
  }
  sqlite3_free(buf.p);

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;
}

/*
** Context object used by sqlite3Fts5StorageIntegrity().
*/

    if( bCorrupt && rc==SQLITE_OK ){
      rc = SQLITE_CORRUPT_VTAB;
    }
  }
  return rc;
}

int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg){
  int rc = fts5StorageLoadTotals(p);
  if( rc==SQLITE_OK ){
    int nAvg = 1;
    if( p->nTotalRow ){
      nAvg = (int)((p->aTotalSize[iCol] + (p->nTotalRow/2)) / p->nTotalRow);
      if( nAvg<1 ) nAvg = 1;
      *pnAvg = nAvg;
    }
  }
  return rc;
}

  do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
  do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
  if {[set_test_counter errors]} break
}

#-------------------------------------------------------------------------
#
breakpoint
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
  INSERT INTO t1(t1) VALUES('pgsz=32');
}

do_execsql_test 6.1 {

do_execsql_test 4.1 {
  SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
  1 {0.5 {} {}} 
}

#-------------------------------------------------------------------------
# Test that the xColumnSize() and xColumnAvgsize() APIs work.
#

reset_db
do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE t5 USING fts5(x, y);
  INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
  INSERT INTO t5 VALUES('', 'a');

  SELECT rowid, fts5_test(t5, 'columntext') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {a {}}
  2 {{} a}
  1 {{a b c d} {e f g h i j}}
}

do_execsql_test 5.3 {
  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {2 2}
  2 {2 2}
  1 {2 2}
}

do_execsql_test 5.4 {
  INSERT INTO t5 VALUES('x y z', 'v w x y z');
  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {2 3}
  2 {2 3}
  1 {2 3}
}



finish_test