Index: ext/fts5/fts5.c ================================================================== --- ext/fts5/fts5.c +++ ext/fts5/fts5.c @@ -617,12 +617,13 @@ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol; } static int fts5ApiColumnAvgSize(Fts5Context *pCtx, int iCol, int *pnToken){ - assert( 0 ); - return 0; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + return sqlite3Fts5StorageAvgsize(pTab->pStorage, iCol, pnToken); } static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -265,10 +265,13 @@ /* Called during startup to register a UDF with SQLite */ int sqlite3Fts5IndexInit(sqlite3*); void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf); +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** @@ -295,11 +298,11 @@ int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol); +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg); /* ** End of interface to code in fts5_storage.c. **************************************************************************/ Index: ext/fts5/fts5_aux.c ================================================================== --- ext/fts5/fts5_aux.c +++ ext/fts5/fts5_aux.c @@ -41,15 +41,28 @@ if( nVal>=1 ){ zReq = (const char*)sqlite3_value_text(apVal[0]); } memset(&s, 0, sizeof(Fts5Buffer)); + nCol = pApi->xColumnCount(pFts); + + if( zReq==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnavgsize "); + } + if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnavgsize") ){ + if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{"); + for(i=0; rc==SQLITE_OK && ixColumnAvgSize(pFts, i, &colsz); + sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz); + } + if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}"); + } if( zReq==0 ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount "); } - nCol = pApi->xColumnCount(pFts); if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){ sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol); } if( zReq==0 ){ Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -732,11 +732,11 @@ } /* ** INSERT OR REPLACE a record into the %_data table. */ -static void fts5DataWrite(Fts5Index *p, i64 iRowid, u8 *pData, int nData){ +static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->rc!=SQLITE_OK ) return; if( p->pWriter==0 ){ int rc; Fts5Config *pConfig = p->pConfig; @@ -2732,10 +2732,11 @@ for(i=0; inPrefix+1; i++){ fts5StructureWrite(p, i, &s); } rc = p->rc; } + sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); } if( rc ){ sqlite3Fts5IndexClose(p, 0); *pp = 0; @@ -3616,6 +3617,25 @@ } fts5CloseReader(pIter->pIndex); sqlite3_free(pIter); } } + +/* +** Read the "averages" record into the buffer supplied as the second +** argument. Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){ + fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID); + return p->rc; +} + +/* +** Replace the current "averages" record with the contents of the buffer +** supplied as the second argument. +*/ +int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ + fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); + return p->rc; +} Index: ext/fts5/fts5_storage.c ================================================================== --- ext/fts5/fts5_storage.c +++ ext/fts5/fts5_storage.c @@ -15,10 +15,12 @@ #include "fts5Int.h" struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; + i64 nTotalRow; /* Total number of rows in FTS table */ + i64 *aTotalSize; /* Total sizes of each column */ sqlite3_stmt *aStmt[9]; }; #if FTS5_STMT_SCAN_ASC!=0 @@ -166,15 +168,19 @@ Fts5Storage **pp, char **pzErr /* OUT: Error message */ ){ int rc; Fts5Storage *p; /* New object */ + int nByte; /* Bytes of space to allocate */ - *pp = p = (Fts5Storage*)sqlite3_malloc(sizeof(Fts5Storage)); + nByte = sizeof(Fts5Storage) /* Fts5Storage object */ + + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ + *pp = p = (Fts5Storage*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, sizeof(Fts5Storage)); + memset(p, 0, nByte); + p->aTotalSize = (i64*)&p[1]; p->pConfig = pConfig; p->pIndex = pIndex; if( bCreate ){ int i; @@ -283,11 +289,13 @@ (const char*)sqlite3_column_text(pSeek, iCol), sqlite3_column_bytes(pSeek, iCol), (void*)&ctx, fts5StorageInsertCallback ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; } + p->nTotalRow--; } rc2 = sqlite3_reset(pSeek); if( rc==SQLITE_OK ) rc = rc2; } @@ -312,10 +320,66 @@ sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } return rc; } + +/* +** Load the contents of the "averages" record from disk into the +** p->nTotalRow and p->aTotalSize[] variables. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageLoadTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + Fts5Buffer buf; + int rc; + memset(&buf, 0, sizeof(buf)); + + memset(p->aTotalSize, 0, sizeof(i64) * nCol); + p->nTotalRow = 0; + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf); + if( rc==SQLITE_OK && buf.n ){ + int i = 0; + int iCol; + i += getVarint(&buf.p[i], (u64*)&p->nTotalRow); + for(iCol=0; iaTotalSize[iCol]); + } + } + sqlite3_free(buf.p); + + return rc; +} + +/* +** Store the current contents of the p->nTotalRow and p->aTotalSize[] +** variables in the "averages" record on disk. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5StorageSaveTotals(Fts5Storage *p){ + int nCol = p->pConfig->nCol; + int i; + Fts5Buffer buf; + int rc = SQLITE_OK; + memset(&buf, 0, sizeof(buf)); + + sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow); + for(i=0; iaTotalSize[i]); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n); + } + sqlite3_free(buf.p); + + return rc; +} + /* ** Insert a new row into the FTS table. */ int sqlite3Fts5StorageInsert( @@ -331,19 +395,22 @@ int i; /* Counter variable */ Fts5InsertCtx ctx; /* Tokenization callback context object */ Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ memset(&buf, 0, sizeof(Fts5Buffer)); + rc = fts5StorageLoadTotals(p); /* Insert the new row into the %_content table. */ - if( eConflict==SQLITE_REPLACE ){ - eStmt = FTS5_STMT_REPLACE_CONTENT; - if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ - rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); + if( rc==SQLITE_OK ){ + if( eConflict==SQLITE_REPLACE ){ + eStmt = FTS5_STMT_REPLACE_CONTENT; + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ + rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); + } + }else{ + eStmt = FTS5_STMT_INSERT_CONTENT; } - }else{ - eStmt = FTS5_STMT_INSERT_CONTENT; } if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, eStmt, &pInsert); } for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ @@ -365,17 +432,24 @@ sqlite3_value_bytes(apVal[ctx.iCol+2]), (void*)&ctx, fts5StorageInsertCallback ); sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); + p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; } + p->nTotalRow++; /* Write the %_docsize record */ if( rc==SQLITE_OK ){ rc = fts5StorageInsertDocsize(p, *piRowid, &buf); } sqlite3_free(buf.p); + + /* Write the averages record */ + if( rc==SQLITE_OK ){ + rc = fts5StorageSaveTotals(p); + } return rc; } /* @@ -536,9 +610,18 @@ } } return rc; } -int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol){ - return 0; +int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg){ + int rc = fts5StorageLoadTotals(p); + if( rc==SQLITE_OK ){ + int nAvg = 1; + if( p->nTotalRow ){ + nAvg = (int)((p->aTotalSize[iCol] + (p->nTotalRow/2)) / p->nTotalRow); + if( nAvg<1 ) nAvg = 1; + *pnAvg = nAvg; + } + } + return rc; } Index: test/fts5aa.test ================================================================== --- test/fts5aa.test +++ test/fts5aa.test @@ -127,10 +127,11 @@ if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # +breakpoint reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); INSERT INTO t1(t1) VALUES('pgsz=32'); } Index: test/fts5ae.test ================================================================== --- test/fts5ae.test +++ test/fts5ae.test @@ -126,11 +126,11 @@ } { 1 {0.5 {} {}} } #------------------------------------------------------------------------- -# Test that the xColumnSize() API works. +# Test that the xColumnSize() and xColumnAvgsize() APIs work. # reset_db do_execsql_test 5.1 { CREATE VIRTUAL TABLE t5 USING fts5(x, y); @@ -153,9 +153,29 @@ } { 3 {a {}} 2 {{} a} 1 {{a b c d} {e f g h i j}} } + +do_execsql_test 5.3 { + SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {2 2} + 2 {2 2} + 1 {2 2} +} + +do_execsql_test 5.4 { + INSERT INTO t5 VALUES('x y z', 'v w x y z'); + SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a' + ORDER BY rowid DESC; +} { + 3 {2 3} + 2 {2 3} + 1 {2 3} +} + finish_test