SQLite

**
******************************************************************************
*/

#include "fts5Int.h"
#include <math.h>

typedef struct SnipPhrase SnipPhrase;
typedef struct SnipIter SnipIter;
typedef struct SnippetCtx SnippetCtx;

struct SnipPhrase {
  u64 mask;                       /* Current mask */
  int nToken;                     /* Tokens in this phrase */
  int i;                          /* Current offset in phrase poslist */
  i64 iPos;                       /* Next position in phrase (-ve -> EOF) */
};

struct SnipIter {
  i64 iLast;                      /* Last token position of current snippet */
  int nScore;                     /* Score of current snippet */

  const Fts5ExtensionApi *pApi;
  Fts5Context *pFts;
  u64 szmask;                     /* Mask used to on SnipPhrase.mask */
  int nPhrase;                    /* Number of phrases */
  SnipPhrase aPhrase[0];       /* Array of size nPhrase */
};

struct SnippetCtx {
  int iFirst;                     /* Offset of first token to record */
  int nToken;                     /* Size of aiStart[] and aiEnd[] arrays */
  int iSeen;                      /* Set to largest offset seen */
  int *aiStart; 

  return rc;
}

/*
** Set pIter->nScore to the score for the current entry.
*/
static void fts5SnippetCalculateScore(SnipIter *pIter){
  int i;
  int nScore = 0;
  assert( pIter->iLast>=0 );

  for(i=0; i<pIter->nPhrase; i++){
    SnipPhrase *p = &pIter->aPhrase[i];
    u64 mask = p->mask;
    if( mask ){
      u64 j;
      nScore += 1000;
      for(j=1; j & pIter->szmask; j<<=1){
        if( mask & j ) nScore++;
      }
    }
  }

  pIter->nScore = nScore;
}

/*
** Allocate a new snippet iter.
*/
static int fts5SnipIterNew(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  int nToken,                     /* Number of tokens in snippets */
  SnipIter **ppIter            /* OUT: New object */
){
  int i;                          /* Counter variable */
  SnipIter *pIter;             /* New iterator object */
  int nByte;                      /* Bytes of space to allocate */
  int nPhrase;                    /* Number of phrases in query */

  *ppIter = 0;
  nPhrase = pApi->xPhraseCount(pFts);
  nByte = sizeof(SnipIter) + nPhrase * sizeof(SnipPhrase);
  pIter = (SnipIter*)sqlite3_malloc(nByte);
  if( pIter==0 ) return SQLITE_NOMEM;
  memset(pIter, 0, nByte);

  pIter->nPhrase = nPhrase;
  pIter->pApi = pApi;
  pIter->pFts = pFts;
  pIter->szmask = ((u64)1 << nToken) - 1;
  assert( nToken<=63 );

  for(i=0; i<nPhrase; i++){
    pIter->aPhrase[i].nToken = pApi->xPhraseSize(pFts, i);
  }

  *ppIter = pIter;
  return SQLITE_OK;
}

/*
** Set the iterator to point to the first candidate snippet.
*/
static void fts5SnipIterFirst(SnipIter *pIter){
  const Fts5ExtensionApi *pApi = pIter->pApi;
  Fts5Context *pFts = pIter->pFts;
  int i;                          /* Used to iterate through phrases */
  SnipPhrase *pMin = 0;        /* Phrase with first match */

  memset(pIter->aPhrase, 0, sizeof(SnipPhrase) * pIter->nPhrase);

  for(i=0; i<pIter->nPhrase; i++){
    SnipPhrase *p = &pIter->aPhrase[i];
    p->nToken = pApi->xPhraseSize(pFts, i);
    pApi->xPoslist(pFts, i, &p->i, &p->iPos);
    if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ){
      pMin = p;
    }
  }
  assert( pMin );

  pIter->iLast = pMin->iPos + pMin->nToken - 1;
  pMin->mask = 0x01;
  pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
  fts5SnippetCalculateScore(pIter);
}

/*
** Advance the snippet iterator to the next candidate snippet.
*/
static void fts5SnipIterNext(SnipIter *pIter){
  const Fts5ExtensionApi *pApi = pIter->pApi;
  Fts5Context *pFts = pIter->pFts;
  int nPhrase = pIter->nPhrase;
  int i;                          /* Used to iterate through phrases */
  SnipPhrase *pMin = 0;

  for(i=0; i<nPhrase; i++){
    SnipPhrase *p = &pIter->aPhrase[i];
    if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ) pMin = p;
  }

  if( pMin==0 ){
    /* pMin==0 indicates that the SnipIter is at EOF. */
    pIter->iLast = -1;
  }else{
    i64 nShift = pMin->iPos - pIter->iLast;
    assert( nShift>=0 );
    for(i=0; i<nPhrase; i++){
      SnipPhrase *p = &pIter->aPhrase[i];
      if( nShift>=63 ){
        p->mask = 0;
      }else{
        p->mask = p->mask << (int)nShift;
        p->mask &= pIter->szmask;
      }
    }

    pIter->iLast = pMin->iPos;
    pMin->mask |= 0x01;
    fts5SnippetCalculateScore(pIter);
    pApi->xPoslist(pFts, pMin - pIter->aPhrase, &pMin->i, &pMin->iPos);
  }
}

static void fts5SnipIterFree(SnipIter *pIter){
  if( pIter ){
    sqlite3_free(pIter);
  }
}

static int fts5SnippetText(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  SnipIter *pIter,             /* Snippet to write to buffer */
  int nToken,                     /* Size of desired snippet in tokens */
  const char *zStart,
  const char *zFinal,
  const char *zEllip,
  Fts5Buffer *pBuf                /* Write output to this buffer */
){
  SnippetCtx ctx;

    iMatchto = -1;

    for(i=i1; i<=i2; i++){

      /* Check if this is the first token of any phrase match. */
      int ip;
      for(ip=0; ip<pIter->nPhrase; ip++){
        SnipPhrase *pPhrase = &pIter->aPhrase[ip];
        u64 m = (1 << (iLast - i - pPhrase->nToken + 1));

        if( i<=iLast && (pPhrase->mask & m) ){
          if( iMatchto<0 ){
            sqlite3Fts5BufferAppendPrintf(&rc, pBuf, "%.*s%s",
                aiStart[i] - aiStart[iPrint],
                &zCol[aiStart[iPrint]],

){
  const char *zStart = "<b>";
  const char *zFinal = "</b>";
  const char *zEllip = "<b>...</b>";
  int nToken = -15;
  int nAbs;
  int rc;
  SnipIter *pIter = 0;

  if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
  if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
  if( nVal>=3 ) zEllip = (const char*)sqlite3_value_text(apVal[2]);
  if( nVal>=4 ){
    nToken = sqlite3_value_int(apVal[3]);
    if( nToken==0 ) nToken = -15;
  }
  nAbs = nToken * (nToken<0 ? -1 : 1);

  rc = fts5SnipIterNew(pApi, pFts, nAbs, &pIter);
  if( rc==SQLITE_OK ){
    Fts5Buffer buf;               /* Result buffer */
    int nBestScore = 0;           /* Score of best snippet found */

    for(fts5SnipIterFirst(pIter); 
        pIter->iLast>=0; 
        fts5SnipIterNext(pIter)
    ){
      if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
    }
    for(fts5SnipIterFirst(pIter); 
        pIter->iLast>=0; 
        fts5SnipIterNext(pIter)
    ){
      if( pIter->nScore==nBestScore ) break;
    }

    memset(&buf, 0, sizeof(Fts5Buffer));
    rc = fts5SnippetText(pApi, pFts, pIter, nAbs, zStart, zFinal, zEllip, &buf);
    if( rc==SQLITE_OK ){
      sqlite3_result_text(pCtx, (const char*)buf.p, buf.n, SQLITE_TRANSIENT);
    }
    sqlite3_free(buf.p);
  }

  fts5SnipIterFree(pIter);
  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}


/*

**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(const char *z){
  return sqlite3_mprintf("%s", z);
}

void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);

/*
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
  const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
  sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance */
  int rc;                         /* Return code */

  sqlite3Fts3SimpleTokenizerModule(&pMod);
  rc = pMod->xCreate(0, 0, &pTokenizer);
  if( rc==SQLITE_OK ){
    pTokenizer->pModule = pMod;

** Argument pIn points to a buffer of nIn bytes. This function allocates
** and returns a new buffer populated with a copy of (pIn/nIn) with a 
** nul-terminator byte appended to it.
**
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. 
*/
static char *fts5Strndup(const char *pIn, int nIn){
  char *zRet = (char*)sqlite3_malloc(nIn+1);
  if( zRet ){
    memcpy(zRet, pIn, nIn);
    zRet[nIn] = '\0';
  }
  return zRet;
}

static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
  *pz = fts5Strndup(pToken->p, pToken->n);
  if( *pz==0 ) return SQLITE_NOMEM;
  return SQLITE_OK;
}

/*
** Free the phrase object passed as the only argument.
*/

    if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
    pCtx->pPhrase = pPhrase = pNew;
    pNew->nTerm = nNew - SZALLOC;
  }

  pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
  memset(pTerm, 0, sizeof(Fts5ExprTerm));
  pTerm->zTerm = fts5Strndup(pToken, nToken);

  return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
}


/*
** Free the phrase object passed as the only argument.

  $(TOP)/ext/icu/sqliteicu.h \
  $(TOP)/ext/icu/icu.c
SRC += \
  $(TOP)/ext/rtree/sqlite3rtree.h \
  $(TOP)/ext/rtree/rtree.h \
  $(TOP)/ext/rtree/rtree.c

SRC += \
   $(TOP)/ext/fts5/fts5.h \
   $(TOP)/ext/fts5/fts5Int.h \
   $(TOP)/ext/fts5/fts5_aux.c \
   $(TOP)/ext/fts5/fts5_buffer.c \
   $(TOP)/ext/fts5/fts5.c \
   $(TOP)/ext/fts5/fts5_config.c \
   $(TOP)/ext/fts5/fts5_expr.c \
   $(TOP)/ext/fts5/fts5_index.c \
   fts5parse.c \
   $(TOP)/ext/fts5/fts5_storage.c 


# Generated source code files
#
SRC += \
  keywordhash.h \
  opcodes.c \
  opcodes.h \

wordcount$(EXE):	$(TOP)/test/wordcount.c sqlite3.c
	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o wordcount$(EXE) \
		$(TOP)/test/wordcount.c sqlite3.c

speedtest1$(EXE):	$(TOP)/test/speedtest1.c sqlite3.o
	$(TCC) -I. -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB)

loadfts: $(TOP)/tool/loadfts.c libsqlite3.a
	$(TCC) $(TOP)/tool/loadfts.c libsqlite3.a -o loadfts $(THREADLIB)

# This target will fail if the SQLite amalgamation contains any exported
# symbols that do not begin with "sqlite3_". It is run as part of the
# releasetest.tcl script.
#
checksymbols: sqlite3.o
	nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0

/*
** 2013-06-10
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
*/

#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <assert.h>
#include <string.h>
#include <errno.h>
#include <dirent.h>
#include "sqlite3.h"

/*
** Implementation of the "readtext(X)" SQL function.  The entire content
** of the file named X is read and returned as a TEXT value. It is assumed
** the file contains UTF-8 text. NULL is returned if the file does not 
** exist or is unreadable.
*/
static void readfileFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zName;
  FILE *in;
  long nIn;
  void *pBuf;

  zName = (const char*)sqlite3_value_text(argv[0]);
  if( zName==0 ) return;
  in = fopen(zName, "rb");
  if( in==0 ) return;
  fseek(in, 0, SEEK_END);
  nIn = ftell(in);
  rewind(in);
  pBuf = sqlite3_malloc( nIn );
  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
    sqlite3_result_text(context, pBuf, nIn, sqlite3_free);
  }else{
    sqlite3_free(pBuf);
  }
  fclose(in);
}

/*
** Print usage text for this program and exit.
*/
static void showHelp(const char *zArgv0){
  printf("\n"
"Usage: %s SWITCHES... DB\n"
"\n"
"  This program opens the database named on the command line and attempts to\n"
"  create an FTS table named \"fts\" with a single column. If successful, it\n"
"  recursively traverses the directory named by the -dir option and inserts\n"
"  the contents of each file into the fts table. All files are assumed to\n"
"  contain UTF-8 text.\n"
"\n"
"Switches are:\n"
"  -fts [345]       FTS version to use (default=5)\n"
"  -idx [01]        Create a mapping from filename to rowid (default=0)\n"
"  -dir <path>      Root of directory tree to load data from (default=.)\n"
, zArgv0
);
  exit(1);
}

/*
** Exit with a message based on the argument and the current value of errno.
*/
static void error_out(const char *zText){
  fprintf(stderr, "%s: %s\n", zText, strerror(errno));
  exit(-1);
}

/*
** Exit with a message based on the first argument and the error message
** currently stored in database handle db.
*/
static void sqlite_error_out(const char *zText, sqlite3 *db){
  fprintf(stderr, "%s: %s\n", zText, sqlite3_errmsg(db));
  exit(-1);
}

/*
** Context object for visit_file().
*/
typedef struct VisitContext VisitContext;
struct VisitContext {
  sqlite3 *db;                    /* Database handle */
  sqlite3_stmt *pInsert;          /* INSERT INTO fts VALUES(readtext(:1)) */
};

/*
** Callback used with traverse(). The first argument points to an object
** of type VisitContext. This function inserts the contents of the text
** file zPath into the FTS table.
*/
void visit_file(void *pCtx, const char *zPath){
  int rc;
  VisitContext *p = (VisitContext*)pCtx;
  /* printf("%s\n", zPath); */
  sqlite3_bind_text(p->pInsert, 1, zPath, -1, SQLITE_STATIC);
  sqlite3_step(p->pInsert);
  rc = sqlite3_reset(p->pInsert);
  if( rc!=SQLITE_OK ) sqlite_error_out("insert", p->db);
}

/*
** Recursively traverse directory zDir. For each file that is not a 
** directory, invoke the supplied callback with its path.
*/
static void traverse(
  const char *zDir,               /* Directory to traverse */
  void *pCtx,                     /* First argument passed to callback */
  void (*xCallback)(void*, const char *zPath)
){
  DIR *d;
  struct dirent *e;

  d = opendir(zDir);
  if( d==0 ) error_out("opendir()");

  for(e=readdir(d); e; e=readdir(d)){
    if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue;
    char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name);
    if (e->d_type & DT_DIR) {
      traverse(zPath, pCtx, xCallback);
    }else{
      xCallback(pCtx, zPath);
    }
    sqlite3_free(zPath);
  }

  closedir(d);
}

int main(int argc, char **argv){
  int iFts = 5;                   /* Value of -fts option */
  int bMap = 0;                   /* True to create mapping table */
  const char *zDir = ".";         /* Directory to scan */
  int i;
  int rc;
  sqlite3 *db;
  char *zSql;
  VisitContext sCtx;

  if( argc % 2 ) showHelp(argv[0]);

  for(i=1; i<(argc-1); i+=2){
    char *zOpt = argv[i];
    char *zArg = argv[i+1];
    if( strcmp(zOpt, "-fts")==0 ){
      iFts = atoi(zArg);
      if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]);
    }
    else if( strcmp(zOpt, "-idx")==0 ){
      bMap = atoi(zArg);
      if( bMap!=0 && bMap!=1 ) showHelp(argv[0]);
    }
    else if( strcmp(zOpt, "-dir")==0 ){
      zDir = zArg;
    }
  }

  /* Open the database file */
  rc = sqlite3_open(argv[argc-1], &db);
  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_open()", db);

  rc = sqlite3_create_function(db, "readtext", 1, SQLITE_UTF8, 0,
                               readfileFunc, 0, 0);
  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_create_function()", db);

  /* Create the FTS table */
  zSql = sqlite3_mprintf("CREATE VIRTUAL TABLE fts USING fts%d(content)", iFts);
  rc = sqlite3_exec(db, zSql, 0, 0, 0);
  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db);
  sqlite3_free(zSql);

  /* Compile the INSERT statement to write data to the FTS table. */
  memset(&sCtx, 0, sizeof(VisitContext));
  sCtx.db = db;
  rc = sqlite3_prepare_v2(db, 
      "INSERT INTO fts VALUES(readtext(?))", -1, &sCtx.pInsert, 0
  );
  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_prepare_v2(1)", db);

  /* Load all files in the directory hierarchy into the FTS table. */
  traverse(zDir, (void*)&sCtx, visit_file);

  /* Clean up and exit. */
  sqlite3_finalize(sCtx.pInsert);
  sqlite3_close(db);
  return 0;
}

foreach hdr {
   btree.h
   btreeInt.h
   fts3.h
   fts3Int.h
   fts3_hash.h
   fts3_tokenizer.h
   fts5.h
   fts5Int.h
   hash.h
   hwtime.h
   keywordhash.h
   mutex.h
   opcodes.h
   os_common.h
   os_setup.h

   fts3_tokenizer1.c
   fts3_tokenize_vtab.c
   fts3_write.c
   fts3_snippet.c
   fts3_unicode.c
   fts3_unicode2.c

   fts5_aux.c
   fts5_buffer.c
   fts5.c
   fts5_config.c
   fts5_expr.c
   fts5_index.c
   fts5parse.c
   fts5_storage.c

   rtree.c
   icu.c
   fts3_icu.c
} {
  copy_file tsrc/$file
}

close $out