Documentation Source Text

    foreach { tag hdr text } $section {}
    if {[string trim $text]==""} continue
    incr i
    set url "${doc}#${tag}"
    insert_entry -rowid $i -url $url -title1 $title -title2 $hdr -content $text
  }
}

proc node_innerhtml {n} {
  set ret ""
  foreach c [$n children] { append ret [$c html] }
  set ret
}

proc changelog_document_import {doc} {

  set content [readfile $doc]
  set end [string first "Changes carried forward from version " $content]
  if {$end>0} { set content [string range $content 0 $end] }

  set dom [::hdom::parse $content]

  foreach n [[$dom root] search p] {
    set c [lindex [$n children] 0]
    if {$c!="" && [$c tag]=="b"} { $n detach }
  }

  # Extract the version number from the document name.
  set version 0.0.0
  regexp {releaselog/(.*).html} $doc -> version
  set version [string map {_ .} $version]

  # Find each of the <li> nodes in the document.
  foreach li [[$dom root] search li] {
    if {0==[releaselog_filterscript $li]} continue

    set i 1
    set ol [$li parent]
    if {$ol=="" || [$ol tag]!="ol"} {error UNTHINKABLE!}
    foreach c [$ol children] {
      if {$c==$li} break
      if {[$c tag]=="li"} {incr i}
    }

    #set t [$li text]
    set t [node_innerhtml $li]
    db eval { INSERT INTO change VALUES($doc, $version, $i, $t) }
  }
}

proc rebuild_database {} {

  db transaction {

      INSERT INTO page(page, rank) VALUES('rank', 'bm25(10.0,10.0,20.0,20.0)');

      DROP TABLE IF EXISTS change;
      CREATE VIRTUAL TABLE change USING fts5(
          url UNINDEXED,          -- Path to document
          version UNINDEXED,      -- SQLite version number
          idx UNINDEXED,          -- Bullet point number
          text,                   -- Text of change log entry
          tokenize='html stoken unicode61 tokenchars _' -- Tokenizer definition
      );
    }

    foreach doc [document_list changelog] { 
      puts "Indexing $doc..."
      changelog_document_import $doc 
    }

  if( rc==SQLITE_OK && nToken>8 && 0==memcmp("sqlite3_", pToken, 8) ){
    rc = p->xToken(
        p->pCtx, FTS5_TOKEN_COLOCATED, pToken+8, nToken-8, iStart, iEnd);
  }

  return rc;
}

/*
** Tokenizer type for "html" tokenizer. Casts to Fts5Tokenizer.
*/
typedef struct HtmlTokenizer HtmlTokenizer;
struct HtmlTokenizer {
  fts5_tokenizer tokenizer;
  Fts5Tokenizer *pTokenizer;
};

static int htmlCreate(
  void *pCtx, 
  const char **azArg, int nArg, 
  Fts5Tokenizer **ppOut
){
  fts5_api *pApi = (fts5_api*)pCtx;
  HtmlTokenizer *p = 0;
  int rc = SQLITE_OK;

  if( nArg==0 ){
    rc = SQLITE_ERROR;
  }else{
    /* Allocate the Fts5Tokenizer object for this tokenizer. */
    p = sqlite3_malloc(sizeof(HtmlTokenizer));
    if( p ){
      memset(p, 0, sizeof(HtmlTokenizer));
    }else{
      return SQLITE_NOMEM;
    }
  }

  if( rc==SQLITE_OK ){
    /* Locate and allocate the next tokenizer */
    void *pNextCtx = 0;
    rc = pApi->xFindTokenizer(pApi, azArg[0], &pNextCtx, &p->tokenizer);
    if( rc==SQLITE_OK ){
      rc = p->tokenizer.xCreate(pNextCtx, &azArg[1], nArg-1, &p->pTokenizer);
    }
  }

  /* Return the new tokenizer to the caller */
  if( rc!=SQLITE_OK ){
    sqlite3_free(p);
    p = 0;
  }
  *ppOut = (Fts5Tokenizer*)p;
  return rc;
}

static void htmlDelete(Fts5Tokenizer *pTokenizer){
  HtmlTokenizer *p = (HtmlTokenizer*)pTokenizer;
  p->tokenizer.xDelete(p->pTokenizer);
  sqlite3_free(p);
}

static int htmlTokenize(
  Fts5Tokenizer *pTokenizer, 
  void *pCtx,
  int flags,            /* Mask of FTS5_TOKENIZE_* flags */
  const char *pText, int nText, 
  int (*xToken)(
    void *pCtx,         /* Copy of 2nd argument to xTokenize() */
    int tflags,         /* Mask of FTS5_TOKEN_* flags */
    const char *pToken, /* Pointer to buffer containing token */
    int nToken,         /* Size of token in bytes */
    int iStart,         /* Byte offset of token within input text */
    int iEnd            /* Byte offset of end of token within input text */
  )
){
  HtmlTokenizer *p = (HtmlTokenizer*)pTokenizer;
  char *zOut;
  int i;
  int bTag=0;
  int rc;
  
  zOut = sqlite3_malloc(nText+1);
  if( zOut==0 ){
    return SQLITE_NOMEM;
  }
  for(i=0; i<nText; i++){
    char c = pText[i];
    if( bTag==0 && c=='<' ) bTag = 1;
    zOut[i] = bTag ? ' ' : c;
    if( bTag==1 && c=='>' ) bTag = 0;
  }

  rc = p->tokenizer.xTokenize(p->pTokenizer, pCtx, flags, zOut, nText, xToken);
  sqlite3_free(zOut);
  return rc;
}

static int stokenTokenize(
  Fts5Tokenizer *pTokenizer, 
  void *pCtx,
  int flags,            /* Mask of FTS5_TOKENIZE_* flags */
  const char *pText, int nText, 
  int (*xToken)(

    );
  }else{
    rc = p->porter.xTokenize(p->pPorter, pCtx, flags, pText, nText, xToken);
  }

  return rc;
}


static int register_tokenizer(sqlite3 *db, char **pzErr, void *p){
  fts5_api *pApi;
  fts5_tokenizer t;
  int rc;

  pApi = fts5_api_from_db(db);
  if( pApi==0 ){
    *pzErr = sqlite3_mprintf("fts5_api_from_db: %s", sqlite3_errmsg(db));
    return SQLITE_ERROR;
  }

  t.xCreate = stokenCreate;
  t.xDelete = stokenDelete;
  t.xTokenize = stokenTokenize;
  rc = pApi->xCreateTokenizer(pApi, "stoken", (void*)pApi, &t, 0);

  if( rc==SQLITE_OK ){
    t.xCreate = htmlCreate;
    t.xDelete = htmlDelete;
    t.xTokenize = htmlTokenize;
    rc = pApi->xCreateTokenizer(pApi, "html", (void*)pApi, &t, 0);
  }

  if( rc==SQLITE_OK ){
    rc = pApi->xCreateFunction(pApi, "srank", 0, srankFunc, 0);
  }

  return rc;
}