Documentation Source Text

CFLAGS = -g -DSQLITE_MEMDEBUG -DSQLITE_DEBUG
CFLAGS = -g -O2 -DNDEBUG
#CFLAGS = -Os -DNDEBUG

tclsqlite3.fts3: searchc.c /home/dan/work/sqliteG/bld/tclsqlite3.c
	gcc $(CFLAGS) -DSQLITE_ENABLE_FTS3 -I/home/dan/work/sqliteG/sqlite/ext/fts3/ -static searchc.c /home/dan/work/sqliteG/bld/tclsqlite3.c -L/home/dan/tcl86/lib -ltcl8.6 -ldl -lm -o tclsqlite3.fts3 -lz -lpthread

clean:
	rm tclsqlite3.fts3

#!/usr/bin/tclsqlite3.fts3

#=========================================================================
# This proc is called to parse the arguments passed to this invocation of
# the CGI program (via either the GET or POST method). It returns a
# key/value list containing the arguments suitable for passing to [array
# set]. For example, if the CGI is invoked via a GET request on the URI:
#
#   http://www.sqlite.org/search?query=fts3+table&results=10
#
# then the returned list value is: 
#
#   {query {fts3 table} results 10}
#
proc cgi_parse_args {} {
  global env

  set charmap {
    + { } %20 { } %21 ! %22 \x22 %23 # %24 {$} %25 % %26 & %27 ' %28 ( 
    %29 ) %2A * %2B + %2C , %2D - %2E . %2F / %30 0 %31 1 %32 2 %33 3 
    %34 4 %35 5 %36 6 %37 7 %38 8 %39 9 %3A : %3B {;} %3C < %3D = %3E > 
    %3F ? %40 @ %41 A %42 B %43 C %44 D %45 E %46 F %47 G %48 H %49 I 
    %4A J %4B K %4C L %4D M %4E N %4F O %50 P %51 Q %52 R %53 S %54 T 
    %55 U %56 V %57 W %58 X %59 Y %5A Z %5B {[} %5C \\ %5D \] %5E ^ 
    %5F _ %60 ` %61 a %62 b %63 c %64 d %65 e %66 f %67 g %68 h %69 i 
    %6A j %6B k %6C l %6D m %6E n %6F o %70 p %71 q %72 r %73 s %74 t 
    %75 u %76 v %77 w %78 x %79 y %7A z %7B \{ %7C | %7D \} %7E ~
  }

  if {$env(REQUEST_METHOD) == "GET"} {
    foreach q [split $::env(QUERY_STRING) &] {
      if {0==[regexp {([a-z]*)=(.*)} $q -> var value]} {
        error "Failed to parse: $q"
      }

      set A($var) [string map $charmap $value]
    }
  } elseif {$env(REQUEST_METHOD) == "POST"} {
    # TODO!
  } else {
    error "Unrecognized method: $env(REQUEST_METHOD)"
  }

  array get A
}


#=========================================================================
# Redirect the web-browser to URL $url. This command does not return.
#
proc cgi_redirect {url} {
  set server $::env(SERVER_NAME)
  set path [file dirname $::env(REQUEST_URI)]
  if {[string range $path end end]!="/"} {
    append path /
  }

  puts "Status: 302 Redirect"
  puts "Location: http://${server}${path}${url}"
  puts "Content-Length: 0"
  puts ""
  exit
}

#=========================================================================
# The argument contains a key value list. The values in the list are
# transformed to an HTTP query key value list. For example:
#
#   % cgi_encode_args {s "search string" t "search \"type\""}
#   s=search+string&t=search+%22type%22
#
proc cgi_encode_args {list} {
  set reslist [list]
  foreach {key value} $list {
    set value [string map {
      \x20 +   \x21 %21 \x2A %2A \x22 %22 \x27 %27 \x28 %28 \x29 %29 \x3B %3B 
      \x3A %3A \x40 %40 \x26 %26 \x3D %3D \x2B %2B \x24 %24 \x2C %2C \x2F %2F 
      \x3F %3F \x25 %25 \x23 %23 \x5B %5B \x5D %5D
    } $value]

    lappend reslist "$key=$value"
  }
  join $reslist &
}

proc htmlize {str} { string map {< &lt; > &gt;} $str }
proc attrize {str} { string map {< &lt; > &gt; \x22 &quot;} $str }

#=========================================================================

proc cgi_env_dump {} {

  set ret "<h1>Arguments</h1><table>"
  foreach {key value} [array get ::A] {
    append ret "<tr><td>[htmlize $key]<td>[htmlize $value]"
  }
  append ret "</table>"

  append ret "<h1>Environment</h1><table>"
  foreach {key value} [array get ::env] {
    append ret "<tr><td>[htmlize $key]<td>[htmlize $value]"
  }
  append ret "</table>"
  return $ret
}

proc searchform {} {
  set initial "Enter search term:"
  catch { set initial $::A(q) }
  return [subst {
    <table style="margin: 1em auto"> <tr><td>Search SQLite docs for:<td>
      <form name=f method=GET action=search>
        <input name=q type=text width=35 value="[attrize $initial]"></input>
        <input name=s type=submit value="Search"></input>
        <input name=s type=submit value="Lucky"></input>
      </form>
    </table>
    <script> 
      document.forms.f.q.focus()
      document.forms.f.q.select()
    </script>
  }]
}

proc rebuildform {} {
  return {
    <hr>
    <table align=right>
    <td><form name=r method=GET action=search>
      <input name=rebuild type=submit value="Rebuild Db"></input>
    </form>
    </table>
  }
}


#-------------------------------------------------------------------------
# This command is similar to the builtin Tcl [time] command, except that
# it only ever runs the supplied script once. Also, instead of returning
# a string like "xxx microseconds per iteration", it returns "x.yy ms" or
# "x.yy s", depending on the magnitude of the time spent running the 
# command. For example:
#
#   % ttime {after 1500}
#   1.50 s
#   % ttime {after 45}
#   45.02 ms
#
proc ttime {script} {
  set t [lindex [time [list uplevel $script]] 0]
  if {$t>1000000} { return [format "%.2f s" [expr {$t/1000000.0}]] }
  return [format "%.2f ms" [expr {$t/1000.0}]]
}


proc searchresults {} {
  if {![info exists ::A(q)]} return ""

  set ::TITLE "Results for: \"[htmlize $::A(q)]\""

  # If the user has clicked the "Lucky" button and the query returns one or
  # more results, redirect the browser to the highest ranked result. If the
  # query returns zero results, fall through and display the "No results"
  # page as if the user had clicked "Search".
  #
  if {[info exists ::A(s)] && $::A(s) == "Lucky"} {
    set url [db one {
      SELECT url FROM page, pagedata 
      WHERE page MATCH $::A(q) AND page.docid = pagedata.docid
      ORDER BY rank(matchinfo(page), nk, nt, nc) DESC
    }]
    if {$url != ""} { cgi_redirect $url }
  }

  # Set nRes to the total number of documents that the users query matches.
  # If nRes is 0, then the users query returned zero results. Return a short 
  # message to that effect.
  #
  set nRes [db one { SELECT count(*) FROM page WHERE page MATCH $::A(q) }]
  if {$nRes == 0} {
    return [subst { No results for: <b>[htmlize $::A(q)]</b> }]
  }

  # Set iStart to the index of the first result to display. Results are
  # indexed starting at zero from most to least relevant.
  #
  set iStart [expr {([info exists ::A(i)] ? $::A(i) : 0)*10}]

  # HTML markup used to highlight keywords within FTS3 generated snippets.
  #
  set open {<span style="font-weight:bold; color:navy">}
  set close {</span>}
  set ellipsis {<b>&nbsp;...&nbsp;</b>}

  set ret [subst {
    <table border=0>
    <p>Search results 
       [expr $iStart+1]..[expr {($nRes < $iStart+10) ? $nRes : $iStart+10}] 
       of $nRes for: <b>[htmlize $::A(q)]</b>
  }]
  db eval {
    SELECT 
      result.rowid+$iStart AS resnum,
      COALESCE(title, 'No Title.') AS title,
      snippet2(page, $open, $close, $ellipsis, 0, 10) AS keywords,
      snippet2(page, $open, $close, $ellipsis, 2, 40) AS snippet,
      url,
      erank(matchinfo(page), nk, nt, nc) AS report
    FROM page, (
        SELECT 
          page.docid AS docid, url, nk, nt, nc
        FROM page, pagedata 
          WHERE page MATCH $::A(q) AND page.docid = pagedata.docid
          ORDER BY rank(matchinfo(page), nk, nt, nc)
          DESC LIMIT 10 OFFSET $iStart
      ) AS result
    WHERE page MATCH $::A(q) AND page.docid = result.docid
    ORDER BY resnum;
  } {
    append ret [subst -nocommands {<tr>
      <td valign=top>${resnum}.</td>
      <td valign=top>
        <div style="white-space:nowrap">
          <a href="$url">$title</a>
          <span style="font-size:small:margin-left:2ex">$keywords</span>
        </div>
        <div style="font-size:small;margin-left: 2ex">
          <div style="width:80ex"> $snippet </div>
          <div style="margin-bottom:1em"><a href="$url">$url</a></div>
        </div>
      </td>

      <td width=100%>
      <td valign=top style="color:grey;white-space:nowrap"> $report </td>
    }]
  }
  append ret { </table> }


  # If the query returned more than 10 results, add up to 10 links to 
  # each set of 10 results (first link to results 1-10, second to 11-20, 
  # third to 21-30, as required).
  #
  if {$nRes>10} {
    set s(0) {border: solid #80a796 1px ; padding: 1ex ; margin: 1ex}
    set s(1) "$s(0);background:#80a796;color:white"
    append ret <center><p>
    for {set i 0} {$i < 10 && ($i*10)<$nRes} {incr i} {
      append ret [subst {
        <a style="$s([expr {($iStart/10)==$i}])" 
           href="search?[cgi_encode_args [list q $::A(q) i $i]]">[expr $i+1]</a>
      }]
    }
    append ret </center>
  }

  return $ret
}

# Return a list of relative paths to documents that should be included 
# in the index.
#
proc document_list {} {
  set files [list]
  foreach f [glob *.html c3ref/*.html releaselog/*.html] {
    if {![string match *crossref* $f]} { lappend files $f }
  }
  return $files
}

proc readfile {zFile} {
  set fd [open $zFile]
  set ret [read $fd]
  close $fd
  return $ret
}

proc keywordparse_callback {tag details} {
  global K P
  switch -- [string tolower $tag] {
    "" {
      if {[info exists K(hyperlink)]} {
        append K($K(hyperlink)) $details
      }
    }
    "a" {
      array set D $details
      if {[info exists D(href)]} { set K(hyperlink) $D(href) }
    }
    "/a" {
      unset -nocomplain P(hyperlink)
    }
  }
}

#-------------------------------------------------------------------------
# This function is used as the callback when parsing ordinary documents 
# (not the keywords document).
#
# Rules for extracting fragment "titles". A fragment title consists of
# all text that follows the tag that opens the fragment until either:
#
#   1. 80 characters have been parsed, or
#   2. 8 characters have been parsed and one of the following is 
#        encountered:
#      a) A block element opening or closing tag, or
#      b) A <br> element, or
#      c) A "." character.
#
#   3. 8 characters have been parsed and a <br> tag or "." character is
#      encountered
#
proc docparse_callback {tag details} {
  global P
  set tag [string tolower $tag]
  switch -glob -- $tag {
    "" {
      append P(text) " $details"
      if {$P(isTitle)} { append P(title) $details }
      if {[llength $P(fragments)]} { 
        append P(ftext) " $details" 
      }
    }

    "/table" {
      if {[info exists P(skipheader)]} {
        set P(text) ""
        unset P(skipheader)
      }
    }

    "title"  { set P(isTitle) 1 }
    "/title" { set P(isTitle) 0 }

    "a" { 
      array set D $details
      if {[info exists D(name)]} {
        if {[llength $P(fragments)]} { 
          lappend P(fragments) $P(ftitle) $P(ftext) 
        }
        lappend P(fragments) $D(name)
        set P(ftext) ""
        set P(ftitle) ""
        catch { unset P(ftitleclose) }
      }
    }
    "h*" {
      array set D $details
      if {[info exists D(id)]} {
        if {[llength $P(fragments)]} { 
          lappend P(fragments) $P(ftitle) $P(ftext) 
        }
        lappend P(fragments) $D(id)
        set P(ftext) ""
        set P(ftitle) ""
      }
    }
  }

  set ftext [string trim $P(ftext) " \v\n"]
  if {[string length $ftext]>4 && $P(ftitle) == ""} {
    set blocktags [list                               \
      br td /td th /th p /p                           \
      h1 h2 h3 h4 h5 h /h1 /h2 /h3 /h4 /h5 /h
    ]
    if {[lsearch $blocktags $tag]>=0} {
      set P(ftitle) $ftext
      set P(ftext)  ""
      puts "setting title to \"$P(ftitle)\""
    } elseif {[string length $ftext]>80} {
      set idx [string last " " [string range $ftext 0 79]]
      if {$idx<0} { set idx 80 }
      set P(ftitle) [string range $ftext 0 [expr $idx-1]]
      set P(ftext)  [string range $ftext $idx end]
      puts "setting title to \"$P(ftitle)\""
    } 
  }
}

proc insert_entry {url keywords title content} {
  set nTitle    [tokencount $::tokenizer $title]
  set nKeywords [tokencount $::tokenizer $keywords]
  set nContent  [tokencount $::tokenizer $content]
  db eval {
    INSERT INTO page VALUES($keywords, $title, $content);
    INSERT INTO pagedata VALUES(NULL, $nKeywords, $nTitle, $nContent, $url);
  }
}

proc rebuild_database {} {
  set report [list]

  db transaction {
    # Create the database schema. If the schema already exists, then those
    # tables that contain document data are dropped and recreated by this
    # proc. The 'config' table is left untouched.
    #
    db eval {
      CREATE TABLE IF NOT EXISTS config(item TEXT, value TEXT);

      DROP TABLE IF EXISTS page;
      DROP TABLE IF EXISTS pagedata;
      CREATE VIRTUAL TABLE page USING fts3(
        keywords,                           -- Document keywords
        title,                              -- Title (or first heading)
        content,                            -- Complete document text
        tokenize porter                     -- Built-in porter tokenizer
      );
      CREATE TABLE pagedata(
        docid INTEGER PRIMARY KEY,          -- Maps to docid of page
        nk INTEGER, nt INTEGER, nc INTEGER, -- Tokens in each ft field
        url TEXT                            -- Relative path to document
      );
    }

    set zHtml [readfile keyword_index.html]
    parsehtml $zHtml keywordparse_callback

    # Scan the file-system for HTML documents. Add each document found to
    # the page and pagedata tables.
    foreach file [document_list] {
      set zHtml [readfile $file]

      array unset ::P
      set ::P(text) ""                 ;# The full document text
      set ::P(isTitle) 0               ;# True while parsing contents of <title>
      set ::P(fragments) [list]        ;# List of document fragments parsed
      set ::P(ftext) ""                ;# Text of current document fragment 

      # If the document appears to contain something that looks like the 
      # header on all website pages, set the P(skipheader) variable. This
      # tells the document parser to ignore all text (except for the document
      # title) up to the first </table> tag in the HTML.
      set tagline {<div class="tagline">Small. Fast. Reliable.}
      if {[string first $tagline $zHtml]>0} {
        set ::P(skipheader) 1
      }

      parsehtml $zHtml docparse_callback
      if {[info exists ::P(ftitle)]} {
        lappend ::P(fragments) $::P(ftitle) $::P(ftext)
      }

      set len [string length $::P(text)]
      set keyword ""
      catch { set keyword $::K($file) }
      if {![info exists ::P(title)]} {set ::P(title) "No Title"}
      insert_entry $file $keyword $::P(title) $::P(text)

      foreach {name title text} $::P(fragments) {
        set url "$file#$name"
        set keyword ""
        catch { set keyword [string trim $::K($url) " \n\v"] }
        insert_entry $url $keyword $title $text
      }
    }

    db eval { INSERT INTO page(page) VALUES('optimize') }
  }
  db eval VACUUM

  set report
}

proc main {} {
  sqlite3 db search.db

  array set ::A [cgi_parse_args]

  set ::tokenizer [db one {SELECT fts3_tokenizer('porter')}]

  # If ::A(rebuild) is set and this is a POST request, it is a command to
  # rebuild the full-text database.
  if {[info exists ::A(rebuild)] && (1 || $::env(REQUEST_METHOD) == "POST")} {
    set t [ttime {set report [rebuild_database]}]
    set msg "<p><b>Database rebuilt. ($t)</b>"
    return "[searchform] $msg $report [rebuildform]"
  }

  # If ::A(q) is set, this is a query. Search for documents that MATCH $::A(q).
  db transaction {
    set t [ttime { set doc "[searchform] [searchresults] [rebuildform]" }]
  }
  append doc "<p>Page generated in $t"
  return $doc

  return [cgi_env_dump]
}

#=========================================================================

set ::HEADER {
  <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
  "http://www.w3.org/TR/html4/strict.dtd">
  <html><head>
  <title>$TITLE</title>
  <style type="text/css">
  body {
    margin: auto;
    font-family: Verdana, sans-serif;
    padding: 8px 1%;
  }

  a { color: #45735f }
  a:visited { color: #734559 }

  .logo { position:absolute; margin:3px; }
  .tagline {
    float:right;
    text-align:right;
    font-style:italic;
    width:300px;
    margin:12px;
    margin-top:58px;
  }

  .toolbar {
    font-variant: small-caps;
    text-align: center;
    line-height: 1.6em;
    margin: 0;
    padding:1px 8px;
  }
  .toolbar a { color: white;
  text-decoration: none; padding: 6px
  12px; }
  .toolbar a:visited { color: white; }
  .toolbar a:hover { color: #80a796;
  background: white; }

  .content    { margin: 5%; }
  .content dt { font-weight:bold; }
  .content dd { margin-bottom: 25px; margin-left:20%; }
  .content ul { padding:0px; padding-left: 15px; margin:0px; }

  /* rounded corners */
  .se  { background: url(images/se.png) 100% 100% no-repeat #80a796 }
  .sw  { background: url(images/sw.png) 0% 100% no-repeat }
  .ne  { background: url(images/ne.png) 100% 0% no-repeat }
  .nw  { background: url(images/nw.png) 0% 0% no-repeat }
  </style>
  <meta http-equiv="content-type" content="text/html; charset=UTF-8">
    
  </head>
  <body>
  <div><!-- container div to satisfy validator -->

  <a href="index.html">
  <img class="logo" src="images/SQLite.gif" alt="SQLite Logo" border="0"></a>
    <div><!-- IE hack to prevent disappearing logo--></div>
    <div class="tagline">Small. Fast. Reliable.<br>Choose any three.</div>

    <table width=100% style="clear:both"><tr><td>
      <div class="se"><div class="sw"><div class="ne"><div class="nw">
        <div class="toolbar">
        <a href="about.html">About</a>
        <a href="sitemap.html">Sitemap</a>
        <a href="docs.html">Documentation</a>
        <a href="download.html">Download</a>

        <a href="copyright.html">License</a>
        <a href="news.html">News</a>
        <!-- <a href="dev.html">Developers</a> -->
        <a href="support.html">Support</a>
        </div></div></div></div></div>
    </td></tr></table>
}

if {![info exists env(REQUEST_METHOD)]} {
  set env(REQUEST_METHOD) GET
  set env(QUERY_STRING) rebuild=1
  set ::HEADER ""

  set env(QUERY_STRING) {q=the}
  set ::HEADER ""
}


set TITLE "Search SQLite Documentation"

if {0==[catch main res]} {
  set document [subst -nocommands $::HEADER]
  append document $res
} else {
  set document "<pre>"
  append document "Error: $res\n\n"
  append document $::errorInfo
  append document "</pre>"
}

puts "Content-type: text/html" 
puts "Content-Length: [string length $document]"
puts ""
puts $document
puts ""
flush stdout
close stdout

exit

/*
** This file contains some functions implemented in C used by the search.tcl
** script. There are two Tcl commands:
**
**   parsehtml
**   tokencount
**
** Tcl command [parsehtml] is a helper command used to extract text and 
** markup tags from the HTML documents in the SQLite documentation. The
** [tokencount] command uses an FTS3 tokenizer to count the number of
** tokens in a document. Both of these are used while building the database
** only.
**
** There are also two SQL user functions registered:
**
**   rank()
**   erank()
**
** rank() interprets the return value of the FTS3 matchinfo() function and
** returns a score for the match (a real number). The higher the score, the
** more relevant the document is considered. This is used to order query
** results when the user searchs the database. The rank() function takes
** (nCol+1) arguments, where nCol is the number of columns in the FTS3
** table. The first argument is the return value of matchinfo(). The
** second argument is the number of tokens in column 0 of the current FTS3 
** table row. The third argument is the number of tokens in column 1, and
** so on.
**
** Function erank() is called in exactly the same way as rank(). Instead
** of returning a score, it returns an HTML formatted table containing
** data that may be used to understand how the score for the current row
** was calculated.
*/

#include <tcl.h>
#include <string.h>
#include <assert.h>
#include <ctype.h>
#include <math.h>

#define ISSPACE(c) (((c)&0x80)==0 && isspace(c))

#include "fts3_tokenizer.h"
#include "sqlite3.h"

typedef unsigned int u32;
typedef unsigned char u8;
typedef sqlite3_uint64 u64;

typedef struct STokenizer STokenizer;
struct STokenizer {
  sqlite3_tokenizer_module *pMod;
  sqlite3_tokenizer *pTokenizer;
  sqlite3_tokenizer_cursor *pCursor;
};

static void tokenizerClose(STokenizer *p){
  if( p->pCursor ){ p->pMod->xClose(p->pCursor); }
  if( p->pTokenizer ){ p->pMod->xDestroy(p->pTokenizer); }
  memset(p, 0, sizeof(STokenizer));
}

static int tokenizerOpen(
  Tcl_Interp *interp,
  Tcl_Obj *pTokenizer,
  Tcl_Obj *pDocument,
  STokenizer *pOut                /* OUT: Structure containing tokenizer */
){
  sqlite3_tokenizer_module *pMod; /* Tokenizer module */
  int rc;                         /* Return code */
  const char *zDoc;               /* Pointer to pDocument string buffer */
  int nDoc;                       /* Number of bytes in buffer zDoc */
  const char *zFail;              /* Error message (if an error occurs) */

  memset(pOut, 0, sizeof(STokenizer));
  memcpy(&pMod, Tcl_GetByteArrayFromObj(pTokenizer, 0), sizeof(pMod));

  rc = pMod->xCreate(0, 0, &pOut->pTokenizer);
  if( rc!=SQLITE_OK ){ zFail = "Error in xCreate()"; goto failed; }
  pOut->pMod = pMod;

  zDoc = Tcl_GetStringFromObj(pDocument, &nDoc);
  rc = pMod->xOpen(pOut->pTokenizer, zDoc, nDoc, &pOut->pCursor);
  if( rc!=SQLITE_OK ){ zFail = "Error in xOpen()"; goto failed; }

  pOut->pCursor->pTokenizer = pOut->pTokenizer;
  return TCL_OK;

 failed:
  tokenizerClose(pOut);
  Tcl_AppendResult(interp, zFail, 0);
  return TCL_ERROR;
}

static int tokenizerNext(
  STokenizer *p,                  /* Tokenizer wrapper object */
  int *piStart,                   /* OUT: Byte offset of start of token */
  int *piEnd,                     /* OUT: Byte offset of end of token */
  int *piCurrent                  /* OUT: Token number */  
){
  const char *z; int n;
  return p->pMod->xNext(p->pCursor, &z, &n, piStart, piEnd, piCurrent);
}

/*
** Tcl command: tokencount TOKENIZER DOCUMENT
*/
static int tokencountcmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *const objv[]
){
  STokenizer sToken;
  int rc;
  int nToken = 0;
  int i1, i2, i3;

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "TOKENIZER DOCUMENT");
    return TCL_ERROR;
  }

  if( tokenizerOpen(interp, objv[1], objv[2], &sToken) ) return TCL_ERROR;

  while( SQLITE_OK==(rc = tokenizerNext(&sToken, &i1, &i2, &i3)) ) nToken++;
  if( rc!=SQLITE_DONE ){
    Tcl_AppendResult(interp, "Error in xNext() 3", 0);
    rc = TCL_ERROR;
  }else{
    Tcl_SetObjResult(interp, Tcl_NewIntObj(nToken));
    rc = TCL_OK;
  }

  tokenizerClose(&sToken);
  return rc;
}


/*
** Tcl command: parsehtml HTML SCRIPT
*/
static int parsehtmlcmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj * const objv[]
){
  char *zHtml;
  char *z;
  Tcl_Obj **aCall;
  int nElem;
  Tcl_Obj **aElem;
  int rc;

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "HTML SCRIPT");
    return TCL_ERROR;
  }
  zHtml = Tcl_GetString(objv[1]);

  rc = Tcl_ListObjGetElements(interp, objv[2], &nElem, &aElem);
  if( rc!=TCL_OK ) return rc;
  aCall = (Tcl_Obj **)ckalloc(sizeof(Tcl_Obj *)*(nElem+2));
  memcpy(aCall, aElem, sizeof(Tcl_Obj *)*nElem);
  aCall[nElem] = 0;
  aCall[nElem+1] = 0;

  z = zHtml;
  while( *z ){
    char *zText = z;
    while( *z && *z!='<' ) z++;

    /* Invoke the callback script for the chunk of text just parsed. */
    Tcl_IncrRefCount( aCall[nElem]   = Tcl_NewObj() );
    Tcl_IncrRefCount( aCall[nElem+1] = Tcl_NewStringObj(zText, z-zText) );
    rc = Tcl_EvalObjv(interp, nElem+2, aCall, 0);
    Tcl_DecrRefCount( aCall[nElem] );
    Tcl_DecrRefCount( aCall[nElem+1] );
    if( rc!=TCL_OK ) return rc;

    /* Unless is at the end of the document, z now points to the start of a
    ** markup tag. Either an opening or a closing tag. Parse it up and 
    ** invoke the callback script. */
    if( *z ){
      int nTag;
      char *zTag;
      z++;

      while( ISSPACE(*z) ) z++;
      zTag = z;

      while( *z && !ISSPACE(*z) && *z!='>' ) z++;
      nTag = z-zTag;

      if( nTag==5 && 0==sqlite3_strnicmp("style", zTag, 5) ){
        while( *z && sqlite3_strnicmp("/style>", z, 6 ) ) z++;
      } else if( nTag>=3 && 0==memcmp("!--", zTag, 3) ){
        while( *z && sqlite3_strnicmp("-->", z, 3 ) ) z++;
      } else {
        Tcl_Obj *pParam = Tcl_NewObj();

        while( *z && *z!='>' ){
          char *zAttr;

          /* Gobble up white-space */
          while( ISSPACE(*z) ) z++;
          zAttr = z;

          /* Advance to the end of the attribute name */
          while( *z && *z!='>' && !ISSPACE(*z) && *z!='=' ) z++;
          if( z==zAttr ) zAttr = 0;

          if( zAttr ){
            Tcl_Obj *pAttr = Tcl_NewStringObj(zAttr, z-zAttr);
            Tcl_ListObjAppendElement(interp, pParam, pAttr);
          }
          while( ISSPACE(*z) ) z++;

          if( *z=='=' ){
            int nVal;
            char *zVal;
            z++;
            while( ISSPACE(*z) ) z++;
            zVal = z;

            if( *zVal=='"' ){
              zVal++;
              z++;
              while( *z && *z!='"' ) z++;
              nVal = z-zVal;
              z++;
            }else{
              while( *z && !ISSPACE(*z) && *z!='>' ) z++;
              nVal = z-zVal;
            }
            Tcl_ListObjAppendElement(interp,pParam,Tcl_NewStringObj(zVal,nVal));
          }else if( zAttr ){
            Tcl_ListObjAppendElement(interp, pParam, Tcl_NewIntObj(1));
          }
        }
        
        Tcl_IncrRefCount( aCall[nElem]   = Tcl_NewStringObj(zTag, nTag) );
        Tcl_IncrRefCount( aCall[nElem+1] = pParam );
        rc = Tcl_EvalObjv(interp, nElem+2, aCall, 0);
        Tcl_DecrRefCount( aCall[nElem] );
        Tcl_DecrRefCount( aCall[nElem+1] );
        if( rc!=TCL_OK ) return rc;
      }

      while( *z && !ISSPACE(*z) && *z!='>' ) z++;
      if( *z ) z++;
    }

  }

  return TCL_OK;
}


/*
** Implementation of search result ranking function.
*/
static void rankfunc(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
  u32 *aMatchinfo;
  double score = 0.0;
  int iCol;
  int iPhrase;
  int nCol;
  int nPhrase;

  int isExplain = sqlite3_user_data(pCtx);
  char *zExplain = 0;

  if( nVal==0 ) goto wna;
  aMatchinfo = (u32 *)sqlite3_value_blob(apVal[0]);
  nPhrase = aMatchinfo[0];
  nCol = aMatchinfo[1];
  if( nVal!=nCol+1 ) goto wna;

  if( isExplain ) zExplain = sqlite3_mprintf("<table width=100%%>");

  for(iCol=0; iCol<nCol; iCol++){
    int nToken = sqlite3_value_int(apVal[iCol+1]);
    double colscore = 0.0;
    if( isExplain ){
      zExplain = sqlite3_mprintf("%z<tr><td>%d.<td>( ", zExplain, iCol);
    }
    for(iPhrase=0; iPhrase<nPhrase; iPhrase++){
      u32 nGlobal = aMatchinfo[2 + iPhrase*nCol + iCol];
      u32 nHit = aMatchinfo[2 + nPhrase*nCol + iPhrase*nCol + iCol];

      if( nHit ) colscore += (double)nHit / (double)nGlobal;
      if( isExplain ){
        const char *zDiv = (iPhrase==0 ? "" : "+ ");
        zExplain = sqlite3_mprintf("%z%s%d/%d ", zExplain, zDiv, nHit, nGlobal);
      }
    }
    colscore = colscore / (log(100+nToken)/log(10)); 
    score += colscore;
    if( isExplain ){
      zExplain = sqlite3_mprintf(
          "%z) / log(100+%d)<td> = %.4f", zExplain, nToken, colscore);
    }
  }

  if( isExplain ){
    sqlite3_result_text(pCtx, sqlite3_mprintf(
        "%z<tr><td><td width=100%%><td>= <b>%.4f</b></table>", zExplain, score
    ), -1, sqlite3_free);
  }else{
    sqlite3_result_double(pCtx, score);
  }
  return;

 wna:
  sqlite3_result_error(pCtx,"wrong number of arguments to function rank()",-1);
}

int Sqlite3_Init(Tcl_Interp *interp);

static int initDb(sqlite3 *db, char **pzErr, void *p){
  sqlite3_create_function(db, "rank",-1, SQLITE_UTF8, 0, rankfunc,0,0);
  sqlite3_create_function(db, "erank", -1, SQLITE_UTF8, (void*)1, rankfunc,0,0);
}

int Parsehtml_Init(Tcl_Interp *interp){
#ifdef USE_TCL_STUBS
  if (Tcl_InitStubs(interp, "8.4", 0) == 0) {
    return TCL_ERROR;
  }
#endif

  Tcl_CreateObjCommand(interp, "parsehtml",  parsehtmlcmd, 0, 0);
  Tcl_CreateObjCommand(interp, "tokencount", tokencountcmd, 0, 0);
  sqlite3_auto_extension(initDb);

  return TCL_OK;
}


static int AppInit(Tcl_Interp *interp) {
  int rc;
  rc = Sqlite3_Init(interp);
  if( rc!=TCL_OK ) return rc;
  rc = Parsehtml_Init(interp);
  return rc;
}

int main(int argc, char *argv[]) {
  Tcl_Main(argc, argv, AppInit);
  return 0;
}