Documentation Source Text

private:	base evidence private_evidence matrix doc

fast:	base doc

tclsh:	$(TCLSQLITE3C)
	$(CC) -g -o tclsh -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS5 -DTCLSH=1 -DSQLITE_TCLMD5 $(TCLINC) $(TCLSQLITE3C) $(TCLFLAGS)

tclsqlite3.search:	$(TCLSQLITE3C) $(DOC)/search/searchc.c
	$(CC) -g -o tclsqlite3.search -I. -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS5 $(TCLINC) $(DOC)/search/searchc.c $(TCLSQLITE3C) $(TCLFLAGS)

sqlite3.h:	tclsh $(SRC)/src/sqlite.h.in $(SRC)/manifest.uuid $(SRC)/VERSION
	./tclsh $(SRC)/tool/mksqlite3h.tcl $(SRC) | \
	sed 's/^SQLITE_API //' >sqlite3.h

# Generate the directory into which generated documentation files will
# be written.

<tcl>hd_keywords {foreign key constraints} {foreign key constraint} {FOREIGN KEY constraints}</tcl>

<title>SQLite Foreign Key Support</title>
<table_of_contents>

<h2 style="margin-left:1.0em" notoc id=overview> Overview</h2>

<p>This document describes the support for SQL foreign key constraints
   introduced in SQLite version 3.6.19. 

<p>The first section introduces the 
   concept of an SQL foreign key by example and defines the terminology 
   used for the remainder of the document. Section 2 describes the steps 

load ./parsehtml.so
source [file join [file dirname [info script]] hdom.tcl]

set ::G(rowid) 1

# Return a list of relative paths to documents that should be included 
# in the index.
proc document_list {type} {
  global weight
  set lFiles [list]
  switch -- $type {
    lang {
      foreach f [glob lang_*.html] { lappend lFiles $f }
    }

    c3ref {
      set blacklist(objlist.html) 1
      set blacklist(constlist.html) 1
      set blacklist(funclist.html) 1

      lappend lFiles c3ref/free.html
      lappend lFiles c3ref/mprintf.html
      lappend lFiles c3ref/io_methods.html

      foreach f [glob c3ref/*.html] { 
        if {[info exists blacklist([file tail $f])]} continue
        if {[lsearch $lFiles $f]<0} { lappend lFiles $f }
      }
    }

    generic {
      set nosearch(doc_keyword_crossref.html) 1
      set nosearch(doc_backlink_crossref.html) 1
      set nosearch(doc_pagelink_crossref.html) 1
      set nosearch(doc_target_crossref.html) 1
      set nosearch(doclist.html) 1
      set nosearch(keyword_index.html) 1
      set nosearch(requirements.html) 1
      set nosearch(sitemap.html) 1
      set nosearch(fileio.html) 1
      set nosearch(btreemodule.html) 1
      set nosearch(capi3ref.html) 1
      set nosearch(changes.html) 1
      set nosearch(fileformat2.html) 1
      set nosearch(index.html) 1
      set nosearch(docs.html) 1

      set weight(chronology.html) 25

      foreach f [glob *.html] { 
        if {[string match lang_* $f]==0 && [info exists nosearch($f)]==0} {
          lappend lFiles $f 
        }
      }

      # "current.html" is a duplicate of the most recent release. Don't
      # index it at all.
      set nosearch(releaselog/current.html) 1


      # As of version 3.7.16, sub-release changelogs duplicated the entries
      # from the major release. This block does the following:
      #
      #   * sets the weight of a changelog containing superceded content
      #     to 25%
      #   * sets the weights of other changelogs to 50%.
      #
      foreach f [glob releaselog/*.html] { 
        set tail [file tail $f]
        set ::weight($f) 50
        if {[regexp {^(3_8_[0-9]*).*} $tail -> prefix]
         || [regexp {^(3_7_16).*} $tail -> prefix]
         || [regexp {^(3_9_).*} $tail -> prefix]
         || [regexp {^(3_[1-9][0-9]).*} $tail -> prefix]
        } {
          set f1 [lindex [lsort -decreasing [glob releaselog/$prefix*.html]] 0]
          if {$f!=$f1} { set ::weight($f) 25 }
        } 
      }

      foreach f [glob releaselog/*.html] { 
        if {[info exists nosearch($f)]==0} { 
          lappend lFiles $f 
        }
      }
    }

    default {
      error "document_list: unknown file type $type"
    }
  }
  return $lFiles
}

proc readfile {zFile} {
  set fd [open $zFile]
  set ret [read $fd]
  close $fd
  return $ret
}

# Insert a new entry into the main "page" table of the search database.
# Values are determined by switches passed to this function:
#
#   -apis      List of APIs
#   -rowid     Rowid to use
#   -title1    Document title
#   -title2    Heading title (or NULL)
#   -content   Document content
#   -url       URL of this document
#
# Return the rowid of the row just inserted into the table.
# 
proc insert_entry {args} {
  global G
  if {[llength $args] % 2} { error "Bad arguments passed to insert_entry (1)" }

  set switches {
    -apis -title1 -title2 -content -url -keywords -rowid
  }
  set V(content) ""

  foreach {k v} $args {
    set idx [lsearch -all $switches $k*] 
    if {[llength $idx]!=1} { error "Bad switch passed to insert_entry: $k" }
    set V([string range [lindex $switches $idx] 1 end]) $v
  }
  
  set V(content) [string trim $V(content)]
  if {[info exists V(rowid)]==0} {
    set V(rowid) [incr G(rowid)];
  }

  db eval {
    INSERT INTO page(rowid, apis, keywords, title1, title2, content, url) 
    VALUES($V(rowid),
        $V(apis), $V(keywords), $V(title1), $V(title2), $V(content), $V(url)
    );
  }

  return [db last_insert_rowid]
}

# Extract a document title from DOM object $dom passed as the first
# argument. If no <title> node can be found in the DOM, use $fallback
# as the title.
#
proc extract_title {dom fallback} {
  set title_node [lindex [[$dom root] search title] 0]
  if {$title_node==""} {
    set title $fallback
  } else {
    set title [$title_node text]
  }

  set title
}

proc c3ref_document_apis {dom} {
  global c3ref_blacklist

  set res [list]
  foreach N [[$dom root] search blockquote] {
    set text [$N text]
    while {[regexp {(sqlite3[0-9a-z_]*) *\((.*)} $text -> api text]} {
      if {[info exists c3ref_blacklist($api)]==0} {
        lappend res "${api}()"
        set c3ref_blacklist($api) 1
      }
    }

    set text [$N text]
    set pattern {struct +(sqlite3[0-9a-z_]*)(.*)}
    while {[regexp $pattern $text -> api text]} {
      if {[info exists c3ref_blacklist($api)]==0} {
        lappend res "struct ${api}"
        set c3ref_blacklist($api) 1
      }
    }

    set text [$N text]
    set pattern {#define +(SQLITE_[0-9A-Z_]*)(.*)}
    while {[regexp $pattern $text -> api text]} {
      if {[info exists c3ref_blacklist($api)]==0} {
        lappend res "${api}"
        set c3ref_blacklist($api) 1
      }
    }
  }



  return [join $res ", "]
}

proc c3ref_filterscript {N} {
  for {set P [$N parent]} {$P!=""} {set P [$P parent]} {
    if {[$P attr -default "" class]=="nosearch"} { return 0 }
    if {[$P tag]=="blockquote" } { return 0 }

    if {[$N tag]==""} {
      if {[eval $filterscript $N]} { append text [$N text] }
    }
  }
  return $text
}

# This is a specialized command for extracting sections from the
# pragma.html document.
#
proc extract_sections_from_pragma_dom {dom} {
  set body [lindex [[$dom root] search body] 0]
  set res [list]

  # Find the section titles.
  foreach T [[$dom root] search table] {
    foreach A [$T search a] {
      if {[string match #pragma_* [$A attr -default "" href]]} {
        set name [$A attr href]
        lappend title($name) [$A text]
      }
    }
  }

  set url     ""
  set hdr     ""
  set content ""

  $body foreach_descendent N {
    set tag  [$N tag]
    if {$tag=="a" && [string match "pragma_*" [$N attr -default "" name]]} {
      lappend res [list $url $hdr $content]

      set name [$N attr name]
      set url "$name"
      set hdr "PRAGMA [join $title(#$name) ,]"
      set content ""
    }

    if {$tag=="" && [generic_filterscript $N]} {
      append content [$N text]
    }
  }
  lappend res [list $url $hdr $content]

  set res
}

proc extract_sections_from_dom {dom filterscript} {
  set body [lindex [[$dom root] search body] 0]

  set h(h) 1
  set h(h1) 1
  set h(h2) 1
  set h(h3) 1

  set res [list]

  $body foreach_descendent N {
    set tag [$N tag]

    if {[info exists h($tag)]} {
      set id [$N attr -default "" id]
      if {$id != ""} {
        if {[info exists H]} {
          lappend res [list [$H attr id] [$H text] $content]
        }
        set H $N
        set content ""
      }
    }

    if {[info exists H] && $tag==""} {
      for {set P [$N parent]} {$P!=""} {set P [$P parent]} {
        if {$P==$H} break
      }
      if {$P==""} { append content [$N text] }
    }
  }

  return $res
}

proc lang_document_import {doc} {
  set dom [::hdom::parse [readfile $doc]]

  # Find the <title> tag and extract the title.
  set title [extract_title $dom $doc]

  # Extract the entire document text.
  set text [extract_text_from_dom $dom lang_filterscript]

  # Insert into the database.
  insert_entry -url $doc -title1 $title -content $text

  $dom destroy
}

proc c3ref_document_import {doc} {
  set dom [::hdom::parse [readfile $doc]]
  
  # Find the <title> tag and extract the title.
  set title [extract_title $dom $doc]
  set title "C API: $title"

  set text [extract_text_from_dom $dom c3ref_filterscript]
  set apis [c3ref_document_apis $dom]

  # Insert into the database.
  insert_entry -url $doc -apis $apis -title1 $title -content $text
}

proc generic_document_import {doc} {
  set dom [::hdom::parse [readfile $doc]]
  
  # Find the <title> tag and extract the title.
  set title [extract_title $dom $doc]

  # Extract the document text
  set text [extract_text_from_dom $dom generic_filterscript]

  # Insert into the database.
  set rowid [insert_entry -url $doc -title1 $title -content $text]

  if {$doc=="pragma.html"} {
    set lSection [extract_sections_from_pragma_dom $dom]
  } else {
    set lSection [extract_sections_from_dom $dom generic_filterscript]
  }

  set i [expr $rowid*1000]
  foreach section $lSection {
    incr i
    foreach { tag hdr text } $section {}
    set url "${doc}#${tag}"
    insert_entry -rowid $i -url $url -title1 $title -title2 $hdr -content $text
  }
}

proc rebuild_database {} {

  db transaction {
    # Create the database schema. If the schema already exists, then those
    # tables that contain document data are dropped and recreated by this
    # proc. The 'config' table is left untouched.
    #
    db eval {

      DROP TABLE IF EXISTS page;
      CREATE VIRTUAL TABLE page USING fts5(
        apis,                               -- C APIs 
        keywords,                           -- Keywords
        title1,                             -- Document title
        title2,                             -- Heading title, if any
        content,                            -- Document text

        url UNINDEXED,                      -- Indexed URL
        tokenize='porter unicode61 tokenchars _' -- Tokenizer definition
      );

      DROP TABLE IF EXISTS weight;
      CREATE TABLE weight(id INTEGER PRIMARY KEY, percent FLOAT);
    }

    foreach doc [document_list lang] {
      puts "Indexing $doc..."
      lang_document_import $doc
    }

    foreach doc [document_list c3ref] {
      puts "Indexing $doc..."
      c3ref_document_import $doc
    }

    foreach doc [document_list generic] { 
      puts "Indexing $doc..."
      generic_document_import $doc 
    }

    db eval { INSERT INTO page(page) VALUES('optimize') }

    foreach f [array names ::weight] {
      set w $::weight($f)
      db eval {SELECT rowid FROM page WHERE url=$f} {
        db eval { INSERT INTO weight VALUES($rowid, $w); }
      }
    }
  }

  db eval VACUUM
}

cd doc
sqlite3 db search.db
rebuild_database

    }
  } else {
    error "Unrecognized method: $env(REQUEST_METHOD)"
  }
}



















#=========================================================================
# The argument contains a key value list. The values in the list are
# transformed to an HTTP query key value list. For example:
#
#   % cgi_encode_args {s "search string" t "search \"type\""}
#   s=search+string&t=search+%22type%22
#

    lappend reslist "$key=$value"
  }
  join $reslist &
}

proc htmlize {str} { string map {< &lt; > &gt;} $str }
proc attrize {str} { string map {< &lt; > &gt; \x22 &quot;} $str }

#=========================================================================

proc cgi_env_dump {} {

  set ret "<h1>Arguments</h1><table>"
  foreach {key value} [array get ::A] {
    append ret "<tr><td>[htmlize $key]<td>[htmlize $value]"
  }
  append ret "</table>"

  append ret "<h1>Environment</h1><table>"
  foreach {key value} [array get ::env] {
    append ret "<tr><td>[htmlize $key]<td>[htmlize $value]"
  }
  append ret "</table>"
  return $ret
}




















proc footer {} {
  return {
    <hr>
    <table align=right>
    <td>
      <i>Powered by <a href="http://www.sqlite.org/fts5.html">FTS5</a>.</i>
    </table>

#   45.02 ms
#
proc ttime {script} {
  set t [lindex [time [list uplevel $script]] 0]
  if {$t>1000000} { return [format "%.2f s" [expr {$t/1000000.0}]] }
  return [format "%.2f ms" [expr {$t/1000.0}]]
}































proc searchresults {} {
  if {![info exists ::A(q)]} return ""
  #set ::A(q) [string map {' ''} $A(q)]
  #regsub -all {[^-/"A-Za-z0-9]} $::A(q) { } ::A(q)

  # Count the '"' characters in $::A(q). If there is an odd number of
  # occurences, add a " to the end of the query so that fts5 can parse
  # it without error.
  if {[regexp -all \x22 $::A(q)] % 2} { append ::A(q) \x22 }













  db one { INSERT INTO page(page, rank) VALUES('rank', 'bm25(20.0, 10.0)') }
























  # Set iStart to the index of the first result to display. Results are
  # indexed starting at zero from most to least relevant.
  #
  set iStart [expr {([info exists ::A(i)] ? $::A(i) : 0)*10}]

  # Grab a list of rowid results.
  #
  set q {
    SELECT rowid FROM page WHERE page MATCH $::A(q) 
    ORDER BY rank * COALESCE(
      (SELECT percent FROM weight WHERE id=page.rowid), 100
    );
  }
  if {[catch { set lRowid [db eval $q] }]} {
    set x ""
    foreach word [split $::A(q) " "] {
      append x " \"[string map [list "\"" "\"\""] $word]\""
    }
    set ::A(q) [string trim $x]
    set lRowid [db eval $q]
  }

  set lRes [list]
  foreach rowid $lRowid {
    if {$rowid > 1000} {
      set parent [expr $rowid / 1000]
      lappend subsections($parent) $rowid
    } else {
      lappend lRes $rowid
    }
  }

  set nRes [llength $lRes]
  set lRes [lrange $lRes $iStart [expr $iStart+9]]

  # If there are no results, return a message to that effect.
  #
  if {[llength $lRes] == 0} {
    return [subst { No results for: <b>[htmlize $::A(q)]</b> }]
  }
  
  # HTML markup used to highlight keywords within FTS5 generated snippets.
  #
  set open {<span style="background-color:#d9f2e6">}
  set close {</span>}
  set ellipsis {<b>&nbsp;...&nbsp;</b>}

  # Grab the required data
  #
  db eval [string map [list %LIST% [join $lRowid ,]] {
    SELECT 
      rowid AS parentid, 
      snippet(page, 0, $open, $close, $ellipsis, 6)  AS s_apis,
      snippet(page, 2, $open, $close, '', 40)        AS s_title1,
      snippet(page, 3, $open, $close, $ellipsis, 40) AS s_title2,
      snippet(page, 4, $open, $close, $ellipsis, 40) AS s_content,
      url, rank
    FROM page($::A(q))
    WHERE rowid IN (%LIST%)
  }] X {
    foreach k [array names X] { set data($X(parentid),$k) [set X($k)] }
  }

  set ret [subst {
    <table border=0>
    <p>Search results 
       [expr $iStart+1]..[expr {($nRes < $iStart+10) ? $nRes : $iStart+10}] 
       of $nRes for: <b>[htmlize $::A(q)]</b>
  }]

  foreach rowid $lRes {


    foreach a {parentid s_apis s_title1 s_content url rank} {
      set $a $data($rowid,$a)
    }

    if {[info exists subsections($parentid)]} {
      set childid [lindex $subsections($parentid) 0]
      set link $data($childid,url)
      set hdr $data($childid,s_title2)


      if {$hdr==""} {
        set s_content ""
      } else {
        set s_content [subst {
          <b><a style=color:#044a64 href=$link>$hdr</a></b>:
        }]
      }

      append s_content " $data($childid,s_content)"
    }

    append ret [subst -nocommands {<tr>
      <td valign=top style="line-height:150%">
        <div style="white-space:wrap;font-size:larger" class=nounderline>
          <a href="$url">$s_title1</a>
        </div>
          <div style="margin-left: 10ex; font:larger monospace">$s_apis</div>
        <div style="ffont-size:small;margin-left: 2ex">
          <div class=nounderline> $s_content </div>
          <div style="margin-left:1em; margin-bottom:1em">
            <a href="$url">$url</a>
          </div>
        </div>
      </td>

    }]
  }
  append ret { </table> }


  # If the query returned more than 10 results, add up to 10 links to 
  # each set of 10 results (first link to results 1-10, second to 11-20, 

  db transaction {
    set t [ttime { 
      if {[catch searchresults srchout]} {
        set A(q) [string tolower $A(q)]
        set srchout [searchresults]
      }
      set doc "$srchout [footer]" 
    }]
  }
  append doc "<p>Page generated in $t."
  return $doc

  # return [cgi_env_dump]
}

  set env(QUERY_STRING) rebuild=1
  set ::HEADER ""

  #set env(QUERY_STRING) {q="one+two+three+four"+eleven}
  set env(QUERY_STRING) {q=windows}
  set ::HEADER ""
}



if {0==[catch main res]} {
  set title "Search SQLite Documentation"
  if {[info exists ::A(q)]} {
    set initsearch [attrize $::A(q)]
    append title " - [htmlize $::A(q)]"
  } else {

#include <sqlite3.h>






/*



*/

/*

** Return a pointer to the fts5_api pointer for database connection db.
** If an error occurs, return NULL and leave an error in the database 
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
fts5_api *fts5_api_from_db(sqlite3 *db){

  fts5_api *pRet = 0;




  sqlite3_stmt *pStmt = 0;








  if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)

      && SQLITE_ROW==sqlite3_step(pStmt) 









      && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)

    ){







    memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));

  }





  sqlite3_finalize(pStmt);



  return pRet;



}

int Sqlite3_Init(Tcl_Interp *interp);

static int register_search_extensions(sqlite3 *db, char **pzErr, void *p){
  fts5_api *pApi = fts5_api_from_db(db);

  return SQLITE_OK;
}

static int AppInit(Tcl_Interp *interp) {
  int rc;
  rc = Sqlite3_Init(interp);
  if( rc!=TCL_OK ) return rc;
  sqlite3_auto_extension(register_search_extensions);
  return TCL_OK;
}

int main(int argc, char *argv[]) {
  Tcl_Main(argc, argv, AppInit);
  return 0;
}