Documentation Source Text

Check-in [e8a8c227d5]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the "searchdb" target to the Makefile. For building the doc/search.db database used by the search cgi script.
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: e8a8c227d53cfabc64a6805a5409a846b1a221ec
User & Date: dan 2010-01-07 18:09:51
Context
2010-01-08
00:31
Update the makefiles so that the TCL extensions used to build the FTS3 tables for the search function are built using STUBS. check-in: 17ee5a34e9 user: drh tags: trunk
2010-01-07
18:09
Add the "searchdb" target to the Makefile. For building the doc/search.db database used by the search cgi script. check-in: e8a8c227d5 user: dan tags: trunk
16:27
Add a <title> to the copyright-release.html document. check-in: 1ecb24abf0 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to main.mk.

28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
...
110
111
112
113
114
115
116








117
118
119
120
121
	@echo 'make all;        # Do all of the above'

all:	base evidence format_evidence matrix doc

private:	base evidence private_evidence matrix doc

tclsh:	$(TCLSQLITE3C)
	$(CC) -g -o tclsh -DTCLSH=1 -DSQLITE_TCLMD5 $(TCLINC) $(TCLSQLITE3C) $(TCLFLAGS)

sqlite3.h:	tclsh $(SRC)/src/sqlite.h.in $(SRC)/manifest.uuid $(SRC)/VERSION
	./tclsh $(SRC)/tool/mksqlite3h.tcl $(SRC) | \
	sed 's/^SQLITE_API //' >sqlite3.h

# Generate the directory into which generated documentation files will
# be written.
................................................................................

# Generate the traceability matrix
#
matrix:	
	rm -rf doc/matrix/images
	cp -r doc/images doc/matrix
	./tclsh $(DOC)/matrix.tcl









always:	

clean:	
	rm -rf tclsh doc sqlite3.h docinfo.db







|







 







>
>
>
>
>
>
>
>





28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
...
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
	@echo 'make all;        # Do all of the above'

all:	base evidence format_evidence matrix doc

private:	base evidence private_evidence matrix doc

tclsh:	$(TCLSQLITE3C)
	$(CC) -g -o tclsh -DTCLSH=1 -DSQLITE_TCLMD5 $(TCLINC) $(TCLSQLITE3C) $(TCLFLAGS) -DSQLITE_ENABLE_FTS3

sqlite3.h:	tclsh $(SRC)/src/sqlite.h.in $(SRC)/manifest.uuid $(SRC)/VERSION
	./tclsh $(SRC)/tool/mksqlite3h.tcl $(SRC) | \
	sed 's/^SQLITE_API //' >sqlite3.h

# Generate the directory into which generated documentation files will
# be written.
................................................................................

# Generate the traceability matrix
#
matrix:	
	rm -rf doc/matrix/images
	cp -r doc/images doc/matrix
	./tclsh $(DOC)/matrix.tcl

# Build the fts3 database used by the search script
#
parsehtml.so: $(DOC)/search/parsehtml.c
	gcc -shared $(TCLINC) $(TCLFLAGS) $(DOC)/search/parsehtml.c -o parsehtml.so -I. -I$(SRC)/ext/fts3

searchdb: parsehtml.so tclsh
	./tclsh $(DOC)/search/buildsearchdb.tcl

always:	

clean:	
	rm -rf tclsh doc sqlite3.h docinfo.db

Changes to search/Makefile.

6
7
8
9
10
11
12
13
14
15
16
17
18
19

LIBSQLITE3A = /home/dan/work/sqliteG/bld/libsqlite3.a
TCLSQLITEC = /home/dan/work/sqliteG/sqlite/src/tclsqlite.c

#SQLITELIB = $(TCLSQLITEC) $(LIBSQLITE3A)
SQLITELIB = $(TCLSQLITE3C)

tclsqlite3.fts3: searchc.c $(TCLSQLITEC) $(LIBSQLITE3A)
	gcc $(CFLAGS) -DSQLITE_ENABLE_FTS3 searchc.c $(SQLITELIB) -I/home/dan/work/sqliteG/sqlite/ext/fts3/ -static -L/home/dan/tcl86/lib -ltcl8.6 -ldl -lm -o tclsqlite3.fts3 -lz -lpthread

clean:
	rm tclsqlite3.fts3

	#gcc $(CFLAGS) -DSQLITE_ENABLE_FTS3 -I/home/dan/work/sqliteG/sqlite/ext/fts3/ -static searchc.c /home/dan/work/sqliteG/bld/tclsqlite3.c -L/home/dan/tcl86/lib -ltcl8.6 -ldl -lm -o tclsqlite3.fts3 -lz -lpthread







|






6
7
8
9
10
11
12
13
14
15
16
17
18
19

LIBSQLITE3A = /home/dan/work/sqliteG/bld/libsqlite3.a
TCLSQLITEC = /home/dan/work/sqliteG/sqlite/src/tclsqlite.c

#SQLITELIB = $(TCLSQLITEC) $(LIBSQLITE3A)
SQLITELIB = $(TCLSQLITE3C)

tclsqlite3.fts3: searchc.c $(SQLITELIB)
	gcc $(CFLAGS) -DSQLITE_ENABLE_FTS3 searchc.c $(SQLITELIB) -I/home/dan/work/sqliteG/sqlite/ext/fts3/ -static -L/home/dan/tcl86/lib -ltcl8.6 -ldl -lm -o tclsqlite3.fts3 -lz -lpthread

clean:
	rm tclsqlite3.fts3

	#gcc $(CFLAGS) -DSQLITE_ENABLE_FTS3 -I/home/dan/work/sqliteG/sqlite/ext/fts3/ -static searchc.c /home/dan/work/sqliteG/bld/tclsqlite3.c -L/home/dan/tcl86/lib -ltcl8.6 -ldl -lm -o tclsqlite3.fts3 -lz -lpthread

Added search/buildsearchdb.tcl.













































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214

load ./parsehtml.so

# Return a list of relative paths to documents that should be included 
# in the index.
#
proc document_list {} {
  set files [list]
  foreach f [glob *.html c3ref/*.html releaselog/*.html] {
    if {![string match *crossref* $f]
     && ![string match fileio.html $f]
     && ![string match btreemodule.html $f]
    } { lappend files $f }
  }
  return $files
}

proc readfile {zFile} {
  set fd [open $zFile]
  set ret [read $fd]
  close $fd
  return $ret
}

proc keywordparse_callback {tag details} {
  global K P
  switch -- [string tolower $tag] {
    "" {
      if {[info exists K(hyperlink)]} {
        append K($K(hyperlink)) $details
      }
    }
    "a" {
      array set D $details
      if {[info exists D(href)]} { set K(hyperlink) $D(href) }
    }
    "/a" {
      unset -nocomplain P(hyperlink)
    }
  }
}

#-------------------------------------------------------------------------
# This function is used as the callback when parsing ordinary documents 
# (not the keywords document).
#
# Rules for extracting fragment "titles". A fragment title consists of
# all text that follows the tag that opens the fragment until either:
#
#   1. 80 characters have been parsed, or
#   2. 8 characters have been parsed and one of the following is 
#        encountered:
#      a) A block element opening or closing tag, or
#      b) A <br> element, or
#      c) A "." character.
#
#   3. 8 characters have been parsed and a <br> tag or "." character is
#      encountered
#
proc docparse_callback {tag details} {
  global P
  set tag [string tolower $tag]
  switch -glob -- $tag {
    "" {
      append P(text) " $details"
      if {$P(isTitle)} { append P(title) $details }
      if {[llength $P(fragments)]} { 
        append P(ftext) " $details" 
      }
    }

    "/table" {
      if {[info exists P(skipheader)]} {
        set P(text) ""
        unset P(skipheader)
      }
    }

    "title"  { set P(isTitle) 1 }
    "/title" { set P(isTitle) 0 }

    "a" { 
      array set D $details
      if {[info exists D(name)]} {
        if {[llength $P(fragments)]} { 
          lappend P(fragments) $P(ftitle) $P(ftext) 
        }
        lappend P(fragments) $D(name)
        set P(ftext) ""
        set P(ftitle) ""
        catch { unset P(ftitleclose) }
      }
    }
    "h*" {
      array set D $details
      if {[info exists D(id)]} {
        if {[llength $P(fragments)]} { 
          lappend P(fragments) $P(ftitle) $P(ftext) 
        }
        lappend P(fragments) $D(id)
        set P(ftext) ""
        set P(ftitle) ""
      }
    }
  }

  set ftext [string trim $P(ftext) " \v\n"]
  if {[string length $ftext]>4 && $P(ftitle) == ""} {
    set blocktags [list                               \
      br td /td th /th p /p                           \
      h1 h2 h3 h4 h5 h /h1 /h2 /h3 /h4 /h5 /h
    ]
    if {[lsearch $blocktags $tag]>=0} {
      set P(ftitle) $ftext
      set P(ftext)  ""
    } elseif {[string length $ftext]>80} {
      set idx [string last " " [string range $ftext 0 79]]
      if {$idx<0} { set idx 80 }
      set P(ftitle) [string range $ftext 0 [expr $idx-1]]
      set P(ftext)  [string range $ftext $idx end]
    } 
  }
}

proc insert_entry {url keywords title content} {
  set nTitle    [tokencount $::tokenizer $title]
  set nKeywords [tokencount $::tokenizer $keywords]
  set nContent  [tokencount $::tokenizer $content]
  db eval {
    INSERT INTO page VALUES($keywords, $title, $content);
    INSERT INTO pagedata VALUES(NULL, $nKeywords, $nTitle, $nContent, $url);
  }
}

proc rebuild_database {} {
  set report [list]

  db transaction {
    # Create the database schema. If the schema already exists, then those
    # tables that contain document data are dropped and recreated by this
    # proc. The 'config' table is left untouched.
    #
    db eval {
      CREATE TABLE IF NOT EXISTS config(item TEXT, value TEXT);

      DROP TABLE IF EXISTS page;
      DROP TABLE IF EXISTS pagedata;
      CREATE VIRTUAL TABLE page USING fts3(
        keywords,                           -- Document keywords
        title,                              -- Title (or first heading)
        content,                            -- Complete document text
        tokenize porter                     -- Built-in porter tokenizer
      );
      CREATE TABLE pagedata(
        docid INTEGER PRIMARY KEY,          -- Maps to docid of page
        nk INTEGER, nt INTEGER, nc INTEGER, -- Tokens in each ft field
        url TEXT                            -- Relative path to document
      );
    }

    set zHtml [readfile keyword_index.html]
    parsehtml $zHtml keywordparse_callback

    # Scan the file-system for HTML documents. Add each document found to
    # the page and pagedata tables.
    foreach file [document_list] {
      set zHtml [readfile $file]

      array unset ::P
      set ::P(text) ""                 ;# The full document text
      set ::P(isTitle) 0               ;# True while parsing contents of <title>
      set ::P(fragments) [list]        ;# List of document fragments parsed
      set ::P(ftext) ""                ;# Text of current document fragment 

      # If the document appears to contain something that looks like the 
      # header on all website pages, set the P(skipheader) variable. This
      # tells the document parser to ignore all text (except for the document
      # title) up to the first </table> tag in the HTML.
      set tagline {<div class="tagline">Small. Fast. Reliable.}
      if {[string first $tagline $zHtml]>0} {
        set ::P(skipheader) 1
      }

      parsehtml $zHtml docparse_callback
      if {[info exists ::P(ftitle)]} {
        lappend ::P(fragments) $::P(ftitle) $::P(ftext)
      }

      set len [string length $::P(text)]
      set keyword ""
      catch { set keyword $::K($file) }
      if {![info exists ::P(title)]} {set ::P(title) "No Title"}
      insert_entry $file $keyword $::P(title) $::P(text)

      foreach {name title text} $::P(fragments) {
        set url "$file#$name"
        set keyword ""
        catch { set keyword [string trim $::K($url) " \n\v"] }
        insert_entry $url $keyword $title $text
      }
    }

    db eval { INSERT INTO page(page) VALUES('optimize') }
  }
  db eval VACUUM

  set report
}

cd doc
sqlite3 db search.db
set ::tokenizer [db one {SELECT fts3_tokenizer('porter')}]
rebuild_database

Added search/parsehtml.c.































































































































































































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255

/*
** This file contains some functions implemented in C used by the 
** buildsearchdb.tcl script. There are two Tcl commands:
**
**   parsehtml
**   tokencount
**
** Tcl command [parsehtml] is a helper command used to extract text and 
** markup tags from the HTML documents in the SQLite documentation. The
** [tokencount] command uses an FTS3 tokenizer to count the number of
** tokens in a document. Both of these are used while building the database
** only.
*/

#include <tcl.h>
#include <string.h>
#include <strings.h>
#include <assert.h>
#include <ctype.h>
#include <math.h>

#define ISSPACE(c) (((c)&0x80)==0 && isspace(c))

#include "fts3_tokenizer.h"
#include "sqlite3.h"

typedef unsigned int u32;
typedef unsigned char u8;
typedef sqlite3_uint64 u64;

typedef struct STokenizer STokenizer;
struct STokenizer {
  sqlite3_tokenizer_module *pMod;
  sqlite3_tokenizer *pTokenizer;
  sqlite3_tokenizer_cursor *pCursor;
};

static void tokenizerClose(STokenizer *p){
  if( p->pCursor ){ p->pMod->xClose(p->pCursor); }
  if( p->pTokenizer ){ p->pMod->xDestroy(p->pTokenizer); }
  memset(p, 0, sizeof(STokenizer));
}

static int tokenizerOpen(
  Tcl_Interp *interp,
  Tcl_Obj *pTokenizer,
  Tcl_Obj *pDocument,
  STokenizer *pOut                /* OUT: Structure containing tokenizer */
){
  sqlite3_tokenizer_module *pMod; /* Tokenizer module */
  int rc;                         /* Return code */
  const char *zDoc;               /* Pointer to pDocument string buffer */
  int nDoc;                       /* Number of bytes in buffer zDoc */
  const char *zFail;              /* Error message (if an error occurs) */

  memset(pOut, 0, sizeof(STokenizer));
  memcpy(&pMod, Tcl_GetByteArrayFromObj(pTokenizer, 0), sizeof(pMod));

  rc = pMod->xCreate(0, 0, &pOut->pTokenizer);
  if( rc!=SQLITE_OK ){ zFail = "Error in xCreate()"; goto failed; }
  pOut->pMod = pMod;

  zDoc = Tcl_GetStringFromObj(pDocument, &nDoc);
  rc = pMod->xOpen(pOut->pTokenizer, zDoc, nDoc, &pOut->pCursor);
  if( rc!=SQLITE_OK ){ zFail = "Error in xOpen()"; goto failed; }

  pOut->pCursor->pTokenizer = pOut->pTokenizer;
  return TCL_OK;

 failed:
  tokenizerClose(pOut);
  Tcl_AppendResult(interp, zFail, 0);
  return TCL_ERROR;
}

static int tokenizerNext(
  STokenizer *p,                  /* Tokenizer wrapper object */
  int *piStart,                   /* OUT: Byte offset of start of token */
  int *piEnd,                     /* OUT: Byte offset of end of token */
  int *piCurrent                  /* OUT: Token number */  
){
  const char *z; int n;
  return p->pMod->xNext(p->pCursor, &z, &n, piStart, piEnd, piCurrent);
}

/*
** Tcl command: tokencount TOKENIZER DOCUMENT
*/
static int tokencountcmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *const objv[]
){
  STokenizer sToken;
  int rc;
  int nToken = 0;
  int i1, i2, i3;

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "TOKENIZER DOCUMENT");
    return TCL_ERROR;
  }

  if( tokenizerOpen(interp, objv[1], objv[2], &sToken) ) return TCL_ERROR;

  while( SQLITE_OK==(rc = tokenizerNext(&sToken, &i1, &i2, &i3)) ) nToken++;
  if( rc!=SQLITE_DONE ){
    Tcl_AppendResult(interp, "Error in xNext() 3", 0);
    rc = TCL_ERROR;
  }else{
    Tcl_SetObjResult(interp, Tcl_NewIntObj(nToken));
    rc = TCL_OK;
  }

  tokenizerClose(&sToken);
  return rc;
}


/*
** Tcl command: parsehtml HTML SCRIPT
*/
static int parsehtmlcmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj * const objv[]
){
  char *zHtml;
  char *z;
  Tcl_Obj **aCall;
  int nElem;
  Tcl_Obj **aElem;
  int rc;

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "HTML SCRIPT");
    return TCL_ERROR;
  }
  zHtml = Tcl_GetString(objv[1]);

  rc = Tcl_ListObjGetElements(interp, objv[2], &nElem, &aElem);
  if( rc!=TCL_OK ) return rc;
  aCall = (Tcl_Obj **)ckalloc(sizeof(Tcl_Obj *)*(nElem+2));
  memcpy(aCall, aElem, sizeof(Tcl_Obj *)*nElem);
  aCall[nElem] = 0;
  aCall[nElem+1] = 0;

  z = zHtml;
  while( *z ){
    char *zText = z;
    while( *z && *z!='<' ) z++;

    /* Invoke the callback script for the chunk of text just parsed. */
    Tcl_IncrRefCount( aCall[nElem]   = Tcl_NewObj() );
    Tcl_IncrRefCount( aCall[nElem+1] = Tcl_NewStringObj(zText, z-zText) );
    rc = Tcl_EvalObjv(interp, nElem+2, aCall, 0);
    Tcl_DecrRefCount( aCall[nElem] );
    Tcl_DecrRefCount( aCall[nElem+1] );
    if( rc!=TCL_OK ) return rc;

    /* Unless is at the end of the document, z now points to the start of a
    ** markup tag. Either an opening or a closing tag. Parse it up and 
    ** invoke the callback script. */
    if( *z ){
      int nTag;
      char *zTag;
      z++;

      while( ISSPACE(*z) ) z++;
      zTag = z;

      while( *z && !ISSPACE(*z) && *z!='>' ) z++;
      nTag = z-zTag;

      if( nTag==5 && 0==strncasecmp("style", zTag, 5) ){
        while( *z && strncasecmp("/style>", z, 6 ) ) z++;
      } else if( nTag>=3 && 0==memcmp("!--", zTag, 3) ){
        while( *z && strncasecmp("-->", z, 3 ) ) z++;
      } else {
        Tcl_Obj *pParam = Tcl_NewObj();

        while( *z && *z!='>' ){
          char *zAttr;

          /* Gobble up white-space */
          while( ISSPACE(*z) ) z++;
          zAttr = z;

          /* Advance to the end of the attribute name */
          while( *z && *z!='>' && !ISSPACE(*z) && *z!='=' ) z++;
          if( z==zAttr ) zAttr = 0;

          if( zAttr ){
            Tcl_Obj *pAttr = Tcl_NewStringObj(zAttr, z-zAttr);
            Tcl_ListObjAppendElement(interp, pParam, pAttr);
          }
          while( ISSPACE(*z) ) z++;

          if( *z=='=' ){
            int nVal;
            char *zVal;
            z++;
            while( ISSPACE(*z) ) z++;
            zVal = z;

            if( *zVal=='"' ){
              zVal++;
              z++;
              while( *z && *z!='"' ) z++;
              nVal = z-zVal;
              z++;
            }else{
              while( *z && !ISSPACE(*z) && *z!='>' ) z++;
              nVal = z-zVal;
            }
            Tcl_ListObjAppendElement(interp,pParam,Tcl_NewStringObj(zVal,nVal));
          }else if( zAttr ){
            Tcl_ListObjAppendElement(interp, pParam, Tcl_NewIntObj(1));
          }
        }
        
        Tcl_IncrRefCount( aCall[nElem]   = Tcl_NewStringObj(zTag, nTag) );
        Tcl_IncrRefCount( aCall[nElem+1] = pParam );
        rc = Tcl_EvalObjv(interp, nElem+2, aCall, 0);
        Tcl_DecrRefCount( aCall[nElem] );
        Tcl_DecrRefCount( aCall[nElem+1] );
        if( rc!=TCL_OK ) return rc;
      }

      while( *z && !ISSPACE(*z) && *z!='>' ) z++;
      if( *z ) z++;
    }

  }

  return TCL_OK;
}

int Parsehtml_Init(Tcl_Interp *interp){

#ifdef USE_TCL_STUBS
  if (Tcl_InitStubs(interp, "8.4", 0) == 0) {
    return TCL_ERROR;
  }
#endif

  Tcl_CreateObjCommand(interp, "parsehtml",  parsehtmlcmd, 0, 0);
  Tcl_CreateObjCommand(interp, "tokencount", tokencountcmd, 0, 0);

  return TCL_OK;
}

Changes to search/search.tcl.

118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
...
168
169
170
171
172
173
174



175
176
177
178
179
180
181
...
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
...
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
    <script> 
      document.forms.f.q.focus()
      document.forms.f.q.select()
    </script>
  }]
}

proc rebuildform {} {
  return {
    <hr>
    <table align=right>
    <td><form name=r method=GET action=search>
      <input name=rebuild type=submit value="Rebuild Db"></input>
    </form>
    </table>
  }
}


#-------------------------------------------------------------------------
# This command is similar to the builtin Tcl [time] command, except that
................................................................................
      SELECT url FROM page, pagedata 
      WHERE page MATCH $::A(q) AND page.docid = pagedata.docid
      ORDER BY rank(matchinfo(page), nk, nt, nc) DESC
    }]
    if {$url != ""} { cgi_redirect $url }
  }




  # Set nRes to the total number of documents that the users query matches.
  # If nRes is 0, then the users query returned zero results. Return a short 
  # message to that effect.
  #
  set nRes [db one { SELECT count(*) FROM page WHERE page MATCH $::A(q) }]
  if {$nRes == 0} {
    return [subst { No results for: <b>[htmlize $::A(q)]</b> }]
................................................................................
  db eval {
    SELECT 
      result.rowid+$iStart AS resnum,
      COALESCE(title, 'No Title.') AS title,
      snippet(page, $open, $close, $ellipsis, 0, 10) AS keywords,
      snippet(page, $open, $close, $ellipsis, 2, 40) AS snippet,
      url,
      erank(matchinfo(page), nk, nt, nc) AS report
    FROM page, (
        SELECT 
          page.docid AS docid, url, nk, nt, nc
        FROM page, pagedata 
          WHERE page MATCH $::A(q) AND page.docid = pagedata.docid
          ORDER BY rank(matchinfo(page), nk, nt, nc)
          DESC LIMIT 10 OFFSET $iStart
................................................................................
    }
    append ret </center>
  }

  return $ret
}

# Return a list of relative paths to documents that should be included 
# in the index.
#
proc document_list {} {
  set files [list]
  foreach f [glob *.html c3ref/*.html releaselog/*.html] {
    if {![string match *crossref* $f]} { lappend files $f }
  }
  return $files
}

proc readfile {zFile} {
  set fd [open $zFile]
  set ret [read $fd]
  close $fd
  return $ret
}

proc keywordparse_callback {tag details} {
  global K P
  switch -- [string tolower $tag] {
    "" {
      if {[info exists K(hyperlink)]} {
        append K($K(hyperlink)) $details
      }
    }
    "a" {
      array set D $details
      if {[info exists D(href)]} { set K(hyperlink) $D(href) }
    }
    "/a" {
      unset -nocomplain P(hyperlink)
    }
  }
}

#-------------------------------------------------------------------------
# This function is used as the callback when parsing ordinary documents 
# (not the keywords document).
#
# Rules for extracting fragment "titles". A fragment title consists of
# all text that follows the tag that opens the fragment until either:
#
#   1. 80 characters have been parsed, or
#   2. 8 characters have been parsed and one of the following is 
#        encountered:
#      a) A block element opening or closing tag, or
#      b) A <br> element, or
#      c) A "." character.
#
#   3. 8 characters have been parsed and a <br> tag or "." character is
#      encountered
#
proc docparse_callback {tag details} {
  global P
  set tag [string tolower $tag]
  switch -glob -- $tag {
    "" {
      append P(text) " $details"
      if {$P(isTitle)} { append P(title) $details }
      if {[llength $P(fragments)]} { 
        append P(ftext) " $details" 
      }
    }

    "/table" {
      if {[info exists P(skipheader)]} {
        set P(text) ""
        unset P(skipheader)
      }
    }

    "title"  { set P(isTitle) 1 }
    "/title" { set P(isTitle) 0 }

    "a" { 
      array set D $details
      if {[info exists D(name)]} {
        if {[llength $P(fragments)]} { 
          lappend P(fragments) $P(ftitle) $P(ftext) 
        }
        lappend P(fragments) $D(name)
        set P(ftext) ""
        set P(ftitle) ""
        catch { unset P(ftitleclose) }
      }
    }
    "h*" {
      array set D $details
      if {[info exists D(id)]} {
        if {[llength $P(fragments)]} { 
          lappend P(fragments) $P(ftitle) $P(ftext) 
        }
        lappend P(fragments) $D(id)
        set P(ftext) ""
        set P(ftitle) ""
      }
    }
  }

  set ftext [string trim $P(ftext) " \v\n"]
  if {[string length $ftext]>4 && $P(ftitle) == ""} {
    set blocktags [list                               \
      br td /td th /th p /p                           \
      h1 h2 h3 h4 h5 h /h1 /h2 /h3 /h4 /h5 /h
    ]
    if {[lsearch $blocktags $tag]>=0} {
      set P(ftitle) $ftext
      set P(ftext)  ""
      puts "setting title to \"$P(ftitle)\""
    } elseif {[string length $ftext]>80} {
      set idx [string last " " [string range $ftext 0 79]]
      if {$idx<0} { set idx 80 }
      set P(ftitle) [string range $ftext 0 [expr $idx-1]]
      set P(ftext)  [string range $ftext $idx end]
      puts "setting title to \"$P(ftitle)\""
    } 
  }
}

proc insert_entry {url keywords title content} {
  set nTitle    [tokencount $::tokenizer $title]
  set nKeywords [tokencount $::tokenizer $keywords]
  set nContent  [tokencount $::tokenizer $content]
  db eval {
    INSERT INTO page VALUES($keywords, $title, $content);
    INSERT INTO pagedata VALUES(NULL, $nKeywords, $nTitle, $nContent, $url);
  }
}

proc rebuild_database {} {
  set report [list]

  db transaction {
    # Create the database schema. If the schema already exists, then those
    # tables that contain document data are dropped and recreated by this
    # proc. The 'config' table is left untouched.
    #
    db eval {
      CREATE TABLE IF NOT EXISTS config(item TEXT, value TEXT);

      DROP TABLE IF EXISTS page;
      DROP TABLE IF EXISTS pagedata;
      CREATE VIRTUAL TABLE page USING fts3(
        keywords,                           -- Document keywords
        title,                              -- Title (or first heading)
        content,                            -- Complete document text
        tokenize porter                     -- Built-in porter tokenizer
      );
      CREATE TABLE pagedata(
        docid INTEGER PRIMARY KEY,          -- Maps to docid of page
        nk INTEGER, nt INTEGER, nc INTEGER, -- Tokens in each ft field
        url TEXT                            -- Relative path to document
      );
    }

    set zHtml [readfile keyword_index.html]
    parsehtml $zHtml keywordparse_callback

    # Scan the file-system for HTML documents. Add each document found to
    # the page and pagedata tables.
    foreach file [document_list] {
      set zHtml [readfile $file]

      array unset ::P
      set ::P(text) ""                 ;# The full document text
      set ::P(isTitle) 0               ;# True while parsing contents of <title>
      set ::P(fragments) [list]        ;# List of document fragments parsed
      set ::P(ftext) ""                ;# Text of current document fragment 

      # If the document appears to contain something that looks like the 
      # header on all website pages, set the P(skipheader) variable. This
      # tells the document parser to ignore all text (except for the document
      # title) up to the first </table> tag in the HTML.
      set tagline {<div class="tagline">Small. Fast. Reliable.}
      if {[string first $tagline $zHtml]>0} {
        set ::P(skipheader) 1
      }

      parsehtml $zHtml docparse_callback
      if {[info exists ::P(ftitle)]} {
        lappend ::P(fragments) $::P(ftitle) $::P(ftext)
      }

      set len [string length $::P(text)]
      set keyword ""
      catch { set keyword $::K($file) }
      if {![info exists ::P(title)]} {set ::P(title) "No Title"}
      insert_entry $file $keyword $::P(title) $::P(text)

      foreach {name title text} $::P(fragments) {
        set url "$file#$name"
        set keyword ""
        catch { set keyword [string trim $::K($url) " \n\v"] }
        insert_entry $url $keyword $title $text
      }
    }

    db eval { INSERT INTO page(page) VALUES('optimize') }
  }
  db eval VACUUM

  set report
}

proc main {} {
  sqlite3 db search.db

  array set ::A [cgi_parse_args]

  set ::tokenizer [db one {SELECT fts3_tokenizer('porter')}]

  # If ::A(rebuild) is set and this is a POST request, it is a command to
  # rebuild the full-text database.
  if {[info exists ::A(rebuild)] && (1 || $::env(REQUEST_METHOD) == "POST")} {
    set t [ttime {set report [rebuild_database]}]
    set msg "<p><b>Database rebuilt. ($t)</b>"
    return "[searchform] $msg $report [rebuildform]"
  }

  # If ::A(q) is set, this is a query. Search for documents that MATCH $::A(q).
  db transaction {
    set t [ttime { set doc "[searchform] [searchresults] [rebuildform]" }]
  }
  append doc "<p>Page generated in $t"
  return $doc

  return [cgi_env_dump]
}

#=========================================================================








|



|
|
<







 







>
>
>







 







|







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<





<
<
<
<
<
<
<
<
<
<
<

|

|







118
119
120
121
122
123
124
125
126
127
128
129
130

131
132
133
134
135
136
137
...
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
...
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
...
255
256
257
258
259
260
261













































































































































































































262
263
264
265
266











267
268
269
270
271
272
273
274
275
276
277
    <script> 
      document.forms.f.q.focus()
      document.forms.f.q.select()
    </script>
  }]
}

proc footer {} {
  return {
    <hr>
    <table align=right>
    <td>
      <i>Powered by SQLite/FTS3.</i>

    </table>
  }
}


#-------------------------------------------------------------------------
# This command is similar to the builtin Tcl [time] command, except that
................................................................................
      SELECT url FROM page, pagedata 
      WHERE page MATCH $::A(q) AND page.docid = pagedata.docid
      ORDER BY rank(matchinfo(page), nk, nt, nc) DESC
    }]
    if {$url != ""} { cgi_redirect $url }
  }

  set score 0
  catch {set score $::A(score)}

  # Set nRes to the total number of documents that the users query matches.
  # If nRes is 0, then the users query returned zero results. Return a short 
  # message to that effect.
  #
  set nRes [db one { SELECT count(*) FROM page WHERE page MATCH $::A(q) }]
  if {$nRes == 0} {
    return [subst { No results for: <b>[htmlize $::A(q)]</b> }]
................................................................................
  db eval {
    SELECT 
      result.rowid+$iStart AS resnum,
      COALESCE(title, 'No Title.') AS title,
      snippet(page, $open, $close, $ellipsis, 0, 10) AS keywords,
      snippet(page, $open, $close, $ellipsis, 2, 40) AS snippet,
      url,
      CASE WHEN $score THEN erank(matchinfo(page), nk, nt, nc) ELSE '' END AS report
    FROM page, (
        SELECT 
          page.docid AS docid, url, nk, nt, nc
        FROM page, pagedata 
          WHERE page MATCH $::A(q) AND page.docid = pagedata.docid
          ORDER BY rank(matchinfo(page), nk, nt, nc)
          DESC LIMIT 10 OFFSET $iStart
................................................................................
    }
    append ret </center>
  }

  return $ret
}














































































































































































































proc main {} {
  sqlite3 db search.db

  array set ::A [cgi_parse_args]












  db transaction {
    set t [ttime { set doc "[searchform] [searchresults] [footer]" }]
  }
  append doc "<p>Page generated in $t."
  return $doc

  return [cgi_env_dump]
}

#=========================================================================

Changes to search/searchc.c.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
..
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
...
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355

/*
** This file contains some functions implemented in C used by the search.tcl
** script. There are two Tcl commands:
**
**   parsehtml
**   tokencount
**
** Tcl command [parsehtml] is a helper command used to extract text and 
** markup tags from the HTML documents in the SQLite documentation. The
** [tokencount] command uses an FTS3 tokenizer to count the number of
** tokens in a document. Both of these are used while building the database
** only.
**
** There are also two SQL user functions registered:
**
**   rank()
**   erank()
**
** rank() interprets the return value of the FTS3 matchinfo() function and
** returns a score for the match (a real number). The higher the score, the
................................................................................

#include <tcl.h>
#include <string.h>
#include <assert.h>
#include <ctype.h>
#include <math.h>

#define ISSPACE(c) (((c)&0x80)==0 && isspace(c))

#include "fts3_tokenizer.h"
#include "sqlite3.h"

typedef unsigned int u32;
typedef unsigned char u8;
typedef sqlite3_uint64 u64;

typedef struct STokenizer STokenizer;
struct STokenizer {
  sqlite3_tokenizer_module *pMod;
  sqlite3_tokenizer *pTokenizer;
  sqlite3_tokenizer_cursor *pCursor;
};

static void tokenizerClose(STokenizer *p){
  if( p->pCursor ){ p->pMod->xClose(p->pCursor); }
  if( p->pTokenizer ){ p->pMod->xDestroy(p->pTokenizer); }
  memset(p, 0, sizeof(STokenizer));
}

static int tokenizerOpen(
  Tcl_Interp *interp,
  Tcl_Obj *pTokenizer,
  Tcl_Obj *pDocument,
  STokenizer *pOut                /* OUT: Structure containing tokenizer */
){
  sqlite3_tokenizer_module *pMod; /* Tokenizer module */
  int rc;                         /* Return code */
  const char *zDoc;               /* Pointer to pDocument string buffer */
  int nDoc;                       /* Number of bytes in buffer zDoc */
  const char *zFail;              /* Error message (if an error occurs) */

  memset(pOut, 0, sizeof(STokenizer));
  memcpy(&pMod, Tcl_GetByteArrayFromObj(pTokenizer, 0), sizeof(pMod));

  rc = pMod->xCreate(0, 0, &pOut->pTokenizer);
  if( rc!=SQLITE_OK ){ zFail = "Error in xCreate()"; goto failed; }
  pOut->pMod = pMod;

  zDoc = Tcl_GetStringFromObj(pDocument, &nDoc);
  rc = pMod->xOpen(pOut->pTokenizer, zDoc, nDoc, &pOut->pCursor);
  if( rc!=SQLITE_OK ){ zFail = "Error in xOpen()"; goto failed; }

  pOut->pCursor->pTokenizer = pOut->pTokenizer;
  return TCL_OK;

 failed:
  tokenizerClose(pOut);
  Tcl_AppendResult(interp, zFail, 0);
  return TCL_ERROR;
}

static int tokenizerNext(
  STokenizer *p,                  /* Tokenizer wrapper object */
  int *piStart,                   /* OUT: Byte offset of start of token */
  int *piEnd,                     /* OUT: Byte offset of end of token */
  int *piCurrent                  /* OUT: Token number */  
){
  const char *z; int n;
  return p->pMod->xNext(p->pCursor, &z, &n, piStart, piEnd, piCurrent);
}

/*
** Tcl command: tokencount TOKENIZER DOCUMENT
*/
static int tokencountcmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *const objv[]
){
  STokenizer sToken;
  int rc;
  int nToken = 0;
  int i1, i2, i3;

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "TOKENIZER DOCUMENT");
    return TCL_ERROR;
  }

  if( tokenizerOpen(interp, objv[1], objv[2], &sToken) ) return TCL_ERROR;

  while( SQLITE_OK==(rc = tokenizerNext(&sToken, &i1, &i2, &i3)) ) nToken++;
  if( rc!=SQLITE_DONE ){
    Tcl_AppendResult(interp, "Error in xNext() 3", 0);
    rc = TCL_ERROR;
  }else{
    Tcl_SetObjResult(interp, Tcl_NewIntObj(nToken));
    rc = TCL_OK;
  }

  tokenizerClose(&sToken);
  return rc;
}


/*
** Tcl command: parsehtml HTML SCRIPT
*/
static int parsehtmlcmd(
  ClientData clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj * const objv[]
){
  char *zHtml;
  char *z;
  Tcl_Obj **aCall;
  int nElem;
  Tcl_Obj **aElem;
  int rc;

  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 1, objv, "HTML SCRIPT");
    return TCL_ERROR;
  }
  zHtml = Tcl_GetString(objv[1]);

  rc = Tcl_ListObjGetElements(interp, objv[2], &nElem, &aElem);
  if( rc!=TCL_OK ) return rc;
  aCall = (Tcl_Obj **)ckalloc(sizeof(Tcl_Obj *)*(nElem+2));
  memcpy(aCall, aElem, sizeof(Tcl_Obj *)*nElem);
  aCall[nElem] = 0;
  aCall[nElem+1] = 0;

  z = zHtml;
  while( *z ){
    char *zText = z;
    while( *z && *z!='<' ) z++;

    /* Invoke the callback script for the chunk of text just parsed. */
    Tcl_IncrRefCount( aCall[nElem]   = Tcl_NewObj() );
    Tcl_IncrRefCount( aCall[nElem+1] = Tcl_NewStringObj(zText, z-zText) );
    rc = Tcl_EvalObjv(interp, nElem+2, aCall, 0);
    Tcl_DecrRefCount( aCall[nElem] );
    Tcl_DecrRefCount( aCall[nElem+1] );
    if( rc!=TCL_OK ) return rc;

    /* Unless is at the end of the document, z now points to the start of a
    ** markup tag. Either an opening or a closing tag. Parse it up and 
    ** invoke the callback script. */
    if( *z ){
      int nTag;
      char *zTag;
      z++;

      while( ISSPACE(*z) ) z++;
      zTag = z;

      while( *z && !ISSPACE(*z) && *z!='>' ) z++;
      nTag = z-zTag;

      if( nTag==5 && 0==sqlite3_strnicmp("style", zTag, 5) ){
        while( *z && sqlite3_strnicmp("/style>", z, 6 ) ) z++;
      } else if( nTag>=3 && 0==memcmp("!--", zTag, 3) ){
        while( *z && sqlite3_strnicmp("-->", z, 3 ) ) z++;
      } else {
        Tcl_Obj *pParam = Tcl_NewObj();

        while( *z && *z!='>' ){
          char *zAttr;

          /* Gobble up white-space */
          while( ISSPACE(*z) ) z++;
          zAttr = z;

          /* Advance to the end of the attribute name */
          while( *z && *z!='>' && !ISSPACE(*z) && *z!='=' ) z++;
          if( z==zAttr ) zAttr = 0;

          if( zAttr ){
            Tcl_Obj *pAttr = Tcl_NewStringObj(zAttr, z-zAttr);
            Tcl_ListObjAppendElement(interp, pParam, pAttr);
          }
          while( ISSPACE(*z) ) z++;

          if( *z=='=' ){
            int nVal;
            char *zVal;
            z++;
            while( ISSPACE(*z) ) z++;
            zVal = z;

            if( *zVal=='"' ){
              zVal++;
              z++;
              while( *z && *z!='"' ) z++;
              nVal = z-zVal;
              z++;
            }else{
              while( *z && !ISSPACE(*z) && *z!='>' ) z++;
              nVal = z-zVal;
            }
            Tcl_ListObjAppendElement(interp,pParam,Tcl_NewStringObj(zVal,nVal));
          }else if( zAttr ){
            Tcl_ListObjAppendElement(interp, pParam, Tcl_NewIntObj(1));
          }
        }
        
        Tcl_IncrRefCount( aCall[nElem]   = Tcl_NewStringObj(zTag, nTag) );
        Tcl_IncrRefCount( aCall[nElem+1] = pParam );
        rc = Tcl_EvalObjv(interp, nElem+2, aCall, 0);
        Tcl_DecrRefCount( aCall[nElem] );
        Tcl_DecrRefCount( aCall[nElem+1] );
        if( rc!=TCL_OK ) return rc;
      }

      while( *z && !ISSPACE(*z) && *z!='>' ) z++;
      if( *z ) z++;
    }

  }

  return TCL_OK;
}


/*
** Implementation of search result ranking function.
*/
static void rankfunc(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
  u32 *aMatchinfo;
  double score = 0.0;
................................................................................

int Sqlite3_Init(Tcl_Interp *interp);

static int initDb(sqlite3 *db, char **pzErr, void *p){
  sqlite3_create_function(db, "rank",-1, SQLITE_UTF8, 0, rankfunc,0,0);
  sqlite3_create_function(db, "erank", -1, SQLITE_UTF8, (void*)1, rankfunc,0,0);
}

int Parsehtml_Init(Tcl_Interp *interp){
#ifdef USE_TCL_STUBS
  if (Tcl_InitStubs(interp, "8.4", 0) == 0) {
    return TCL_ERROR;
  }
#endif

  Tcl_CreateObjCommand(interp, "parsehtml",  parsehtmlcmd, 0, 0);
  Tcl_CreateObjCommand(interp, "tokencount", tokencountcmd, 0, 0);
  sqlite3_auto_extension(initDb);

  return TCL_OK;
}


static int AppInit(Tcl_Interp *interp) {
  int rc;
  rc = Sqlite3_Init(interp);
  if( rc!=TCL_OK ) return rc;
  rc = Parsehtml_Init(interp);
  return rc;
}

int main(int argc, char *argv[]) {
  Tcl_Main(argc, argv, AppInit);
  return 0;
}



<
<
<
<
<
<
<
<
<
<
<
<







 







<
<
<





<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<





|
|







1
2












3
4
5
6
7
8
9
..
23
24
25
26
27
28
29



30
31
32
33
34



















































































































































































































35
36
37
38
39
40
41
..
94
95
96
97
98
99
100















101
102
103
104
105
106
107
108
109
110
111
112
113
114

/*












** There are also two SQL user functions registered:
**
**   rank()
**   erank()
**
** rank() interprets the return value of the FTS3 matchinfo() function and
** returns a score for the match (a real number). The higher the score, the
................................................................................

#include <tcl.h>
#include <string.h>
#include <assert.h>
#include <ctype.h>
#include <math.h>




#include "sqlite3.h"

typedef unsigned int u32;
typedef unsigned char u8;
typedef sqlite3_uint64 u64;




















































































































































































































/*
** Implementation of search result ranking function.
*/
static void rankfunc(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
  u32 *aMatchinfo;
  double score = 0.0;
................................................................................

int Sqlite3_Init(Tcl_Interp *interp);

static int initDb(sqlite3 *db, char **pzErr, void *p){
  sqlite3_create_function(db, "rank",-1, SQLITE_UTF8, 0, rankfunc,0,0);
  sqlite3_create_function(db, "erank", -1, SQLITE_UTF8, (void*)1, rankfunc,0,0);
}
















static int AppInit(Tcl_Interp *interp) {
  int rc;
  rc = Sqlite3_Init(interp);
  if( rc!=TCL_OK ) return rc;
  sqlite3_auto_extension(initDb);
  return TCL_OK;
}

int main(int argc, char *argv[]) {
  Tcl_Main(argc, argv, AppInit);
  return 0;
}