Documentation Source Text

Check-in [cc51dec17e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further updates to search database and script.
Timelines: family | ancestors | descendants | both | experimental
Files: files | file ages | folders
SHA1: cc51dec17e8d87bcb2d99bcfccec396ed9e7c26b
User & Date: dan 2016-08-24 19:26:13
Context
2016-08-25
12:18
Add <fancy_format> or <table_of_contents> markup to a few more documents. To ensure that there are enough anchors in the longer documents for the search script to use. check-in: 066e5931ce user: dan tags: experimental
2016-08-24
19:26
Further updates to search database and script. check-in: cc51dec17e user: dan tags: experimental
2016-08-20
17:22
Add "jump to" links to relevant sections of large documents in search results. check-in: 9d49a78f9b user: dan tags: experimental
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to document_header.tcl.

   139    139           <a href="${path}copyright.html">License</a>
   140    140           <a href="${path}support.html">Support</a>
   141    141           <a href="http://www.hwaci.com/sw/sqlite/prosupport.html">Purchase</a>
   142    142         </div>
   143    143     }]
   144    144   
   145    145     if {$search==""} {
   146         -    set initval   "Search SQLite Docs..."
          146  +    set initval   "Search with FTS5..."
   147    147       set initstyle {font-style:italic;color:#044a64}
   148    148     } else {
   149    149       set initval   $search
   150    150       set initstyle {font-style:normal;color:black}
   151    151     }
   152    152   
   153    153     append ret [subst -nocommands {
   154    154       <script>
   155         -      gMsg = "Search SQLite Docs..."
          155  +      gMsg = "Search with FTS5..."
   156    156         function entersearch() {
   157    157           var q = document.getElementById("q");
   158    158           if( q.value == gMsg ) { q.value = "" }
   159    159           q.style.color = "black"
   160    160           q.style.fontStyle = "normal"
   161    161         }
   162    162         function leavesearch() {

Changes to main.mk.

    40     40   private:	base evidence private_evidence matrix doc
    41     41   
    42     42   fast:	base doc
    43     43   
    44     44   tclsh:	$(TCLSQLITE3C)
    45     45   	$(CC) -g -o tclsh -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS5 -DTCLSH=1 -DSQLITE_TCLMD5 $(TCLINC) $(TCLSQLITE3C) $(TCLFLAGS)
    46     46   
    47         -tclsqlite3.search:	$(TCLSQLITE3C) $(DOC)/search/searchc.c
    48         -	$(CC) -g -o tclsqlite3.search -I. -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS5 $(TCLINC) $(DOC)/search/searchc.c $(TCLSQLITE3C) $(TCLFLAGS)
           47  +
    49     48   
    50     49   sqlite3.h:	tclsh $(SRC)/src/sqlite.h.in $(SRC)/manifest.uuid $(SRC)/VERSION
    51     50   	./tclsh $(SRC)/tool/mksqlite3h.tcl $(SRC) | \
    52     51   	sed 's/^SQLITE_API //' >sqlite3.h
    53     52   
    54     53   # Generate the directory into which generated documentation files will
    55     54   # be written.
................................................................................
   136    135   # Generate the traceability matrix
   137    136   #
   138    137   matrix:	
   139    138   	rm -rf doc/matrix/images
   140    139   	cp -r doc/images doc/matrix
   141    140   	./tclsh $(DOC)/matrix.tcl
   142    141   
   143         -# Build the fts3 database used by the search script
          142  +
          143  +#-------------------------------------------------------------------------
          144  +
          145  +# Source files for the [tclsqlite3.search] executable. 
          146  +#
          147  +SSRC = $(DOC)/search/searchc.c \
          148  +	    $(DOC)/search/parsehtml.c \
          149  +	    $(DOC)/search/fts5ext.c \
          150  +	    $(TCLSQLITE3C)
          151  +
          152  +# Flags to build [tclsqlite3.search] with.
   144    153   #
   145         -parsehtml.so: $(DOC)/search/parsehtml.c
   146         -	gcc -g -shared -fPIC $(TCLINC) -I. -I$(SRC)/ext/fts3 $(DOC)/search/parsehtml.c $(TCLSTUBFLAGS) -o parsehtml.so
          154  +SFLAGS = -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS5
          155  +
          156  +tclsqlite3.search: $(SSRC)
          157  +	$(CC) -static -O2 -o $@ -I. $(SFLAGS) $(SSRC) $(STATICTCLFLAGS)
   147    158   
   148         -searchdb: parsehtml.so tclsh
   149         -	./tclsh $(DOC)/search/buildsearchdb.tcl
          159  +searchdb: tclsqlite3.search
          160  +	./tclsqlite3.search $(DOC)/search/buildsearchdb.tcl
   150    161   	cp $(DOC)/document_header.tcl doc/document_header.tcl
   151    162   	cp $(DOC)/search/search.tcl doc/search
   152    163   	chmod +x doc/search
   153    164   
   154    165   always:	
   155    166   
   156    167   clean:	
   157    168   	rm -rf tclsh doc sqlite3.h

Changes to pages/amalgamation.in.

     1      1   <title>The SQLite Amalgamation</title>
     2      2   <tcl>hd_keywords {amalgamation} {the amalgamation}</tcl>
     3      3   
     4         -<h2>1.0 The SQLite Amalgamation</h2>
            4  +<fancy_format>
            5  +
            6  +<h1>The SQLite Amalgamation</h1>
     5      7   
     6      8   <p>The SQLite library consists of 102 files of C code
     7      9   (as of [Version 3.9.0]) in the core with 32 additional files that
     8     10   implement the [FTS3], [FTS5], [RTREE], [dbstat|DBSTAT], [json1|JSON1], and
     9     11   [RBU] extensions.
    10     12   Most of these are "source" files in the sense that they are stored 
    11     13   in the [https://www.sqlite.org/src | SQLite version control system]
................................................................................
    57     59   of between 5 and 10% when we use the amalgamation to compile 
    58     60   SQLite rather than individual source files.  The downside of this
    59     61   is that the additional optimizations often take the form of 
    60     62   function inlining which tends to make the size of the resulting
    61     63   binary image larger.</p>
    62     64   
    63     65   <tcl>hd_fragment amal32k {split amalgamation}</tcl>
    64         -<h2>2.0 The Split Amalgamation</h2>
           66  +<h1>The Split Amalgamation</h1>
    65     67   
    66     68   <p>Developers sometimes experience trouble debugging the
    67     69   185,000-line-long amalgamation source file because some debuggers
    68     70   are only able to handle source code line numbers less than 32,768.
    69     71   The amalgamation source code runs fine.  One just cannot single-step
    70     72   through it in a debugger.
    71     73   
................................................................................
    89     91   <p>Applications using the split amalgamation simply compile against
    90     92   "sqlite3-all.c" instead of "sqlite3.c".  The two files work exactly
    91     93   the same.  But with "sqlite3-all.c", no single source file contains more
    92     94   than 32,767 lines of code, and so it is more convenient to use some
    93     95   debuggers.  The downside of the split amalgamation is that it consists
    94     96   of 6 C source code files instead of just 1.
    95     97   
    96         -<h2>3.0 Download Copies Of The Amalgamation</h2>
           98  +<h1>Download Copies Of The Amalgamation</h1>
    97     99   
    98    100   <p>The amalgamation and
    99    101   the sqlite3.h header file are available on
   100    102   the <a href="download.html">download page</a> as a file 
   101    103   named sqlite-amalgamation-X.zip
   102    104   where the X is replaced by the appropriate version number.</p>
   103    105   
   104    106   <tcl>hd_fragment amalgbuild</tcl>
   105         -<h2>4.0 Building The Amalgamation From Canonical Source Code</h2>
          107  +<h1>Building The Amalgamation From Canonical Source Code</h1>
   106    108   
   107    109   <p>To build the amalgamation (either the full amalgamation or the
   108    110   split amalgamation), first
   109    111   [get the canonical source code] from one of the three servers.
   110    112   Then, on both unix-like systems and on Windows systems that have the
   111    113   free [http://mingw.org/wiki/msys|MinGW] development environment
   112    114   installed, the amalgamation can be built using the
................................................................................
   122    124   <blockquote><pre>
   123    125   nmake /f makefile.msc sqlite3.c
   124    126   </pre></blockquote>
   125    127   
   126    128   <p>In both cases, the split amalgamation can be obtained by
   127    129   substituting "sqlite3-all.c" for "sqlite3.c" as the make target.
   128    130   
   129         -<h3>4.1 Dependencies</h3>
          131  +<h2>Dependencies</h2>
   130    132   
   131    133   <p>The build process makes extensive use of the 
   132    134   [http://www.tcl-lang.org/|Tcl] scripting language.  You will need to have a
   133    135   copy of TCL installed in order for the make targets above to work.
   134    136   Easy-to-use installers can be obtained from [http://www.tcl-lang.org/].
   135    137   Many unix workstations have Tcl installed by default.
   136    138   
   137         -<h3>4.2 See Also</h3>
          139  +<h2>See Also</h2>
   138    140   
   139    141   <p>Additional notes on compiling SQLite can be found on the
   140    142   [How To Compile SQLite] page.

Changes to pages/autoinc.in.

     1      1   <title>SQLite Autoincrement</title>
     2      2   <tcl>hd_keywords AUTOINCREMENT</tcl>
     3         -<h1 align=center>Autoincrement In SQLite</h1>
            3  +<fancy_format>
     4      4   
     5         -<h2>Summary</h2>
            5  +<h1>Summary</h1>
     6      6   
     7      7   <ol type="1">
     8      8   <li><p>
     9      9     The AUTOINCREMENT keyword imposes extra CPU, memory, disk space,
    10     10     and disk I/O overhead and should be avoided if not strictly needed.
    11     11     It is usually not needed.
    12     12   <li><p>
................................................................................
    22     22     If the AUTOINCREMENT keyword appears after INTEGER PRIMARY KEY, that
    23     23     changes the automatic ROWID assignment algorithm to prevent
    24     24     the reuse of ROWIDs over the lifetime of the database.  In other words,
    25     25     the purpose of AUTOINCREMENT is to prevent the reuse of ROWIDs from
    26     26     previously deleted rows.
    27     27   </ol>
    28     28   
    29         -<h2>Background</h2>
           29  +<h1>Background</h1>
    30     30   
    31     31   <p>
    32     32   ^In SQLite, table rows normally have a 64-bit signed integer [ROWID]
    33     33   which is unique among all rows in the same table.
    34     34   ([WITHOUT ROWID] tables are the exception.)
    35     35   </p>
    36     36   
................................................................................
    85     85   delete the entry in the table with the largest ROWID. 
    86     86   ^If you ever delete rows or if you ever create a row with the maximum possible
    87     87   ROWID, then ROWIDs from previously deleted rows might be reused when creating
    88     88   new rows and newly created ROWIDs might not be in strictly ascending order.
    89     89   </p>
    90     90   
    91     91   
    92         -<h2>The AUTOINCREMENT Keyword</h2>
           92  +<h1>The AUTOINCREMENT Keyword</h1>
    93     93   
    94     94   <p>
    95     95   ^If a column has the type INTEGER PRIMARY KEY AUTOINCREMENT then a slightly
    96     96   different ROWID selection algorithm is used.  
    97     97   ^The ROWID chosen for the new row is at least one larger than the largest ROWID
    98     98   that has ever before existed in that same table.  ^If the table has never
    99     99   before contained any data, then a ROWID of 1 is used.  ^If the table

Changes to pages/partialindex.in.

     1      1   <title>Partial Indexes</title>
     2      2   <tcl>
     3      3   hd_keywords {partial index} {partial indexes} {partial indices}
     4      4   </tcl>
     5         -<h1 align="center">Partial Indexes</h1>
            5  +<fancy_format>
     6      6   
     7         -<h2>1.0 Introduction</h2>
            7  +<h1>Introduction</h1>
     8      8   
     9      9   <p>
    10     10   A partial index is an index over a subset of the rows of a table.
    11     11   </p>
    12     12   
    13     13   <p>
    14     14   ^In ordinary indexes, there is exactly one entry in the index for every
................................................................................
    15     15   row in the table.  ^In partial indexes, only some subset of the rows in the
    16     16   table have corresponding index entries.  ^For example, a partial index might
    17     17   omit entries for which the column being indexed is NULL.  When used 
    18     18   judiciously, partial indexes can result in smaller database files and
    19     19   improvements in both query and write performance.
    20     20   </p>
    21     21   
    22         -<h2>2.0 Creating Partial Indexes</h2>
           22  +<h1>Creating Partial Indexes</h1>
    23     23   
    24     24   <p>
    25     25   ^Create a partial index by adding a WHERE clause to the end of an 
    26     26   ordinary [CREATE INDEX] statement.
    27     27   </p>
    28     28   
    29     29   <tcl>RecursiveBubbleDiagram create-index-stmt</tcl>
................................................................................
    78     78   </blockquote>)^
    79     79   
    80     80   <p>^The query above will use the po_parent index to help find the answer,
    81     81   since the po_parent index contains entries for all rows of interest.
    82     82   Note that since po_parent is smaller than a full index, the query will
    83     83   likely run faster too.</p>
    84     84   
    85         -<h3>2.1 Unique Partial Indexes</h3>
           85  +<h2>Unique Partial Indexes</h2>
    86     86   
    87     87   <p>^A partial index definition may include the UNIQUE keyword.  ^If it
    88     88   does, then SQLite requires every entry <em>in the index</em> to be unique.
    89     89   This provides a mechanism for enforcing uniqueness across some subset of
    90     90   the rows in a table.</p>
    91     91   
    92     92   <p>For example, suppose you have a database of the members of a large
................................................................................
   117    117   ^(<p>Coincidentally, that same index is useful for locating the team leader
   118    118   of a particular team:</p>
   119    119   
   120    120   <blockquote>
   121    121   SELECT person_id FROM person WHERE is_team_leader AND team_id=?1;
   122    122   </blockquote>)^
   123    123   
   124         -<h2>3.0 Queries Using Partial Indexes</h2>
          124  +<h1>Queries Using Partial Indexes</h1>
   125    125   
   126    126   <p>Let X be the expression in the WHERE clause of a partial
   127    127   index, and let W be the WHERE clause of a query that uses the
   128    128   table that is indexed.  Then, the query is permitted to use 
   129    129   the partial index if W&#x21d2;X, where the &#x21d2; operator
   130    130   (usually pronounced "implies") is the logic operator 
   131    131   equivalent to "X or not W".
................................................................................
   189    189   
   190    190   <p>These two rules describe how the query planner for SQLite works as of
   191    191   this writing (2013-08-01).  And the rules above will always be honored.
   192    192   However, future versions of SQLite might incorporate a better theorem prover
   193    193   that can find other cases where W&#x21d2;X is true and thus may
   194    194   find more instances where partial indexes are useful.</p>
   195    195   
   196         -<h2>4.0 Supported Versions</h2>
          196  +<h1>Supported Versions</h1>
   197    197   
   198    198   <p>
   199    199   Partial indexes have been supported in SQLite since version 3.8.0.
   200    200   </p>
   201    201   
   202    202   <p>Database files that contain partial indices are not readable or writable
   203    203   by versions of SQLite prior to 3.8.0.  However, a database file created
   204    204   by SQLite 3.8.0 is still readable and writable by prior versions as long
   205    205   as its schema contains no partial indexes.  A database that is unreadable
   206    206   by legacy versions of SQLite can be made readable simply by running
   207    207   [DROP INDEX] on the partial indexes.</p>

Changes to search/buildsearchdb.tcl.

     1      1   
     2      2   
     3         -load ./parsehtml.so
            3  +#load ./parsehtml.so
            4  +#load ./tokenize.so
            5  +
     4      6   source [file join [file dirname [info script]] hdom.tcl]
     5      7   
     6      8   set ::G(rowid) 1
     7      9   
     8     10   # Return a list of relative paths to documents that should be included 
     9     11   # in the index.
    10     12   proc document_list {type} {
................................................................................
   353    355       set lSection [extract_sections_from_dom $dom generic_filterscript]
   354    356     }
   355    357   
   356    358     set i [expr $rowid*1000]
   357    359     foreach section $lSection {
   358    360       incr i
   359    361       foreach { tag hdr text } $section {}
          362  +    if {[string trim $text]==""} continue
   360    363       set url "${doc}#${tag}"
   361    364       insert_entry -rowid $i -url $url -title1 $title -title2 $hdr -content $text
   362    365     }
   363    366   }
   364    367   
   365    368   proc rebuild_database {} {
   366    369   
................................................................................
   375    378           apis,                               -- C APIs 
   376    379           keywords,                           -- Keywords
   377    380           title1,                             -- Document title
   378    381           title2,                             -- Heading title, if any
   379    382           content,                            -- Document text
   380    383   
   381    384           url UNINDEXED,                      -- Indexed URL
   382         -        tokenize='porter unicode61 tokenchars _' -- Tokenizer definition
          385  +        tokenize='stoken unicode61 tokenchars _' -- Tokenizer definition
   383    386         );
   384    387   
   385    388         DROP TABLE IF EXISTS weight;
   386    389         CREATE TABLE weight(id INTEGER PRIMARY KEY, percent FLOAT);
          390  +
          391  +      INSERT INTO page(page, rank) VALUES('rank', 'bm25(20.0,20.0,10.0,10.0)');
   387    392       }
   388    393   
   389    394       foreach doc [document_list lang] {
   390    395         puts "Indexing $doc..."
   391    396         lang_document_import $doc
   392    397       }
   393    398   

Added search/fts5ext.c.

            1  +
            2  +/*
            3  +** This file contains the implementation of a custom FTS5 tokenizer. This
            4  +** tokenizer implements the following special features:
            5  +**
            6  +**   * For all tokens that match the pattern "SQLITE_XXX" (case sensitive),
            7  +**     "XXX" is added as a synonym for SQLITE_XXX.
            8  +**
            9  +**   * For all tokens that match the pattern "sqlite3_xxx" (case sensitive),
           10  +**     "xxx" is added as a synonym for sqlite3_xxx.
           11  +*/
           12  +
           13  +#include <sqlite3.h>
           14  +#include <tcl.h>
           15  +#include <string.h>
           16  +
           17  +
           18  +/*************************************************************************
           19  +** This is generic code copied from the FTS5 documentation.
           20  +**
           21  +** Return a pointer to the fts5_api pointer for database connection db.
           22  +** If an error occurs, return NULL and leave an error in the database 
           23  +** handle (accessible using sqlite3_errcode()/errmsg()).
           24  +*/
           25  +fts5_api *fts5_api_from_db(sqlite3 *db){
           26  +  fts5_api *pRet = 0;
           27  +  sqlite3_stmt *pStmt = 0;
           28  +  if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
           29  +   && SQLITE_ROW==sqlite3_step(pStmt) 
           30  +   && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
           31  +  ){
           32  +    memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
           33  +  }
           34  +  sqlite3_finalize(pStmt);
           35  +  return pRet;
           36  +}
           37  +/************************************************************************/
           38  +
           39  +
           40  +typedef struct STokenizer STokenizer;
           41  +typedef struct STokenCtx STokenCtx;
           42  +
           43  +/*
           44  +** Tokenizer type. Casts to Fts5Tokenizer.
           45  +*/
           46  +struct STokenizer {
           47  +  fts5_tokenizer porter;
           48  +  Fts5Tokenizer *pPorter;
           49  +};
           50  +
           51  +/*
           52  +** Context passed through underlying tokenizer to wrapper callback.
           53  +*/
           54  +struct STokenCtx {
           55  +  void *pCtx;
           56  +  int (*xToken)(void*, int, const char*, int, int, int);
           57  +};
           58  +
           59  +static int stokenCreate(
           60  +  void *pCtx, 
           61  +  const char **azArg, int nArg, 
           62  +  Fts5Tokenizer **ppOut
           63  +){
           64  +  fts5_api *pApi = (fts5_api*)pCtx;
           65  +  STokenizer *p;
           66  +  void *pPorterCtx;
           67  +  int rc;
           68  +
           69  +  /* Allocate the Fts5Tokenizer object for this tokenizer. */
           70  +  p = sqlite3_malloc(sizeof(STokenizer));
           71  +  if( p ){
           72  +    memset(p, 0, sizeof(STokenizer));
           73  +  }else{
           74  +    return SQLITE_NOMEM;
           75  +  }
           76  +
           77  +  /* Locate and allocate the porter tokenizer */
           78  +  rc = pApi->xFindTokenizer(pApi, "porter", &pPorterCtx, &p->porter);
           79  +  if( rc==SQLITE_OK ){
           80  +    rc = p->porter.xCreate(pPorterCtx, azArg, nArg, &p->pPorter);
           81  +  }
           82  +
           83  +  /* Return the new tokenizer to the caller */
           84  +  if( rc!=SQLITE_OK ){
           85  +    sqlite3_free(p);
           86  +    p = 0;
           87  +  }
           88  +  *ppOut = (Fts5Tokenizer*)p;
           89  +  return rc;
           90  +}
           91  +
           92  +static void stokenDelete(Fts5Tokenizer *pTokenizer){
           93  +  STokenizer *p = (STokenizer*)pTokenizer;
           94  +  p->porter.xDelete(p->pPorter);
           95  +  sqlite3_free(p);
           96  +}
           97  +
           98  +static int stokenTokenizeCb(
           99  +  void *pCtx,         /* Copy of 2nd argument to xTokenize() */
          100  +  int tflags,         /* Mask of FTS5_TOKEN_* flags */
          101  +  const char *pToken, /* Pointer to buffer containing token */
          102  +  int nToken,         /* Size of token in bytes */
          103  +  int iStart,         /* Byte offset of token within input text */
          104  +  int iEnd            /* Byte offset of end of token within input text */
          105  +){
          106  +  STokenCtx *p = (STokenCtx*)pCtx;
          107  +  int rc = p->xToken(p->pCtx, 0, pToken, nToken, iStart, iEnd);
          108  +  if( rc==SQLITE_OK && nToken>7 && 0==memcmp("sqlite_", pToken, 7) ){
          109  +    rc = p->xToken(
          110  +        p->pCtx, FTS5_TOKEN_COLOCATED, pToken+7, nToken-7, iStart, iEnd);
          111  +  }
          112  +
          113  +  if( rc==SQLITE_OK && nToken>8 && 0==memcmp("sqlite3_", pToken, 8) ){
          114  +    rc = p->xToken(
          115  +        p->pCtx, FTS5_TOKEN_COLOCATED, pToken+8, nToken-8, iStart, iEnd);
          116  +  }
          117  +
          118  +  return rc;
          119  +}
          120  +
          121  +static int stokenTokenize(
          122  +  Fts5Tokenizer *pTokenizer, 
          123  +  void *pCtx,
          124  +  int flags,            /* Mask of FTS5_TOKENIZE_* flags */
          125  +  const char *pText, int nText, 
          126  +  int (*xToken)(
          127  +    void *pCtx,         /* Copy of 2nd argument to xTokenize() */
          128  +    int tflags,         /* Mask of FTS5_TOKEN_* flags */
          129  +    const char *pToken, /* Pointer to buffer containing token */
          130  +    int nToken,         /* Size of token in bytes */
          131  +    int iStart,         /* Byte offset of token within input text */
          132  +    int iEnd            /* Byte offset of end of token within input text */
          133  +  )
          134  +){
          135  +  STokenizer *p = (STokenizer*)pTokenizer;
          136  +  int rc;
          137  +
          138  +  if( flags==FTS5_TOKENIZE_DOCUMENT ){
          139  +    STokenCtx ctx;
          140  +    ctx.xToken = xToken;
          141  +    ctx.pCtx = pCtx;
          142  +    rc = p->porter.xTokenize(
          143  +        p->pPorter, (void*)&ctx, flags, pText, nText, stokenTokenizeCb
          144  +    );
          145  +  }else{
          146  +    rc = p->porter.xTokenize(p->pPorter, pCtx, flags, pText, nText, xToken);
          147  +  }
          148  +
          149  +  return rc;
          150  +}
          151  +
          152  +static int register_tokenizer(sqlite3 *db, char **pzErr, void *p){
          153  +  fts5_api *pApi;
          154  +  fts5_tokenizer t;
          155  +
          156  +  pApi = fts5_api_from_db(db);
          157  +  if( pApi==0 ){
          158  +    *pzErr = sqlite3_mprintf("fts5_api_from_db: %s", sqlite3_errmsg(db));
          159  +    return SQLITE_ERROR;
          160  +  }
          161  +
          162  +  t.xCreate = stokenCreate;
          163  +  t.xDelete = stokenDelete;
          164  +  t.xTokenize = stokenTokenize;
          165  +
          166  +  return pApi->xCreateTokenizer(pApi, "stoken", (void*)pApi, &t, 0);
          167  +}
          168  +
          169  +int Fts5ext_Init(Tcl_Interp *interp){
          170  +#ifdef USE_TCL_STUBS
          171  +  if (Tcl_InitStubs(interp, "8.4", 0) == 0) {
          172  +    return TCL_ERROR;
          173  +  }
          174  +#endif
          175  +  sqlite3_auto_extension((void (*)(void))register_tokenizer);
          176  +  return TCL_OK;
          177  +}
          178  +

Changes to search/hdom.tcl.

    38     38   #
    39     39   #    $node attr ?-default VALUE? ATTR
    40     40   #
    41     41   #    $node search PATTERN
    42     42   #
    43     43   
    44     44   
    45         -load ./parsehtml.so
           45  +catch { load ./parsehtml.so }
    46     46   
    47     47   #-------------------------------------------------------------------------
    48     48   # Throw an exception if the expression passed as the only argument does
    49     49   # not evaluate to true.
    50     50   #
    51     51   proc assert {condition} {
    52     52     uplevel [list if "! ($condition)" [list error "assert failed: $condition"]]

Changes to search/search.tcl.

     1         -#!/home/dan/bin/tclsqlite3
            1  +#!/usr/bin/tclsqlite3.search
     2      2   
     3      3   source [file dirname [info script]]/document_header.tcl
     4      4   
     5      5   # Decode an HTTP %-encoded string
     6      6   #
     7      7   proc percent_decode {str} {
     8      8       # rewrite "+" back to space
................................................................................
   129    129     #regsub -all {[^-/"A-Za-z0-9]} $::A(q) { } ::A(q)
   130    130   
   131    131     # Count the '"' characters in $::A(q). If there is an odd number of
   132    132     # occurences, add a " to the end of the query so that fts5 can parse
   133    133     # it without error.
   134    134     if {[regexp -all \x22 $::A(q)] % 2} { append ::A(q) \x22 }
   135    135   
   136         -  db one { INSERT INTO page(page, rank) VALUES('rank', 'bm25(20.0, 10.0)') }
   137         -
   138    136     # Set iStart to the index of the first result to display. Results are
   139    137     # indexed starting at zero from most to least relevant.
   140    138     #
   141    139     set iStart [expr {([info exists ::A(i)] ? $::A(i) : 0)*10}]
   142    140   
   143    141     # Grab a list of rowid results.
   144    142     #
................................................................................
   216    214         set link $data($childid,url)
   217    215         set hdr $data($childid,s_title2)
   218    216   
   219    217         if {$hdr==""} {
   220    218           set s_content ""
   221    219         } else {
   222    220           set s_content [subst {
   223         -          <b><a style=color:#044a64 href=$link>$hdr</a></b>:
          221  +          <b><a style=color:#044a64 href=$link>$hdr</a></b>
   224    222           }]
   225    223         }
   226    224   
   227    225         append s_content " $data($childid,s_content)"
   228    226       }
   229    227   
   230    228       append ret [subst -nocommands {<tr>
   231    229         <td valign=top style="line-height:150%">
   232    230           <div style="white-space:wrap;font-size:larger" class=nounderline>
   233         -          <a href="$url">$s_title1</a>
          231  +          <a href="$url">$s_title1</a> 
          232  +          <div style="float:right;font-size:smaller;color:#BBB">($url)</div>
   234    233           </div>
   235    234             <div style="margin-left: 10ex; font:larger monospace">$s_apis</div>
   236         -        <div style="ffont-size:small;margin-left: 2ex">
   237         -          <div class=nounderline> $s_content </div>
   238         -          <div style="margin-left:1em; margin-bottom:1em">
   239         -            <a href="$url">$url</a>
   240         -          </div>
          235  +        <div style="margin-left: 4ex; margin-bottom:1.5em">
          236  +           $s_content 
   241    237           </div>
   242    238         </td>
   243    239       }]
   244    240     }
   245    241     append ret { </table> }
   246    242   
   247    243   

Changes to search/searchc.c.

     1      1   
     2         -#include <sqlite3.h>
     3      2   
     4         -/*
     5         -*/
            3  +#include <sqlite3.h>
            4  +#include <tcl.h>
     6      5   
     7         -/*
     8         -** Return a pointer to the fts5_api pointer for database connection db.
     9         -** If an error occurs, return NULL and leave an error in the database 
    10         -** handle (accessible using sqlite3_errcode()/errmsg()).
    11         -*/
    12         -fts5_api *fts5_api_from_db(sqlite3 *db){
    13         -  fts5_api *pRet = 0;
    14         -  sqlite3_stmt *pStmt = 0;
    15         -
    16         -  if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
    17         -      && SQLITE_ROW==sqlite3_step(pStmt) 
    18         -      && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
    19         -    ){
    20         -    memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
    21         -  }
    22         -  sqlite3_finalize(pStmt);
    23         -  return pRet;
    24         -}
    25         -
    26         -int Sqlite3_Init(Tcl_Interp *interp);
    27         -
    28         -static int register_search_extensions(sqlite3 *db, char **pzErr, void *p){
    29         -  fts5_api *pApi = fts5_api_from_db(db);
    30         -
    31         -  return SQLITE_OK;
    32         -}
            6  +int Sqlite3_Init(Tcl_Interp*);
            7  +int Parsehtml_Init(Tcl_Interp*);
            8  +int Fts5ext_Init(Tcl_Interp*);
    33      9   
    34     10   static int AppInit(Tcl_Interp *interp) {
    35     11     int rc;
    36     12     rc = Sqlite3_Init(interp);
    37     13     if( rc!=TCL_OK ) return rc;
    38         -  sqlite3_auto_extension(register_search_extensions);
           14  +
           15  +  rc = Parsehtml_Init(interp);
           16  +  if( rc!=TCL_OK ) return rc;
           17  +
           18  +  rc = Fts5ext_Init(interp);
           19  +  if( rc!=TCL_OK ) return rc;
           20  +
    39     21     return TCL_OK;
    40     22   }
    41     23   
    42     24   int main(int argc, char *argv[]) {
    43     25     Tcl_Main(argc, argv, AppInit);
    44     26     return 0;
    45     27   }
    46     28