Documentation Source Text

Check-in [368f8fa51f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a website option to search the changelog instead of the documentation. Do not add any changelog files to the documentation index.
Timelines: family | ancestors | descendants | both | changelog-search
Files: files | file ages | folders
SHA3-256: 368f8fa51fdf51c2b99bcd57b230f7dd73ece512cd442591f31d410c6d218e6c
User & Date: dan 2017-08-01 19:24:32
Context
2017-08-01
19:29
Add a website option to search the changelog instead of the documentation. Do not add any changelog files to the documentation index. check-in: 8f15082015 user: dan tags: trunk
19:24
Add a website option to search the changelog instead of the documentation. Do not add any changelog files to the documentation index. Closed-Leaf check-in: 368f8fa51f user: dan tags: changelog-search
18:42
Fix a typo in the changelog for release 3.4.0. check-in: 0a8c13265d user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to document_header.tcl.

34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54




55
56
57
58
59
60
61
62
..
65
66
67
68
69
70
71













72
73
74
75
76
77
78
          <li class='wideonly'><a href='${path}about.html'>About</a>
          <li class='desktoponly'><a href="${path}docs.html">Documentation</a>
          <li class='desktoponly'><a href="${path}download.html">Download</a>
          <li class='wideonly'><a href='${path}copyright.html'>License</a>
          <li class='desktoponly'><a href="${path}support.html">Support</a>
          <li class='desktoponly'><a href="${path}prosupport.html">Purchase</a>
          <li class='search' id='search_menubutton'>
            <a href="javascript:void(0)" onclick='toggle_div("searchmenu")'>Search</a>
        </ul>
      </div>
      <div class="menu submenu" id="submenu">
        <ul>
          <li><a href='${path}about.html'>About</a>
          <li><a href='${path}docs.html'>Documentation</a>
          <li><a href='${path}download.html'>Download</a>
          <li><a href='${path}support.html'>Support</a>
          <li><a href='${path}prosupport.html'>Purchase</a>
        </ul>
      </div>
      <div class="searchmenu" id="searchmenu">
        <form method="GET" action="${path}search">




          <span class="desktoponly">Search for:</span> <input type="text" name="q">
          <input type="submit" value="Go">
        </form>
      </div>
    </div>
  }]

  append ret [subst -nocommands {
................................................................................
        var w = document.getElementById(nm);
        if( w.style.display=="block" ){
          w.style.display = "none";
        }else{
          w.style.display = "block";
        }
      }













      function div_off(nm){document.getElementById(nm).style.display="none";}
      window.onbeforeunload = function(e){div_off("submenu");}

      /* Disable the Search feature if we are not operating from CGI, since */
      /* Search is accomplished using CGI and will not work without it. */
      if( !location.origin.match || !location.origin.match(/http/) ){
        document.getElementById("search_menubutton").style.display = "none";







|













>
>
>
>
|







 







>
>
>
>
>
>
>
>
>
>
>
>
>







34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
..
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
          <li class='wideonly'><a href='${path}about.html'>About</a>
          <li class='desktoponly'><a href="${path}docs.html">Documentation</a>
          <li class='desktoponly'><a href="${path}download.html">Download</a>
          <li class='wideonly'><a href='${path}copyright.html'>License</a>
          <li class='desktoponly'><a href="${path}support.html">Support</a>
          <li class='desktoponly'><a href="${path}prosupport.html">Purchase</a>
          <li class='search' id='search_menubutton'>
            <a href="javascript:void(0)" onclick='toggle_search()'>Search</a>
        </ul>
      </div>
      <div class="menu submenu" id="submenu">
        <ul>
          <li><a href='${path}about.html'>About</a>
          <li><a href='${path}docs.html'>Documentation</a>
          <li><a href='${path}download.html'>Download</a>
          <li><a href='${path}support.html'>Support</a>
          <li><a href='${path}prosupport.html'>Purchase</a>
        </ul>
      </div>
      <div class="searchmenu" id="searchmenu">
        <form method="GET" action="${path}search">
          <select name="s" id="searchtype">
            <option value="d">Search Documentation</option>
            <option value="c">Search Changelog</option>
          </select>
          <input type="text" name="q" id="searchbox" value="$search">
          <input type="submit" value="Go">
        </form>
      </div>
    </div>
  }]

  append ret [subst -nocommands {
................................................................................
        var w = document.getElementById(nm);
        if( w.style.display=="block" ){
          w.style.display = "none";
        }else{
          w.style.display = "block";
        }
      }

      function toggle_search() {
        var w = document.getElementById("searchmenu");
        if( w.style.display=="block" ){
          w.style.display = "none";
        } else {
          w.style.display = "block";
          setTimeout(function(){
            document.getElementById("searchbox").focus()
          }, 30);
        }
      }

      function div_off(nm){document.getElementById(nm).style.display="none";}
      window.onbeforeunload = function(e){div_off("submenu");}

      /* Disable the Search feature if we are not operating from CGI, since */
      /* Search is accomplished using CGI and will not work without it. */
      if( !location.origin.match || !location.origin.match(/http/) ){
        document.getElementById("search_menubutton").style.display = "none";

Changes to search/buildsearchdb.tcl.

2
3
4
5
6
7
8


















9
10
11
12
13
14
15
..
79
80
81
82
83
84
85
86
87
88
89
90



91
92
93
94
95
96
97
...
218
219
220
221
222
223
224








225
226
227
228
229
230
231
...
333
334
335
336
337
338
339






























340
341
342
343
344
345
346
...
358
359
360
361
362
363
364













365
366
367
368
369
370
371
...
374
375
376
377
378
379
380
381

382

383
384
385
386
387
388
389

#load ./parsehtml.so
#load ./tokenize.so

source [file join [file dirname [info script]] hdom.tcl]

set ::G(rowid) 1



















# Return a list of relative paths to documents that should be included 
# in the index.
proc document_list {type} {
  global weight
  set lFiles [list]
  switch -- $type {
................................................................................
         || [regexp {^(3_9_).*} $tail -> prefix]
         || [regexp {^(3_[1-9][0-9]).*} $tail -> prefix]
        } {
          set f1 [lindex [lsort -decreasing [glob releaselog/$prefix*.html]] 0]
          if {$f!=$f1} { set ::weight($f) 10 }
        } 
      }

      foreach f [glob releaselog/*.html] { 
        if {[info exists nosearch($f)]==0} { 
          lappend lFiles $f 
        }



      }
    }

    default {
      error "document_list: unknown file type $type"
    }
  }
................................................................................

proc generic_filterscript {N} {
  for {set P [$N parent]} {$P!=""} {set P [$P parent]} {
    if {[$P attr -default "" class]=="nosearch"} { return 0 }
  }
  return 1
}









proc extract_text_from_dom {dom filterscript} {
  set text ""
  set body [lindex [[$dom root] search body] 0]
  $body foreach_descendent N {
    if {[$N tag]==""} {
      if {[eval $filterscript $N]} { append text [$N text] }
................................................................................
    foreach { tag hdr text } $section {}
    if {[string trim $text]==""} continue
    incr i
    set url "${doc}#${tag}"
    insert_entry -rowid $i -url $url -title1 $title -title2 $hdr -content $text
  }
}































proc rebuild_database {} {

  db transaction {
    # Create the database schema. If the schema already exists, then those
    # tables that contain document data are dropped and recreated by this
    # proc. The 'config' table is left untouched.
................................................................................
        tokenize='stoken unicode61 tokenchars _' -- Tokenizer definition
      );

      DROP TABLE IF EXISTS weight;
      CREATE TABLE weight(id INTEGER PRIMARY KEY, percent FLOAT);

      INSERT INTO page(page, rank) VALUES('rank', 'bm25(10.0,10.0,20.0,20.0)');













    }

    foreach doc [document_list lang] {
      puts "Indexing $doc..."
      lang_document_import $doc
    }

................................................................................
      c3ref_document_import $doc
    }

    foreach doc [document_list generic] { 
      puts "Indexing $doc..."
      generic_document_import $doc 
    }


    db eval { INSERT INTO page(page) VALUES('optimize') }


    foreach f [array names ::weight] {
      set w $::weight($f)
      db eval {SELECT rowid FROM page WHERE url=$f} {
        db eval { INSERT INTO weight VALUES($rowid, $w); }
      }
    }







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
<
<
<
|
>
>
>







 







>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>







 








>

>







2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
..
97
98
99
100
101
102
103
104



105
106
107
108
109
110
111
112
113
114
115
...
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
...
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
...
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
...
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460

#load ./parsehtml.so
#load ./tokenize.so

source [file join [file dirname [info script]] hdom.tcl]

set ::G(rowid) 1

proc releaselog_to_version {doc} {
  set version 0_0_0
  regexp {releaselog/(.*).html} $doc -> version
  split $version _
}

proc release_cmp {a b} {
  set v1 [releaselog_to_version $a]
  set v2 [releaselog_to_version $b]
  lappend v1 {*}[lrange {-1 -1 -1 -1 -1} [llength $v1] end]
  lappend v2 {*}[lrange {-1 -1 -1 -1 -1} [llength $v2] end]
  foreach x $v1 y $v2 {
    if {$x < $y} { return -1 }
    if {$x > $y} { return +1 }
  }
  return 0
}

# Return a list of relative paths to documents that should be included 
# in the index.
proc document_list {type} {
  global weight
  set lFiles [list]
  switch -- $type {
................................................................................
         || [regexp {^(3_9_).*} $tail -> prefix]
         || [regexp {^(3_[1-9][0-9]).*} $tail -> prefix]
        } {
          set f1 [lindex [lsort -decreasing [glob releaselog/$prefix*.html]] 0]
          if {$f!=$f1} { set ::weight($f) 10 }
        } 
      }
    }




    changelog {
      foreach f [lsort -decr -command release_cmp [glob releaselog/*.html]] { 
        if {$f != "releaselog/current.html"} { lappend lFiles $f }
      }
    }

    default {
      error "document_list: unknown file type $type"
    }
  }
................................................................................

proc generic_filterscript {N} {
  for {set P [$N parent]} {$P!=""} {set P [$P parent]} {
    if {[$P attr -default "" class]=="nosearch"} { return 0 }
  }
  return 1
}

proc releaselog_filterscript {N} {
  for {set P [$N parent]} {$P!=""} {set P [$P parent]} {
    if {[$P attr -default "" class]=="nosearch"} { return 0 }
    if {[$P tag]=="li"} { return 0 }
  }
  return 1
}

proc extract_text_from_dom {dom filterscript} {
  set text ""
  set body [lindex [[$dom root] search body] 0]
  $body foreach_descendent N {
    if {[$N tag]==""} {
      if {[eval $filterscript $N]} { append text [$N text] }
................................................................................
    foreach { tag hdr text } $section {}
    if {[string trim $text]==""} continue
    incr i
    set url "${doc}#${tag}"
    insert_entry -rowid $i -url $url -title1 $title -title2 $hdr -content $text
  }
}

proc changelog_document_import {doc} {

  set content [readfile $doc]
  set end [string first "Changes carried forward from version " $content]
  if {$end>0} { set content [string range $content 0 $end] }

  set dom [::hdom::parse $content]

  # Extract the version number from the document name.
  set version 0.0.0
  regexp {releaselog/(.*).html} $doc -> version
  set version [string map {_ .} $version]

  # Find each of the <li> nodes in the document.
  foreach li [[$dom root] search li] {
    if {0==[releaselog_filterscript $li]} continue

    set i 1
    set ol [$li parent]
    if {$ol=="" || [$ol tag]!="ol"} {error UNTHINKABLE!}
    foreach c [$ol children] {
      if {$c==$li} break
      if {[$c tag]=="li"} {incr i}
    }

    set t [$li text]
    db eval { INSERT INTO change VALUES($doc, $version, $i, $t) }
  }
}

proc rebuild_database {} {

  db transaction {
    # Create the database schema. If the schema already exists, then those
    # tables that contain document data are dropped and recreated by this
    # proc. The 'config' table is left untouched.
................................................................................
        tokenize='stoken unicode61 tokenchars _' -- Tokenizer definition
      );

      DROP TABLE IF EXISTS weight;
      CREATE TABLE weight(id INTEGER PRIMARY KEY, percent FLOAT);

      INSERT INTO page(page, rank) VALUES('rank', 'bm25(10.0,10.0,20.0,20.0)');

      DROP TABLE IF EXISTS change;
      CREATE VIRTUAL TABLE change USING fts5(
          url UNINDEXED,          -- Path to document
          version UNINDEXED,      -- SQLite version number
          idx UNINDEXED,          -- Bullet point number
          text                    -- Text of change log entry
      );
    }

    foreach doc [document_list changelog] { 
      puts "Indexing $doc..."
      changelog_document_import $doc 
    }

    foreach doc [document_list lang] {
      puts "Indexing $doc..."
      lang_document_import $doc
    }

................................................................................
      c3ref_document_import $doc
    }

    foreach doc [document_list generic] { 
      puts "Indexing $doc..."
      generic_document_import $doc 
    }


    db eval { INSERT INTO page(page) VALUES('optimize') }
    db eval { INSERT INTO change(change) VALUES('optimize') }

    foreach f [array names ::weight] {
      set w $::weight($f)
      db eval {SELECT rowid FROM page WHERE url=$f} {
        db eval { INSERT INTO weight VALUES($rowid, $w); }
      }
    }

Changes to search/search.tcl.

113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
...
219
220
221
222
223
224
225




































226
227
228
229
230
231
232
...
378
379
380
381
382
383
384





385
386
387
388
389
390
391
392
393
394

395

396
397
398
399
400
401
402
...
418
419
420
421
422
423
424
















425
426
427
428
429
430
431
    BEGIN;
      CREATE TABLE IF NOT EXISTS log(
        ip,                  -- IP query was made from
        query,               -- Fts5 query string
        nres,                -- Number of results
        timestamp DEFAULT CURRENT_TIMESTAMP
      );

      INSERT INTO log(ip, query, nres) VALUES($ip, $query, $nRes);
    COMMIT;
  }

  db2 close
}

................................................................................
#   45.02 ms
#
proc ttime {script} {
  set t [lindex [time [list uplevel $script]] 0]
  if {$t>1000000} { return [format "%.2f s" [expr {$t/1000000.0}]] }
  return [format "%.2f ms" [expr {$t/1000.0}]]
}





































proc searchresults {} {
  if {![info exists ::A(q)]} return ""
  #set ::A(q) [string map {' ''} $A(q)]
  #regsub -all {[^-/"A-Za-z0-9]} $::A(q) { } ::A(q)

  # Count the '"' characters in $::A(q). If there is an odd number of
................................................................................
  # If "admin=1" is specified, jump to the admin screen.
  if {[string match *admin* $::env(REQUEST_URI)]} {
    set ::PATH ../
    return [admin_list]
  }

  sqlite3 db search.d/search.db






  db transaction {
    set t [ttime { 
      if {[catch searchresults srchout]} {
        set A(q) [string tolower $A(q)]
        set srchout [searchresults]
      }
      set doc $srchout
    }]
  }

  append doc "<p>Page generated by <a href='fts5.html'>FTS5</a> in about $t."

  return $doc

  # return [cgi_env_dump]
}

#=========================================================================

................................................................................
  if {[info exists ::A(q)]} {
    set initsearch [attrize $::A(q)]
    append title " - [htmlize $::A(q)]"
  } else {
    set initsearch {}
  }
  set document [document_header $title $::PATH $initsearch]
















  append document $res
} else {
  set document "<pre>"
  append document "Error: $res\n\n"
  append document $::errorInfo
  append document "</pre>"
}







<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>



|

|




>

>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







113
114
115
116
117
118
119

120
121
122
123
124
125
126
...
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
...
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
...
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
    BEGIN;
      CREATE TABLE IF NOT EXISTS log(
        ip,                  -- IP query was made from
        query,               -- Fts5 query string
        nres,                -- Number of results
        timestamp DEFAULT CURRENT_TIMESTAMP
      );

      INSERT INTO log(ip, query, nres) VALUES($ip, $query, $nRes);
    COMMIT;
  }

  db2 close
}

................................................................................
#   45.02 ms
#
proc ttime {script} {
  set t [lindex [time [list uplevel $script]] 0]
  if {$t>1000000} { return [format "%.2f s" [expr {$t/1000000.0}]] }
  return [format "%.2f ms" [expr {$t/1000.0}]]
}

proc searchchanges {} {
  global A
  if {![info exists A(q)]} return ""

  set open {<span style="background-color:#d9f2e6">}
  set close {</span>}
  set query {
    SELECT url, version, idx, highlight(change, 3, $open, $close) AS text 
    FROM change($A(q)) ORDER BY rowid ASC
  }

  set ret [subst {
    <p>Change log entries mentioning: <b>[htmlize $::A(q)]</b>
    <table border=0>
  }]

  set s2 "style=\"margin-top:0\""
  set s1 "style=\"font-size:larger; text-align:left\" class=nounderline"
  set prev ""
  db eval $query {
    if {$prev!=$version} {
      append ret [subst {
        <tr> <td $s1 valign=top> <a href=$url>$version</a> <td> <ul $s2>
      }]
      set prev $version
    }
    append ret [subst { <li value=$idx> ($idx) $text }]
  }

  append ret "</table>"
  append ret "<center><p>You can also see the <a href=changes.html>entire"
  append ret " changelog as a single page</a> if you wish.</center>"

  return $ret
}

proc searchresults {} {
  if {![info exists ::A(q)]} return ""
  #set ::A(q) [string map {' ''} $A(q)]
  #regsub -all {[^-/"A-Za-z0-9]} $::A(q) { } ::A(q)

  # Count the '"' characters in $::A(q). If there is an odd number of
................................................................................
  # If "admin=1" is specified, jump to the admin screen.
  if {[string match *admin* $::env(REQUEST_URI)]} {
    set ::PATH ../
    return [admin_list]
  }

  sqlite3 db search.d/search.db

  set cmd searchresults
  if {[info exists A(s)] && $A(s)=="c"} {
    set cmd searchchanges
  }

  db transaction {
    set t [ttime { 
      if {[catch $cmd srchout]} {
        set A(q) [string tolower $A(q)]
        set srchout [$cmd]
      }
      set doc $srchout
    }]
  }
  append doc "<center>"
  append doc "<p>Page generated by <a href='fts5.html'>FTS5</a> in about $t."
  append doc "</center>"
  return $doc

  # return [cgi_env_dump]
}

#=========================================================================

................................................................................
  if {[info exists ::A(q)]} {
    set initsearch [attrize $::A(q)]
    append title " - [htmlize $::A(q)]"
  } else {
    set initsearch {}
  }
  set document [document_header $title $::PATH $initsearch]
  append document [subst {
    <script>
      window.addEventListener('load', function() {
        var w = document.getElementById("searchmenu");
        w.style.display = "block";

        document.getElementById("searchtype").value = "$::A(s)"

        setTimeout(function(){
          var s = document.getElementById("searchbox");
          s.focus();
          s.select();
        }, 30);
      });
    </script>
  }]
  append document $res
} else {
  set document "<pre>"
  append document "Error: $res\n\n"
  append document $::errorInfo
  append document "</pre>"
}