/ Check-in [ec257348]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add fts5_source_id() SQL function, similar to sqlite_source_id().
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: ec2573487cb86664f7f056168a764c28ca8236fc
User & Date: dan 2015-07-02 18:16:02
Context
2015-07-02
18:52
Minor change to mkfts5c.tcl so that fts5.c can be used according to the instructions in loadext.html. check-in: edbcccd3 user: dan tags: trunk
18:16
Add fts5_source_id() SQL function, similar to sqlite_source_id(). check-in: ec257348 user: dan tags: trunk
16:52
Fix harmless compiler warnings and one macro VVA_ONLY macro usage issue. check-in: bcc8a755 user: mistachkin tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_main.c.

2218
2219
2220
2221
2222
2223
2224












2225
2226
2227
2228
2229
2230
2231
....
2279
2280
2281
2282
2283
2284
2285





2286
2287
2288
2289
2290
2291
2292
  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
  char buf[8];
  assert( nArg==0 );
  assert( sizeof(buf)>=sizeof(pGlobal) );
  memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
  sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
}













#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_fts5_init(
  sqlite3 *db,
  char **pzErrMsg,
................................................................................
    if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_function(
          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
      );





    }
  }
  return rc;
}

#ifdef _WIN32
__declspec(dllexport)







>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>







2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
....
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
  Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
  char buf[8];
  assert( nArg==0 );
  assert( sizeof(buf)>=sizeof(pGlobal) );
  memcpy(buf, (void*)&pGlobal, sizeof(pGlobal));
  sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT);
}

/*
** Implementation of fts5_source_id() function.
*/
static void fts5SourceIdFunc(
  sqlite3_context *pCtx,          /* Function call context */
  int nArg,                       /* Number of args */
  sqlite3_value **apVal           /* Function arguments */
){
  assert( nArg==0 );
  sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
}

#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_fts5_init(
  sqlite3 *db,
  char **pzErrMsg,
................................................................................
    if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
    if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_function(
          db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
      );
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_function(
          db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0
      );
    }
  }
  return rc;
}

#ifdef _WIN32
__declspec(dllexport)

Changes to ext/fts5/test/fts5unicode2.test.

376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
proc do_isspace_test {tn tokenizer lCp} {
  set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] 
  set txt "${whitespace}hello${whitespace}world${whitespace}"
  uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}]
}

set tokenizers [list unicode61]
ifcapable icu { lappend tokenizers icu }

# Some tests to check that the tokenizers can both identify white-space 
# codepoints. All codepoints tested below are of type "Zs" in the
# UnicodeData.txt file.
foreach T $tokenizers {
  do_isspace_test 6.$T.1 $T    32
  do_isspace_test 6.$T.2 $T    160







|







376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
proc do_isspace_test {tn tokenizer lCp} {
  set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp] 
  set txt "${whitespace}hello${whitespace}world${whitespace}"
  uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}]
}

set tokenizers [list unicode61]
#ifcapable icu { lappend tokenizers icu }

# Some tests to check that the tokenizers can both identify white-space 
# codepoints. All codepoints tested below are of type "Zs" in the
# UnicodeData.txt file.
foreach T $tokenizers {
  do_isspace_test 6.$T.1 $T    32
  do_isspace_test 6.$T.2 $T    160

Changes to ext/fts5/tool/mkfts5c.tcl.

36
37
38
39
40
41
42



43
44
45
46
47
48
















49
50
51
52
53
54
55
56
57
58
59
60
61


62
63
64
65
66

67
68
69
70
71
72
73
}

set G(footer) {
    
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */
}




proc readfile {zFile} {
  set fd [open $zFile]
  set data [read $fd]
  close $fd
  return $data
}

















proc fts5c_init {zOut} {
  global G
  set G(fd) stdout
  set G(fd) [open $zOut w]

  puts -nonewline $G(fd) $G(hdr)
}

proc fts5c_printfile {zIn} {
  global G
  set data [readfile $zIn]
  puts $G(fd) "#line 1 \"[file tail $zIn]\""


  foreach line [split $data "\n"] {
    if {[regexp {^#include.*fts5} $line]} continue
    if {[regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line]} {
      set line "static $line"
    }

    puts $G(fd) $line
  }
}

proc fts5c_close {} {
  global G
  puts -nonewline $G(fd) $G(footer)







>
>
>






>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>













>
>





>







36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
}

set G(footer) {
    
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */
}

#-------------------------------------------------------------------------
# Read and return the entire contents of text file $zFile from disk.
#
proc readfile {zFile} {
  set fd [open $zFile]
  set data [read $fd]
  close $fd
  return $data
}

#-------------------------------------------------------------------------
# This command returns a string identifying the current sqlite version -
# the equivalent of the SQLITE_SOURCE_ID string.
#
proc fts5_source_id {zDir} {
  set top [file dirname [file dirname $zDir]]
  set uuid [string trim [readfile [file join $top manifest.uuid]]]

  set L [split [readfile [file join $top manifest]]] 
  set date [lindex $L [expr [lsearch -exact $L D]+1]]
  set date [string range $date 0 [string last . $date]-1]
  set date [string map {T { }} $date]

  return "fts5: $date $uuid"
}

proc fts5c_init {zOut} {
  global G
  set G(fd) stdout
  set G(fd) [open $zOut w]

  puts -nonewline $G(fd) $G(hdr)
}

proc fts5c_printfile {zIn} {
  global G
  set data [readfile $zIn]
  puts $G(fd) "#line 1 \"[file tail $zIn]\""

  set srcid_map [list --FTS5-SOURCE-ID-- [fts5_source_id $::srcdir]]
  foreach line [split $data "\n"] {
    if {[regexp {^#include.*fts5} $line]} continue
    if {[regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line]} {
      set line "static $line"
    }
    set line [string map $srcid_map $line]
    puts $G(fd) $line
  }
}

proc fts5c_close {} {
  global G
  puts -nonewline $G(fd) $G(footer)

Changes to main.mk.

67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
...
303
304
305
306
307
308
309
310

311
312
313
314
315
316
317
...
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
         pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \
         random.o resolve.o rowset.o rtree.o select.o sqlite3ota.o status.o \
         table.o threads.o tokenize.o treeview.o trigger.o \
         update.o userauth.o util.o vacuum.o \
         vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
	 vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \
         utf.o vtab.o

LIBOBJ += fts5.o



# All of the source code files.
#
SRC = \
  $(TOP)/src/alter.c \
  $(TOP)/src/analyze.c \
................................................................................
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/wholenumber.c \
  $(TOP)/ext/misc/vfslog.c \
  $(TOP)/ext/fts5/fts5_tcl.c 



#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c

TESTSRC2 = \
  $(TOP)/src/attach.c \
................................................................................
	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c

fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl

fts5.o:	fts5.c $(HDR) $(EXTHDR)
	$(TCCX) -DSQLITE_CORE -c fts5.c



userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c

sqlite3ota.o:	$(TOP)/ext/ota/sqlite3ota.c $(HDR) $(EXTHDR)
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/ota/sqlite3ota.c







<
<
<







 







|
>







 







<
<
<
<







67
68
69
70
71
72
73



74
75
76
77
78
79
80
...
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
...
659
660
661
662
663
664
665




666
667
668
669
670
671
672
         pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \
         random.o resolve.o rowset.o rtree.o select.o sqlite3ota.o status.o \
         table.o threads.o tokenize.o treeview.o trigger.o \
         update.o userauth.o util.o vacuum.o \
         vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \
	 vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \
         utf.o vtab.o





# All of the source code files.
#
SRC = \
  $(TOP)/src/alter.c \
  $(TOP)/src/analyze.c \
................................................................................
  $(TOP)/ext/misc/nextchar.c \
  $(TOP)/ext/misc/percentile.c \
  $(TOP)/ext/misc/regexp.c \
  $(TOP)/ext/misc/spellfix.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/wholenumber.c \
  $(TOP)/ext/misc/vfslog.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
  fts5.c


#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c

TESTSRC2 = \
  $(TOP)/src/attach.c \
................................................................................
	cat fts5parse.c.orig | sed 's/yy/fts5yy/g' | sed 's/YY/fts5YY/g' \
		| sed 's/TOKEN/FTS5TOKEN/g' >> fts5parse.c

fts5parse.h: fts5parse.c

fts5.c: $(FTS5_SRC)
	tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl






userauth.o:	$(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c

sqlite3ota.o:	$(TOP)/ext/ota/sqlite3ota.c $(HDR) $(EXTHDR)
	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/ota/sqlite3ota.c