SQLite

Check-in [80225abe79]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Extra tests for fts4 compress/uncompress hooks. Fix some minor problems with the same.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 80225abe79b0a7723b922ec129954438af95855a
User & Date: dan 2011-02-03 12:48:17.458
Context
2011-02-04
00:51
Fix the ATTACH command so that the filename argument can be any expression and so that if authorizer callback gets a NULL pointer for the filename if the filename argument is anything other than a string literal. Ticket [9013e13dba5b58c7] (check-in: e64e1453a9 user: drh tags: trunk)
2011-02-03
12:48
Extra tests for fts4 compress/uncompress hooks. Fix some minor problems with the same. (check-in: 80225abe79 user: dan tags: trunk)
10:56
Extra tests for the fts4aux module. (check-in: cfc475690d user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
687
688
689
690
691
692
693





























694
695
696
697
698
699
700
      z = z2;
    }
    if( z==0 ) *pRc = SQLITE_NOMEM;
    sqlite3_free(*pz);
    *pz = z;
  }
}






























/*
** Return a list of comma separated SQL expressions that could be used
** in a SELECT statement such as the following:
**
**     SELECT <list of expressions> FROM %_content AS x ...
**







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
      z = z2;
    }
    if( z==0 ) *pRc = SQLITE_NOMEM;
    sqlite3_free(*pz);
    *pz = z;
  }
}

/*
** Return a copy of input string zInput enclosed in double-quotes (") and
** with all double quote characters escaped. For example:
**
**     fts3QuoteId("un \"zip\"")   ->    "un \"\"zip\"\""
**
** The pointer returned points to memory obtained from sqlite3_malloc(). It
** is the callers responsibility to call sqlite3_free() to release this
** memory.
*/
static char *fts3QuoteId(char const *zInput){
  int nRet;
  char *zRet;
  nRet = 2 + strlen(zInput)*2 + 1;
  zRet = sqlite3_malloc(nRet);
  if( zRet ){
    int i;
    char *z = zRet;
    *(z++) = '"';
    for(i=0; zInput[i]; i++){
      if( zInput[i]=='"' ) *(z++) = '"';
      *(z++) = zInput[i];
    }
    *(z++) = '"';
    *(z++) = '\0';
  }
  return zRet;
}

/*
** Return a list of comma separated SQL expressions that could be used
** in a SELECT statement such as the following:
**
**     SELECT <list of expressions> FROM %_content AS x ...
**
713
714
715
716
717
718
719


720

721




722
723
724
725

726
727
728
729
730
731
732
** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
** a NULL pointer is returned). Otherwise, if an OOM error is encountered
** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
** no error occurs, *pRc is left unmodified.
*/
static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
  char *zRet = 0;


  int i;

  if( !zFunc ) zFunc = "";




  fts3Appendf(pRc, &zRet, "docid");
  for(i=0; i<p->nColumn; i++){
    fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunc, i, p->azColumn[i]);
  }

  return zRet;
}

/*
** Return a list of N comma separated question marks, where N is the number
** of columns in the %_content table (one for the docid plus one for each
** user-defined text column).







>
>

>
|
>
>
>
>


|

>







742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
** a NULL pointer is returned). Otherwise, if an OOM error is encountered
** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
** no error occurs, *pRc is left unmodified.
*/
static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
  char *zRet = 0;
  char *zFree = 0;
  char *zFunction;
  int i;

  if( !zFunc ){
    zFunction = "";
  }else{
    zFree = zFunction = fts3QuoteId(zFunc);
  }
  fts3Appendf(pRc, &zRet, "docid");
  for(i=0; i<p->nColumn; i++){
    fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
  }
  sqlite3_free(zFree);
  return zRet;
}

/*
** Return a list of N comma separated question marks, where N is the number
** of columns in the %_content table (one for the docid plus one for each
** user-defined text column).
744
745
746
747
748
749
750


751

752




753
754
755
756

757
758
759
760
761
762
763
** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
** a NULL pointer is returned). Otherwise, if an OOM error is encountered
** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
** no error occurs, *pRc is left unmodified.
*/
static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){
  char *zRet = 0;


  int i;

  if( !zFunc ) zFunc = "";




  fts3Appendf(pRc, &zRet, "?");
  for(i=0; i<p->nColumn; i++){
    fts3Appendf(pRc, &zRet, ",%s(?)", zFunc);
  }

  return zRet;
}

/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**







>
>

>
|
>
>
>
>


|

>







781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and
** a NULL pointer is returned). Otherwise, if an OOM error is encountered
** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If
** no error occurs, *pRc is left unmodified.
*/
static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){
  char *zRet = 0;
  char *zFree = 0;
  char *zFunction;
  int i;

  if( !zFunc ){
    zFunction = "";
  }else{
    zFree = zFunction = fts3QuoteId(zFunc);
  }
  fts3Appendf(pRc, &zRet, "?");
  for(i=0; i<p->nColumn; i++){
    fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
  }
  sqlite3_free(zFree);
  return zRet;
}

/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
    sqlite3Fts3Dequote(zCsr);
    p->azColumn[iCol] = zCsr;
    zCsr += n+1;
    assert( zCsr <= &((char *)p)[nByte] );
  }

  if( (zCompress==0)!=(zUncompress==0) ){
    char const *zMissing = (zCompress==0 ? "compress" : "uncompress");
    rc = SQLITE_ERROR;
    *pzErr = sqlite3_mprintf("missing %s parameter", zMissing);
  }
  p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
  p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
  if( rc!=SQLITE_OK ) goto fts3_init_out;

  /* If this is an xCreate call, create the underlying tables in the 
  ** database. TODO: For xConnect(), it could verify that said tables exist.







|

|







968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
    sqlite3Fts3Dequote(zCsr);
    p->azColumn[iCol] = zCsr;
    zCsr += n+1;
    assert( zCsr <= &((char *)p)[nByte] );
  }

  if( (zCompress==0)!=(zUncompress==0) ){
    char const *zMiss = (zCompress==0 ? "compress" : "uncompress");
    rc = SQLITE_ERROR;
    *pzErr = sqlite3_mprintf("missing %s parameter in fts4 constructor", zMiss);
  }
  p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc);
  p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc);
  if( rc!=SQLITE_OK ) goto fts3_init_out;

  /* If this is an xCreate call, create the underlying tables in the 
  ** database. TODO: For xConnect(), it could verify that said tables exist.
Changes to test/fts3comp1.test.
13
14
15
16
17
18
19







20
21
22
23
24
25
26
27
28
29









30
31
32




33
34
35
36
37
38
39
40
41

42
43
44
45
46
47
48




49
50
51
52
53
54
55
56
57
58










59
60
61
62
63
64
65
66
67
68
69
70
71
72

73



74
75

76



77
78
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
set ::testprefix fts3comp1








set next_x 0
proc zip {x} {
 incr ::next_x
 set ::strings($::next_x) $x
 return $::next_x
}
proc unzip {x} {
  return $::strings($x)
}










db func zip zip
db func unzip unzip





do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts4(
    a, b, 
    compress=zip, uncompress=unzip
  );
  INSERT INTO t1 VALUES('one two three', 'two four six');
}

do_execsql_test 1.1 {

  SELECT a, b FROM t1;
} {{one two three} {two four six}}

do_execsql_test 1.2 {
  SELECT c0a, c1b FROM t1_content;
} {1 2}





do_execsql_test 1.3 {
  INSERT INTO t1 VALUES('three six nine', 'four eight twelve');
  SELECT a, b FROM t1;
} {{one two three} {two four six} {three six nine} {four eight twelve}}

do_execsql_test 1.4 {
  SELECT c0a, c1b FROM t1_content;
} {1 2 3 4}

do_execsql_test 1.5 {










  CREATE VIRTUAL TABLE terms USING fts4aux(t1);
  SELECT * FROM terms;
} {
  eight 1 1    four 2 2    nine 1 1    one 1 1 
  six 2 2      three 2 2   twelve 1 1  two 1 2
}

do_execsql_test 1.6 {
  DELETE FROM t1 WHERE docid = 1;
  SELECT * FROM terms;
} {
  eight 1 1   four 1 1    nine 1 1 
  six 1 1     three 1 1   twelve 1 1
}





do_execsql_test 1.7 {
  SELECT c0a, c1b FROM t1_content;

} {3 4}




finish_test







>
>
>
>
>
>
>










>
>
>
>
>
>
>
>
>
|
|
|
>
>
>
>
|
|
|
|
|
<
<
|
|
>
|
|
<
|
|
|
|
>
>
>
>
|
|
|
|
<
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
<
|
|
|
|
|
|
|
>
|
>
>
>
|
<
>
|
>
>
>


13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57


58
59
60
61
62

63
64
65
66
67
68
69
70
71
72
73
74

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

96
97
98
99
100
101
102
103
104
105
106
107
108

109
110
111
112
113
114
115
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
set ::testprefix fts3comp1

# Create a pretend compression system. 
#
# Each time the [zip] function is called, an entry is added to the ::strings
# array mapping from an integer key to the string argument to zip. The key
# is returned. Later on, when the key is passed to [unzip], the original
# string is retrieved from the ::strings array and returned.
#
set next_x 0
proc zip {x} {
 incr ::next_x
 set ::strings($::next_x) $x
 return $::next_x
}
proc unzip {x} {
  return $::strings($x)
}

foreach {tn zip unzip} {
  1   zip   unzip
  2   {z.i.p!!}    {un "zip"}
} {

  set next_x 0
  catch {db close}
  forcedelete test.db
  sqlite3 db test.db
  db func $zip zip
  db func $unzip unzip
  
  # Create a table that uses zip/unzip. Check that content inserted into
  # the table can be read back (using a full-scan query). Check that the
  # underlying %_content table contains the compressed (integer) values.
  #
  do_execsql_test 1.$tn.0 "
    CREATE VIRTUAL TABLE t1 USING fts4(
      a, b, 
      compress='$zip', uncompress='$unzip'
    );


  "
  do_execsql_test 1.$tn.1 {
    INSERT INTO t1 VALUES('one two three', 'two four six');
    SELECT a, b FROM t1;
  } {{one two three} {two four six}}

  do_execsql_test 1.$tn.2 {
    SELECT c0a, c1b FROM t1_content;
  } {1 2}
  
  # Insert another row and check that it can be read back. Also that the
  # %_content table still contains all compressed content. This time, try
  # full-text index and by-docid queries too.
  #
  do_execsql_test 1.$tn.3 {
    INSERT INTO t1 VALUES('three six nine', 'four eight twelve');
    SELECT a, b FROM t1;
  } {{one two three} {two four six} {three six nine} {four eight twelve}}

  do_execsql_test 1.$tn.4 {
    SELECT c0a, c1b FROM t1_content;
  } {1 2 3 4}
  
  do_execsql_test 1.$tn.5 {
    SELECT a, b FROM t1 WHERE docid = 2
  } {{three six nine} {four eight twelve}}
  do_execsql_test 1.$tn.6 {
    SELECT a, b FROM t1 WHERE t1 MATCH 'two'
  } {{one two three} {two four six}}
  
  # Delete a row and check that the full-text index is correctly updated.
  # Inspect the full-text index using an fts4aux table.
  #
  do_execsql_test 1.$tn.7 {
    CREATE VIRTUAL TABLE terms USING fts4aux(t1);
    SELECT * FROM terms;
  } {
    eight 1 1    four 2 2    nine 1 1    one 1 1 
    six 2 2      three 2 2   twelve 1 1  two 1 2
  }

  do_execsql_test 1.$tn.8 {
    DELETE FROM t1 WHERE docid = 1;
    SELECT * FROM terms;
  } {
    eight 1 1   four 1 1    nine 1 1 
    six 1 1     three 1 1   twelve 1 1
  }
  do_execsql_test 1.$tn.9 { SELECT c0a, c1b FROM t1_content } {3 4}
}

# Test that is an error to specify just one of compress and uncompress.
#
do_catchsql_test 2.1 {

  CREATE VIRTUAL TABLE t2 USING fts4(x, compress=zip)
} {1 {missing uncompress parameter in fts4 constructor}}
do_catchsql_test 2.2 {
  CREATE VIRTUAL TABLE t2 USING fts4(x, uncompress=unzip)
} {1 {missing compress parameter in fts4 constructor}}

finish_test