/ Check-in [33e6606f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Have fts5 load its configuration from the xConnect() method is invoked. This ensures that the very first query run uses the correct value of the 'rank' option.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:33e6606f5e497e81119ec491cf2370f60bddafc0
User & Date: dan 2015-11-06 12:50:57
Context
2015-11-06
17:01
Test cases for the ieee754 extension. check-in: 840cbda8 user: drh tags: trunk
12:50
Have fts5 load its configuration from the xConnect() method is invoked. This ensures that the very first query run uses the correct value of the 'rank' option. check-in: 33e6606f user: dan tags: trunk
2015-11-05
21:49
Fix typo in help information for an FTS5 script. check-in: 777ae800 user: mistachkin tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_config.c.

703
704
705
706
707
708
709



710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736

737
738
739
740
741
742
743
  char *zRank = 0;
  char *zRankArgs = 0;
  int rc = SQLITE_OK;

  *pzRank = 0;
  *pzRankArgs = 0;




  p = fts5ConfigSkipWhitespace(p);
  pRank = p;
  p = fts5ConfigSkipBareword(p);

  if( p ){
    zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
    if( zRank ) memcpy(zRank, pRank, p-pRank);
  }else{
    rc = SQLITE_ERROR;
  }

  if( rc==SQLITE_OK ){
    p = fts5ConfigSkipWhitespace(p);
    if( *p!='(' ) rc = SQLITE_ERROR;
    p++;
  }
  if( rc==SQLITE_OK ){
    const char *pArgs; 
    p = fts5ConfigSkipWhitespace(p);
    pArgs = p;
    if( *p!=')' ){
      p = fts5ConfigSkipArgs(p);
      if( p==0 ){
        rc = SQLITE_ERROR;
      }else{
        zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
        if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);

      }
    }
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(zRank);
    assert( zRankArgs==0 );







>
>
>
|
|
|

|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>







703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
  char *zRank = 0;
  char *zRankArgs = 0;
  int rc = SQLITE_OK;

  *pzRank = 0;
  *pzRankArgs = 0;

  if( p==0 ){
    rc = SQLITE_ERROR;
  }else{
    p = fts5ConfigSkipWhitespace(p);
    pRank = p;
    p = fts5ConfigSkipBareword(p);

    if( p ){
      zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
      if( zRank ) memcpy(zRank, pRank, p-pRank);
    }else{
      rc = SQLITE_ERROR;
    }

    if( rc==SQLITE_OK ){
      p = fts5ConfigSkipWhitespace(p);
      if( *p!='(' ) rc = SQLITE_ERROR;
      p++;
    }
    if( rc==SQLITE_OK ){
      const char *pArgs; 
      p = fts5ConfigSkipWhitespace(p);
      pArgs = p;
      if( *p!=')' ){
        p = fts5ConfigSkipArgs(p);
        if( p==0 ){
          rc = SQLITE_ERROR;
        }else{
          zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
          if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
        }
      }
    }
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(zRank);
    assert( zRankArgs==0 );

Changes to ext/fts5/fts5_index.c.

4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
*/
int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  assert( p->rc==SQLITE_OK );

  /* Allocate the hash table if it has not already been allocated */
  if( p->pHash==0 ){
    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);

    /* Force the configuration to be loaded */
    fts5StructureRelease(fts5StructureRead(p));
  }

  /* Flush the hash table to disk if required */
  if( iRowid<p->iWriteRowid 
   || (iRowid==p->iWriteRowid && p->bDelete==0)
   || (p->nPendingData > p->pConfig->nHashSize) 
  ){







<
<
<







4443
4444
4445
4446
4447
4448
4449



4450
4451
4452
4453
4454
4455
4456
*/
int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  assert( p->rc==SQLITE_OK );

  /* Allocate the hash table if it has not already been allocated */
  if( p->pHash==0 ){
    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);



  }

  /* Flush the hash table to disk if required */
  if( iRowid<p->iWriteRowid 
   || (iRowid==p->iWriteRowid && p->bDelete==0)
   || (p->nPendingData > p->pConfig->nHashSize) 
  ){

Changes to ext/fts5/fts5_main.c.

390
391
392
393
394
395
396









397
398
399
400
401
402
403
...
825
826
827
828
829
830
831






























832
833
834
835
836
837
838
...
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
    );
  }

  /* Call sqlite3_declare_vtab() */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
  }










  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab);
    pTab = 0;
  }else if( bCreate ){
    fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
  }
................................................................................
        break;
    }
  }
  
  return rc;
}































static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
  Fts5Config *pConfig = pTab->pConfig;
  Fts5Sorter *pSorter;
  int nPhrase;
  int nByte;
  int rc = SQLITE_OK;
  char *zSql;
................................................................................
  /* TODO: It would be better to have some system for reusing statement
  ** handles here, rather than preparing a new one for each query. But that
  ** is not possible as SQLite reference counts the virtual table objects.
  ** And since the statement required here reads from this very virtual 
  ** table, saving it creates a circular reference.
  **
  ** If SQLite a built-in statement cache, this wouldn't be a problem. */
  zSql = sqlite3Fts5Mprintf(&rc, 
      "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
      pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
      (zRankArgs ? ", " : ""),
      (zRankArgs ? zRankArgs : ""),
      bDesc ? "DESC" : "ASC"
  );
  if( zSql ){
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pSorter->pStmt, 0);
    sqlite3_free(zSql);
  }

  pCsr->pSorter = pSorter;
  if( rc==SQLITE_OK ){
    assert( pTab->pSortCsr==0 );
    pTab->pSortCsr = pCsr;
    rc = fts5SorterNext(pCsr);
    pTab->pSortCsr = 0;







>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|






<
<
<
<







390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
...
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
...
888
889
890
891
892
893
894
895
896
897
898
899
900
901




902
903
904
905
906
907
908
    );
  }

  /* Call sqlite3_declare_vtab() */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
  }

  /* Load the initial configuration */
  if( rc==SQLITE_OK ){
    assert( pConfig->pzErrmsg==0 );
    pConfig->pzErrmsg = pzErr;
    rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex);
    sqlite3Fts5IndexRollback(pTab->pIndex);
    pConfig->pzErrmsg = 0;
  }

  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab);
    pTab = 0;
  }else if( bCreate ){
    fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
  }
................................................................................
        break;
    }
  }
  
  return rc;
}


static sqlite3_stmt *fts5PrepareStatement(
  int *pRc,
  Fts5Config *pConfig, 
  const char *zFmt,
  ...
){
  sqlite3_stmt *pRet = 0;
  va_list ap;
  va_start(ap, zFmt);

  if( *pRc==SQLITE_OK ){
    int rc;
    char *zSql = sqlite3_vmprintf(zFmt, ap);
    if( zSql==0 ){
      rc = SQLITE_NOMEM; 
    }else{
      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0);
      if( rc!=SQLITE_OK ){
        *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db));
      }
      sqlite3_free(zSql);
    }
    *pRc = rc;
  }

  va_end(ap);
  return pRet;
} 

static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
  Fts5Config *pConfig = pTab->pConfig;
  Fts5Sorter *pSorter;
  int nPhrase;
  int nByte;
  int rc = SQLITE_OK;
  char *zSql;
................................................................................
  /* TODO: It would be better to have some system for reusing statement
  ** handles here, rather than preparing a new one for each query. But that
  ** is not possible as SQLite reference counts the virtual table objects.
  ** And since the statement required here reads from this very virtual 
  ** table, saving it creates a circular reference.
  **
  ** If SQLite a built-in statement cache, this wouldn't be a problem. */
  pSorter->pStmt = fts5PrepareStatement(&rc, pConfig,
      "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s",
      pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
      (zRankArgs ? ", " : ""),
      (zRankArgs ? zRankArgs : ""),
      bDesc ? "DESC" : "ASC"
  );





  pCsr->pSorter = pSorter;
  if( rc==SQLITE_OK ){
    assert( pTab->pSortCsr==0 );
    pTab->pSortCsr = pCsr;
    rc = fts5SorterNext(pCsr);
    pTab->pSortCsr = 0;

Changes to ext/fts5/test/fts5fault1.test.

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
..
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
do_faultsim_test 2 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { 
    INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
  }
} -test {
  faultsim_test_result {0 {}} 
}

reset_db
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
  INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
}
faultsim_save_and_close
do_faultsim_test 3 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { DELETE FROM t1 }
} -test {
  faultsim_test_result {0 {}} 
}

reset_db
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
  INSERT INTO t2 VALUES('m f a jj th q jr ar',   'hj n h h sg j i m');
  INSERT INTO t2 VALUES('nr s t g od j kf h',    'sb h aq rg op rb n nl');
................................................................................
  9 { m* + f* }       {1 8}
} {
  do_faultsim_test 4.$tn -prep {
    faultsim_restore_and_reopen
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]}
  "
}


#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#







|













|







 







|







48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
..
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
do_faultsim_test 2 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { 
    INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
  }
} -test {
  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
}

reset_db
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
  INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
}
faultsim_save_and_close
do_faultsim_test 3 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { DELETE FROM t1 }
} -test {
  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
}

reset_db
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b);
  INSERT INTO t2 VALUES('m f a jj th q jr ar',   'hj n h h sg j i m');
  INSERT INTO t2 VALUES('nr s t g od j kf h',    'sb h aq rg op rb n nl');
................................................................................
  9 { m* + f* }       {1 8}
} {
  do_faultsim_test 4.$tn -prep {
    faultsim_restore_and_reopen
  } -body "
    execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
  " -test "
    faultsim_test_result {[list 0 $res]} {1 {vtable constructor failed: t2}}
  "
}


#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#

Changes to ext/fts5/test/fts5fault4.test.

103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
}

set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]

do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
  faultsim_test_result {0 {0 {} 3}}
}

#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {







|







103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
}

set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]

do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
  faultsim_test_result {0 {0 {} 4}}
}

#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {

Changes to ext/fts5/test/fts5rank.test.

36
37
38
39
40
41
42
43























































44
45
  SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
} [list [string map {x [x]} $doc]]

do_execsql_test 1.3 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz
  WHERE xyz MATCH 'x AND y' ORDER BY rank
} [list [string map {x [x] y [y]} $doc]]
























































finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
  SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
} [list [string map {x [x]} $doc]]

do_execsql_test 1.3 {
  SELECT highlight(xyz, 0, '[', ']') FROM xyz
  WHERE xyz MATCH 'x AND y' ORDER BY rank
} [list [string map {x [x] y [y]} $doc]]

#-------------------------------------------------------------------------
# Check that the 'rank' option really is persistent.
#
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE tt USING fts5(a);
  INSERT INTO tt VALUES('a x x x x');
  INSERT INTO tt VALUES('x x a a a');
  INSERT INTO tt VALUES('x a a x x');
}

proc firstinst {cmd} { 
  foreach {p c o} [$cmd xInst 0] {}
  return $o
}
sqlite3_fts5_create_function db firstinst firstinst

do_execsql_test 2.1 {
  SELECT rowid FROM tt('a') ORDER BY rank;
} {2 3 1}

do_execsql_test 2.2 {
  SELECT rowid FROM tt('a', 'firstinst()') ORDER BY rank;
} {1 3 2}

do_execsql_test 2.3 {
  INSERT INTO tt(tt, rank) VALUES('rank', 'firstinst()');
  SELECT rowid FROM tt('a') ORDER BY rank;
} {1 3 2}

do_test 2.4 {
  sqlite3 db2 test.db
  catchsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
} {1 {no such function: firstinst}}

do_test 2.5 {
  db2 close
  sqlite3 db2 test.db
  sqlite3_fts5_create_function db2 firstinst firstinst
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
} {1 3 2}

do_test 2.6 {
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db2
} {1 3 2}

do_test 2.7 {
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
} {1 3 2}







finish_test

Changes to ext/fts5/test/fts5simple.test.

331
332
333
334
335
336
337















338
339
340
    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
  COMMIT;
}
do_test 14.2 { 
  fts5_level_segs ttt 
} {1}
















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
  COMMIT;
}
do_test 14.2 { 
  fts5_level_segs ttt 
} {1}

#-------------------------------------------------------------------------
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(x);
  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);

  WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
  INSERT INTO x1 SELECT rnddoc(5) FROM ii;
}

do_execsql_test 4.1 {
  SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
} {0 {} 4}


finish_test