/ Check-in [edfc2acf]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Merge enhancements and the ALTER TABLE bug fix from trunk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | apple-osx
Files: files | file ages | folders
SHA3-256: edfc2acfcd161d962996e2b4c74c6c6e6db1473d9e98ae8f533e26583d346248
User & Date: drh 2018-12-05 18:28:17
Context
2018-12-10
02:00
Merge enhancements from trunk. check-in: b1bbc718 user: drh tags: apple-osx
2018-12-05
18:28
Merge enhancements and the ALTER TABLE bug fix from trunk. check-in: edfc2acf user: drh tags: apple-osx
17:48
Simplifications to the sqlite3_normalized_sql() implementation. check-in: 94ea6379 user: drh tags: trunk
2018-11-30
18:22
Merge the pre-3.26.0 fixes from trunk. check-in: 2c76ce4f user: drh tags: apple-osx
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to Makefile.in.

  1315   1315   
  1316   1316   showwal$(TEXE):	$(TOP)/tool/showwal.c sqlite3.lo
  1317   1317   	$(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS)
  1318   1318   
  1319   1319   showshm$(TEXE):	$(TOP)/tool/showshm.c
  1320   1320   	$(LTLINK) -o $@ $(TOP)/tool/showshm.c
  1321   1321   
         1322  +index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo
         1323  +	$(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS)
         1324  +
  1322   1325   changeset$(TEXE):	$(TOP)/ext/session/changeset.c sqlite3.lo
  1323   1326   	$(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS)
  1324   1327   
  1325   1328   changesetfuzz$(TEXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.lo
  1326   1329   	$(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS)
  1327   1330   
  1328   1331   rollback-test$(TEXE):	$(TOP)/tool/rollback-test.c sqlite3.lo

Changes to Makefile.msc.

  2440   2440   showwal.exe:	$(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H)
  2441   2441   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2442   2442   		$(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2443   2443   
  2444   2444   showshm.exe:	$(TOP)\tool\showshm.c
  2445   2445   	$(LTLINK) $(NO_WARN)	$(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS)
  2446   2446   
         2447  +index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H)
         2448  +	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
         2449  +		$(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
         2450  +
  2447   2451   changeset.exe:	$(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H)
  2448   2452   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
  2449   2453   		-DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \
  2450   2454   		$(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
  2451   2455   
  2452   2456   changesetfuzz.exe:	$(TOP)\ext\session\changesetfuzz.c $(SQLITE3C) $(SQLITE3H)
  2453   2457   	$(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \

Changes to autoconf/Makefile.msc.

   279    279   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
   280    280   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
   281    281   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
   282    282   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
   283    283   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
   284    284   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
   285    285   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1
          286  +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1
   286    287   !ENDIF
   287    288   OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
   288    289   !ENDIF
   289    290   
   290    291   # Should the session extension be enabled?  If so, add compilation options
   291    292   # to enable it.
   292    293   #
................................................................................
   933    934   # Additional compiler options for the shell.  These are only effective
   934    935   # when the shell is not being dynamically linked.
   935    936   #
   936    937   !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
   937    938   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
   938    939   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
   939    940   SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
          941  +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1
   940    942   !ENDIF
   941    943   
   942    944   
   943    945   # This is the default Makefile target.  The objects listed here
   944    946   # are what get build when you type just "make" with no arguments.
   945    947   #
   946    948   core:	dll shell

Changes to ext/fts3/fts3_unicode.c.

    78     78   #endif /* ifndef SQLITE_AMALGAMATION */
    79     79   
    80     80   typedef struct unicode_tokenizer unicode_tokenizer;
    81     81   typedef struct unicode_cursor unicode_cursor;
    82     82   
    83     83   struct unicode_tokenizer {
    84     84     sqlite3_tokenizer base;
    85         -  int bRemoveDiacritic;
           85  +  int eRemoveDiacritic;
    86     86     int nException;
    87     87     int *aiException;
    88     88   };
    89     89   
    90     90   struct unicode_cursor {
    91     91     sqlite3_tokenizer_cursor base;
    92     92     const unsigned char *aInput;    /* Input text being tokenized */
................................................................................
   223    223     unicode_tokenizer *pNew;        /* New tokenizer object */
   224    224     int i;
   225    225     int rc = SQLITE_OK;
   226    226   
   227    227     pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
   228    228     if( pNew==NULL ) return SQLITE_NOMEM;
   229    229     memset(pNew, 0, sizeof(unicode_tokenizer));
   230         -  pNew->bRemoveDiacritic = 1;
          230  +  pNew->eRemoveDiacritic = 1;
   231    231   
   232    232     for(i=0; rc==SQLITE_OK && i<nArg; i++){
   233    233       const char *z = azArg[i];
   234    234       int n = (int)strlen(z);
   235    235   
   236    236       if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
   237         -      pNew->bRemoveDiacritic = 1;
          237  +      pNew->eRemoveDiacritic = 1;
   238    238       }
   239    239       else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
   240         -      pNew->bRemoveDiacritic = 0;
          240  +      pNew->eRemoveDiacritic = 0;
          241  +    }
          242  +    else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
          243  +      pNew->eRemoveDiacritic = 2;
   241    244       }
   242    245       else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
   243    246         rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
   244    247       }
   245    248       else if( n>=11 && memcmp("separators=", z, 11)==0 ){
   246    249         rc = unicodeAddExceptions(pNew, 0, &z[11], n-11);
   247    250       }
................................................................................
   346    349         zOut = &zNew[zOut - pCsr->zToken];
   347    350         pCsr->zToken = zNew;
   348    351         pCsr->nAlloc += 64;
   349    352       }
   350    353   
   351    354       /* Write the folded case of the last character read to the output */
   352    355       zEnd = z;
   353         -    iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
          356  +    iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
   354    357       if( iOut ){
   355    358         WRITE_UTF8(zOut, iOut);
   356    359       }
   357    360   
   358    361       /* If the cursor is not at EOF, read the next character */
   359    362       if( z>=zTerm ) break;
   360    363       READ_UTF8(z, zTerm, iCode);

Changes to ext/fts3/fts3_unicode2.c.

   155    155   ** If the argument is a codepoint corresponding to a lowercase letter
   156    156   ** in the ASCII range with a diacritic added, return the codepoint
   157    157   ** of the ASCII letter only. For example, if passed 235 - "LATIN
   158    158   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
   159    159   ** E"). The resuls of passing a codepoint that corresponds to an
   160    160   ** uppercase letter are undefined.
   161    161   */
   162         -static int remove_diacritic(int c){
          162  +static int remove_diacritic(int c, int bComplex){
   163    163     unsigned short aDia[] = {
   164    164           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
   165    165        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
   166    166        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
   167    167        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
   168         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
   169         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
   170         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
   171         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
   172         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
   173         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
   174         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
   175         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
   176         -    62924, 63050, 63082, 63274, 63390, 
          168  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
          169  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
          170  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
          171  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
          172  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
          173  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
          174  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
          175  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
          176  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
          177  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
          178  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
          179  +    63182, 63242, 63274, 63310, 63368, 63390, 
   177    180     };
   178    181     char aChar[] = {
   179         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
   180         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
   181         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
   182         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
   183         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
   184         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
   185         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
   186         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
   187         -    'e',  'i',  'o',  'u',  'y',  
          182  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
          183  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
          184  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
          185  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
          186  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
          187  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
          188  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
          189  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
          190  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
          191  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
          192  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
          193  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
          194  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
          195  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
          196  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
          197  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
          198  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
          199  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
          200  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
          201  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
          202  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
   188    203     };
   189    204   
   190    205     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
   191    206     int iRes = 0;
   192    207     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
   193    208     int iLo = 0;
   194    209     while( iHi>=iLo ){
................................................................................
   197    212         iRes = iTest;
   198    213         iLo = iTest+1;
   199    214       }else{
   200    215         iHi = iTest-1;
   201    216       }
   202    217     }
   203    218     assert( key>=aDia[iRes] );
   204         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
          219  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
          220  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
   205    221   }
   206    222   
   207    223   
   208    224   /*
   209    225   ** Return true if the argument interpreted as a unicode codepoint
   210    226   ** is a diacritical modifier character.
   211    227   */
................................................................................
   224    240   ** is an upper case character that has a lower case equivalent,
   225    241   ** return the codepoint corresponding to the lower case version.
   226    242   ** Otherwise, return a copy of the argument.
   227    243   **
   228    244   ** The results are undefined if the value passed to this function
   229    245   ** is less than zero.
   230    246   */
   231         -int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
          247  +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
   232    248     /* Each entry in the following array defines a rule for folding a range
   233    249     ** of codepoints to lower case. The rule applies to a range of nRange
   234    250     ** codepoints starting at codepoint iCode.
   235    251     **
   236    252     ** If the least significant bit in flags is clear, then the rule applies
   237    253     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   238    254     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   347    363       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   348    364       p = &aEntry[iRes];
   349    365       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   350    366         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   351    367         assert( ret>0 );
   352    368       }
   353    369   
   354         -    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
          370  +    if( eRemoveDiacritic ){
          371  +      ret = remove_diacritic(ret, eRemoveDiacritic==2);
          372  +    }
   355    373     }
   356    374     
   357    375     else if( c>=66560 && c<66600 ){
   358    376       ret = c + 40;
   359    377     }
   360    378   
   361    379     return ret;
   362    380   }
   363    381   #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
   364    382   #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */

Changes to ext/fts3/unicode/mkunicode.tcl.

     5      5     global tl_lookup_table
     6      6     set aChar [list]
     7      7     set lRange [list]
     8      8   
     9      9     set nRange 1
    10     10     set iFirst  [lindex $map 0 0]
    11     11     set cPrev   [lindex $map 0 1]
           12  +  set fPrev   [lindex $map 0 2]
    12     13   
    13     14     foreach m [lrange $map 1 end] {
    14         -    foreach {i c} $m {}
           15  +    foreach {i c f} $m {}
    15     16   
    16         -    if {$cPrev == $c} {
           17  +    if {$cPrev == $c && $fPrev==$f} {
    17     18         for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
    18     19           if {[info exists tl_lookup_table($j)]==0} break
    19     20         }
    20     21   
    21     22         if {$j==$i} {
    22     23           set nNew [expr {(1 + $i - $iFirst)}]
    23     24           if {$nNew<=8} {
................................................................................
    25     26             continue
    26     27           }
    27     28         }
    28     29       }
    29     30   
    30     31       lappend lRange [list $iFirst $nRange]
    31     32       lappend aChar  $cPrev
           33  +    lappend aFlag  $fPrev
    32     34   
    33     35       set iFirst $i
    34     36       set cPrev  $c
           37  +    set fPrev  $f
    35     38       set nRange 1
    36     39     }
    37     40     lappend lRange [list $iFirst $nRange]
    38     41     lappend aChar $cPrev
           42  +  lappend aFlag $fPrev
    39     43   
    40     44     puts "/*"
    41     45     puts "** If the argument is a codepoint corresponding to a lowercase letter"
    42     46     puts "** in the ASCII range with a diacritic added, return the codepoint"
    43     47     puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
    44     48     puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
    45     49     puts "** E\"). The resuls of passing a codepoint that corresponds to an"
    46     50     puts "** uppercase letter are undefined."
    47     51     puts "*/"
    48         -  puts "static int ${::remove_diacritic}(int c)\{"
           52  +  puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
    49     53     puts "  unsigned short aDia\[\] = \{"
    50     54     puts -nonewline "        0, "
    51     55     set i 1
    52     56     foreach r $lRange {
    53     57       foreach {iCode nRange} $r {}
    54     58       if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
    55     59       incr i
................................................................................
    56     60   
    57     61       puts -nonewline [format "%5d" [expr ($iCode<<3) + $nRange-1]]
    58     62       puts -nonewline ", "
    59     63     }
    60     64     puts ""
    61     65     puts "  \};"
    62     66     puts "  char aChar\[\] = \{"
    63         -  puts -nonewline "    '\\0', "
           67  +  puts -nonewline "    '\\0',      "
    64     68     set i 1
    65         -  foreach c $aChar {
    66         -    set str "'$c',  "
    67         -    if {$c == ""} { set str "'\\0', " }
           69  +  foreach c $aChar f $aFlag {
           70  +    if { $f } {
           71  +      set str "'$c'|0x80,  "
           72  +    } else {
           73  +      set str "'$c'|0x00,  "
           74  +    }
           75  +    if {$c == ""} { set str "'\\0',      " }
    68     76   
    69         -    if {($i % 12)==0} {puts "" ; puts -nonewline "    " }
           77  +    if {($i % 6)==0} {puts "" ; puts -nonewline "    " }
    70     78       incr i
    71     79       puts -nonewline "$str"
    72     80     }
    73     81     puts ""
    74     82     puts "  \};"
    75     83     puts {
    76     84     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
................................................................................
    83     91         iRes = iTest;
    84     92         iLo = iTest+1;
    85     93       }else{
    86     94         iHi = iTest-1;
    87     95       }
    88     96     }
    89     97     assert( key>=aDia[iRes] );
    90         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
           98  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           99  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
    91    100     puts "\}"
    92    101   }
    93    102   
    94    103   proc print_isdiacritic {zFunc map} {
    95    104   
    96    105     set lCode [list]
    97    106     foreach m $map {
    98         -    foreach {code char} $m {}
          107  +    foreach {code char flag} $m {}
          108  +    if {$flag} continue
    99    109       if {$code && $char == ""} { lappend lCode $code }
   100    110     }
   101    111     set lCode [lsort -integer $lCode]
   102    112     set iFirst [lindex $lCode 0]
   103    113     set iLast [lindex $lCode end]
   104    114   
   105    115     set i1 0
................................................................................
   468    478     puts "** is an upper case character that has a lower case equivalent,"
   469    479     puts "** return the codepoint corresponding to the lower case version."
   470    480     puts "** Otherwise, return a copy of the argument."
   471    481     puts "**"
   472    482     puts "** The results are undefined if the value passed to this function"
   473    483     puts "** is less than zero."
   474    484     puts "*/"
   475         -  puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
          485  +  puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
   476    486   
   477    487     set liOff [tl_generate_ioff_table $lRecord]
   478    488     tl_print_table_header
   479    489     foreach entry $lRecord { 
   480    490       if {[tl_print_table_entry toggle $entry $liOff]} { 
   481    491         lappend lHigh $entry 
   482    492       } 
................................................................................
   512    522       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   513    523       p = &aEntry[iRes];
   514    524       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   515    525         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   516    526         assert( ret>0 );
   517    527       }
   518    528   
   519         -    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
          529  +    if( eRemoveDiacritic ){
          530  +      ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
          531  +    }
   520    532     }
   521    533     }]
   522    534   
   523    535     foreach entry $lHigh {
   524    536       tl_print_if_entry $entry
   525    537     }
   526    538   
................................................................................
   601    613     set caseN [categories_switch C N {d l o}]
   602    614     set caseP [categories_switch C P {c d e f i o s}]
   603    615     set caseS [categories_switch C S {c k m o}]
   604    616     set caseZ [categories_switch C Z {l p s}]
   605    617   
   606    618     set nCat [expr [llength [array names C]] + 1]
   607    619     puts [code {
   608         -    int sqlite3Fts5UnicodeNCat(void) { 
   609         -      return $nCat;
   610         -    }
   611         -
   612    620       int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   613    621         aArray[0] = 1;
   614    622         switch( zCat[0] ){
   615    623           $caseC
   616    624           $caseL
   617    625           $caseM
   618    626           $caseN

Changes to ext/fts3/unicode/parseunicode.tcl.

     3      3   # Parameter $zName must be a path to the file UnicodeData.txt. This command
     4      4   # reads the file and returns a list of mappings required to remove all
     5      5   # diacritical marks from a unicode string. Each mapping is itself a list
     6      6   # consisting of two elements - the unicode codepoint and the single ASCII
     7      7   # character that it should be replaced with, or an empty string if the 
     8      8   # codepoint should simply be removed from the input. Examples:
     9      9   #
    10         -#   { 224 a  }     (replace codepoint 224 to "a")
    11         -#   { 769 "" }     (remove codepoint 769 from input)
           10  +#   { 224 a  0 }     (replace codepoint 224 to "a")
           11  +#   { 769 "" 0 }     (remove codepoint 769 from input)
    12     12   #
    13     13   # Mappings are only returned for non-upper case codepoints. It is assumed
    14     14   # that the input has already been folded to lower case.
           15  +#
           16  +# The third value in the list is always either 0 or 1. 0 if the 
           17  +# UnicodeData.txt file maps the codepoint to a single ASCII character and
           18  +# a diacritic, or 1 if the mapping is indirect. For example, consider the 
           19  +# two entries:
           20  +#
           21  +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
           22  +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
           23  +#
           24  +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a 
           25  +# diacritic). The second is an indirect mapping, as it maps to the
           26  +# first codepoint plus 0302 (a diacritic).
    15     27   #
    16     28   proc rd_load_unicodedata_text {zName} {
    17     29     global tl_lookup_table
    18     30   
    19     31     set fd [open $zName]
    20     32     set lField {
    21     33       code
................................................................................
    49     61         continue
    50     62       }
    51     63   
    52     64       set iCode  [expr "0x$code"]
    53     65       set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
    54     66       set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
    55     67   
           68  +    # Filter out upper-case characters, as they will be mapped to their
           69  +    # lower-case equivalents before this data is used.
    56     70       if {[info exists tl_lookup_table($iCode)]} continue
           71  +
           72  +    # Check if this is an indirect mapping. If so, set bIndirect to true
           73  +    # and change $iAscii to the indirectly mappped ASCII character.
           74  +    set bIndirect 0
           75  +    if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
           76  +      set iAscii $mapping($iAscii)
           77  +      set bIndirect 1
           78  +    }
    57     79   
    58     80       if { ($iAscii >= 97 && $iAscii <= 122)
    59     81         || ($iAscii >= 65 && $iAscii <= 90)
    60     82       } {
    61         -      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
           83  +      lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
           84  +      set mapping($iCode) $iAscii
    62     85         set dia($iDia) 1
    63     86       }
    64     87     }
    65     88   
    66     89     foreach d [array names dia] {
    67         -    lappend lRet [list $d ""]
           90  +    lappend lRet [list $d "" 0]
    68     91     }
    69     92     set lRet [lsort -integer -index 0 $lRet]
    70     93   
    71     94     close $fd
    72     95     set lRet
    73     96   }
    74     97   

Changes to ext/fts5/fts5_tokenize.c.

   230    230   #endif /* ifndef SQLITE_AMALGAMATION */
   231    231   
   232    232   typedef struct Unicode61Tokenizer Unicode61Tokenizer;
   233    233   struct Unicode61Tokenizer {
   234    234     unsigned char aTokenChar[128];  /* ASCII range token characters */
   235    235     char *aFold;                    /* Buffer to fold text into */
   236    236     int nFold;                      /* Size of aFold[] in bytes */
   237         -  int bRemoveDiacritic;           /* True if remove_diacritics=1 is set */
          237  +  int eRemoveDiacritic;           /* True if remove_diacritics=1 is set */
   238    238     int nException;
   239    239     int *aiException;
   240    240   
   241    241     unsigned char aCategory[32];    /* True for token char categories */
   242    242   };
          243  +
          244  +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
          245  +#define FTS5_REMOVE_DIACRITICS_NONE    0
          246  +#define FTS5_REMOVE_DIACRITICS_SIMPLE  1
          247  +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2
   243    248   
   244    249   static int fts5UnicodeAddExceptions(
   245    250     Unicode61Tokenizer *p,          /* Tokenizer object */
   246    251     const char *z,                  /* Characters to treat as exceptions */
   247    252     int bTokenChars                 /* 1 for 'tokenchars', 0 for 'separators' */
   248    253   ){
   249    254     int rc = SQLITE_OK;
................................................................................
   357    362     }else{
   358    363       p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
   359    364       if( p ){
   360    365         const char *zCat = "L* N* Co";
   361    366         int i;
   362    367         memset(p, 0, sizeof(Unicode61Tokenizer));
   363    368   
   364         -      p->bRemoveDiacritic = 1;
          369  +      p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
   365    370         p->nFold = 64;
   366    371         p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
   367    372         if( p->aFold==0 ){
   368    373           rc = SQLITE_NOMEM;
   369    374         }
   370    375   
   371    376         /* Search for a "categories" argument */
................................................................................
   378    383         if( rc==SQLITE_OK ){
   379    384           rc = unicodeSetCategories(p, zCat);
   380    385         }
   381    386   
   382    387         for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
   383    388           const char *zArg = azArg[i+1];
   384    389           if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
   385         -          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
          390  +          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
   386    391               rc = SQLITE_ERROR;
          392  +          }else{
          393  +            p->eRemoveDiacritic = (zArg[0] - '0');
          394  +            assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
          395  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
          396  +                 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
          397  +            );
   387    398             }
   388         -          p->bRemoveDiacritic = (zArg[0]=='1');
   389    399           }else
   390    400           if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
   391    401             rc = fts5UnicodeAddExceptions(p, zArg, 1);
   392    402           }else
   393    403           if( 0==sqlite3_stricmp(azArg[i], "separators") ){
   394    404             rc = fts5UnicodeAddExceptions(p, zArg, 0);
   395    405           }else
................................................................................
   495    505   
   496    506         if( *zCsr & 0x80 ){
   497    507           /* An non-ascii-range character. Fold it into the output buffer if
   498    508           ** it is a token character, or break out of the loop if it is not. */
   499    509           READ_UTF8(zCsr, zTerm, iCode);
   500    510           if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
   501    511    non_ascii_tokenchar:
   502         -          iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic);
          512  +          iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
   503    513             if( iCode ) WRITE_UTF8(zOut, iCode);
   504    514           }else{
   505    515             break;
   506    516           }
   507    517         }else if( a[*zCsr]==0 ){
   508    518           /* An ascii-range separator character. End of token. */
   509    519           break; 

Changes to ext/fts5/fts5_unicode2.c.

    24     24   ** If the argument is a codepoint corresponding to a lowercase letter
    25     25   ** in the ASCII range with a diacritic added, return the codepoint
    26     26   ** of the ASCII letter only. For example, if passed 235 - "LATIN
    27     27   ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
    28     28   ** E"). The resuls of passing a codepoint that corresponds to an
    29     29   ** uppercase letter are undefined.
    30     30   */
    31         -static int fts5_remove_diacritic(int c){
           31  +static int fts5_remove_diacritic(int c, int bComplex){
    32     32     unsigned short aDia[] = {
    33     33           0,  1797,  1848,  1859,  1891,  1928,  1940,  1995, 
    34     34        2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286, 
    35     35        2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732, 
    36     36        2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336, 
    37         -     3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928, 
    38         -     3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234, 
    39         -     4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504, 
    40         -     6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529, 
    41         -    61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, 
    42         -    61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, 
    43         -    62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, 
    44         -    62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, 
    45         -    62924, 63050, 63082, 63274, 63390, 
           37  +     3456,  3696,  3712,  3728,  3744,  3766,  3832,  3896, 
           38  +     3912,  3928,  3944,  3968,  4008,  4040,  4056,  4106, 
           39  +     4138,  4170,  4202,  4234,  4266,  4296,  4312,  4344, 
           40  +     4408,  4424,  4442,  4472,  4488,  4504,  6148,  6198, 
           41  +     6264,  6280,  6360,  6429,  6505,  6529, 61448, 61468, 
           42  +    61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, 
           43  +    61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, 
           44  +    61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, 
           45  +    62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, 
           46  +    62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, 
           47  +    62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, 
           48  +    63182, 63242, 63274, 63310, 63368, 63390, 
    46     49     };
    47     50     char aChar[] = {
    48         -    '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',  
    49         -    'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',  
    50         -    's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',  
    51         -    'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',  
    52         -    'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0', 
    53         -    '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',  
    54         -    'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',  
    55         -    'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',  
    56         -    'e',  'i',  'o',  'u',  'y',  
           51  +    '\0',      'a'|0x00,  'c'|0x00,  'e'|0x00,  'i'|0x00,  'n'|0x00,  
           52  +    'o'|0x00,  'u'|0x00,  'y'|0x00,  'y'|0x00,  'a'|0x00,  'c'|0x00,  
           53  +    'd'|0x00,  'e'|0x00,  'e'|0x00,  'g'|0x00,  'h'|0x00,  'i'|0x00,  
           54  +    'j'|0x00,  'k'|0x00,  'l'|0x00,  'n'|0x00,  'o'|0x00,  'r'|0x00,  
           55  +    's'|0x00,  't'|0x00,  'u'|0x00,  'u'|0x00,  'w'|0x00,  'y'|0x00,  
           56  +    'z'|0x00,  'o'|0x00,  'u'|0x00,  'a'|0x00,  'i'|0x00,  'o'|0x00,  
           57  +    'u'|0x00,  'u'|0x80,  'a'|0x80,  'g'|0x00,  'k'|0x00,  'o'|0x00,  
           58  +    'o'|0x80,  'j'|0x00,  'g'|0x00,  'n'|0x00,  'a'|0x80,  'a'|0x00,  
           59  +    'e'|0x00,  'i'|0x00,  'o'|0x00,  'r'|0x00,  'u'|0x00,  's'|0x00,  
           60  +    't'|0x00,  'h'|0x00,  'a'|0x00,  'e'|0x00,  'o'|0x80,  'o'|0x00,  
           61  +    'o'|0x80,  'y'|0x00,  '\0',      '\0',      '\0',      '\0',      
           62  +    '\0',      '\0',      '\0',      '\0',      'a'|0x00,  'b'|0x00,  
           63  +    'c'|0x80,  'd'|0x00,  'd'|0x00,  'e'|0x80,  'e'|0x00,  'e'|0x80,  
           64  +    'f'|0x00,  'g'|0x00,  'h'|0x00,  'h'|0x00,  'i'|0x00,  'i'|0x80,  
           65  +    'k'|0x00,  'l'|0x00,  'l'|0x80,  'l'|0x00,  'm'|0x00,  'n'|0x00,  
           66  +    'o'|0x80,  'p'|0x00,  'r'|0x00,  'r'|0x80,  'r'|0x00,  's'|0x00,  
           67  +    's'|0x80,  't'|0x00,  'u'|0x00,  'u'|0x80,  'v'|0x00,  'w'|0x00,  
           68  +    'w'|0x00,  'x'|0x00,  'y'|0x00,  'z'|0x00,  'h'|0x00,  't'|0x00,  
           69  +    'w'|0x00,  'y'|0x00,  'a'|0x00,  'a'|0x80,  'a'|0x80,  'a'|0x80,  
           70  +    'e'|0x00,  'e'|0x80,  'e'|0x80,  'i'|0x00,  'o'|0x00,  'o'|0x80,  
           71  +    'o'|0x80,  'o'|0x80,  'u'|0x00,  'u'|0x80,  'u'|0x80,  'y'|0x00,  
    57     72     };
    58     73   
    59     74     unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
    60     75     int iRes = 0;
    61     76     int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
    62     77     int iLo = 0;
    63     78     while( iHi>=iLo ){
................................................................................
    66     81         iRes = iTest;
    67     82         iLo = iTest+1;
    68     83       }else{
    69     84         iHi = iTest-1;
    70     85       }
    71     86     }
    72     87     assert( key>=aDia[iRes] );
    73         -  return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
           88  +  if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
           89  +  return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
    74     90   }
    75     91   
    76     92   
    77     93   /*
    78     94   ** Return true if the argument interpreted as a unicode codepoint
    79     95   ** is a diacritical modifier character.
    80     96   */
................................................................................
    93    109   ** is an upper case character that has a lower case equivalent,
    94    110   ** return the codepoint corresponding to the lower case version.
    95    111   ** Otherwise, return a copy of the argument.
    96    112   **
    97    113   ** The results are undefined if the value passed to this function
    98    114   ** is less than zero.
    99    115   */
   100         -int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
          116  +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   101    117     /* Each entry in the following array defines a rule for folding a range
   102    118     ** of codepoints to lower case. The rule applies to a range of nRange
   103    119     ** codepoints starting at codepoint iCode.
   104    120     **
   105    121     ** If the least significant bit in flags is clear, then the rule applies
   106    122     ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
   107    123     ** need to be folded). Or, if it is set, then the rule only applies to
................................................................................
   216    232       assert( iRes>=0 && c>=aEntry[iRes].iCode );
   217    233       p = &aEntry[iRes];
   218    234       if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
   219    235         ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
   220    236         assert( ret>0 );
   221    237       }
   222    238   
   223         -    if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
          239  +    if( eRemoveDiacritic ){
          240  +      ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
          241  +    }
   224    242     }
   225    243     
   226    244     else if( c>=66560 && c<66600 ){
   227    245       ret = c + 40;
   228    246     }
   229    247   
   230    248     return ret;
   231    249   }
   232    250   
   233         -
   234         -#if 0
   235         -int sqlite3Fts5UnicodeNCat(void) { 
   236         -  return 32;
   237         -}
   238         -#endif
   239         -
   240    251   int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   241    252     aArray[0] = 1;
   242    253     switch( zCat[0] ){
   243    254       case 'C':
   244    255             switch( zCat[1] ){
   245    256               case 'c': aArray[1] = 1; break;
   246    257               case 'f': aArray[2] = 1; break;
................................................................................
   752    763   void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
   753    764     int i = 0;
   754    765     int iTbl = 0;
   755    766     while( i<128 ){
   756    767       int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
   757    768       int n = (aFts5UnicodeData[iTbl] >> 5) + i;
   758    769       for(; i<128 && i<n; i++){
   759         -      aAscii[i] = (u8)bToken;
          770  +      aAscii[i] = bToken;
   760    771       }
   761    772       iTbl++;
   762    773     }
   763    774   }
   764         -

Changes to ext/fts5/test/fts5tokenizer.test.

   185    185     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 tokenchars');
   186    186   } {1 {error in tokenizer constructor}}
   187    187   do_catchsql_test 6.2 {
   188    188     CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'unicode61 a b');
   189    189   } {1 {error in tokenizer constructor}}
   190    190   do_catchsql_test 6.3 {
   191    191     CREATE VIRTUAL TABLE a3 USING fts5(
   192         -    x, y, tokenize = 'unicode61 remove_diacritics 2'
          192  +    x, y, tokenize = 'unicode61 remove_diacritics 3'
   193    193     );
   194    194   } {1 {error in tokenizer constructor}}
   195    195   do_catchsql_test 6.4 {
   196    196     CREATE VIRTUAL TABLE a3 USING fts5(
   197    197       x, y, tokenize = 'unicode61 remove_diacritics 10'
   198    198     );
   199    199   } {1 {error in tokenizer constructor}}

Added ext/fts5/test/fts5umlaut.test.

            1  +# 2014 June 17
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5umlaut
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts5(
           27  +      x, 
           28  +      tokenize="unicode61 remove_diacritics 2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1($q)
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1('Ha Noi')
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2($q)
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2('Ha Noi')
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to ext/fts5/test/fts5unicode3.test.

    32     32   
    33     33   tl_load_casefolding_txt $CF
    34     34   foreach x [an_load_unicodedata_text $UD] {
    35     35     set aNotAlnum($x) 1
    36     36   }
    37     37   
    38     38   foreach {y} [rd_load_unicodedata_text $UD] {
    39         -  foreach {code ascii} $y {}
           39  +  foreach {code ascii f} $y {}
    40     40     if {$ascii==""} {
    41     41       set int 0
    42     42     } else {
    43     43       binary scan $ascii c int
    44     44     }
    45         -  set aDiacritic($code) $int
           45  +  set aDiacritic($code,$f) $int
           46  +  if {$f==0} { set aDiacritic($code,1) $int }
    46     47   }
    47     48   
    48     49   proc tcl_fold {i {bRemoveDiacritic 0}} {
    49     50     global tl_lookup_table
    50     51     global aDiacritic
           52  +  set f [expr $bRemoveDiacritic==2]
    51     53   
    52     54     if {[info exists tl_lookup_table($i)]} {
    53     55       set i $tl_lookup_table($i)
    54     56     }
    55         -  if {$bRemoveDiacritic && [info exists aDiacritic($i)]} {
    56         -    set i $aDiacritic($i)
           57  +  if {$bRemoveDiacritic && [info exists aDiacritic($i,$f)]} {
           58  +    set i $aDiacritic($i,$f)
    57     59     }
    58     60     expr $i
    59     61   }
    60     62   db func tcl_fold tcl_fold
    61     63   
    62     64   proc tcl_isalnum {i} {
    63     65     global aNotAlnum
................................................................................
    81     83       SELECT -1
    82     84       UNION ALL
    83     85       SELECT i+1 FROM ii WHERE i<100000
    84     86     )
    85     87     SELECT count(*), min(i) FROM ii WHERE fts5_fold(i)!=CAST(tcl_fold(i) AS int);
    86     88   } {0 {}}
    87     89   
    88         -do_execsql_test 1.2 {
           90  +do_execsql_test 1.2.1 {
    89     91     WITH ii(i) AS (
    90     92       SELECT -1
    91     93       UNION ALL
    92     94       SELECT i+1 FROM ii WHERE i<100000
    93     95     )
    94     96     SELECT count(*), min(i) FROM ii 
    95     97     WHERE fts5_fold(i,1)!=CAST(tcl_fold(i,1) AS int);
    96     98   } {0 {}}
           99  +
          100  +do_execsql_test 1.2.2 {
          101  +  WITH ii(i) AS (
          102  +    SELECT -1
          103  +    UNION ALL
          104  +    SELECT i+1 FROM ii WHERE i<100000
          105  +  )
          106  +  SELECT count(*), min(i) FROM ii 
          107  +  WHERE fts5_fold(i,2)!=CAST(tcl_fold(i,2) AS int);
          108  +} {0 {}}
    97    109   
    98    110   do_execsql_test 1.3 {
    99    111     WITH ii(i) AS (
   100    112       SELECT -1
   101    113       UNION ALL
   102    114       SELECT i+1 FROM ii WHERE i<100000
   103    115     )

Changes to ext/rtree/sqlite3rtree.h.

    92     92     unsigned int *anQueue;            /* Number of pending entries in the queue */
    93     93     int nCoord;                       /* Number of coordinates */
    94     94     int iLevel;                       /* Level of current node or entry */
    95     95     int mxLevel;                      /* The largest iLevel value in the tree */
    96     96     sqlite3_int64 iRowid;             /* Rowid for current entry */
    97     97     sqlite3_rtree_dbl rParentScore;   /* Score of parent node */
    98     98     int eParentWithin;                /* Visibility of parent node */
    99         -  int eWithin;                      /* OUT: Visiblity */
           99  +  int eWithin;                      /* OUT: Visibility */
   100    100     sqlite3_rtree_dbl rScore;         /* OUT: Write the score here */
   101    101     /* The following fields are only available in 3.8.11 and later */
   102    102     sqlite3_value **apSqlParam;       /* Original SQL values of parameters */
   103    103   };
   104    104   
   105    105   /*
   106    106   ** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin.

Changes to main.mk.

   995    995   
   996    996   showwal$(EXE):	$(TOP)/tool/showwal.c sqlite3.o
   997    997   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \
   998    998   		$(TOP)/tool/showwal.c sqlite3.o $(THREADLIB)
   999    999   
  1000   1000   showshm$(EXE):	$(TOP)/tool/showshm.c
  1001   1001   	$(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c
         1002  +
         1003  +index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o
         1004  +	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \
         1005  +		$(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB)
  1002   1006   
  1003   1007   changeset$(EXE):	$(TOP)/ext/session/changeset.c sqlite3.o
  1004   1008   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \
  1005   1009   		$(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB)
  1006   1010   
  1007   1011   changesetfuzz$(EXE):	$(TOP)/ext/session/changesetfuzz.c sqlite3.o
  1008   1012   	$(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changesetfuzz$(EXE) \

Changes to src/alter.c.

   775    775         pToken->pNext = pCtx->pList;
   776    776         pCtx->pList = pToken;
   777    777         pCtx->nList++;
   778    778         break;
   779    779       }
   780    780     }
   781    781   }
          782  +
          783  +/*
          784  +** Iterate through the Select objects that are part of WITH clauses attached
          785  +** to select statement pSelect.
          786  +*/
          787  +static void renameWalkWith(Walker *pWalker, Select *pSelect){
          788  +  if( pSelect->pWith ){
          789  +    int i;
          790  +    for(i=0; i<pSelect->pWith->nCte; i++){
          791  +      Select *p = pSelect->pWith->a[i].pSelect;
          792  +      NameContext sNC;
          793  +      memset(&sNC, 0, sizeof(sNC));
          794  +      sNC.pParse = pWalker->pParse;
          795  +      sqlite3SelectPrep(sNC.pParse, p, &sNC);
          796  +      sqlite3WalkSelect(pWalker, p);
          797  +    }
          798  +  }
          799  +}
   782    800   
   783    801   /*
   784    802   ** This is a Walker select callback. It does nothing. It is only required
   785    803   ** because without a dummy callback, sqlite3WalkExpr() and similar do not
   786    804   ** descend into sub-select statements.
   787    805   */
   788    806   static int renameColumnSelectCb(Walker *pWalker, Select *p){
   789         -  UNUSED_PARAMETER(pWalker);
   790         -  UNUSED_PARAMETER(p);
          807  +  renameWalkWith(pWalker, p);
   791    808     return WRC_Continue;
   792    809   }
   793    810   
   794    811   /*
   795    812   ** This is a Walker expression callback.
   796    813   **
   797    814   ** For every TK_COLUMN node in the expression tree, search to see
................................................................................
  1360   1377     SrcList *pSrc = pSelect->pSrc;
  1361   1378     for(i=0; i<pSrc->nSrc; i++){
  1362   1379       struct SrcList_item *pItem = &pSrc->a[i];
  1363   1380       if( pItem->pTab==p->pTab ){
  1364   1381         renameTokenFind(pWalker->pParse, p, pItem->zName);
  1365   1382       }
  1366   1383     }
         1384  +  renameWalkWith(pWalker, pSelect);
  1367   1385   
  1368   1386     return WRC_Continue;
  1369   1387   }
  1370   1388   
  1371   1389   
  1372   1390   /*
  1373   1391   ** This C function implements an SQL user function that is used by SQL code

Changes to src/build.c.

   223    223   
   224    224   
   225    225     /* Get the VDBE program ready for execution
   226    226     */
   227    227     if( v && pParse->nErr==0 && !db->mallocFailed ){
   228    228       /* A minimum of one cursor is required if autoincrement is used
   229    229       *  See ticket [a696379c1f08866] */
   230         -    if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1;
          230  +    assert( pParse->pAinc==0 || pParse->nTab>0 );
   231    231       sqlite3VdbeMakeReady(v, pParse);
   232    232       pParse->rc = SQLITE_DONE;
   233    233     }else{
   234    234       pParse->rc = SQLITE_ERROR;
   235    235     }
   236    236   }
   237    237   

Changes to src/callback.c.

   279    279     return match;
   280    280   }
   281    281   
   282    282   /*
   283    283   ** Search a FuncDefHash for a function with the given name.  Return
   284    284   ** a pointer to the matching FuncDef if found, or 0 if there is no match.
   285    285   */
   286         -static FuncDef *functionSearch(
          286  +FuncDef *sqlite3FunctionSearch(
   287    287     int h,               /* Hash of the name */
   288    288     const char *zFunc    /* Name of function */
   289    289   ){
   290    290     FuncDef *p;
   291    291     for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
   292    292       if( sqlite3StrICmp(p->zName, zFunc)==0 ){
   293    293         return p;
   294    294       }
   295    295     }
   296    296     return 0;
   297    297   }
   298         -#ifdef SQLITE_ENABLE_NORMALIZE
   299         -FuncDef *sqlite3FunctionSearchN(
   300         -  int h,               /* Hash of the name */
   301         -  const char *zFunc,   /* Name of function */
   302         -  int nFunc            /* Length of the name */
   303         -){
   304         -  FuncDef *p;
   305         -  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
   306         -    if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
   307         -      return p;
   308         -    }
   309         -  }
   310         -  return 0;
   311         -}
   312         -#endif /* SQLITE_ENABLE_NORMALIZE */
   313    298   
   314    299   /*
   315    300   ** Insert a new FuncDef into a FuncDefHash hash table.
   316    301   */
   317    302   void sqlite3InsertBuiltinFuncs(
   318    303     FuncDef *aDef,      /* List of global functions to be inserted */
   319    304     int nDef            /* Length of the apDef[] list */
................................................................................
   321    306     int i;
   322    307     for(i=0; i<nDef; i++){
   323    308       FuncDef *pOther;
   324    309       const char *zName = aDef[i].zName;
   325    310       int nName = sqlite3Strlen30(zName);
   326    311       int h = SQLITE_FUNC_HASH(zName[0], nName);
   327    312       assert( zName[0]>='a' && zName[0]<='z' );
   328         -    pOther = functionSearch(h, zName);
          313  +    pOther = sqlite3FunctionSearch(h, zName);
   329    314       if( pOther ){
   330    315         assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] );
   331    316         aDef[i].pNext = pOther->pNext;
   332    317         pOther->pNext = &aDef[i];
   333    318       }else{
   334    319         aDef[i].pNext = 0;
   335    320         aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h];
................................................................................
   399    384     ** have fields overwritten with new information appropriate for the
   400    385     ** new function.  But the FuncDefs for built-in functions are read-only.
   401    386     ** So we must not search for built-ins when creating a new function.
   402    387     */ 
   403    388     if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
   404    389       bestScore = 0;
   405    390       h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
   406         -    p = functionSearch(h, zName);
          391  +    p = sqlite3FunctionSearch(h, zName);
   407    392       while( p ){
   408    393         int score = matchQuality(p, nArg, enc);
   409    394         if( score>bestScore ){
   410    395           pBest = p;
   411    396           bestScore = score;
   412    397         }
   413    398         p = p->pNext;

Changes to src/hash.c.

    60     60       ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    61     61       ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    62     62       h += sqlite3UpperToLower[c];
    63     63       h *= 0x9e3779b1;
    64     64     }
    65     65     return h;
    66     66   }
    67         -#ifdef SQLITE_ENABLE_NORMALIZE
    68         -static unsigned int strHashN(const char *z, int n){
    69         -  unsigned int h = 0;
    70         -  int i;
    71         -  for(i=0; i<n; i++){
    72         -    /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
    73         -    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
    74         -    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
    75         -    h += sqlite3UpperToLower[z[i]];
    76         -    h *= 0x9e3779b1;
    77         -  }
    78         -  return h;
    79         -}
    80         -#endif /* SQLITE_ENABLE_NORMALIZE */
    81     67   
    82     68   
    83     69   /* Link pNew element into the hash table pH.  If pEntry!=0 then also
    84     70   ** insert pNew into the pEntry hash bucket.
    85     71   */
    86     72   static void insertElement(
    87     73     Hash *pH,              /* The complete hash table */
................................................................................
   185    171       if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ 
   186    172         return elem;
   187    173       }
   188    174       elem = elem->next;
   189    175     }
   190    176     return &nullElement;
   191    177   }
   192         -#ifdef SQLITE_ENABLE_NORMALIZE
   193         -static HashElem *findElementWithHashN(
   194         -  const Hash *pH,     /* The pH to be searched */
   195         -  const char *pKey,   /* The key we are searching for */
   196         -  int nKey,           /* Number of key bytes to use */
   197         -  unsigned int *pHash /* Write the hash value here */
   198         -){
   199         -  HashElem *elem;                /* Used to loop thru the element list */
   200         -  int count;                     /* Number of elements left to test */
   201         -  unsigned int h;                /* The computed hash */
   202         -  static HashElem nullElement = { 0, 0, 0, 0 };
   203         -
   204         -  if( pH->ht ){   /*OPTIMIZATION-IF-TRUE*/
   205         -    struct _ht *pEntry;
   206         -    h = strHashN(pKey, nKey) % pH->htsize;
   207         -    pEntry = &pH->ht[h];
   208         -    elem = pEntry->chain;
   209         -    count = pEntry->count;
   210         -  }else{
   211         -    h = 0;
   212         -    elem = pH->first;
   213         -    count = pH->count;
   214         -  }
   215         -  if( pHash ) *pHash = h;
   216         -  while( count-- ){
   217         -    assert( elem!=0 );
   218         -    if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ 
   219         -      return elem;
   220         -    }
   221         -    elem = elem->next;
   222         -  }
   223         -  return &nullElement;
   224         -}
   225         -#endif /* SQLITE_ENABLE_NORMALIZE */
   226    178   
   227    179   /* Remove a single entry from the hash table given a pointer to that
   228    180   ** element and a hash on the element's key.
   229    181   */
   230    182   static void removeElementGivenHash(
   231    183     Hash *pH,         /* The pH containing "elem" */
   232    184     HashElem* elem,   /* The element to be removed from the pH */
................................................................................
   263    215   ** found, or NULL if there is no match.
   264    216   */
   265    217   void *sqlite3HashFind(const Hash *pH, const char *pKey){
   266    218     assert( pH!=0 );
   267    219     assert( pKey!=0 );
   268    220     return findElementWithHash(pH, pKey, 0)->data;
   269    221   }
   270         -#ifdef SQLITE_ENABLE_NORMALIZE
   271         -void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
   272         -  assert( pH!=0 );
   273         -  assert( pKey!=0 );
   274         -  assert( nKey>=0 );
   275         -  return findElementWithHashN(pH, pKey, nKey, 0)->data;
   276         -}
   277         -#endif /* SQLITE_ENABLE_NORMALIZE */
   278    222   
   279    223   /* Insert an element into the hash table pH.  The key is pKey
   280    224   ** and the data is "data".
   281    225   **
   282    226   ** If no element exists with a matching key, then a new
   283    227   ** element is created and NULL is returned.
   284    228   **

Changes to src/insert.c.

   315    315       aOp[3].p5 = SQLITE_JUMPIFNULL;
   316    316       aOp[4].p2 = memId+1;
   317    317       aOp[5].p3 = memId;
   318    318       aOp[6].p1 = memId;
   319    319       aOp[7].p2 = memId+2;
   320    320       aOp[7].p1 = memId;
   321    321       aOp[10].p2 = memId;
          322  +    if( pParse->nTab==0 ) pParse->nTab = 1;
   322    323     }
   323    324   }
   324    325   
   325    326   /*
   326    327   ** Update the maximum rowid for an autoincrement calculation.
   327    328   **
   328    329   ** This routine should be called when the regRowid register holds a

Changes to src/main.c.

  2000   2000       (void)SQLITE_MISUSE_BKPT;
  2001   2001       return 0;
  2002   2002     }
  2003   2003   #endif
  2004   2004     sqlite3_mutex_enter(db->mutex);
  2005   2005     pOld = db->pTraceArg;
  2006   2006     db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0;
         2007  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  2007   2008     db->xTrace = (int(*)(u32,void*,void*,void*))xTrace;
  2008   2009     db->pTraceArg = pArg;
  2009   2010     sqlite3_mutex_leave(db->mutex);
  2010   2011     return pOld;
  2011   2012   }
  2012   2013   #endif /* SQLITE_OMIT_DEPRECATED */
  2013   2014   
................................................................................
  2024   2025       return SQLITE_MISUSE_BKPT;
  2025   2026     }
  2026   2027   #endif
  2027   2028     sqlite3_mutex_enter(db->mutex);
  2028   2029     if( mTrace==0 ) xTrace = 0;
  2029   2030     if( xTrace==0 ) mTrace = 0;
  2030   2031     db->mTrace = mTrace;
         2032  +#ifndef SQLITE_OMIT_DEPRECATED
         2033  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
         2034  +#endif
  2031   2035     db->xTrace = xTrace;
  2032   2036     db->pTraceArg = pArg;
  2033   2037     sqlite3_mutex_leave(db->mutex);
  2034   2038     return SQLITE_OK;
  2035   2039   }
  2036   2040   
  2037   2041   #ifndef SQLITE_OMIT_DEPRECATED
................................................................................
  2056   2060       return 0;
  2057   2061     }
  2058   2062   #endif
  2059   2063     sqlite3_mutex_enter(db->mutex);
  2060   2064     pOld = db->pProfileArg;
  2061   2065     db->xProfile = xProfile;
  2062   2066     db->pProfileArg = pArg;
         2067  +  db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK;
         2068  +  if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE;
  2063   2069     sqlite3_mutex_leave(db->mutex);
  2064   2070     return pOld;
  2065   2071   }
  2066   2072   #endif /* SQLITE_OMIT_DEPRECATED */
  2067   2073   #endif /* SQLITE_OMIT_TRACE */
  2068   2074   
  2069   2075   /*

Changes to src/parse.y.

   660    660         A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,N,U);
   661    661         if( A ){
   662    662           struct SrcList_item *pNew = &A->a[A->nSrc-1];
   663    663           struct SrcList_item *pOld = F->a;
   664    664           pNew->zName = pOld->zName;
   665    665           pNew->zDatabase = pOld->zDatabase;
   666    666           pNew->pSelect = pOld->pSelect;
          667  +        if( pOld->fg.isTabFunc ){
          668  +          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
          669  +          pOld->u1.pFuncArg = 0;
          670  +          pOld->fg.isTabFunc = 0;
          671  +          pNew->fg.isTabFunc = 1;
          672  +        }
   667    673           pOld->zName = pOld->zDatabase = 0;
   668    674           pOld->pSelect = 0;
   669    675         }
   670    676         sqlite3SrcListDelete(pParse->db, F);
   671    677       }else{
   672    678         Select *pSubquery;
   673    679         sqlite3SrcListShiftJoinType(F);

Changes to src/pcache1.c.

    98     98     unsigned int iKey;             /* Key value (page number) */
    99     99     u8 isBulkLocal;                /* This page from bulk local storage */
   100    100     u8 isAnchor;                   /* This is the PGroup.lru element */
   101    101     PgHdr1 *pNext;                 /* Next in hash table chain */
   102    102     PCache1 *pCache;               /* Cache that currently owns this page */
   103    103     PgHdr1 *pLruNext;              /* Next in LRU list of unpinned pages */
   104    104     PgHdr1 *pLruPrev;              /* Previous in LRU list of unpinned pages */
          105  +                                 /* NB: pLruPrev is only valid if pLruNext!=0 */
   105    106   };
   106    107   
   107    108   /*
   108    109   ** A page is pinned if it is not on the LRU list.  To be "pinned" means
   109    110   ** that the page is in active use and must not be deallocated.
   110    111   */
   111    112   #define PAGE_IS_PINNED(p)    ((p)->pLruNext==0)
................................................................................
   565    566     assert( PAGE_IS_UNPINNED(pPage) );
   566    567     assert( pPage->pLruNext );
   567    568     assert( pPage->pLruPrev );
   568    569     assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) );
   569    570     pPage->pLruPrev->pLruNext = pPage->pLruNext;
   570    571     pPage->pLruNext->pLruPrev = pPage->pLruPrev;
   571    572     pPage->pLruNext = 0;
   572         -  pPage->pLruPrev = 0;
          573  +  /* pPage->pLruPrev = 0;
          574  +  ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */
   573    575     assert( pPage->isAnchor==0 );
   574    576     assert( pPage->pCache->pGroup->lru.isAnchor==1 );
   575    577     pPage->pCache->nRecyclable--;
   576    578     return pPage;
   577    579   }
   578    580   
   579    581   
................................................................................
   903    905   
   904    906     if( pPage ){
   905    907       unsigned int h = iKey % pCache->nHash;
   906    908       pCache->nPage++;
   907    909       pPage->iKey = iKey;
   908    910       pPage->pNext = pCache->apHash[h];
   909    911       pPage->pCache = pCache;
   910         -    pPage->pLruPrev = 0;
   911    912       pPage->pLruNext = 0;
          913  +    /* pPage->pLruPrev = 0;
          914  +    ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */
   912    915       *(void **)pPage->page.pExtra = 0;
   913    916       pCache->apHash[h] = pPage;
   914    917       if( iKey>pCache->iMaxKey ){
   915    918         pCache->iMaxKey = iKey;
   916    919       }
   917    920     }
   918    921     return pPage;
................................................................................
  1064   1067    
  1065   1068     assert( pPage->pCache==pCache );
  1066   1069     pcache1EnterMutex(pGroup);
  1067   1070   
  1068   1071     /* It is an error to call this function if the page is already 
  1069   1072     ** part of the PGroup LRU list.
  1070   1073     */
  1071         -  assert( pPage->pLruPrev==0 && pPage->pLruNext==0 );
         1074  +  assert( pPage->pLruNext==0 );
  1072   1075     assert( PAGE_IS_PINNED(pPage) );
  1073   1076   
  1074   1077     if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){
  1075   1078       pcache1RemoveFromHash(pPage, 1);
  1076   1079     }else{
  1077   1080       /* Add the page to the PGroup LRU list. */
  1078   1081       PgHdr1 **ppFirst = &pGroup->lru.pLruNext;

Changes to src/prepare.c.

   727    727   ){
   728    728     int bFound = 0;     /* Non-zero if token is an identifier name. */
   729    729     int i, j;           /* Database and column loop indexes. */
   730    730     Schema *pSchema;    /* Schema for current database. */
   731    731     Hash *pHash;        /* Hash table of tables for current database. */
   732    732     HashElem *e;        /* Hash element for hash table iteration. */
   733    733     Table *pTab;        /* Database table for columns being checked. */
          734  +  char *zId;          /* Zero terminated name of the identifier */
          735  +  char zSpace[50];    /* Static space for the zero-terminated name */
   734    736   
   735         -  if( sqlite3IsRowidN(zToken, nToken) ){
   736         -    return 1;
          737  +  if( nToken<sizeof(zSpace) ){
          738  +    memcpy(zSpace, zToken, nToken);
          739  +    zSpace[nToken] = 0;
          740  +    zId = zSpace;
          741  +  }else{
          742  +    zId = sqlite3DbStrNDup(db, zToken, nToken);
          743  +    if( zId==0 ) return 0;
          744  +  }
          745  +  if( sqlite3IsRowid(zId) ){
          746  +    bFound = 1;
          747  +    goto done;
   737    748     }
   738    749     if( nToken>0 ){
   739    750       int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
   740         -    if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
          751  +    if( sqlite3FunctionSearch(hash, zId) ){
          752  +      bFound = 1;
          753  +      goto done;
          754  +    }
   741    755     }
   742    756     assert( db!=0 );
   743    757     sqlite3_mutex_enter(db->mutex);
   744    758     sqlite3BtreeEnterAll(db);
   745    759     for(i=0; i<db->nDb; i++){
   746    760       pHash = &db->aFunc;
   747         -    if( sqlite3HashFindN(pHash, zToken, nToken) ){
          761  +    if( sqlite3HashFind(pHash, zId) ){
   748    762         bFound = 1;
   749    763         break;
   750    764       }
   751    765       pSchema = db->aDb[i].pSchema;
   752    766       if( pSchema==0 ) continue;
   753    767       pHash = &pSchema->tblHash;
   754         -    if( sqlite3HashFindN(pHash, zToken, nToken) ){
          768  +    if( sqlite3HashFind(pHash, zId) ){
   755    769         bFound = 1;
   756    770         break;
   757    771       }
   758    772       for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
   759    773         pTab = sqliteHashData(e);
   760    774         if( pTab==0 ) continue;
   761    775         pHash = pTab->pColHash;
................................................................................
   769    783             }
   770    784           }else{
   771    785             *pRc = SQLITE_NOMEM_BKPT;
   772    786             bFound = 0;
   773    787             goto done;
   774    788           }
   775    789         }
   776         -      if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
          790  +      if( pHash && sqlite3HashFind(pHash, zId) ){
   777    791           bFound = 1;
   778    792           goto done;
   779    793         }
   780    794       }
   781    795     }
   782    796   done:
   783    797     sqlite3BtreeLeaveAll(db);
   784    798     sqlite3_mutex_leave(db->mutex);
          799  +  if( zId!=zSpace ) sqlite3DbFree(db, zId);
   785    800     return bFound;
   786    801   }
   787    802   
   788    803   /*
   789    804   ** Attempt to estimate the final output buffer size needed for the fully
   790    805   ** normalized version of the specified SQL string.  This should take into
   791    806   ** account any potential expansion that could occur (e.g. via IN clauses
   792    807   ** being expanded, etc).  This size returned is the total number of bytes
   793    808   ** including the NUL terminator.
   794    809   */
   795    810   static int estimateNormalizedSize(
   796    811     const char *zSql, /* The original SQL string */
   797         -  int nSql,         /* Length of original SQL string */
   798         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
          812  +  int nSql          /* Length of original SQL string */
   799    813   ){
   800    814     int nOut = nSql + 4;
   801    815     const char *z = zSql;
   802    816     while( nOut<nSql*5 ){
   803    817       while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
   804    818       if( z[0]==0 ) break;
   805    819       z++;
................................................................................
   846    860         zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
   847    861       }
   848    862     }
   849    863     *piOut = j;
   850    864   }
   851    865   
   852    866   /*
   853         -** Perform normalization of the SQL contained in the prepared statement and
   854         -** store the result in the zNormSql field.  The schema for the associated
   855         -** databases are consulted while performing the normalization in order to
   856         -** determine if a token appears to be an identifier.  All identifiers are
   857         -** left intact in the normalized SQL and all literals are replaced with a
   858         -** single '?'.
          867  +** Compute a normalization of the SQL given by zSql[0..nSql-1].  Return
          868  +** the normalization in space obtained from sqlite3DbMalloc().  Or return
          869  +** NULL if anything goes wrong or if zSql is NULL.
   859    870   */
   860         -void sqlite3Normalize(
          871  +char *sqlite3Normalize(
   861    872     Vdbe *pVdbe,      /* VM being reprepared */
   862    873     const char *zSql, /* The original SQL string */
   863         -  int nSql,         /* Size of the input string in bytes */
   864         -  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
          874  +  int nSql          /* Size of the input string in bytes */
   865    875   ){
   866    876     sqlite3 *db;           /* Database handle. */
   867    877     char *z;               /* The output string */
   868    878     int nZ;                /* Size of the output string in bytes */
   869    879     int i;                 /* Next character to read from zSql[] */
   870    880     int j;                 /* Next character to fill in on z[] */
   871    881     int tokenType = 0;     /* Type of the next token */
................................................................................
   872    882     int prevTokenType = 0; /* Type of the previous token, except spaces */
   873    883     int n;                 /* Size of the next token */
   874    884     int nParen = 0;        /* Nesting level of parenthesis */
   875    885     Hash inHash;           /* Table of parenthesis levels to output index. */
   876    886   
   877    887     db = sqlite3VdbeDb(pVdbe);
   878    888     assert( db!=0 );
   879         -  assert( pVdbe->zNormSql==0 );
   880         -  if( zSql==0 ) return;
   881         -  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
          889  +  if( zSql==0 ) return 0;
          890  +  nZ = estimateNormalizedSize(zSql, nSql);
   882    891     z = sqlite3DbMallocRawNN(db, nZ);
   883         -  if( z==0 ) return;
          892  +  if( z==0 ) goto normalizeError;
   884    893     sqlite3HashInit(&inHash);
   885    894     for(i=j=0; i<nSql && zSql[i]; i+=n){
   886    895       int flags = 0;
   887    896       if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
   888    897       n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
   889    898       switch( tokenType ){
   890    899         case TK_SPACE: {
   891    900           break;
   892    901         }
   893    902         case TK_ILLEGAL: {
   894         -        sqlite3DbFree(db, z);
   895         -        sqlite3HashClear(&inHash);
   896         -        return;
          903  +        goto normalizeError;
   897    904         }
   898    905         case TK_STRING:
   899    906         case TK_INTEGER:
   900    907         case TK_FLOAT:
   901    908         case TK_VARIABLE:
   902    909         case TK_BLOB: {
   903    910           z[j++] = '?';
................................................................................
   970    977             int i2 = i, n2 = n, rc = SQLITE_OK;
   971    978             if( nParen>0 ){
   972    979               assert( nParen<nSql );
   973    980               sqlite3HashInsert(&inHash, zSql+nParen, 0);
   974    981             }
   975    982             if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
   976    983             if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
   977         -            if( rc!=SQLITE_OK ){
   978         -              sqlite3DbFree(db, z);
   979         -              sqlite3HashClear(&inHash);
   980         -              return;
   981         -            }
          984  +            if( rc!=SQLITE_OK ) goto normalizeError;
   982    985               if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
   983    986                 z[j++] = '?';
   984    987                 break;
   985    988               }
   986    989             }
   987    990           }
   988    991           copyNormalizedToken(zSql, i, n, flags, z, &j);
................................................................................
   991    994       }
   992    995     }
   993    996     assert( j<nZ && "one" );
   994    997     while( j>0 && z[j-1]==' ' ){ j--; }
   995    998     if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
   996    999     z[j] = 0;
   997   1000     assert( j<nZ && "two" );
   998         -  pVdbe->zNormSql = z;
         1001  +  sqlite3HashClear(&inHash);
         1002  +  return z;
         1003  +
         1004  +normalizeError:
         1005  +  sqlite3DbFree(db, z);
   999   1006     sqlite3HashClear(&inHash);
         1007  +  return 0;
  1000   1008   }
  1001   1009   #endif /* SQLITE_ENABLE_NORMALIZE */
  1002   1010   
  1003   1011   /*
  1004   1012   ** Rerun the compilation of a statement after a schema change.
  1005   1013   **
  1006   1014   ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise,

Changes to src/resolve.c.

    76     76     db = pParse->db;
    77     77     pDup = sqlite3ExprDup(db, pOrig, 0);
    78     78     if( pDup!=0 ){
    79     79       if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery);
    80     80       if( pExpr->op==TK_COLLATE ){
    81     81         pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken);
    82     82       }
    83         -    ExprSetProperty(pDup, EP_Alias);
           83  +//    ExprSetProperty(pDup, EP_Alias);
    84     84   
    85     85       /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This 
    86     86       ** prevents ExprDelete() from deleting the Expr structure itself,
    87     87       ** allowing it to be repopulated by the memcpy() on the following line.
    88     88       ** The pExpr->u.zToken might point into memory that will be freed by the
    89     89       ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to
    90     90       ** make a copy of the token before doing the sqlite3DbFree().

Changes to src/shell.c.in.

  1003   1003     u8 autoEQP;            /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  1004   1004     u8 autoEQPtest;        /* autoEQP is in test mode */
  1005   1005     u8 statsOn;            /* True to display memory stats before each finalize */
  1006   1006     u8 scanstatsOn;        /* True to display scan stats before each finalize */
  1007   1007     u8 openMode;           /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */
  1008   1008     u8 doXdgOpen;          /* Invoke start/open/xdg-open in output_reset() */
  1009   1009     u8 nEqpLevel;          /* Depth of the EQP output graph */
         1010  +  u8 eTraceType;         /* SHELL_TRACE_* value for type of trace */
  1010   1011     unsigned mEqpLines;    /* Mask of veritical lines in the EQP output graph */
  1011   1012     int outCount;          /* Revert to stdout when reaching zero */
  1012   1013     int cnt;               /* Number of records displayed so far */
  1013   1014     FILE *out;             /* Write results here */
  1014   1015     FILE *traceOut;        /* Output for sqlite3_trace() */
  1015   1016     int nErr;              /* Number of errors seen */
  1016   1017     int mode;              /* An output mode setting */
................................................................................
  1062   1063   #define SHELL_OPEN_UNSPEC      0      /* No open-mode specified */
  1063   1064   #define SHELL_OPEN_NORMAL      1      /* Normal database file */
  1064   1065   #define SHELL_OPEN_APPENDVFS   2      /* Use appendvfs */
  1065   1066   #define SHELL_OPEN_ZIPFILE     3      /* Use the zipfile virtual table */
  1066   1067   #define SHELL_OPEN_READONLY    4      /* Open a normal database read-only */
  1067   1068   #define SHELL_OPEN_DESERIALIZE 5      /* Open using sqlite3_deserialize() */
  1068   1069   
         1070  +/* Allowed values for ShellState.eTraceType
         1071  +*/
         1072  +#define SHELL_TRACE_PLAIN      0      /* Show input SQL text */
         1073  +#define SHELL_TRACE_EXPANDED   1      /* Show expanded SQL text */
         1074  +#define SHELL_TRACE_NORMALIZED 2      /* Show normalized SQL text */
         1075  +
  1069   1076   /*
  1070   1077   ** These are the allowed shellFlgs values
  1071   1078   */
  1072   1079   #define SHFLG_Pagecache      0x00000001 /* The --pagecache option is used */
  1073   1080   #define SHFLG_Lookaside      0x00000002 /* Lookaside memory is used */
  1074   1081   #define SHFLG_Backslash      0x00000004 /* The --backslash option is used */
  1075   1082   #define SHFLG_PreserveRowid  0x00000008 /* .dump preserves rowid values */
................................................................................
  3488   3495   #ifndef SQLITE_NOHAVE_SYSTEM
  3489   3496     ".system CMD ARGS...      Run CMD ARGS... in a system shell",
  3490   3497   #endif
  3491   3498     ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
  3492   3499     ".testcase NAME           Begin redirecting output to 'testcase-out.txt'",
  3493   3500     ".timeout MS              Try opening locked tables for MS milliseconds",
  3494   3501     ".timer on|off            Turn SQL timer on or off",
  3495         -  ".trace FILE|off          Output each SQL statement as it is run",
         3502  +#ifndef SQLITE_OMIT_TRACE
         3503  +  ".trace ?OPTIONS?         Output each SQL statement as it is run",
         3504  +  "    FILE                    Send output to FILE",
         3505  +  "    stdout                  Send output to stdout",
         3506  +  "    stderr                  Send output to stderr",
         3507  +  "    off                     Disable tracing",
         3508  +  "    --expanded              Expand query parameters",
         3509  +#ifdef SQLITE_ENABLE_NORMALIZE
         3510  +  "    --normalized            Normal the SQL statements",
         3511  +#endif
         3512  +  "    --plain                 Show SQL as it is input",
         3513  +  "    --stmt                  Trace statement execution (SQLITE_TRACE_STMT)",
         3514  +  "    --profile               Profile statements (SQLITE_TRACE_PROFILE)",
         3515  +  "    --row                   Trace each row (SQLITE_TRACE_ROW)",
         3516  +  "    --close                 Trace connection close (SQLITE_TRACE_CLOSE)",
         3517  +#endif /* SQLITE_OMIT_TRACE */
  3496   3518     ".vfsinfo ?AUX?           Information about the top-level VFS",
  3497   3519     ".vfslist                 List all available VFSes",
  3498   3520     ".vfsname ?AUX?           Print the name of the VFS stack",
  3499   3521     ".width NUM1 NUM2 ...     Set column widths for \"column\" mode",
  3500   3522     "     Negative values right-justify",
  3501   3523   };
  3502   3524   
................................................................................
  3995   4017       if( f==0 ){
  3996   4018         utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
  3997   4019       }
  3998   4020     }
  3999   4021     return f;
  4000   4022   }
  4001   4023   
  4002         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
         4024  +#ifndef SQLITE_OMIT_TRACE
  4003   4025   /*
  4004   4026   ** A routine for handling output from sqlite3_trace().
  4005   4027   */
  4006   4028   static int sql_trace_callback(
  4007         -  unsigned mType,
  4008         -  void *pArg,
  4009         -  void *pP,
  4010         -  void *pX
  4011         -){
  4012         -  FILE *f = (FILE*)pArg;
  4013         -  UNUSED_PARAMETER(mType);
  4014         -  UNUSED_PARAMETER(pP);
  4015         -  if( f ){
  4016         -    const char *z = (const char*)pX;
  4017         -    int i = strlen30(z);
  4018         -    while( i>0 && z[i-1]==';' ){ i--; }
  4019         -    utf8_printf(f, "%.*s;\n", i, z);
         4029  +  unsigned mType,         /* The trace type */
         4030  +  void *pArg,             /* The ShellState pointer */
         4031  +  void *pP,               /* Usually a pointer to sqlite_stmt */
         4032  +  void *pX                /* Auxiliary output */
         4033  +){
         4034  +  ShellState *p = (ShellState*)pArg;
         4035  +  sqlite3_stmt *pStmt;
         4036  +  const char *zSql;
         4037  +  int nSql;
         4038  +  if( p->traceOut==0 ) return 0;
         4039  +  if( mType==SQLITE_TRACE_CLOSE ){
         4040  +    utf8_printf(p->traceOut, "-- closing database connection\n");
         4041  +    return 0;
         4042  +  }
         4043  +  if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){
         4044  +    zSql = (const char*)pX;
         4045  +  }else{
         4046  +    pStmt = (sqlite3_stmt*)pP;
         4047  +    switch( p->eTraceType ){
         4048  +      case SHELL_TRACE_EXPANDED: {
         4049  +        zSql = sqlite3_expanded_sql(pStmt);
         4050  +        break;
         4051  +      }
         4052  +#ifdef SQLITE_ENABLE_NORMALIZE
         4053  +      case SHELL_TRACE_NORMALIZED: {
         4054  +        zSql = sqlite3_normalized_sql(pStmt);
         4055  +        break;
         4056  +      }
         4057  +#endif
         4058  +      default: {
         4059  +        zSql = sqlite3_sql(pStmt);
         4060  +        break;
         4061  +      }
         4062  +    }
         4063  +  }
         4064  +  if( zSql==0 ) return 0;
         4065  +  nSql = strlen30(zSql);
         4066  +  while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; }
         4067  +  switch( mType ){
         4068  +    case SQLITE_TRACE_ROW:
         4069  +    case SQLITE_TRACE_STMT: {
         4070  +      utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql);
         4071  +      break;
         4072  +    }
         4073  +    case SQLITE_TRACE_PROFILE: {
         4074  +      sqlite3_int64 nNanosec = *(sqlite3_int64*)pX;
         4075  +      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec);
         4076  +      break;
         4077  +    }
  4020   4078     }
  4021   4079     return 0;
  4022   4080   }
  4023   4081   #endif
  4024   4082   
  4025   4083   /*
  4026   4084   ** A no-op routine that runs with the ".breakpoint" doc-command.  This is
................................................................................
  7834   7892         }
  7835   7893       }else{
  7836   7894         raw_printf(stderr, "Usage: .timer on|off\n");
  7837   7895         rc = 1;
  7838   7896       }
  7839   7897     }else
  7840   7898   
         7899  +#ifndef SQLITE_OMIT_TRACE
  7841   7900     if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){
         7901  +    int mType = 0;
         7902  +    int jj;
  7842   7903       open_db(p, 0);
  7843         -    if( nArg!=2 ){
  7844         -      raw_printf(stderr, "Usage: .trace FILE|off\n");
  7845         -      rc = 1;
  7846         -      goto meta_command_exit;
         7904  +    for(jj=1; jj<nArg; jj++){
         7905  +      const char *z = azArg[jj];
         7906  +      if( z[0]=='-' ){
         7907  +        if( optionMatch(z, "expanded") ){
         7908  +          p->eTraceType = SHELL_TRACE_EXPANDED;
         7909  +        }
         7910  +#ifdef SQLITE_ENABLE_NORMALIZE
         7911  +        else if( optionMatch(z, "normalized") ){
         7912  +          p->eTraceType = SHELL_TRACE_NORMALIZED;
         7913  +        }
         7914  +#endif
         7915  +        else if( optionMatch(z, "plain") ){
         7916  +          p->eTraceType = SHELL_TRACE_PLAIN;
         7917  +        }
         7918  +        else if( optionMatch(z, "profile") ){
         7919  +          mType |= SQLITE_TRACE_PROFILE;
         7920  +        }
         7921  +        else if( optionMatch(z, "row") ){
         7922  +          mType |= SQLITE_TRACE_ROW;
         7923  +        }
         7924  +        else if( optionMatch(z, "stmt") ){
         7925  +          mType |= SQLITE_TRACE_STMT;
         7926  +        }
         7927  +        else if( optionMatch(z, "close") ){
         7928  +          mType |= SQLITE_TRACE_CLOSE;
         7929  +        }
         7930  +        else {
         7931  +          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
         7932  +          rc = 1;
         7933  +          goto meta_command_exit;
         7934  +        }
         7935  +      }else{
         7936  +        output_file_close(p->traceOut);
         7937  +        p->traceOut = output_file_open(azArg[1], 0);
         7938  +      }
  7847   7939       }
  7848         -    output_file_close(p->traceOut);
  7849         -    p->traceOut = output_file_open(azArg[1], 0);
  7850         -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
  7851   7940       if( p->traceOut==0 ){
  7852   7941         sqlite3_trace_v2(p->db, 0, 0, 0);
  7853   7942       }else{
  7854         -      sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut);
         7943  +      if( mType==0 ) mType = SQLITE_TRACE_STMT;
         7944  +      sqlite3_trace_v2(p->db, mType, sql_trace_callback, p);
  7855   7945       }
  7856         -#endif
  7857   7946     }else
         7947  +#endif /* !defined(SQLITE_OMIT_TRACE) */
  7858   7948   
  7859   7949   #if SQLITE_USER_AUTHENTICATION
  7860   7950     if( c=='u' && strncmp(azArg[0], "user", n)==0 ){
  7861   7951       if( nArg<2 ){
  7862   7952         raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
  7863   7953         rc = 1;
  7864   7954         goto meta_command_exit;

Changes to src/sqliteInt.h.

  1368   1368                                  const char*);
  1369   1369   #endif
  1370   1370   
  1371   1371   #ifndef SQLITE_OMIT_DEPRECATED
  1372   1372   /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
  1373   1373   ** in the style of sqlite3_trace()
  1374   1374   */
  1375         -#define SQLITE_TRACE_LEGACY  0x80
         1375  +#define SQLITE_TRACE_LEGACY          0x40     /* Use the legacy xTrace */
         1376  +#define SQLITE_TRACE_XPROFILE        0x80     /* Use the legacy xProfile */
  1376   1377   #else
  1377         -#define SQLITE_TRACE_LEGACY  0
         1378  +#define SQLITE_TRACE_LEGACY          0
         1379  +#define SQLITE_TRACE_XPROFILE        0
  1378   1380   #endif /* SQLITE_OMIT_DEPRECATED */
         1381  +#define SQLITE_TRACE_NONLEGACY_MASK  0x0f     /* Normal flags */
  1379   1382   
  1380   1383   
  1381   1384   /*
  1382   1385   ** Each database connection is an instance of the following structure.
  1383   1386   */
  1384   1387   struct sqlite3 {
  1385   1388     sqlite3_vfs *pVfs;            /* OS Interface */
................................................................................
  1430   1433     int nVdbeWrite;               /* Number of active VDBEs that read and write */
  1431   1434     int nVdbeExec;                /* Number of nested calls to VdbeExec() */
  1432   1435     int nVDestroy;                /* Number of active OP_VDestroy operations */
  1433   1436     int nExtension;               /* Number of loaded extensions */
  1434   1437     void **aExtension;            /* Array of shared library handles */
  1435   1438     int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
  1436   1439     void *pTraceArg;                          /* Argument to the trace function */
         1440  +#ifndef SQLITE_OMIT_DEPRECATED
  1437   1441     void (*xProfile)(void*,const char*,u64);  /* Profiling function */
  1438   1442     void *pProfileArg;                        /* Argument to profile function */
         1443  +#endif
  1439   1444     void *pCommitArg;                 /* Argument to xCommitCallback() */
  1440   1445     int (*xCommitCallback)(void*);    /* Invoked at every commit. */
  1441   1446     void *pRollbackArg;               /* Argument to xRollbackCallback() */
  1442   1447     void (*xRollbackCallback)(void*); /* Invoked at every commit. */
  1443   1448     void *pUpdateArg;
  1444   1449     void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
  1445   1450   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
................................................................................
  4063   4068   void sqlite3UniqueConstraint(Parse*, int, Index*);
  4064   4069   void sqlite3RowidConstraint(Parse*, int, Table*);
  4065   4070   Expr *sqlite3ExprDup(sqlite3*,Expr*,int);
  4066   4071   ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
  4067   4072   SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
  4068   4073   IdList *sqlite3IdListDup(sqlite3*,IdList*);
  4069   4074   Select *sqlite3SelectDup(sqlite3*,Select*,int);
  4070         -#ifdef SQLITE_ENABLE_NORMALIZE
  4071         -FuncDef *sqlite3FunctionSearchN(int,const char*,int);
  4072         -#endif
         4075  +FuncDef *sqlite3FunctionSearch(int,const char*);
  4073   4076   void sqlite3InsertBuiltinFuncs(FuncDef*,int);
  4074   4077   FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
  4075   4078   void sqlite3RegisterBuiltinFunctions(void);
  4076   4079   void sqlite3RegisterDateTimeFunctions(void);
  4077   4080   void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
  4078   4081   int sqlite3SafetyCheckOk(sqlite3*);
  4079   4082   int sqlite3SafetyCheckSickOrOk(sqlite3*);
................................................................................
  4431   4434   int sqlite3VtabBegin(sqlite3 *, VTable *);
  4432   4435   FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
  4433   4436   sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
  4434   4437   int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
  4435   4438   int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
  4436   4439   void sqlite3ParserReset(Parse*);
  4437   4440   #ifdef SQLITE_ENABLE_NORMALIZE
  4438         -void sqlite3Normalize(Vdbe*, const char*, int, u8);
         4441  +char *sqlite3Normalize(Vdbe*, const char*, int);
  4439   4442   #endif
  4440   4443   int sqlite3Reprepare(Vdbe*);
  4441   4444   void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
  4442   4445   CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
  4443   4446   int sqlite3TempInMemory(const sqlite3*);
  4444   4447   const char *sqlite3JournalModename(int);
  4445   4448   #ifndef SQLITE_OMIT_WAL

Changes to src/vdbeapi.c.

    61     61   ** Invoke the profile callback.  This routine is only called if we already
    62     62   ** know that the profile callback is defined and needs to be invoked.
    63     63   */
    64     64   static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
    65     65     sqlite3_int64 iNow;
    66     66     sqlite3_int64 iElapse;
    67     67     assert( p->startTime>0 );
    68         -  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
           68  +  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
    69     69     assert( db->init.busy==0 );
    70     70     assert( p->zSql!=0 );
    71     71     sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
    72     72     iElapse = (iNow - p->startTime)*1000000;
           73  +#ifndef SQLITE_OMIT_DEPRECATED  	
    73     74     if( db->xProfile ){
    74     75       db->xProfile(db->pProfileArg, p->zSql, iElapse);
    75     76     }
           77  +#endif
    76     78     if( db->mTrace & SQLITE_TRACE_PROFILE ){
    77     79       db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
    78     80     }
    79     81     p->startTime = 0;
    80     82   }
    81     83   /*
    82     84   ** The checkProfileCallback(DB,P) macro checks to see if a profile callback
................................................................................
   614    616       }
   615    617   
   616    618       assert( db->nVdbeWrite>0 || db->autoCommit==0 
   617    619           || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
   618    620       );
   619    621   
   620    622   #ifndef SQLITE_OMIT_TRACE
   621         -    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
          623  +    if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0
   622    624           && !db->init.busy && p->zSql ){
   623    625         sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
   624    626       }else{
   625    627         assert( p->startTime==0 );
   626    628       }
   627    629   #endif
   628    630   
................................................................................
   641    643   #endif /* SQLITE_OMIT_EXPLAIN */
   642    644     {
   643    645       db->nVdbeExec++;
   644    646       rc = sqlite3VdbeExec(p);
   645    647       db->nVdbeExec--;
   646    648     }
   647    649   
          650  +  if( rc!=SQLITE_ROW ){
   648    651   #ifndef SQLITE_OMIT_TRACE
   649         -  /* If the statement completed successfully, invoke the profile callback */
   650         -  if( rc!=SQLITE_ROW ) checkProfileCallback(db, p);
          652  +    /* If the statement completed successfully, invoke the profile callback */
          653  +    checkProfileCallback(db, p);
   651    654   #endif
   652    655   
   653         -  if( rc==SQLITE_DONE && db->autoCommit ){
   654         -    assert( p->rc==SQLITE_OK );
   655         -    p->rc = doWalCallbacks(db);
   656         -    if( p->rc!=SQLITE_OK ){
   657         -      rc = SQLITE_ERROR;
          656  +    if( rc==SQLITE_DONE && db->autoCommit ){
          657  +      assert( p->rc==SQLITE_OK );
          658  +      p->rc = doWalCallbacks(db);
          659  +      if( p->rc!=SQLITE_OK ){
          660  +        rc = SQLITE_ERROR;
          661  +      }
   658    662       }
   659    663     }
   660    664   
   661    665     db->errCode = rc;
   662    666     if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
   663    667       p->rc = SQLITE_NOMEM_BKPT;
   664    668     }
................................................................................
  1747   1751   
  1748   1752   #ifdef SQLITE_ENABLE_NORMALIZE
  1749   1753   /*
  1750   1754   ** Return the normalized SQL associated with a prepared statement.
  1751   1755   */
  1752   1756   const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
  1753   1757     Vdbe *p = (Vdbe *)pStmt;
  1754         -  return p ? p->zNormSql : 0;
         1758  +  if( p==0 ) return 0;
         1759  +  if( p->zNormSql==0 && p->zSql!=0 ){
         1760  +    p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql));
         1761  +  }
         1762  +  return p->zNormSql;
  1755   1763   }
  1756   1764   #endif /* SQLITE_ENABLE_NORMALIZE */
  1757   1765   
  1758   1766   #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
  1759   1767   /*
  1760   1768   ** Allocate and populate an UnpackedRecord structure based on the serialized
  1761   1769   ** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure

Changes to src/vdbeaux.c.

    63     63       p->expmask = 0;
    64     64     }
    65     65     assert( p->zSql==0 );
    66     66     p->zSql = sqlite3DbStrNDup(p->db, z, n);
    67     67   #ifdef SQLITE_ENABLE_NORMALIZE
    68     68     assert( p->zNormSql==0 );
    69     69     if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
    70         -    sqlite3Normalize(p, p->zSql, n, prepFlags);
           70  +    p->zNormSql = sqlite3Normalize(p, p->zSql, n);
    71     71       assert( p->zNormSql!=0 || p->db->mallocFailed );
    72     72     }
    73     73   #endif
    74     74   }
    75     75   
    76     76   /*
    77     77   ** Swap all content between two VDBE structures.

Changes to test/altertab2.test.

     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #*************************************************************************
    11     11   #
    12     12   
    13     13   set testdir [file dirname $argv0]
    14     14   source $testdir/tester.tcl
    15         -set testprefix altertab
           15  +set testprefix altertab2
    16     16   
    17     17   # If SQLITE_OMIT_ALTERTABLE is defined, omit this file.
    18     18   ifcapable !altertable {
    19     19     finish_test
    20     20     return
    21     21   }
    22     22   
................................................................................
    81     81     SELECT sql FROM sqlite_master WHERE name LIKE 'c%';
    82     82   } {
    83     83     {CREATE TABLE c1(x REFERENCES "p3")}
    84     84     {CREATE TABLE c2(x, FOREIGN KEY (x) REFERENCES "p3")}
    85     85     {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))}
    86     86   }
    87     87   
           88  +#-------------------------------------------------------------------------
           89  +# Table name in WITH clauses that are part of views or triggers.
           90  +#
           91  +foreach {tn schema} {
           92  +  1 {
           93  +    CREATE TABLE log_entry(col1, y);
           94  +    CREATE INDEX i1 ON log_entry(col1);
           95  +  }
           96  +
           97  +  2 {
           98  +    CREATE TABLE t1(a, b, c);
           99  +    CREATE TABLE t2(x);
          100  +    CREATE TABLE log_entry(col1);
          101  +    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
          102  +      INSERT INTO t2 SELECT col1 FROM log_entry;
          103  +    END;
          104  +  }
          105  +
          106  +  3 {
          107  +    CREATE TABLE t1(a, b, c);
          108  +    CREATE TABLE t2(x);
          109  +    CREATE TABLE log_entry(col1);
          110  +    CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
          111  +      INSERT INTO t2
          112  +        WITH xyz(x) AS (SELECT col1 FROM log_entry)
          113  +        SELECT x FROM xyz;
          114  +    END;
          115  +  }
          116  +
          117  +  4 {
          118  +    CREATE TABLE log_entry(col1);
          119  +    CREATE VIEW ttt AS
          120  +        WITH xyz(x) AS (SELECT col1 FROM log_entry)
          121  +        SELECT x FROM xyz;
          122  +  }
          123  +} {
          124  +  reset_db
          125  +  do_execsql_test 3.$tn.1 $schema
          126  +  set expect [db eval "SELECT sql FROM sqlite_master"]
          127  +  set expect [string map {log_entry {"newname"}} $expect]
          128  +
          129  +  do_execsql_test 3.$tn.2 {
          130  +    ALTER TABLE log_entry RENAME TO newname;
          131  +    SELECT sql FROM sqlite_master;
          132  +  } $expect
          133  +
          134  +  reset_db
          135  +  do_execsql_test 3.$tn.3 $schema
          136  +  set expect [db eval "SELECT sql FROM sqlite_master"]
          137  +  set expect [string map {col1 newname} $expect]
          138  +
          139  +  do_execsql_test 3.$tn.4 {
          140  +    ALTER TABLE log_entry RENAME col1 TO newname;
          141  +    SELECT sql FROM sqlite_master;
          142  +  } $expect
          143  +}
    88    144   
    89    145   finish_test

Added test/fts4umlaut.test.

            1  +# 2018 December 3
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set testprefix fts4umlaut
           18  +
           19  +ifcapable !fts3 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(x);
           26  +  CREATE VIRTUAL TABLE t2 USING fts4(
           27  +      x, 
           28  +      tokenize=unicode61 "remove_diacritics=2"
           29  +  );
           30  +}
           31  +
           32  +foreach {tn q res1 res2} {
           33  +  1 "Hà Nội"                  0 1
           34  +  2 "Hà Noi"                  1 1
           35  +  3 "Ha Noi"                  1 1
           36  +  4 "Ha N\u1ed9i"             0 1
           37  +  5 "Ha N\u006fi"             1 1
           38  +  6 "Ha N\u006f\u0302i"       1 1
           39  +  7 "Ha N\u006f\u0323\u0302i" 1 1
           40  +} {
           41  +  do_execsql_test 1.$tn.1 {
           42  +    DELETE FROM t1;
           43  +    INSERT INTO t1(rowid, x) VALUES (1, 'Ha Noi');
           44  +    SELECT count(*) FROM t1 WHERE t1 MATCH $q
           45  +  } $res1
           46  +  do_execsql_test 1.$tn.2 {
           47  +    DELETE FROM t1;
           48  +    INSERT INTO t1(rowid, x) VALUES (1, $q);
           49  +    SELECT count(*) FROM t1 WHERE t1 MATCH 'Ha Noi'
           50  +  } $res1
           51  +
           52  +  do_execsql_test 1.$tn.2 {
           53  +    DELETE FROM t2;
           54  +    INSERT INTO t2(rowid, x) VALUES (1, 'Ha Noi');
           55  +    SELECT count(*) FROM t2 WHERE t2 MATCH $q
           56  +  } $res2
           57  +  do_execsql_test 1.$tn.2 {
           58  +    DELETE FROM t2;
           59  +    INSERT INTO t2(rowid, x) VALUES (1, $q);
           60  +    SELECT count(*) FROM t2 WHERE t2 MATCH 'Ha Noi'
           61  +  } $res2
           62  +}
           63  +
           64  +finish_test
           65  +

Changes to test/json101.test.

   809    809   do_execsql_test json-14.160 {
   810    810     SELECT fullkey FROM json_tree('"hello"');
   811    811   } {$}
   812    812   do_execsql_test json-14.170 {
   813    813     SELECT fullkey FROM json_tree('null');
   814    814   } {$}
   815    815   
   816         -
          816  +# 2018-12-03
          817  +# Make sure the table-valued functions contained within parentheses
          818  +# work correctly.
          819  +#
          820  +# Bug reported via private email. See TH3 for more information.
          821  +#
          822  +do_execsql_test json-15.100 {
          823  +  SELECT * FROM JSON_EACH('{"a":1, "b":2}');
          824  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          825  +do_execsql_test json-15.110 {
          826  +  SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
          827  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          828  +do_execsql_test json-15.120 {
          829  +  SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
          830  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
          831  +do_execsql_test json-15.130 {
          832  +  SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
          833  +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
   817    834   
   818    835   finish_test

Changes to test/shell4.test.

   103    103   }]
   104    104     list [regexp {Memory Used} $res] \
   105    105          [regexp {Heap Usage} $res] \
   106    106          [regexp {Autoindex Inserts} $res]
   107    107   } {1 1 1}
   108    108   
   109    109   do_test shell4-2.1 {
   110         -  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace"
   111         -} {1 {Usage: .trace FILE|off}}
          110  +  catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown"
          111  +} {1 {Unknown option "--unknown" on ".trace"}}
   112    112   do_test shell4-2.2 {
   113    113     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n"
   114    114   } {0 {}}
   115    115   do_test shell4-2.3 {
   116         -  catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n"
   117         -} {/^1 {PRAGMA.*Usage:.*}$/}
          116  +  catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
          117  +} {/^0 {PRAGMA.*}$/}
   118    118   ifcapable trace {
   119    119   do_test shell4-2.4 {
   120    120     catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
   121    121   } {0 {CREATE TABLE t1(x);
   122    122   SELECT * FROM t1;}}
   123    123   do_test shell4-2.5 {
   124    124     catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;"

Changes to test/tabfunc01.test.

   120    120     SELECT * FROM temp.generate_series(1,4)
   121    121   } {1 2 3 4}
   122    122   do_execsql_test tabfunc01-4.3 {
   123    123     ATTACH ':memory:' AS aux1;
   124    124     CREATE TABLE aux1.t1(a,b,c);
   125    125     SELECT * FROM aux1.generate_series(1,4)
   126    126   } {1 2 3 4}
          127  +
          128  +# 2018-12-03: Fix bug reported by by private email.
          129  +do_execsql_test tabfunc01-4.4 {
          130  +  SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
          131  +} {1 3 5}
   127    132   
   128    133   # The next series of tests is verifying that virtual table are able
   129    134   # to optimize the IN operator, even on terms that are not marked "omit".
   130    135   # When the generate_series virtual table is compiled for the testfixture,
   131    136   # the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
   132    137   # causes the xBestIndex method of generate_series to leave the
   133    138   # sqlite3_index_constraint_usage.omit flag set to 0, which should cause

Added tool/index_usage.c.

            1  +/*
            2  +** 2018-12-04
            3  +**
            4  +** The author disclaims copyright to this source code.  In place of
            5  +** a legal notice, here is a blessing:
            6  +**
            7  +**    May you do good and not evil.
            8  +**    May you find forgiveness for yourself and forgive others.
            9  +**    May you share freely, never taking more than you give.
           10  +**
           11  +*************************************************************************
           12  +** 
           13  +** This file implements a utility program used to help determine which
           14  +** indexes in a database schema are used and unused, and how often specific
           15  +** indexes are used.
           16  +*/
           17  +#include "sqlite3.h"
           18  +#include <stdio.h>
           19  +#include <stdlib.h>
           20  +#include <assert.h>
           21  +#include <string.h>
           22  +
           23  +static void usage(const char *argv0){
           24  +  printf("Usage: %s DATABASE LOG\n\n", argv0);
           25  +  printf(
           26  +    "DATABASE is an SQLite database against which various statements\n"
           27  +    "have been run.  The SQL text is stored in LOG.  LOG is an SQLite\n"
           28  +    "database with this schema:\n"
           29  +    "\n"
           30  +    "    CREATE TABLE sqllog(sql TEXT);\n"
           31  +    "\n"
           32  +    "This utility program analyzes statements contained in LOG and prints\n"
           33  +    "a report showing how many times each index in DATABASE is used by the\n"
           34  +    "statements in LOG.\n"
           35  +    "\n"
           36  +    "DATABASE only needs to contain the schema used by the statements in\n"
           37  +    "LOG. The content can be removed from DATABASE.\n"
           38  +  );
           39  +  printf("\nAnalysis will be done by SQLite version %s dated %.20s\n"
           40  +         "checkin number %.40s. Different versions\n"
           41  +         "of SQLite might use different indexes.\n",
           42  +         sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21);
           43  +  exit(1);
           44  +}
           45  +
           46  +int main(int argc, char **argv){
           47  +  sqlite3 *db = 0;          /* The main database */
           48  +  sqlite3_stmt *pStmt = 0;  /* a query */
           49  +  char *zSql;
           50  +  int nErr = 0;
           51  +  int rc;
           52  +
           53  +  if( argc!=3 ) usage(argv[0]);
           54  +  rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0);
           55  +  if( rc ){
           56  +    printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db));
           57  +    goto errorOut;
           58  +  }
           59  +  rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0);
           60  +  if( rc ){
           61  +    printf("Cannot read the schema from \"%s\" - %s\n", argv[1],
           62  +           sqlite3_errmsg(db));
           63  +    goto errorOut;
           64  +  }
           65  +  sqlite3_finalize(pStmt);
           66  +  pStmt = 0;
           67  +  rc = sqlite3_exec(db, 
           68  +     "CREATE TABLE temp.idxu(\n"
           69  +     "  tbl TEXT,\n"
           70  +     "  idx TEXT,\n"
           71  +     "  cnt INT,\n"
           72  +     "  PRIMARY KEY(idx)\n"
           73  +     ") WITHOUT ROWID;", 0, 0, 0);
           74  +  if( rc ){
           75  +    printf("Cannot create the result table - %s\n",
           76  +           sqlite3_errmsg(db));
           77  +    goto errorOut;
           78  +  }
           79  +  rc = sqlite3_exec(db,
           80  +     "INSERT INTO temp.idxu(tbl,idx,cnt)"
           81  +     " SELECT tbl_name, name, 0 FROM sqlite_master"
           82  +     " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0);
           83  +
           84  +  /* Open the LOG database */
           85  +  zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]);
           86  +  rc = sqlite3_exec(db, zSql, 0, 0, 0);
           87  +  sqlite3_free(zSql);
           88  +  if( rc ){
           89  +    printf("Cannot open the LOG database \"%s\" - %s\n",
           90  +           argv[2], sqlite3_errmsg(db));
           91  +    goto errorOut;
           92  +  }
           93  +  rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog",
           94  +                          -1, &pStmt, 0);
           95  +  if( rc ){
           96  +    printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n",
           97  +           argv[2], sqlite3_errmsg(db));
           98  +    goto errorOut;
           99  +  }
          100  +
          101  +  /* Update the counts based on LOG */
          102  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          103  +    const char *zLog = (const char*)sqlite3_column_text(pStmt, 0);
          104  +    sqlite3_stmt *pS2;
          105  +    if( zLog==0 ) continue;
          106  +    zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog);
          107  +    rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0);
          108  +    sqlite3_free(zSql);
          109  +    if( rc ){
          110  +      printf("Cannot compile LOG entry %d (%s): %s\n",
          111  +             sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db));
          112  +      nErr++;
          113  +    }else{
          114  +      while( sqlite3_step(pS2)==SQLITE_ROW ){
          115  +        const char *zExplain = (const char*)sqlite3_column_text(pS2,3);
          116  +        const char *z1, *z2;
          117  +        int n;
          118  +        /* printf("EXPLAIN: %s\n", zExplain); */
          119  +        z1 = strstr(zExplain, " USING INDEX ");
          120  +        if( z1==0 ) continue;
          121  +        z1 += 13;
          122  +        for(z2=z1+1; z2[1] && z2[1]!='('; z2++){}
          123  +        n = z2 - z1;
          124  +        zSql = sqlite3_mprintf(
          125  +          "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1
          126  +        );
          127  +        /* printf("sql: %s\n", zSql); */
          128  +        sqlite3_exec(db, zSql, 0, 0, 0);
          129  +        sqlite3_free(zSql);
          130  +      }
          131  +    }
          132  +    sqlite3_finalize(pS2);
          133  +  }
          134  +  sqlite3_finalize(pStmt);
          135  +
          136  +  /* Generate the report */
          137  +  rc = sqlite3_prepare_v2(db,
          138  +     "SELECT tbl, idx, cnt, "
          139  +     "   (SELECT group_concat(name,',') FROM pragma_index_info(idx))"
          140  +     " FROM temp.idxu, main.sqlite_master"
          141  +     " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name"
          142  +     "   AND temp.idxu.idx=main.sqlite_master.name"
          143  +     " ORDER BY cnt DESC, tbl, idx",
          144  +     -1, &pStmt, 0);
          145  +  if( rc ){
          146  +    printf("Cannot query the result table - %s\n",
          147  +           sqlite3_errmsg(db));
          148  +    goto errorOut;
          149  +  }
          150  +  while( sqlite3_step(pStmt)==SQLITE_ROW ){
          151  +    printf("%10d %s on %s(%s)\n", 
          152  +       sqlite3_column_int(pStmt, 2),
          153  +       sqlite3_column_text(pStmt, 1),
          154  +       sqlite3_column_text(pStmt, 0),
          155  +       sqlite3_column_text(pStmt, 3));
          156  +  }
          157  +  sqlite3_finalize(pStmt);
          158  +  pStmt = 0;
          159  +
          160  +errorOut:
          161  +  sqlite3_finalize(pStmt);
          162  +  sqlite3_close(db);
          163  +  return nErr;
          164  +}

Changes to tool/lemon.c.

  4586   4586     }
  4587   4587     tplt_xfer(lemp->name,in,out,&lineno);
  4588   4588   
  4589   4589     /* Generate code which executes whenever the parser stack overflows */
  4590   4590     tplt_print(out,lemp,lemp->overflow,&lineno);
  4591   4591     tplt_xfer(lemp->name,in,out,&lineno);
  4592   4592   
  4593         -  /* Generate the table of rule information
         4593  +  /* Generate the tables of rule information.  yyRuleInfoLhs[] and
         4594  +  ** yyRuleInfoNRhs[].
  4594   4595     **
  4595   4596     ** Note: This code depends on the fact that rules are number
  4596   4597     ** sequentually beginning with 0.
  4597   4598     */
  4598   4599     for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
  4599         -    fprintf(out,"  { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i);
         4600  +    fprintf(out,"  %4d,  /* (%d) ", rp->lhs->index, i);
         4601  +     rule_print(out, rp);
         4602  +    fprintf(out," */\n"); lineno++;
         4603  +  }
         4604  +  tplt_xfer(lemp->name,in,out,&lineno);
         4605  +  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
         4606  +    fprintf(out,"  %3d,  /* (%d) ", -rp->nrhs, i);
  4600   4607       rule_print(out, rp);
  4601   4608       fprintf(out," */\n"); lineno++;
  4602   4609     }
  4603   4610     tplt_xfer(lemp->name,in,out,&lineno);
  4604   4611   
  4605   4612     /* Generate code which execution during each REDUCE action */
  4606   4613     i = 0;

Changes to tool/lempar.c.

   682    682     yytos = yypParser->yytos;
   683    683     yytos->stateno = yyNewState;
   684    684     yytos->major = yyMajor;
   685    685     yytos->minor.yy0 = yyMinor;
   686    686     yyTraceShift(yypParser, yyNewState, "Shift");
   687    687   }
   688    688   
   689         -/* The following table contains information about every rule that
   690         -** is used during the reduce.
   691         -*/
   692         -static const struct {
   693         -  YYCODETYPE lhs;       /* Symbol on the left-hand side of the rule */
   694         -  signed char nrhs;     /* Negative of the number of RHS symbols in the rule */
   695         -} yyRuleInfo[] = {
          689  +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side
          690  +** of that rule */
          691  +static const YYCODETYPE yyRuleInfoLhs[] = {
          692  +%%
          693  +};
          694  +
          695  +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
          696  +** of symbols on the right-hand side of that rule. */
          697  +static const signed char yyRuleInfoNRhs[] = {
   696    698   %%
   697    699   };
   698    700   
   699    701   static void yy_accept(yyParser*);  /* Forward Declaration */
   700    702   
   701    703   /*
   702    704   ** Perform a reduce action and the shift that must immediately
................................................................................
   721    723     int yysize;                     /* Amount to pop the stack */
   722    724     ParseARG_FETCH
   723    725     (void)yyLookahead;
   724    726     (void)yyLookaheadToken;
   725    727     yymsp = yypParser->yytos;
   726    728   #ifndef NDEBUG
   727    729     if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
   728         -    yysize = yyRuleInfo[yyruleno].nrhs;
          730  +    yysize = yyRuleInfoNRhs[yyruleno];
   729    731       if( yysize ){
   730    732         fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n",
   731    733           yyTracePrompt,
   732    734           yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno);
   733    735       }else{
   734    736         fprintf(yyTraceFILE, "%sReduce %d [%s].\n",
   735    737           yyTracePrompt, yyruleno, yyRuleName[yyruleno]);
................................................................................
   736    738       }
   737    739     }
   738    740   #endif /* NDEBUG */
   739    741   
   740    742     /* Check that the stack is large enough to grow by a single entry
   741    743     ** if the RHS of the rule is empty.  This ensures that there is room
   742    744     ** enough on the stack to push the LHS value */
   743         -  if( yyRuleInfo[yyruleno].nrhs==0 ){
          745  +  if( yyRuleInfoNRhs[yyruleno]==0 ){
   744    746   #ifdef YYTRACKMAXSTACKDEPTH
   745    747       if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
   746    748         yypParser->yyhwm++;
   747    749         assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
   748    750       }
   749    751   #endif
   750    752   #if YYSTACKDEPTH>0 
................................................................................
   778    780     **  #line <lineno> <thisfile>
   779    781     **     break;
   780    782     */
   781    783   /********** Begin reduce actions **********************************************/
   782    784   %%
   783    785   /********** End reduce actions ************************************************/
   784    786     };
   785         -  assert( yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
   786         -  yygoto = yyRuleInfo[yyruleno].lhs;
   787         -  yysize = yyRuleInfo[yyruleno].nrhs;
          787  +  assert( yyruleno<sizeof(yyRuleInfoLhs)/sizeof(yyRuleInfoLhs[0]) );
          788  +  yygoto = yyRuleInfoLhs[yyruleno];
          789  +  yysize = yyRuleInfoNRhs[yyruleno];
   788    790     yyact = yy_find_reduce_action(yymsp[yysize].stateno,(YYCODETYPE)yygoto);
   789    791   
   790    792     /* There are no SHIFTREDUCE actions on nonterminals because the table
   791    793     ** generator has simplified them to pure REDUCE actions. */
   792    794     assert( !(yyact>YY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) );
   793    795   
   794    796     /* It is not possible for a REDUCE to be followed by an error */