/ Check-in [7b9886f8]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Tighter encoding of the keyword hash table in the tokenizer. (CVS 2028)
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 7b9886f8d4db366bc7dbf25495f0d3b907d25689
User & Date: drh 2004-10-23 05:10:18
Context
2004-10-25
20:33
Minor optimizations in the pragma module. (CVS 2029) check-in: 63efd50a user: drh tags: trunk
2004-10-23
05:10
Tighter encoding of the keyword hash table in the tokenizer. (CVS 2028) check-in: 7b9886f8 user: drh tags: trunk
2004-10-22
20:29
Add the experimental and scary pragma "writable_schema". (CVS 2027) check-in: 39f7870a user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to src/tokenize.c.

    11     11   *************************************************************************
    12     12   ** An tokenizer for SQL
    13     13   **
    14     14   ** This file contains C code that splits an SQL input string up into
    15     15   ** individual tokens and sends those tokens one-by-one over to the
    16     16   ** parser for analysis.
    17     17   **
    18         -** $Id: tokenize.c,v 1.91 2004/10/07 19:03:01 drh Exp $
           18  +** $Id: tokenize.c,v 1.92 2004/10/23 05:10:18 drh Exp $
    19     19   */
    20     20   #include "sqliteInt.h"
    21     21   #include "os.h"
    22     22   #include <ctype.h>
    23     23   #include <stdlib.h>
    24     24   
    25     25   /*
................................................................................
    31     31   ** mkkeywordhash.c, located in the tool subdirectory of the distribution.
    32     32   ** The output of the mkkeywordhash.c program was manually cut and pasted
    33     33   ** into this file.  When the set of keywords for SQLite changes, you
    34     34   ** must modify the mkkeywordhash.c program (to add or remove keywords from
    35     35   ** the data tables) then rerun that program to regenerate this function.
    36     36   */
    37     37   int sqlite3KeywordCode(const char *z, int n){
    38         -  static const char zText[519] =
    39         -    "ABORTAFTERALLANDASCATTACHBEFOREBEGINBETWEENBYCASCADECASECHECK"
    40         -    "COLLATECOMMITCONFLICTCONSTRAINTCREATECROSSDATABASEDEFAULTDEFERRABLE"
    41         -    "DEFERREDDELETEDESCDETACHDISTINCTDROPEACHELSEENDEXCEPTEXCLUSIVE"
    42         -    "EXPLAINFAILFOREIGNFROMFULLGLOBGROUPHAVINGIGNOREIMMEDIATEINDEX"
    43         -    "INITIALLYINNERINSERTINSTEADINTERSECTINTOISNULLJOINKEYLEFTLIKE"
    44         -    "LIMITMATCHNATURALNOTNULLNULLOFFSETONORDEROUTERPRAGMAPRIMARYRAISE"
    45         -    "REFERENCESREPLACERESTRICTRIGHTROLLBACKROWSELECTSETSTATEMENTTABLE"
    46         -    "TEMPORARYTHENTRANSACTIONTRIGGERUNIONUNIQUEUPDATEUSINGVACUUMVALUES"
    47         -    "VIEWWHENWHERE";
           38  +  static const char zText[443] =
           39  +    "ABORTABLEFTEMPORARYAFTERAISELECTHENDATABASEACHECKEYANDEFAULTRANSACTION"
           40  +    "ATURALIKELSEXCEPTRIGGEREFERENCESTATEMENTATTACHAVINGLOBEFOREIGN"
           41  +    "OREPLACEXCLUSIVEXPLAINDEXBEGINITIALLYBETWEENOTNULLIMITBYCASCADE"
           42  +    "FERRABLECASECOLLATECOMMITCONFLICTCONSTRAINTERSECTCREATECROSSDEFERRED"
           43  +    "ELETEDESCDETACHDISTINCTDROPRAGMATCHFAILFROMFULLGROUPDATEIMMEDIATE"
           44  +    "INNERESTRICTINSERTINSTEADINTOFFSETISNULLJOINORDERIGHTOUTEROLLBACK"
           45  +    "PRIMARYROWHENUNIONUNIQUEUSINGVACUUMVALUESVIEWHERE";
    48     46     static const unsigned char aHash[154] = {
    49         -       0,  75,  82,   0,   0,  97,  80,   0,  83,   0,   0,   0,   0,
    50         -       0,   0,   6,   0,  95,   4,   0,   0,   0,   0,   0,   0,   0,
    51         -       0,  96,  86,   8,   0,  26,  13,   7,  19,  15,   0,   0,  32,
    52         -      25,   0,  21,  31,  41,   0,   0,   0,  34,  27,   0,   0,  30,
    53         -       0,   0,   0,   9,   0,  10,   0,   0,   0,   0,  51,   0,  44,
    54         -      43,   0,  45,  40,   0,  29,  39,  35,   0,   0,  20,   0,  59,
    55         -       0,  16,   0,  17,   0,  18,   0,  55,  42,  72,   0,  33,   0,
    56         -       0,  61,  66,  56,   0,   0,   0,   0,   0,   0,   0,  54,   0,
    57         -       0,   0,   0,   0,  74,  50,  76,  64,  52,   0,   0,   0,   0,
    58         -      68,  84,   0,  47,   0,  58,  60,  92,   0,   0,  48,   0,  93,
    59         -       0,  63,  71,  98,   0,   0,   0,   0,   0,  67,   0,   0,   0,
    60         -       0,  87,   0,   0,   0,   0,   0,  90,  88,   0,  94,
           47  +       0,  26,  82,   0,   0,  91,  90,   0,  27,   0,   0,   0,   0,
           48  +       0,   0,  49,   0,  96,  17,   0,   0,   0,   0,   0,   0,   0,
           49  +       0,  97,   5,  31,   0,  62,  51,  28,  58,  52,   0,   0,  60,
           50  +      61,   0,  12,  41,  50,   0,   0,   0,  36,  63,   0,   0,  15,
           51  +       0,   0,   0,  39,   0,  42,   0,   0,   0,   0,  78,   0,  34,
           52  +      29,   0,  74,  71,   0,  66,  70,  37,   0,   0,  59,   0,  33,
           53  +       0,  53,   0,  54,   0,  55,   0,  83,  72,  67,   0,  24,   0,
           54  +       0,  79,  80,  84,   0,   0,   0,   0,   0,   0,   0,  75,   0,
           55  +       0,   0,   0,   0,  45,  77,  35,  44,  57,   0,   0,   0,   0,
           56  +      20,   2,   0,  38,   0,   3,  46,  93,   0,   0,  40,   0,  94,
           57  +       0,  43,  87,  98,   0,   0,   0,   0,   0,  81,   0,   0,   0,
           58  +       0,  10,   0,   0,   0,   0,   0,  92,  19,   0,  95,
    61     59     };
    62     60     static const unsigned char aNext[98] = {
    63         -       0,   0,   0,   0,   2,   0,   0,   0,   0,   0,   0,   0,   0,
    64         -       0,  12,   0,   0,   0,   0,   0,   0,  11,   0,   0,   0,   0,
    65         -       0,   0,   0,  14,   3,  24,   0,   0,   0,   1,  22,   0,   0,
    66         -      36,  23,  28,   0,   0,   0,   0,   0,   0,   0,   0,   5,   0,
    67         -       0,  49,  37,   0,   0,   0,  38,   0,  53,   0,  57,  62,   0,
    68         -       0,   0,   0,   0,   0,  70,  46,   0,  65,   0,   0,   0,   0,
    69         -      69,  73,   0,  77,   0,   0,   0,   0,   0,   0,  81,  85,   0,
    70         -      91,  79,  78,   0,   0,  89,   0,
           61  +       0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,
           62  +       0,  14,   0,   0,   0,   0,   0,   0,   0,   0,   0,   9,   0,
           63  +       0,   0,   0,   0,   0,  18,  22,   0,   0,   0,   0,   0,   0,
           64  +       0,  23,   0,  16,  21,   8,   0,  32,   0,   0,  30,   0,  48,
           65  +       0,   0,   0,   0,   0,   0,   0,  11,   0,   0,   0,   0,   0,
           66  +       0,  56,   0,   1,   0,  69,  64,   0,   0,  65,   0,   0,  13,
           67  +      68,   0,   0,  76,  47,   0,   0,   0,  85,   6,   0,  89,  25,
           68  +       4,  73,  88,  86,   0,   0,   0,
    71     69     };
    72     70     static const unsigned char aLen[98] = {
    73         -       5,   5,   3,   3,   2,   3,   6,   6,   5,   7,   2,   7,   4,
    74         -       5,   7,   6,   8,  10,   6,   5,   8,   7,  10,   8,   6,   4,
    75         -       6,   8,   4,   4,   4,   3,   6,   9,   7,   4,   3,   7,   4,
    76         -       4,   4,   5,   6,   6,   9,   2,   5,   9,   5,   6,   7,   9,
    77         -       4,   2,   6,   4,   3,   4,   4,   5,   5,   7,   3,   7,   4,
    78         -       2,   6,   2,   2,   5,   5,   6,   7,   5,  10,   7,   8,   5,
    79         -       8,   3,   6,   3,   9,   5,   4,   9,   4,  11,   7,   5,   6,
    80         -       6,   5,   6,   6,   4,   4,   5,
           71  +       5,   5,   4,   4,   9,   2,   5,   5,   6,   4,   3,   8,   2,
           72  +       4,   5,   3,   3,   7,  11,   2,   7,   4,   4,   6,   7,  10,
           73  +       9,   6,   6,   4,   6,   3,   7,   6,   7,   9,   7,   5,   5,
           74  +       9,   3,   7,   3,   7,   4,   5,   2,   7,   3,  10,   4,   7,
           75  +       6,   8,  10,   2,   9,   6,   5,   8,   6,   4,   6,   8,   2,
           76  +       4,   6,   5,   4,   4,   4,   5,   6,   9,   5,   8,   6,   7,
           77  +       4,   2,   6,   3,   6,   4,   5,   5,   5,   8,   7,   3,   4,
           78  +       5,   6,   5,   6,   6,   4,   5,
    81     79     };
    82     80     static const unsigned short int aOffset[98] = {
    83         -       0,   5,  10,  13,  16,  16,  19,  25,  31,  36,  43,  45,  52,
    84         -      56,  61,  68,  74,  82,  92,  98, 103, 111, 118, 128, 136, 142,
    85         -     146, 152, 160, 164, 168, 172, 175, 181, 190, 197, 201, 201, 208,
    86         -     212, 216, 220, 225, 231, 237, 246, 246, 251, 260, 265, 271, 278,
    87         -     287, 291, 291, 297, 301, 304, 308, 312, 317, 322, 329, 329, 336,
    88         -     340, 340, 346, 348, 348, 353, 358, 364, 371, 376, 386, 393, 401,
    89         -     406, 414, 417, 423, 426, 435, 440, 440, 449, 453, 464, 471, 476,
    90         -     482, 488, 493, 499, 505, 509, 513,
           81  +       0,   4,   7,  10,  10,  14,  19,  23,  26,  31,  33,  35,  40,
           82  +      42,  44,  48,  51,  53,  59,  68,  69,  75,  78,  81,  86,  92,
           83  +     101, 110, 115, 120, 123, 125, 125, 129, 133, 139, 147, 152, 157,
           84  +     160, 165, 169, 175, 175, 178, 181, 186, 188, 189, 193, 203, 207,
           85  +     214, 220, 228, 235, 235, 244, 250, 255, 262, 268, 272, 278, 279,
           86  +     286, 289, 293, 298, 302, 306, 310, 313, 319, 328, 332, 340, 346,
           87  +     353, 356, 356, 359, 362, 368, 372, 376, 381, 385, 393, 400, 402,
           88  +     406, 411, 417, 422, 428, 434, 437,
    91     89     };
    92     90     static const unsigned char aCode[98] = {
    93         -    TK_ABORT,      TK_AFTER,      TK_ALL,        TK_AND,        TK_AS,         
    94         -    TK_ASC,        TK_ATTACH,     TK_BEFORE,     TK_BEGIN,      TK_BETWEEN,    
    95         -    TK_BY,         TK_CASCADE,    TK_CASE,       TK_CHECK,      TK_COLLATE,    
    96         -    TK_COMMIT,     TK_CONFLICT,   TK_CONSTRAINT, TK_CREATE,     TK_JOIN_KW,    
    97         -    TK_DATABASE,   TK_DEFAULT,    TK_DEFERRABLE, TK_DEFERRED,   TK_DELETE,     
    98         -    TK_DESC,       TK_DETACH,     TK_DISTINCT,   TK_DROP,       TK_EACH,       
    99         -    TK_ELSE,       TK_END,        TK_EXCEPT,     TK_EXCLUSIVE,  TK_EXPLAIN,    
   100         -    TK_FAIL,       TK_FOR,        TK_FOREIGN,    TK_FROM,       TK_JOIN_KW,    
   101         -    TK_GLOB,       TK_GROUP,      TK_HAVING,     TK_IGNORE,     TK_IMMEDIATE,  
   102         -    TK_IN,         TK_INDEX,      TK_INITIALLY,  TK_JOIN_KW,    TK_INSERT,     
   103         -    TK_INSTEAD,    TK_INTERSECT,  TK_INTO,       TK_IS,         TK_ISNULL,     
   104         -    TK_JOIN,       TK_KEY,        TK_JOIN_KW,    TK_LIKE,       TK_LIMIT,      
   105         -    TK_MATCH,      TK_JOIN_KW,    TK_NOT,        TK_NOTNULL,    TK_NULL,       
   106         -    TK_OF,         TK_OFFSET,     TK_ON,         TK_OR,         TK_ORDER,      
   107         -    TK_JOIN_KW,    TK_PRAGMA,     TK_PRIMARY,    TK_RAISE,      TK_REFERENCES, 
   108         -    TK_REPLACE,    TK_RESTRICT,   TK_JOIN_KW,    TK_ROLLBACK,   TK_ROW,        
   109         -    TK_SELECT,     TK_SET,        TK_STATEMENT,  TK_TABLE,      TK_TEMP,       
   110         -    TK_TEMP,       TK_THEN,       TK_TRANSACTION,TK_TRIGGER,    TK_UNION,      
   111         -    TK_UNIQUE,     TK_UPDATE,     TK_USING,      TK_VACUUM,     TK_VALUES,     
   112         -    TK_VIEW,       TK_WHEN,       TK_WHERE,      
           91  +    TK_ABORT,      TK_TABLE,      TK_JOIN_KW,    TK_TEMP,       TK_TEMP,       
           92  +    TK_OR,         TK_AFTER,      TK_RAISE,      TK_SELECT,     TK_THEN,       
           93  +    TK_END,        TK_DATABASE,   TK_AS,         TK_EACH,       TK_CHECK,      
           94  +    TK_KEY,        TK_AND,        TK_DEFAULT,    TK_TRANSACTION,TK_ON,         
           95  +    TK_JOIN_KW,    TK_LIKE,       TK_ELSE,       TK_EXCEPT,     TK_TRIGGER,    
           96  +    TK_REFERENCES, TK_STATEMENT,  TK_ATTACH,     TK_HAVING,     TK_GLOB,       
           97  +    TK_BEFORE,     TK_FOR,        TK_FOREIGN,    TK_IGNORE,     TK_REPLACE,    
           98  +    TK_EXCLUSIVE,  TK_EXPLAIN,    TK_INDEX,      TK_BEGIN,      TK_INITIALLY,  
           99  +    TK_ALL,        TK_BETWEEN,    TK_NOT,        TK_NOTNULL,    TK_NULL,       
          100  +    TK_LIMIT,      TK_BY,         TK_CASCADE,    TK_ASC,        TK_DEFERRABLE, 
          101  +    TK_CASE,       TK_COLLATE,    TK_COMMIT,     TK_CONFLICT,   TK_CONSTRAINT, 
          102  +    TK_IN,         TK_INTERSECT,  TK_CREATE,     TK_JOIN_KW,    TK_DEFERRED,   
          103  +    TK_DELETE,     TK_DESC,       TK_DETACH,     TK_DISTINCT,   TK_IS,         
          104  +    TK_DROP,       TK_PRAGMA,     TK_MATCH,      TK_FAIL,       TK_FROM,       
          105  +    TK_JOIN_KW,    TK_GROUP,      TK_UPDATE,     TK_IMMEDIATE,  TK_JOIN_KW,    
          106  +    TK_RESTRICT,   TK_INSERT,     TK_INSTEAD,    TK_INTO,       TK_OF,         
          107  +    TK_OFFSET,     TK_SET,        TK_ISNULL,     TK_JOIN,       TK_ORDER,      
          108  +    TK_JOIN_KW,    TK_JOIN_KW,    TK_ROLLBACK,   TK_PRIMARY,    TK_ROW,        
          109  +    TK_WHEN,       TK_UNION,      TK_UNIQUE,     TK_USING,      TK_VACUUM,     
          110  +    TK_VALUES,     TK_VIEW,       TK_WHERE,      
   113    111     };
   114    112     int h, i;
   115    113     if( n<2 ) return TK_ID;
   116    114     h = (sqlite3UpperToLower[((unsigned char*)z)[0]]*5 + 
   117    115         sqlite3UpperToLower[((unsigned char*)z)[n-1]]*3 +
   118    116         n) % 154;
   119    117     for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){
   120    118       if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){
   121    119         return aCode[i];
   122    120       }
   123    121     }
   124    122     return TK_ID;
   125    123   }
          124  +
   126    125   
   127    126   /*
   128    127   ** If X is a character that can be used in an identifier and
   129    128   ** X&0x80==0 then isIdChar[X] will be 1.  If X&0x80==0x80 then
   130    129   ** X is always an identifier character.  (Hence all UTF-8
   131    130   ** characters can be part of an identifier).  isIdChar[X] will
   132    131   ** be 0 for every character in the lower 128 ASCII characters

Changes to tool/mkkeywordhash.c.

    11     11   ** All the keywords of the SQL language are stored as in a hash
    12     12   ** table composed of instances of the following structure.
    13     13   */
    14     14   typedef struct Keyword Keyword;
    15     15   struct Keyword {
    16     16     char *zName;         /* The keyword name */
    17     17     char *zTokenType;    /* Token value for this keyword */
           18  +  int id;              /* Unique ID for this record */
    18     19     int hash;            /* Hash on the keyword */
    19     20     int offset;          /* Offset to start of name string */
    20     21     int len;             /* Length of this keyword, not counting final \000 */
           22  +  int prefix;          /* Number of characters in prefix */
    21     23     int iNext;           /* Index in aKeywordTable[] of next with same hash */
           24  +  int substrId;        /* Id to another keyword this keyword is embedded in */
           25  +  int substrOffset;    /* Offset into substrId for start of this keyword */
    22     26   };
    23     27   
    24     28   /*
    25     29   ** These are the keywords
    26     30   */
    27     31   static Keyword aKeywordTable[] = {
    28     32     { "ABORT",            "TK_ABORT",        },
................................................................................
   149    153       252,253,254,255
   150    154   };
   151    155   #define UpperToLower sqlite3UpperToLower
   152    156   
   153    157   /*
   154    158   ** Comparision function for two Keyword records
   155    159   */
   156         -static int keywordCompare(const void *a, const void *b){
          160  +static int keywordCompare1(const void *a, const void *b){
          161  +  const Keyword *pA = (Keyword*)a;
          162  +  const Keyword *pB = (Keyword*)b;
          163  +  int n = pA->len - pB->len;
          164  +  if( n==0 ){
          165  +    n = strcmp(pA->zName, pB->zName);
          166  +  }
          167  +  return n;
          168  +}
          169  +static int keywordCompare2(const void *a, const void *b){
          170  +  const Keyword *pA = (Keyword*)a;
          171  +  const Keyword *pB = (Keyword*)b;
          172  +  int n = strcmp(pA->zName, pB->zName);
          173  +  return n;
          174  +}
          175  +static int keywordCompare3(const void *a, const void *b){
   157    176     const Keyword *pA = (Keyword*)a;
   158    177     const Keyword *pB = (Keyword*)b;
   159         -  return strcmp(pA->zName, pB->zName);
          178  +  int n = pA->offset - pB->offset;
          179  +  return n;
          180  +}
          181  +
          182  +/*
          183  +** Return a KeywordTable entry with the given id
          184  +*/
          185  +static Keyword *findById(int id){
          186  +  int i;
          187  +  for(i=0; i<NKEYWORD; i++){
          188  +    if( aKeywordTable[i].id==id ) break;
          189  +  }
          190  +  return &aKeywordTable[i];
   160    191   }
   161    192   
   162    193   /*
   163    194   ** This routine does the work.  The generated code is printed on standard
   164    195   ** output.
   165    196   */
   166    197   int main(int argc, char **argv){
   167         -  int i, j, h;
          198  +  int i, j, k, h;
   168    199     int bestSize, bestCount;
   169    200     int count;
   170    201     int nChar;
   171    202     int aHash[1000];  /* 1000 is much bigger than NKEYWORD */
   172    203   
   173         -  /* Make sure the table is sorted */
   174         -  qsort(aKeywordTable, NKEYWORD, sizeof(aKeywordTable[0]), keywordCompare);
   175         -
   176         -  /* Fill in the hash value, length, and offset for all entries */
   177         -  nChar = 0;
          204  +  /* Fill in the lengths of strings and hashes for all entries. */
   178    205     for(i=0; i<NKEYWORD; i++){
   179    206       Keyword *p = &aKeywordTable[i];
   180    207       p->len = strlen(p->zName);
   181         -    /* p->hash = sqlite3HashNoCase(p->zName, p->len); */
   182    208       p->hash = UpperToLower[p->zName[0]]*5 +
   183    209                 UpperToLower[p->zName[p->len-1]]*3 + p->len;
          210  +    p->id = i+1;
          211  +  }
          212  +
          213  +  /* Sort the table from shortest to longest keyword */
          214  +  qsort(aKeywordTable, NKEYWORD, sizeof(aKeywordTable[0]), keywordCompare1);
          215  +
          216  +  /* Look for short keywords embedded in longer keywords */
          217  +  for(i=NKEYWORD-2; i>=0; i--){
          218  +    Keyword *p = &aKeywordTable[i];
          219  +    for(j=NKEYWORD-1; j>i && p->substrId==0; j--){
          220  +      Keyword *pOther = &aKeywordTable[j];
          221  +      if( pOther->substrId ) continue;
          222  +      if( pOther->len<=p->len ) continue;
          223  +      for(k=0; k<=pOther->len-p->len; k++){
          224  +        if( memcmp(p->zName, &pOther->zName[k], p->len)==0 ){
          225  +          p->substrId = pOther->id;
          226  +          p->substrOffset = k;
          227  +          break;
          228  +        }
          229  +      }
          230  +    }
          231  +  }
          232  +
          233  +  /* Sort the table into alphabetical order */
          234  +  qsort(aKeywordTable, NKEYWORD, sizeof(aKeywordTable[0]), keywordCompare2);
          235  +
          236  +  /* Fill in the offset for all entries */
          237  +  nChar = 0;
          238  +  for(i=0; i<NKEYWORD; i++){
          239  +    Keyword *p = &aKeywordTable[i];
          240  +    if( p->offset>0 || p->substrId ) continue;
   184    241       p->offset = nChar;
   185         -    if( i<NKEYWORD-1 && strncmp(p->zName, aKeywordTable[i+1].zName,p->len)==0 ){
   186         -      /* This entry is a prefix of the one that follows.  Do not advance
   187         -      ** the offset */
   188         -    }else{
   189         -      nChar += p->len;
          242  +    nChar += p->len;
          243  +    for(k=p->len-1; k>=1; k--){
          244  +      for(j=i+1; j<NKEYWORD; j++){
          245  +        Keyword *pOther = &aKeywordTable[j];
          246  +        if( pOther->offset>0 || pOther->substrId ) continue;
          247  +        if( pOther->len<=k ) continue;
          248  +        if( memcmp(&p->zName[p->len-k], pOther->zName, k)==0 ){
          249  +          p = pOther;
          250  +          p->offset = nChar - k;
          251  +          nChar = p->offset + p->len;
          252  +          p->zName += k;
          253  +          p->len -= k;
          254  +          p->prefix = k;
          255  +          j = i;
          256  +          k = p->len;
          257  +        }
          258  +      }
          259  +    }
          260  +  }
          261  +  for(i=0; i<NKEYWORD; i++){
          262  +    Keyword *p = &aKeywordTable[i];
          263  +    if( p->substrId ){
          264  +      p->offset = findById(p->substrId)->offset + p->substrOffset;
   190    265       }
   191    266     }
   192    267   
          268  +  /* Sort the table by offset */
          269  +  qsort(aKeywordTable, NKEYWORD, sizeof(aKeywordTable[0]), keywordCompare3);
          270  +
   193    271     /* Figure out how big to make the hash table in order to minimize the
   194    272     ** number of collisions */
   195    273     bestSize = NKEYWORD;
   196    274     bestCount = NKEYWORD*NKEYWORD;
   197    275     for(i=NKEYWORD/2; i<=2*NKEYWORD; i++){
   198    276       for(j=0; j<i; j++) aHash[j] = 0;
   199    277       for(j=0; j<NKEYWORD; j++){
................................................................................
   218    296   
   219    297     /* Begin generating code */
   220    298     printf("int sqlite3KeywordCode(const char *z, int n){\n");
   221    299   
   222    300     printf("  static const char zText[%d] =\n", nChar+1);
   223    301     for(i=j=0; i<NKEYWORD; i++){
   224    302       Keyword *p = &aKeywordTable[i];
   225         -    if( i<NKEYWORD-1 && p->offset==aKeywordTable[i+1].offset ) continue;
          303  +    if( p->substrId ) continue;
   226    304       if( j==0 ) printf("    \"");
   227    305       printf("%s", p->zName);
   228    306       j += p->len;
   229    307       if( j>60 ){
   230    308         printf("\"\n");
   231    309         j = 0;
   232    310       }
................................................................................
   256    334       }
   257    335     }
   258    336     printf("%s  };\n", j==0 ? "" : "\n");    
   259    337   
   260    338     printf("  static const unsigned char aLen[%d] = {\n", NKEYWORD);
   261    339     for(i=j=0; i<NKEYWORD; i++){
   262    340       if( j==0 ) printf("    ");
   263         -    printf(" %3d,", aKeywordTable[i].len);
          341  +    printf(" %3d,", aKeywordTable[i].len+aKeywordTable[i].prefix);
   264    342       j++;
   265    343       if( j>12 ){
   266    344         printf("\n");
   267    345         j = 0;
   268    346       }
   269    347     }
   270    348     printf("%s  };\n", j==0 ? "" : "\n");