SQLite

*************************************************************************
** An tokenizer for SQL
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
**
** $Id: tokenize.c,v 1.92 2004/10/23 05:10:18 drh Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <ctype.h>
#include <stdlib.h>

/*
** This function looks up an identifier to determine if it is a
** keyword.  If it is a keyword, the token code of that keyword is 
** returned.  If the input is not a keyword, TK_ID is returned.
**
** The implementation of this routine was generated by a program,
** mkkeywordhash.c, located in the tool subdirectory of the distribution.
** The output of the mkkeywordhash.c program was manually cut and pasted
** into this file.  When the set of keywords for SQLite changes, you
** must modify the mkkeywordhash.c program (to add or remove keywords from
** the data tables) then rerun that program to regenerate this function.
*/
int sqlite3KeywordCode(const char *z, int n){
  static const char zText[443] =
    "ABORTABLEFTEMPORARYAFTERAISELECTHENDATABASEACHECKEYANDEFAULTRANSACTION"
    "ATURALIKELSEXCEPTRIGGEREFERENCESTATEMENTATTACHAVINGLOBEFOREIGN"
    "OREPLACEXCLUSIVEXPLAINDEXBEGINITIALLYBETWEENOTNULLIMITBYCASCADE"
    "FERRABLECASECOLLATECOMMITCONFLICTCONSTRAINTERSECTCREATECROSSDEFERRED"
    "ELETEDESCDETACHDISTINCTDROPRAGMATCHFAILFROMFULLGROUPDATEIMMEDIATE"



    "INNERESTRICTINSERTINSTEADINTOFFSETISNULLJOINORDERIGHTOUTEROLLBACK"
    "PRIMARYROWHENUNIONUNIQUEUSINGVACUUMVALUESVIEWHERE";

  static const unsigned char aHash[154] = {
       0,  26,  82,   0,   0,  91,  90,   0,  27,   0,   0,   0,   0,
       0,   0,  49,   0,  96,  17,   0,   0,   0,   0,   0,   0,   0,

       0,  97,   5,  31,   0,  62,  51,  28,  58,  52,   0,   0,  60,
      61,   0,  12,  41,  50,   0,   0,   0,  36,  63,   0,   0,  15,
       0,   0,   0,  39,   0,  42,   0,   0,   0,   0,  78,   0,  34,
      29,   0,  74,  71,   0,  66,  70,  37,   0,   0,  59,   0,  33,
       0,  53,   0,  54,   0,  55,   0,  83,  72,  67,   0,  24,   0,
       0,  79,  80,  84,   0,   0,   0,   0,   0,   0,   0,  75,   0,
       0,   0,   0,   0,  45,  77,  35,  44,  57,   0,   0,   0,   0,
      20,   2,   0,  38,   0,   3,  46,  93,   0,   0,  40,   0,  94,
       0,  43,  87,  98,   0,   0,   0,   0,   0,  81,   0,   0,   0,
       0,  10,   0,   0,   0,   0,   0,  92,  19,   0,  95,
  };
  static const unsigned char aNext[98] = {
       0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,


       0,  14,   0,   0,   0,   0,   0,   0,   0,   0,   0,   9,   0,
       0,   0,   0,   0,   0,  18,  22,   0,   0,   0,   0,   0,   0,
       0,  23,   0,  16,  21,   8,   0,  32,   0,   0,  30,   0,  48,
       0,   0,   0,   0,   0,   0,   0,  11,   0,   0,   0,   0,   0,
       0,  56,   0,   1,   0,  69,  64,   0,   0,  65,   0,   0,  13,
      68,   0,   0,  76,  47,   0,   0,   0,  85,   6,   0,  89,  25,
       4,  73,  88,  86,   0,   0,   0,
  };
  static const unsigned char aLen[98] = {
       5,   5,   4,   4,   9,   2,   5,   5,   6,   4,   3,   8,   2,
       4,   5,   3,   3,   7,  11,   2,   7,   4,   4,   6,   7,  10,
       9,   6,   6,   4,   6,   3,   7,   6,   7,   9,   7,   5,   5,
       9,   3,   7,   3,   7,   4,   5,   2,   7,   3,  10,   4,   7,
       6,   8,  10,   2,   9,   6,   5,   8,   6,   4,   6,   8,   2,
       4,   6,   5,   4,   4,   4,   5,   6,   9,   5,   8,   6,   7,
       4,   2,   6,   3,   6,   4,   5,   5,   5,   8,   7,   3,   4,


       5,   6,   5,   6,   6,   4,   5,
  };
  static const unsigned short int aOffset[98] = {
       0,   4,   7,  10,  10,  14,  19,  23,  26,  31,  33,  35,  40,
      42,  44,  48,  51,  53,  59,  68,  69,  75,  78,  81,  86,  92,
     101, 110, 115, 120, 123, 125, 125, 129, 133, 139, 147, 152, 157,
     160, 165, 169, 175, 175, 178, 181, 186, 188, 189, 193, 203, 207,
     214, 220, 228, 235, 235, 244, 250, 255, 262, 268, 272, 278, 279,

     286, 289, 293, 298, 302, 306, 310, 313, 319, 328, 332, 340, 346,
     353, 356, 356, 359, 362, 368, 372, 376, 381, 385, 393, 400, 402,
     406, 411, 417, 422, 428, 434, 437,
  };
  static const unsigned char aCode[98] = {
    TK_ABORT,      TK_TABLE,      TK_JOIN_KW,    TK_TEMP,       TK_TEMP,       
    TK_OR,         TK_AFTER,      TK_RAISE,      TK_SELECT,     TK_THEN,       
    TK_END,        TK_DATABASE,   TK_AS,         TK_EACH,       TK_CHECK,      
    TK_KEY,        TK_AND,        TK_DEFAULT,    TK_TRANSACTION,TK_ON,         
    TK_JOIN_KW,    TK_LIKE,       TK_ELSE,       TK_EXCEPT,     TK_TRIGGER,    
    TK_REFERENCES, TK_STATEMENT,  TK_ATTACH,     TK_HAVING,     TK_GLOB,       
    TK_BEFORE,     TK_FOR,        TK_FOREIGN,    TK_IGNORE,     TK_REPLACE,    
    TK_EXCLUSIVE,  TK_EXPLAIN,    TK_INDEX,      TK_BEGIN,      TK_INITIALLY,  
    TK_ALL,        TK_BETWEEN,    TK_NOT,        TK_NOTNULL,    TK_NULL,       
    TK_LIMIT,      TK_BY,         TK_CASCADE,    TK_ASC,        TK_DEFERRABLE, 
    TK_CASE,       TK_COLLATE,    TK_COMMIT,     TK_CONFLICT,   TK_CONSTRAINT, 
    TK_IN,         TK_INTERSECT,  TK_CREATE,     TK_JOIN_KW,    TK_DEFERRED,   
    TK_DELETE,     TK_DESC,       TK_DETACH,     TK_DISTINCT,   TK_IS,         

    TK_DROP,       TK_PRAGMA,     TK_MATCH,      TK_FAIL,       TK_FROM,       
    TK_JOIN_KW,    TK_GROUP,      TK_UPDATE,     TK_IMMEDIATE,  TK_JOIN_KW,    
    TK_RESTRICT,   TK_INSERT,     TK_INSTEAD,    TK_INTO,       TK_OF,         



    TK_OFFSET,     TK_SET,        TK_ISNULL,     TK_JOIN,       TK_ORDER,      

    TK_JOIN_KW,    TK_JOIN_KW,    TK_ROLLBACK,   TK_PRIMARY,    TK_ROW,        


    TK_WHEN,       TK_UNION,      TK_UNIQUE,     TK_USING,      TK_VACUUM,     
    TK_VALUES,     TK_VIEW,       TK_WHERE,      
  };
  int h, i;
  if( n<2 ) return TK_ID;
  h = (sqlite3UpperToLower[((unsigned char*)z)[0]]*5 + 
      sqlite3UpperToLower[((unsigned char*)z)[n-1]]*3 +
      n) % 154;
  for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){
    if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){
      return aCode[i];
    }
  }
  return TK_ID;
}


/*
** If X is a character that can be used in an identifier and
** X&0x80==0 then isIdChar[X] will be 1.  If X&0x80==0x80 then
** X is always an identifier character.  (Hence all UTF-8
** characters can be part of an identifier).  isIdChar[X] will
** be 0 for every character in the lower 128 ASCII characters

** All the keywords of the SQL language are stored as in a hash
** table composed of instances of the following structure.
*/
typedef struct Keyword Keyword;
struct Keyword {
  char *zName;         /* The keyword name */
  char *zTokenType;    /* Token value for this keyword */
  int id;              /* Unique ID for this record */
  int hash;            /* Hash on the keyword */
  int offset;          /* Offset to start of name string */
  int len;             /* Length of this keyword, not counting final \000 */
  int prefix;          /* Number of characters in prefix */
  int iNext;           /* Index in aKeywordTable[] of next with same hash */
  int substrId;        /* Id to another keyword this keyword is embedded in */
  int substrOffset;    /* Offset into substrId for start of this keyword */
};

/*
** These are the keywords
*/
static Keyword aKeywordTable[] = {
  { "ABORT",            "TK_ABORT",        },

    252,253,254,255
};
#define UpperToLower sqlite3UpperToLower

/*
** Comparision function for two Keyword records
*/
static int keywordCompare1(const void *a, const void *b){
  const Keyword *pA = (Keyword*)a;
  const Keyword *pB = (Keyword*)b;
  int n = pA->len - pB->len;
  if( n==0 ){
    n = strcmp(pA->zName, pB->zName);
  }
  return n;
}
static int keywordCompare2(const void *a, const void *b){
  const Keyword *pA = (Keyword*)a;
  const Keyword *pB = (Keyword*)b;
  int n = strcmp(pA->zName, pB->zName);
  return n;
}
static int keywordCompare3(const void *a, const void *b){
  const Keyword *pA = (Keyword*)a;
  const Keyword *pB = (Keyword*)b;
  int n = pA->offset - pB->offset;
  return n;
}

/*
** Return a KeywordTable entry with the given id
*/
static Keyword *findById(int id){
  int i;
  for(i=0; i<NKEYWORD; i++){
    if( aKeywordTable[i].id==id ) break;
  }
  return &aKeywordTable[i];
}

/*
** This routine does the work.  The generated code is printed on standard
** output.
*/
int main(int argc, char **argv){
  int i, j, k, h;
  int bestSize, bestCount;
  int count;
  int nChar;
  int aHash[1000];  /* 1000 is much bigger than NKEYWORD */



  /* Fill in the lengths of strings and hashes for all entries. */


  for(i=0; i<NKEYWORD; i++){
    Keyword *p = &aKeywordTable[i];
    p->len = strlen(p->zName);

    p->hash = UpperToLower[p->zName[0]]*5 +
              UpperToLower[p->zName[p->len-1]]*3 + p->len;
    p->id = i+1;
  }

  /* Sort the table from shortest to longest keyword */
  qsort(aKeywordTable, NKEYWORD, sizeof(aKeywordTable[0]), keywordCompare1);

  /* Look for short keywords embedded in longer keywords */
  for(i=NKEYWORD-2; i>=0; i--){
    Keyword *p = &aKeywordTable[i];
    for(j=NKEYWORD-1; j>i && p->substrId==0; j--){
      Keyword *pOther = &aKeywordTable[j];
      if( pOther->substrId ) continue;
      if( pOther->len<=p->len ) continue;
      for(k=0; k<=pOther->len-p->len; k++){
        if( memcmp(p->zName, &pOther->zName[k], p->len)==0 ){
          p->substrId = pOther->id;
          p->substrOffset = k;
          break;
        }
      }
    }
  }

  /* Sort the table into alphabetical order */
  qsort(aKeywordTable, NKEYWORD, sizeof(aKeywordTable[0]), keywordCompare2);

  /* Fill in the offset for all entries */
  nChar = 0;
  for(i=0; i<NKEYWORD; i++){
    Keyword *p = &aKeywordTable[i];
    if( p->offset>0 || p->substrId ) continue;
    p->offset = nChar;
    nChar += p->len;
    for(k=p->len-1; k>=1; k--){
      for(j=i+1; j<NKEYWORD; j++){
        Keyword *pOther = &aKeywordTable[j];
        if( pOther->offset>0 || pOther->substrId ) continue;
        if( pOther->len<=k ) continue;
        if( memcmp(&p->zName[p->len-k], pOther->zName, k)==0 ){
          p = pOther;
          p->offset = nChar - k;
          nChar = p->offset + p->len;
          p->zName += k;
          p->len -= k;
          p->prefix = k;
          j = i;
          k = p->len;
        }
      }
    }
  }
  for(i=0; i<NKEYWORD; i++){
    Keyword *p = &aKeywordTable[i];
    if( p->substrId ){
      p->offset = findById(p->substrId)->offset + p->substrOffset;
    }
  }

  /* Sort the table by offset */
  qsort(aKeywordTable, NKEYWORD, sizeof(aKeywordTable[0]), keywordCompare3);

  /* Figure out how big to make the hash table in order to minimize the
  ** number of collisions */
  bestSize = NKEYWORD;
  bestCount = NKEYWORD*NKEYWORD;
  for(i=NKEYWORD/2; i<=2*NKEYWORD; i++){
    for(j=0; j<i; j++) aHash[j] = 0;
    for(j=0; j<NKEYWORD; j++){

  /* Begin generating code */
  printf("int sqlite3KeywordCode(const char *z, int n){\n");

  printf("  static const char zText[%d] =\n", nChar+1);
  for(i=j=0; i<NKEYWORD; i++){
    Keyword *p = &aKeywordTable[i];
    if( p->substrId ) continue;
    if( j==0 ) printf("    \"");
    printf("%s", p->zName);
    j += p->len;
    if( j>60 ){
      printf("\"\n");
      j = 0;
    }

    }
  }
  printf("%s  };\n", j==0 ? "" : "\n");    

  printf("  static const unsigned char aLen[%d] = {\n", NKEYWORD);
  for(i=j=0; i<NKEYWORD; i++){
    if( j==0 ) printf("    ");
    printf(" %3d,", aKeywordTable[i].len+aKeywordTable[i].prefix);
    j++;
    if( j>12 ){
      printf("\n");
      j = 0;
    }
  }
  printf("%s  };\n", j==0 ? "" : "\n");