Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -643,13 +643,13 @@ sqlite3.lo: sqlite3.c $(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c # Rules to build the LEMON compiler generator # -lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/src/lempar.c +lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c $(BCC) -o $@ $(TOP)/tool/lemon.c - cp $(TOP)/src/lempar.c . + cp $(TOP)/tool/lempar.c . # Rules to build individual *.o files from generated *.c files. This # applies to: # # parse.o Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1308,12 +1308,12 @@ sqlite3.lo: $(SQLITE3C) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C) # Rules to build the LEMON compiler generator # -lempar.c: $(TOP)\src\lempar.c - copy $(TOP)\src\lempar.c . +lempar.c: $(TOP)\tool\lempar.c + copy $(TOP)\tool\lempar.c . lemon.exe: $(TOP)\tool\lemon.c lempar.c $(BCC) $(NO_WARN) -Daccess=_access \ -Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) Index: ext/fts5/fts5_main.c ================================================================== --- ext/fts5/fts5_main.c +++ ext/fts5/fts5_main.c @@ -872,11 +872,10 @@ Fts5Config *pConfig = pTab->pConfig; Fts5Sorter *pSorter; int nPhrase; int nByte; int rc = SQLITE_OK; - char *zSql; const char *zRank = pCsr->zRank; const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); Index: ext/fts5/fts5parse.y ================================================================== --- ext/fts5/fts5parse.y +++ ext/fts5/fts5parse.y @@ -56,10 +56,22 @@ /* ** Make yytestcase() the same as testcase() */ #define yytestcase(X) testcase(X) +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENOTNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 + } // end %include %left OR. %left AND. %left NOT. @@ -166,9 +178,5 @@ */ %type star_opt {int} star_opt(A) ::= STAR. { A = 1; } star_opt(A) ::= . { A = 0; } - - - - Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -552,13 +552,13 @@ fts3amal.c: target_source $(TOP)/ext/fts3/mkfts3amal.tcl tclsh $(TOP)/ext/fts3/mkfts3amal.tcl # Rules to build the LEMON compiler generator # -lemon: $(TOP)/tool/lemon.c $(TOP)/src/lempar.c +lemon: $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c $(BCC) -o lemon $(TOP)/tool/lemon.c - cp $(TOP)/src/lempar.c . + cp $(TOP)/tool/lempar.c . # Rules to build individual *.o files from generated *.c files. This # applies to: # # parse.o DELETED src/lempar.c Index: src/lempar.c ================================================================== --- src/lempar.c +++ /dev/null @@ -1,892 +0,0 @@ -/* Driver template for the LEMON parser generator. -** The author disclaims copyright to this source code. -** -** This version of "lempar.c" is modified, slightly, for use by SQLite. -** The only modifications are the addition of a couple of NEVER() -** macros to disable tests that are needed in the case of a general -** LALR(1) grammar but which are always false in the -** specific grammar used by SQLite. -*/ -/* First off, code is included that follows the "include" declaration -** in the input grammar file. */ -#include -%% -/* Next is all token values, in a form suitable for use by makeheaders. -** This section will be null unless lemon is run with the -m switch. -*/ -/* -** These constants (all generated automatically by the parser generator) -** specify the various kinds of tokens (terminals) that the parser -** understands. -** -** Each symbol here is a terminal symbol in the grammar. -*/ -%% -/* Make sure the INTERFACE macro is defined. -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/* The next thing included is series of defines which control -** various aspects of the generated parser. -** YYCODETYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 terminals -** and nonterminals. "int" is used otherwise. -** YYNOCODE is a number of type YYCODETYPE which corresponds -** to no legal terminal or nonterminal number. This -** number is used to fill in empty slots of the hash -** table. -** YYFALLBACK If defined, this indicates that one or more tokens -** have fall-back values which should be used if the -** original value of the token will not parse. -** YYACTIONTYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 rules and -** states combined. "int" is used otherwise. -** ParseTOKENTYPE is the data type used for minor tokens given -** directly to the parser from the tokenizer. -** YYMINORTYPE is the data type used for all minor tokens. -** This is typically a union of many types, one of -** which is ParseTOKENTYPE. The entry in the union -** for base tokens is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** ParseARG_SDECL A static variable declaration for the %extra_argument -** ParseARG_PDECL A parameter declaration for the %extra_argument -** ParseARG_STORE Code to store %extra_argument into yypParser -** ParseARG_FETCH Code to extract %extra_argument from yypParser -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YY_MAX_SHIFT Maximum value for shift actions -** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions -** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_MIN_REDUCE Maximum value for reduce actions -** YY_ERROR_ACTION The yy_action[] code for syntax error -** YY_ACCEPT_ACTION The yy_action[] code for accept -** YY_NO_ACTION The yy_action[] code for no-op -*/ -%% - -/* The yyzerominor constant is used to initialize instances of -** YYMINORTYPE objects to zero. */ -static const YYMINORTYPE yyzerominor = { 0 }; - -/* Define the yytestcase() macro to be a no-op if is not already defined -** otherwise. -** -** Applications can choose to define yytestcase() in the %include section -** to a macro that can assist in verifying code coverage. For production -** code the yytestcase() macro should be turned off. But it is useful -** for testing. -*/ -#ifndef yytestcase -# define yytestcase(X) -#endif - - -/* Next are the tables used to determine what action to take based on the -** current state and lookahead token. These tables are used to implement -** functions that take a state number and lookahead value and return an -** action integer. -** -** Suppose the action integer is N. Then the action is determined as -** follows -** -** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead -** token onto the stack and goto state N. -** -** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then -** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. -** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE - -** N == YY_ERROR_ACTION A syntax error has occurred. -** -** N == YY_ACCEPT_ACTION The parser accepts its input. -** -** N == YY_NO_ACTION No such action. Denotes unused -** slots in the yy_action[] table. -** -** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as -** -** yy_action[ yy_shift_ofst[S] + X ] -** -** If the index value yy_shift_ofst[S]+X is out of range or if the value -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] -** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table -** and that yy_default[S] should be used instead. -** -** The formula above is for computing the action when the lookahead is -** a terminal symbol. If the lookahead is a non-terminal (as occurs after -** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of -** YY_SHIFT_USE_DFLT. -** -** The following are the tables generated in this section: -** -** yy_action[] A single table containing all actions. -** yy_lookahead[] A table containing the lookahead for each entry in -** yy_action. Used to detect hash collisions. -** yy_shift_ofst[] For each state, the offset into yy_action for -** shifting terminals. -** yy_reduce_ofst[] For each state, the offset into yy_action for -** shifting non-terminals after a reduce. -** yy_default[] Default action for each state. -*/ -%% - -/* The next table maps tokens into fallback tokens. If a construct -** like the following: -** -** %fallback ID X Y Z. -** -** appears in the grammar, then ID becomes a fallback token for X, Y, -** and Z. Whenever one of the tokens X, Y, or Z is input to the parser -** but it does not parse, the type of the token is changed to ID and -** the parse is retried before an error is thrown. -*/ -#ifdef YYFALLBACK -static const YYCODETYPE yyFallback[] = { -%% -}; -#endif /* YYFALLBACK */ - -/* The following structure represents a single element of the -** parser's stack. Information stored includes: -** -** + The state number for the parser at this level of the stack. -** -** + The value of the token stored at this level of the stack. -** (In other words, the "major" token.) -** -** + The semantic value stored at this level of the stack. This is -** the information used by the action routines in the grammar. -** It is sometimes called the "minor" token. -** -** After the "shift" half of a SHIFTREDUCE action, the stateno field -** actually contains the reduce action for the second half of the -** SHIFTREDUCE. -*/ -struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ - YYCODETYPE major; /* The major token value. This is the code - ** number for the token at this stack level */ - YYMINORTYPE minor; /* The user-supplied minor token value. This - ** is the value of the token */ -}; -typedef struct yyStackEntry yyStackEntry; - -/* The state of the parser is completely contained in an instance of -** the following structure */ -struct yyParser { - int yyidx; /* Index of top element in stack */ -#ifdef YYTRACKMAXSTACKDEPTH - int yyidxMax; /* Maximum value of yyidx */ -#endif - int yyerrcnt; /* Shifts left before out of the error */ - ParseARG_SDECL /* A place to hold %extra_argument */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ -#endif -}; -typedef struct yyParser yyParser; - -#ifndef NDEBUG -#include -static FILE *yyTraceFILE = 0; -static char *yyTracePrompt = 0; -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* -** Turn parser tracing on by giving a stream to which to write the trace -** and a prompt to preface each trace message. Tracing is turned off -** by making either argument NULL -** -** Inputs: -** -** -** Outputs: -** None. -*/ -void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ - yyTraceFILE = TraceFILE; - yyTracePrompt = zTracePrompt; - if( yyTraceFILE==0 ) yyTracePrompt = 0; - else if( yyTracePrompt==0 ) yyTraceFILE = 0; -} -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* For tracing shifts, the names of all terminals and nonterminals -** are required. The following table supplies these names */ -static const char *const yyTokenName[] = { -%% -}; -#endif /* NDEBUG */ - -#ifndef NDEBUG -/* For tracing reduce actions, the names of all rules are required. -*/ -static const char *const yyRuleName[] = { -%% -}; -#endif /* NDEBUG */ - - -#if YYSTACKDEPTH<=0 -/* -** Try to increase the size of the parser stack. -*/ -static void yyGrowStack(yyParser *p){ - int newSize; - yyStackEntry *pNew; - - newSize = p->yystksz*2 + 100; - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - if( pNew ){ - p->yystack = pNew; - p->yystksz = newSize; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", - yyTracePrompt, p->yystksz); - } -#endif - } -} -#endif - -/* -** This function allocates a new parser. -** The only argument is a pointer to a function which works like -** malloc. -** -** Inputs: -** A pointer to the function used to allocate memory. -** -** Outputs: -** A pointer to a parser. This pointer is used in subsequent calls -** to Parse and ParseFree. -*/ -void *ParseAlloc(void *(*mallocProc)(u64)){ - yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (u64)sizeof(yyParser) ); - if( pParser ){ - pParser->yyidx = -1; -#ifdef YYTRACKMAXSTACKDEPTH - pParser->yyidxMax = 0; -#endif -#if YYSTACKDEPTH<=0 - pParser->yystack = NULL; - pParser->yystksz = 0; - yyGrowStack(pParser); -#endif - } - return pParser; -} - -/* The following function deletes the value associated with a -** symbol. The symbol can be either a terminal or nonterminal. -** "yymajor" is the symbol code, and "yypminor" is a pointer to -** the value. -*/ -static void yy_destructor( - yyParser *yypParser, /* The parser */ - YYCODETYPE yymajor, /* Type code for object to destroy */ - YYMINORTYPE *yypminor /* The object to be destroyed */ -){ - ParseARG_FETCH; - switch( yymajor ){ - /* Here is inserted the actions which take place when a - ** terminal or non-terminal is destroyed. This can happen - ** when the symbol is popped from the stack during a - ** reduce or during error processing or when a parser is - ** being destroyed before it is finished parsing. - ** - ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are not used - ** inside the C code. - */ -%% - default: break; /* If no destructor action specified: do nothing */ - } -} - -/* -** Pop the parser's stack once. -** -** If there is a destructor routine associated with the token which -** is popped from the stack, then call it. -*/ -static void yy_pop_parser_stack(yyParser *pParser){ - yyStackEntry *yytos; - assert( pParser->yyidx>=0 ); - yytos = &pParser->yystack[pParser->yyidx--]; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sPopping %s\n", - yyTracePrompt, - yyTokenName[yytos->major]); - } -#endif - yy_destructor(pParser, yytos->major, &yytos->minor); -} - -/* -** Deallocate and destroy a parser. Destructors are all called for -** all stack elements before shutting the parser down. -** -** Inputs: -** -*/ -void ParseFree( - void *p, /* The parser to be deleted */ - void (*freeProc)(void*) /* Function used to reclaim memory */ -){ - yyParser *pParser = (yyParser*)p; - /* In SQLite, we never try to destroy a parser that was not successfully - ** created in the first place. */ - if( NEVER(pParser==0) ) return; - while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - free(pParser->yystack); -#endif - (*freeProc)((void*)pParser); -} - -/* -** Return the peak depth of the stack for a parser. -*/ -#ifdef YYTRACKMAXSTACKDEPTH -int ParseStackPeak(void *p){ - yyParser *pParser = (yyParser*)p; - return pParser->yyidxMax; -} -#endif - -/* -** Find the appropriate action for a parser given the terminal -** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. -*/ -static int yy_find_shift_action( - yyParser *pParser, /* The parser */ - YYCODETYPE iLookAhead /* The look-ahead token */ -){ - int i; - int stateno = pParser->yystack[pParser->yyidx].stateno; - - if( stateno>=YY_MIN_REDUCE ) return stateno; - assert( stateno <= YY_SHIFT_COUNT ); - do{ - i = yy_shift_ofst[stateno]; - if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - if( iLookAhead>0 ){ -#ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; - } -#endif -#ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); - } -#endif /* NDEBUG */ - return yy_action[j]; - } - } -#endif /* YYWILDCARD */ - } - return yy_default[stateno]; - }else{ - return yy_action[i]; - } - }while(1); -} - -/* -** Find the appropriate action for a parser given the non-terminal -** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. -*/ -static int yy_find_reduce_action( - int stateno, /* Current state number */ - YYCODETYPE iLookAhead /* The look-ahead token */ -){ - int i; -#ifdef YYERRORSYMBOL - if( stateno>YY_REDUCE_COUNT ){ - return yy_default[stateno]; - } -#else - assert( stateno<=YY_REDUCE_COUNT ); -#endif - i = yy_reduce_ofst[stateno]; - assert( i!=YY_REDUCE_USE_DFLT ); - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; -#ifdef YYERRORSYMBOL - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - return yy_default[stateno]; - } -#else - assert( i>=0 && iyyidx--; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); - } -#endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will execute if the parser - ** stack every overflows */ -%% - ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ -} - -/* -** Print tracing information for a SHIFT action -*/ -#ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState){ - if( yyTraceFILE ){ - int i; - if( yyNewStateyyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - }else{ - fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); - } - } -} -#else -# define yyTraceShift(X,Y) -#endif - -/* -** Perform a shift action. Return the number of errors. -*/ -static void yy_shift( - yyParser *yypParser, /* The parser to be shifted */ - int yyNewState, /* The new state to shift in */ - int yyMajor, /* The major token to shift in */ - YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ -){ - yyStackEntry *yytos; - yypParser->yyidx++; -#ifdef YYTRACKMAXSTACKDEPTH - if( yypParser->yyidx>yypParser->yyidxMax ){ - yypParser->yyidxMax = yypParser->yyidx; - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yyidx>=YYSTACKDEPTH ){ - yyStackOverflow(yypParser, yypMinor); - return; - } -#else - if( yypParser->yyidx>=yypParser->yystksz ){ - yyGrowStack(yypParser); - if( yypParser->yyidx>=yypParser->yystksz ){ - yyStackOverflow(yypParser, yypMinor); - return; - } - } -#endif - yytos = &yypParser->yystack[yypParser->yyidx]; - yytos->stateno = (YYACTIONTYPE)yyNewState; - yytos->major = (YYCODETYPE)yyMajor; - yytos->minor = *yypMinor; - yyTraceShift(yypParser, yyNewState); -} - -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - unsigned char nrhs; /* Number of right-hand side symbols in the rule */ -} yyRuleInfo[] = { -%% -}; - -static void yy_accept(yyParser*); /* Forward Declaration */ - -/* -** Perform a reduce action and the shift that must immediately -** follow the reduce. -*/ -static void yy_reduce( - yyParser *yypParser, /* The parser */ - int yyruleno /* Number of the rule by which to reduce */ -){ - int yygoto; /* The next state */ - int yyact; /* The next action */ - YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ - yyStackEntry *yymsp; /* The top of the parser's stack */ - int yysize; /* Amount to pop the stack */ - ParseARG_FETCH; - yymsp = &yypParser->yystack[yypParser->yyidx]; -#ifndef NDEBUG - if( yyTraceFILE && yyruleno>=0 - && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; - fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, - yyRuleName[yyruleno], yymsp[-yysize].stateno); - } -#endif /* NDEBUG */ - - /* Silence complaints from purify about yygotominor being uninitialized - ** in some cases when it is copied into the stack after the following - ** switch. yygotominor is uninitialized when a rule reduces that does - ** not set the value of its left-hand side nonterminal. Leaving the - ** value of the nonterminal uninitialized is utterly harmless as long - ** as the value is never used. So really the only thing this code - ** accomplishes is to quieten purify. - ** - ** 2007-01-16: The wireshark project (www.wireshark.org) reports that - ** without this code, their parser segfaults. I'm not sure what there - ** parser is doing to make this happen. This is the second bug report - ** from wireshark this week. Clearly they are stressing Lemon in ways - ** that it has not been previously stressed... (SQLite ticket #2172) - */ - /*memset(&yygotominor, 0, sizeof(yygotominor));*/ - yygotominor = yyzerominor; - - - switch( yyruleno ){ - /* Beginning here are the reduction cases. A typical example - ** follows: - ** case 0: - ** #line - ** { ... } // User supplied code - ** #line - ** break; - */ -%% - }; - assert( yyruleno>=0 && yyrulenoyyidx -= yysize; - yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - /* If the reduce action popped at least - ** one element off the stack, then we can push the new element back - ** onto the stack here, and skip the stack overflow test in yy_shift(). - ** That gives a significant speed improvement. */ - if( yysize ){ - yypParser->yyidx++; - yymsp -= yysize-1; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yymsp->minor = yygotominor; - yyTraceShift(yypParser, yyact); - }else{ - yy_shift(yypParser,yyact,yygoto,&yygotominor); - } - }else{ - assert( yyact == YY_ACCEPT_ACTION ); - yy_accept(yypParser); - } -} - -/* -** The following code executes when the parse fails -*/ -#ifndef YYNOERRORRECOVERY -static void yy_parse_failed( - yyParser *yypParser /* The parser */ -){ - ParseARG_FETCH; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); - } -#endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser fails */ -%% - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} -#endif /* YYNOERRORRECOVERY */ - -/* -** The following code executes when a syntax error first occurs. -*/ -static void yy_syntax_error( - yyParser *yypParser, /* The parser */ - int yymajor, /* The major type of the error token */ - YYMINORTYPE yyminor /* The minor type of the error token */ -){ - ParseARG_FETCH; -#define TOKEN (yyminor.yy0) -%% - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} - -/* -** The following is executed when the parser accepts -*/ -static void yy_accept( - yyParser *yypParser /* The parser */ -){ - ParseARG_FETCH; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); - } -#endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser accepts */ -%% - ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ -} - -/* The main parser program. -** The first argument is a pointer to a structure obtained from -** "ParseAlloc" which describes the current state of the parser. -** The second argument is the major token number. The third is -** the minor token. The fourth optional argument is whatever the -** user wants (and specified in the grammar) and is available for -** use by the action routines. -** -** Inputs: -**
    -**
  • A pointer to the parser (an opaque structure.) -**
  • The major token number. -**
  • The minor token number. -**
  • An option argument of a grammar-specified type. -**
-** -** Outputs: -** None. -*/ -void Parse( - void *yyp, /* The parser */ - int yymajor, /* The major token code number */ - ParseTOKENTYPE yyminor /* The value for the token */ - ParseARG_PDECL /* Optional %extra_argument parameter */ -){ - YYMINORTYPE yyminorunion; - int yyact; /* The parser action. */ -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - int yyendofinput; /* True if we are at the end of input */ -#endif -#ifdef YYERRORSYMBOL - int yyerrorhit = 0; /* True if yymajor has invoked an error */ -#endif - yyParser *yypParser; /* The parser */ - - /* (re)initialize the parser, if necessary */ - yypParser = (yyParser*)yyp; - if( yypParser->yyidx<0 ){ -#if YYSTACKDEPTH<=0 - if( yypParser->yystksz <=0 ){ - /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ - yyminorunion = yyzerominor; - yyStackOverflow(yypParser, &yyminorunion); - return; - } -#endif - yypParser->yyidx = 0; - yypParser->yyerrcnt = -1; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; - } - yyminorunion.yy0 = yyminor; -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - yyendofinput = (yymajor==0); -#endif - ParseARG_STORE; - -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); - } -#endif - - do{ - yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - yy_shift(yypParser,yyact,yymajor,&yyminorunion); - yypParser->yyerrcnt--; - yymajor = YYNOCODE; - }else if( yyact <= YY_MAX_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE); - }else{ - assert( yyact == YY_ERROR_ACTION ); -#ifdef YYERRORSYMBOL - int yymx; -#endif -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); - } -#endif -#ifdef YYERRORSYMBOL - /* A syntax error has occurred. - ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". - ** - ** This is what we do if the grammar does define ERROR: - ** - ** * Call the %syntax_error function. - ** - ** * Begin popping the stack until we enter a state where - ** it is legal to shift the error symbol, then shift - ** the error symbol. - ** - ** * Set the error count to three. - ** - ** * Begin accepting and shifting new tokens. No new error - ** processing will occur until three tokens have been - ** shifted successfully. - ** - */ - if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); - } - yymx = yypParser->yystack[yypParser->yyidx].major; - if( yymx==YYERRORSYMBOL || yyerrorhit ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sDiscard input token %s\n", - yyTracePrompt,yyTokenName[yymajor]); - } -#endif - yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - }else{ - while( - yypParser->yyidx >= 0 && - yymx != YYERRORSYMBOL && - (yyact = yy_find_reduce_action( - yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YY_MIN_REDUCE - ){ - yy_pop_parser_stack(yypParser); - } - if( yypParser->yyidx < 0 || yymajor==0 ){ - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yy_parse_failed(yypParser); - yymajor = YYNOCODE; - }else if( yymx!=YYERRORSYMBOL ){ - YYMINORTYPE u2; - u2.YYERRSYMDT = 0; - yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); - } - } - yypParser->yyerrcnt = 3; - yyerrorhit = 1; -#elif defined(YYNOERRORRECOVERY) - /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to - ** do any kind of error recovery. Instead, simply invoke the syntax - ** error routine and continue going as if nothing had happened. - ** - ** Applications can set this macro (for example inside %include) if - ** they intend to abandon the parse upon the first syntax error seen. - */ - yy_syntax_error(yypParser,yymajor,yyminorunion); - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - -#else /* YYERRORSYMBOL is not defined */ - /* This is what we do if the grammar does not define ERROR: - ** - ** * Report an error message, and throw away the input token. - ** - ** * If the input token is $, then fail the parse. - ** - ** As before, subsequent error messages are suppressed until - ** three input tokens have been successfully shifted. - */ - if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor,yyminorunion); - } - yypParser->yyerrcnt = 3; - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - if( yyendofinput ){ - yy_parse_failed(yypParser); - } - yymajor = YYNOCODE; -#endif - } - }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); - } -#endif - return; -} Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -58,10 +58,22 @@ /* ** Make yytestcase() the same as testcase() */ #define yytestcase(X) testcase(X) +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENOTNULL 1 + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 + /* ** An instance of this structure holds information about the ** LIMIT clause of a SELECT statement. */ struct LimitVal { @@ -637,11 +649,10 @@ %type fullname {SrcList*} %destructor fullname {sqlite3SrcListDelete(pParse->db, $$);} fullname(A) ::= nm(X) dbnm(Y). {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y);} %type joinop {int} -%type joinop2 {int} joinop(X) ::= COMMA|JOIN. { X = JT_INNER; } joinop(X) ::= JOIN_KW(A) JOIN. { X = sqlite3JoinType(pParse,&A,0,0); } joinop(X) ::= JOIN_KW(A) nm(B) JOIN. { X = sqlite3JoinType(pParse,&A,&B,0); } joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN. { X = sqlite3JoinType(pParse,&A,&B,&C); } Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -367,12 +367,12 @@ default: { if( !IdChar(*z) ){ break; } for(i=1; IdChar(z[i]); i++){} - *tokenType = keywordCode((char*)z, i); - return i; + *tokenType = TK_ID; + return keywordCode((char*)z, i, tokenType); } } *tokenType = TK_ILLEGAL; return 1; } Index: tool/lempar.c ================================================================== --- tool/lempar.c +++ tool/lempar.c @@ -1,51 +1,69 @@ -/* Driver template for the LEMON parser generator. -** The author disclaims copyright to this source code. +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: */ -/* First off, code is included that follows the "include" declaration -** in the input grammar file. */ #include -%% -/* Next is all token values, in a form suitable for use by makeheaders. -** This section will be null unless lemon is run with the -m switch. -*/ -/* -** These constants (all generated automatically by the parser generator) -** specify the various kinds of tokens (terminals) that the parser -** understands. -** -** Each symbol here is a terminal symbol in the grammar. -*/ -%% -/* Make sure the INTERFACE macro is defined. -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/* The next thing included is series of defines which control +/************ Begin %include sections from the grammar ************************/ +%% +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +%% +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. ** various aspects of the generated parser. -** YYCODETYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 terminals -** and nonterminals. "int" is used otherwise. -** YYNOCODE is a number of type YYCODETYPE which corresponds -** to no legal terminal or nonterminal number. This -** number is used to fill in empty slots of the hash -** table. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. ** YYFALLBACK If defined, this indicates that one or more tokens -** have fall-back values which should be used if the -** original value of the token will not parse. -** YYACTIONTYPE is the data type used for storing terminal -** and nonterminal numbers. "unsigned char" is -** used if there are fewer than 250 rules and -** states combined. "int" is used otherwise. -** ParseTOKENTYPE is the data type used for minor tokens given -** directly to the parser from the tokenizer. -** YYMINORTYPE is the data type used for all minor tokens. +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** ParseTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of ** which is ParseTOKENTYPE. The entry in the union -** for base tokens is called "yy0". +** for terminal symbols is called "yy0". ** YYSTACKDEPTH is the maximum depth of the parser's stack. If ** zero the stack is dynamically sized using realloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_STORE Code to store %extra_argument into yypParser @@ -60,11 +78,16 @@ ** YY_MIN_REDUCE Maximum value for reduce actions ** YY_ERROR_ACTION The yy_action[] code for syntax error ** YY_ACCEPT_ACTION The yy_action[] code for accept ** YY_NO_ACTION The yy_action[] code for no-op */ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ %% +/************* End control #defines *******************************************/ /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ static const YYMINORTYPE yyzerominor = { 0 }; @@ -129,22 +152,28 @@ ** yy_shift_ofst[] For each state, the offset into yy_action for ** shifting terminals. ** yy_reduce_ofst[] For each state, the offset into yy_action for ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. -*/ +** +*********** Begin parsing tables **********************************************/ %% +/********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens into fallback tokens. If a construct -** like the following: +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: ** ** %fallback ID X Y Z. ** ** appears in the grammar, then ID becomes a fallback token for X, Y, ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. */ #ifdef YYFALLBACK static const YYCODETYPE yyFallback[] = { %% }; @@ -263,10 +292,19 @@ #endif } } #endif +/* Datatype of the argument to the memory allocated passed as the +** second argument to ParseAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like ** malloc. ** @@ -275,13 +313,13 @@ ** ** Outputs: ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(size_t)){ +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH pParser->yyidxMax = 0; #endif @@ -292,14 +330,16 @@ #endif } return pParser; } -/* The following function deletes the value associated with a -** symbol. The symbol can be either a terminal or nonterminal. -** "yymajor" is the symbol code, and "yypminor" is a pointer to -** the value. +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. */ static void yy_destructor( yyParser *yypParser, /* The parser */ YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ @@ -311,14 +351,16 @@ ** when the symbol is popped from the stack during a ** reduce or during error processing or when a parser is ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are not used + ** which appear on the RHS of the rule, but which are *not* used ** inside the C code. */ +/********* Begin destructor definitions ***************************************/ %% +/********* End destructor definitions *****************************************/ default: break; /* If no destructor action specified: do nothing */ } } /* @@ -340,27 +382,25 @@ #endif yy_destructor(pParser, yytos->major, &yytos->minor); } /* -** Deallocate and destroy a parser. Destructors are all called for +** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. ** -** Inputs: -**
    -**
  • A pointer to the parser. This should be a pointer -** obtained from ParseAlloc. -**
  • A pointer to a function used to reclaim memory obtained -** from malloc. -**
+** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. */ void ParseFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ yyParser *pParser = (yyParser*)p; +#ifndef YYPARSEFREENEVERNULL if( pParser==0 ) return; +#endif while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); #if YYSTACKDEPTH<=0 free(pParser->yystack); #endif (*freeProc)((void*)pParser); @@ -377,14 +417,10 @@ #endif /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ static int yy_find_shift_action( yyParser *pParser, /* The parser */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ @@ -447,14 +483,10 @@ } /* ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. -** -** If the look-ahead token is YYNOCODE, then check to see if the action is -** independent of the look-ahead. If it is, return the action, otherwise -** return YY_NO_ACTION. */ static int yy_find_reduce_action( int stateno, /* Current state number */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ @@ -493,11 +525,13 @@ } #endif while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ %% +/******** End %stack_overflow code ********************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } /* ** Print tracing information for a SHIFT action @@ -520,11 +554,11 @@ #else # define yyTraceShift(X,Y) #endif /* -** Perform a shift action. Return the number of errors. +** Perform a shift action. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -591,28 +625,11 @@ yysize = yyRuleInfo[yyruleno].nrhs; fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ - - /* Silence complaints from purify about yygotominor being uninitialized - ** in some cases when it is copied into the stack after the following - ** switch. yygotominor is uninitialized when a rule reduces that does - ** not set the value of its left-hand side nonterminal. Leaving the - ** value of the nonterminal uninitialized is utterly harmless as long - ** as the value is never used. So really the only thing this code - ** accomplishes is to quieten purify. - ** - ** 2007-01-16: The wireshark project (www.wireshark.org) reports that - ** without this code, their parser segfaults. I'm not sure what there - ** parser is doing to make this happen. This is the second bug report - ** from wireshark this week. Clearly they are stressing Lemon in ways - ** that it has not been previously stressed... (SQLite ticket #2172) - */ - /*memset(&yygotominor, 0, sizeof(yygotominor));*/ yygotominor = yyzerominor; - switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example ** follows: ** case 0: @@ -619,11 +636,13 @@ ** #line ** { ... } // User supplied code ** #line ** break; */ +/********** Begin reduce actions **********************************************/ %% +/********** End reduce actions ************************************************/ }; assert( yyruleno>=0 && yyrulenoyyidx -= yysize; @@ -664,11 +683,13 @@ } #endif while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ +/************ Begin %parse_failure code ***************************************/ %% +/************ End %parse_failure code *****************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } #endif /* YYNOERRORRECOVERY */ /* @@ -679,11 +700,13 @@ int yymajor, /* The major type of the error token */ YYMINORTYPE yyminor /* The minor type of the error token */ ){ ParseARG_FETCH; #define TOKEN (yyminor.yy0) +/************ Begin %syntax_error code ****************************************/ %% +/************ End %syntax_error code ******************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* ** The following is executed when the parser accepts @@ -698,11 +721,13 @@ } #endif while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ %% +/*********** End %parse_accept code *******************************************/ ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* The main parser program. ** The first argument is a pointer to a structure obtained from Index: tool/mkkeywordhash.c ================================================================== --- tool/mkkeywordhash.c +++ tool/mkkeywordhash.c @@ -275,31 +275,12 @@ }; /* Number of keywords */ static int nKeyword = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0])); -/* An array to map all upper-case characters into their corresponding -** lower-case character. -*/ -const unsigned char sqlite3UpperToLower[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, - 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, - 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, - 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, - 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, - 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, - 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, - 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, - 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, - 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, - 252,253,254,255 -}; -#define UpperToLower sqlite3UpperToLower +/* Map all alphabetic characters into the same case */ +#define charMap(X) (0x20|(X)) /* ** Comparision function for two Keyword records */ static int keywordCompare1(const void *a, const void *b){ @@ -345,11 +326,11 @@ /* ** This routine does the work. The generated code is printed on standard ** output. */ int main(int argc, char **argv){ - int i, j, k, h; + int i, j, k, h, m; int bestSize, bestCount; int count; int nChar; int totalLen = 0; int aHash[1000]; /* 1000 is much bigger than nKeyword */ @@ -370,12 +351,12 @@ Keyword *p = &aKeywordTable[i]; p->len = (int)strlen(p->zName); assert( p->lenzOrigName) ); memcpy(p->zOrigName, p->zName, p->len+1); totalLen += p->len; - p->hash = (UpperToLower[(int)p->zName[0]]*4) ^ - (UpperToLower[(int)p->zName[p->len-1]]*3) ^ p->len; + p->hash = (charMap(p->zName[0])*4) ^ + (charMap(p->zName[p->len-1])*3) ^ (p->len*1); p->id = i+1; } /* Sort the table from shortest to longest keyword */ qsort(aKeywordTable, nKeyword, sizeof(aKeywordTable[0]), keywordCompare1); @@ -479,11 +460,11 @@ } /* Begin generating code */ printf("%s", zHdr); printf("/* Hash score: %d */\n", bestCount); - printf("static int keywordCode(const char *z, int n){\n"); + printf("static int keywordCode(const char *z, int n, int *pType){\n"); printf(" /* zText[] encodes %d bytes of keywords in %d bytes */\n", totalLen + nKeyword, nChar+1 ); for(i=j=k=0; isubstrId ) continue; @@ -583,28 +564,31 @@ } } printf("%s };\n", j==0 ? "" : "\n"); printf(" int h, i;\n"); - printf(" if( n<2 ) return TK_ID;\n"); - printf(" h = ((charMap(z[0])*4) ^\n" - " (charMap(z[n-1])*3) ^\n" - " n) %% %d;\n", bestSize); - printf(" for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n"); - printf(" if( aLen[i]==n &&" - " sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n"); + printf(" if( n>=2 ){\n"); + printf(" h = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n", + bestSize); + printf(" for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n"); + printf(" if( aLen[i]==n &&" + " sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n"); for(i=0; i