/ Check-in [c1fb41aa]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improve on the previous checkin. Still a bit slow.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | weak-fallback
Files: files | file ages | folders
SHA3-256:c1fb41aa7b7207b81ee1d5d32da3380b36d694033b87a2873981e0c6437ba956
User & Date: dan 2018-06-29 19:54:51
Context
2018-06-29
20:21
Further tweaks to sqlite3RunParser(). check-in: eef61ffa user: dan tags: weak-fallback
19:54
Improve on the previous checkin. Still a bit slow. check-in: c1fb41aa user: dan tags: weak-fallback
17:44
Instead of using a lemon %fallback directive, have the tokenizer try to figure out whether an instance of "WINDOW" should be TK_WINDOW or TK_ID. check-in: 022079cb user: dan tags: weak-fallback
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/tokenize.c.

218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238

239
240
241
242
243
244
245
...
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
...
590
591
592
593
594
595
596



597
598
599
600
601
602
603
**   * TK_ID, or something else that can be used as a window name, and
**   * TK_AS.
**
** Instead of using sqlite3GetToken() to parse tokens directly, this function
** uses windowGetToken(). This is to avoid recursion if the input is similar
** to "window window window window".
*/
static void analyzeWindowKeyword(const unsigned char *z, int *tokenType){
  int t;
  assert( *tokenType==TK_WINDOW );
  while( (t = windowGetToken(&z))==TK_SPACE );
  if( t!=TK_ID && t!=TK_STRING 
   && t!=TK_JOIN_KW && sqlite3ParserFallback(t)!=TK_ID 
  ){
    *tokenType = TK_ID;
  }else{
    while( (t = windowGetToken(&z))==TK_SPACE );
    if( t!=TK_AS ){
      *tokenType = TK_ID;
    }
  }

}

/*
** Return the length (in bytes) of the token that begins at z[0]. 
** Store the token type in *tokenType before returning.
*/
int sqlite3GetToken(const unsigned char *z, int *tokenType){
................................................................................
        /* This token started out using characters that can appear in keywords,
        ** but z[i] is a character not allowed within keywords, so this must
        ** be an identifier instead */
        i++;
        break;
      }
      *tokenType = TK_ID;
      keywordCode((char*)z, i, tokenType);
      if( *tokenType==TK_WINDOW ){
        assert( i==6 );
        analyzeWindowKeyword(&z[6], tokenType);
      }
      return i;
    }
    case CC_X: {
#ifndef SQLITE_OMIT_BLOB_LITERAL
      testcase( z[0]=='x' ); testcase( z[0]=='X' );
      if( z[1]=='\'' ){
        *tokenType = TK_BLOB;
        for(i=2; sqlite3Isxdigit(z[i]); i++){}
................................................................................
      }
      if( tokenType==TK_ILLEGAL ){
        sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
        break;
      }
      zSql += n;
    }else{



      pParse->sLastToken.z = zSql;
      pParse->sLastToken.n = n;
      sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
      lastTokenParsed = tokenType;
      zSql += n;
      if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
    }







|

|




|



|


>







 







|
<
<
<
<
<







 







>
>
>







218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
...
479
480
481
482
483
484
485
486





487
488
489
490
491
492
493
...
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
**   * TK_ID, or something else that can be used as a window name, and
**   * TK_AS.
**
** Instead of using sqlite3GetToken() to parse tokens directly, this function
** uses windowGetToken(). This is to avoid recursion if the input is similar
** to "window window window window".
*/
static int analyzeWindowKeyword(const unsigned char *z){
  int t;
  int ret = TK_WINDOW;
  while( (t = windowGetToken(&z))==TK_SPACE );
  if( t!=TK_ID && t!=TK_STRING 
   && t!=TK_JOIN_KW && sqlite3ParserFallback(t)!=TK_ID 
  ){
    ret = TK_ID;
  }else{
    while( (t = windowGetToken(&z))==TK_SPACE );
    if( t!=TK_AS ){
      ret = TK_ID;
    }
  }
  return ret;
}

/*
** Return the length (in bytes) of the token that begins at z[0]. 
** Store the token type in *tokenType before returning.
*/
int sqlite3GetToken(const unsigned char *z, int *tokenType){
................................................................................
        /* This token started out using characters that can appear in keywords,
        ** but z[i] is a character not allowed within keywords, so this must
        ** be an identifier instead */
        i++;
        break;
      }
      *tokenType = TK_ID;
      return keywordCode((char*)z, i, tokenType);





    }
    case CC_X: {
#ifndef SQLITE_OMIT_BLOB_LITERAL
      testcase( z[0]=='x' ); testcase( z[0]=='X' );
      if( z[1]=='\'' ){
        *tokenType = TK_BLOB;
        for(i=2; sqlite3Isxdigit(z[i]); i++){}
................................................................................
      }
      if( tokenType==TK_ILLEGAL ){
        sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
        break;
      }
      zSql += n;
    }else{
      if( tokenType==TK_WINDOW ){
        tokenType = analyzeWindowKeyword((const u8*)&zSql[6]);
      }
      pParse->sLastToken.z = zSql;
      pParse->sLastToken.n = n;
      sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
      lastTokenParsed = tokenType;
      zSql += n;
      if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
    }