/ Check-in [d62cd757]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Small size reduction and performance increase in the parser.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:d62cd757a69cc49c2d309e27c948610b5868632f
User & Date: drh 2015-11-09 02:08:09
Context
2015-11-09
12:33
Avoid unnecessary function prologues in the sqlite3VdbeAddOp3() routine. check-in: 7c6a19ba user: drh tags: trunk
02:08
Small size reduction and performance increase in the parser. check-in: d62cd757 user: drh tags: trunk
2015-11-07
18:32
Enhance the sqldiff utility to deal gracefully with ALTER TABLE ADD COLUMN. check-in: 7ea036ac user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/tokenize.c.

412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440

441
442

443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
  assert( pParse->pNewTable==0 );
  assert( pParse->pNewTrigger==0 );
  assert( pParse->nVar==0 );
  assert( pParse->nzVar==0 );
  assert( pParse->azVar==0 );
  enableLookaside = db->lookaside.bEnabled;
  if( db->lookaside.pStart ) db->lookaside.bEnabled = 1;
  while( !db->mallocFailed && zSql[i]!=0 ){
    assert( i>=0 );
    pParse->sLastToken.z = &zSql[i];
    pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
    i += pParse->sLastToken.n;
    if( i>mxSqlLen ){
      pParse->rc = SQLITE_TOOBIG;
      break;
    }
    switch( tokenType ){
      case TK_SPACE: {
        if( db->u1.isInterrupted ){
          sqlite3ErrorMsg(pParse, "interrupt");
          pParse->rc = SQLITE_INTERRUPT;
          goto abort_parse;
        }
        break;
      }
      case TK_ILLEGAL: {
        sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"",
                        &pParse->sLastToken);
        goto abort_parse;

      }
      case TK_SEMI: {

        pParse->zTail = &zSql[i];
        /* Fall thru into the default case */
      }
      default: {
        sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
        lastTokenParsed = tokenType;
        if( pParse->rc!=SQLITE_OK ){
          goto abort_parse;
        }
        break;
      }
    }
  }
abort_parse:
  assert( nErr==0 );
  if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
    assert( zSql[i]==0 );
    if( lastTokenParsed!=TK_SEMI ){
      sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
      pParse->zTail = &zSql[i];
    }







|








|
|
|
|
|
<
<


|


<
>

<
>
|
<
<
<
|
|
|
<
|
<
|
<
<
<







412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432


433
434
435
436
437

438
439

440
441



442
443
444

445

446



447
448
449
450
451
452
453
  assert( pParse->pNewTable==0 );
  assert( pParse->pNewTrigger==0 );
  assert( pParse->nVar==0 );
  assert( pParse->nzVar==0 );
  assert( pParse->azVar==0 );
  enableLookaside = db->lookaside.bEnabled;
  if( db->lookaside.pStart ) db->lookaside.bEnabled = 1;
  while( zSql[i]!=0 ){
    assert( i>=0 );
    pParse->sLastToken.z = &zSql[i];
    pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
    i += pParse->sLastToken.n;
    if( i>mxSqlLen ){
      pParse->rc = SQLITE_TOOBIG;
      break;
    }
    if( tokenType>=TK_SPACE ){
      assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
      if( db->u1.isInterrupted ){
        sqlite3ErrorMsg(pParse, "interrupt");
        pParse->rc = SQLITE_INTERRUPT;


        break;
      }
      if( tokenType==TK_ILLEGAL ){
        sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"",
                        &pParse->sLastToken);

        break;
      }

    }else{
      if( tokenType==TK_SEMI ) pParse->zTail = &zSql[i];



      sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
      lastTokenParsed = tokenType;
      if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;

    }

  }



  assert( nErr==0 );
  if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
    assert( zSql[i]==0 );
    if( lastTokenParsed!=TK_SEMI ){
      sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
      pParse->zTail = &zSql[i];
    }

Changes to tool/addopcodes.tcl.

14
15
16
17
18
19
20
21

22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40






41
42
43
44
45
46
47
    puts $line
    set x [lindex $line 2]
    if {$x>$max} {set max $x}
  }
}
close $in

# The following are the extra token codes to be added

#
set extras {
  TO_TEXT
  TO_BLOB
  TO_NUMERIC
  TO_INT
  TO_REAL
  ISNOT
  END_OF_FILE
  ILLEGAL
  SPACE
  UNCLOSED_STRING
  FUNCTION
  COLUMN
  AGG_FUNCTION
  AGG_COLUMN
  UMINUS
  UPLUS
  REGISTER






}
foreach x $extras {
  incr max
  puts [format "#define TK_%-29s %4d" $x $max]
}

# Some additional #defines related to token codes.







|
>









<
<








>
>
>
>
>
>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31


32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
    puts $line
    set x [lindex $line 2]
    if {$x>$max} {set max $x}
  }
}
close $in

# The following are the extra token codes to be added.  SPACE and 
# ILLEGAL *must* be the last two token codes and they must be in that order.
#
set extras {
  TO_TEXT
  TO_BLOB
  TO_NUMERIC
  TO_INT
  TO_REAL
  ISNOT
  END_OF_FILE


  UNCLOSED_STRING
  FUNCTION
  COLUMN
  AGG_FUNCTION
  AGG_COLUMN
  UMINUS
  UPLUS
  REGISTER
  SPACE
  ILLEGAL
}
if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} {
  error "SPACE and ILLEGAL must be the last two token codes and they\
         must be in that order"
}
foreach x $extras {
  incr max
  puts [format "#define TK_%-29s %4d" $x $max]
}

# Some additional #defines related to token codes.