/ Check-in [5eb47765]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further performance related tweaks for sqlite3RunParser().
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | weak-fallback
Files: files | file ages | folders
SHA3-256:5eb4776598f5bba7ef21a2c58c03105544da73d642d7ffc146f84eff1993d71e
User & Date: dan 2018-06-29 20:43:33
Context
2018-06-30
18:54
Have the tokenizer handle fallback for tokens "OVER" and "FILTER" in the same way as it does for "WINDOW". Leaf check-in: 12d819e1 user: dan tags: weak-fallback
2018-06-29
20:43
Further performance related tweaks for sqlite3RunParser(). check-in: 5eb47765 user: dan tags: weak-fallback
20:21
Further tweaks to sqlite3RunParser(). check-in: eef61ffa user: dan tags: weak-fallback
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to src/parse.y.

1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
....
1689
1690
1691
1692
1693
1694
1695





































































































  A = sqlite3ExprFunction(pParse, 0, &X);
  sqlite3WindowAttach(pParse, A, Z);
}
term(A) ::= CTIME_KW(OP). {
  A = sqlite3ExprFunction(pParse, 0, &OP);
}

%ifndef SQLITE_OMIT_WINDOWFUNC

%type windowdefn_opt {Window*}
%destructor windowdefn_opt {sqlite3WindowDelete(pParse->db, $$);}
windowdefn_opt(A) ::= . { A = 0; }
windowdefn_opt(A) ::= WINDOW windowdefn_list(B). { A = B; }

%type windowdefn_list {Window*}
%destructor windowdefn_list {sqlite3WindowDelete(pParse->db, $$);}
windowdefn_list(A) ::= windowdefn(Z). { A = Z; }
windowdefn_list(A) ::= windowdefn_list(Y) COMMA windowdefn(Z). {
  if( Z ) Z->pNextWin = Y;
  A = Z;
}

%type windowdefn {Window*}
%destructor windowdefn {sqlite3WindowDelete(pParse->db, $$);}
windowdefn(A) ::= nm(X) AS window(Y). {
  if( Y ){
    Y->zName = sqlite3DbStrNDup(pParse->db, X.z, X.n);
  }
  A = Y;
}

%type over_opt {Window*}
%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);}

%type window {Window*}
%destructor window {sqlite3WindowDelete(pParse->db, $$);}

%type frame_opt {Window*}
%destructor frame_opt {sqlite3WindowDelete(pParse->db, $$);}

%type window_or_nm {Window*}
%destructor window_or_nm {
sqlite3WindowDelete(pParse->db, $$);}

%type part_opt {ExprList*}
%destructor part_opt {sqlite3ExprListDelete(pParse->db, $$);}

%type filter_opt {Expr*}
%destructor filter_opt {sqlite3ExprDelete(pParse->db, $$);}

%type range_or_rows {int}

%type frame_bound {struct FrameBound}
%destructor frame_bound {sqlite3ExprDelete(pParse->db, $$.pExpr);}

over_opt(A) ::= . { A = 0; }
over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). {
  A = Z;
  if( A ) A->pFilter = W;
}

window_or_nm(A) ::= window(Z). {A = Z;}
window_or_nm(A) ::= nm(Z). {
  A = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
  if( A ){
    A->zName = sqlite3DbStrNDup(pParse->db, Z.z, Z.n);
  }
}

window(A) ::= LP part_opt(X) orderby_opt(Y) frame_opt(Z) RP. {
  A = Z;
  if( A ){
    A->pPartition = X;
    A->pOrderBy = Y;
  }
}

part_opt(A) ::= PARTITION BY exprlist(X). { A = X; }
part_opt(A) ::= .                         { A = 0; }
filter_opt(A) ::= .                            { A = 0; }
filter_opt(A) ::= FILTER LP WHERE expr(X) RP.  { A = X; }

frame_opt(A) ::= .                             { 
  A = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0);
}
frame_opt(A) ::= range_or_rows(X) frame_bound(Y). { 
  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, TK_CURRENT, 0);
}
frame_opt(A) ::= range_or_rows(X) BETWEEN frame_bound(Y) AND frame_bound(Z). { 
  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, Z.eType, Z.pExpr);
}

range_or_rows(A) ::= RANGE.   { A = TK_RANGE; }
range_or_rows(A) ::= ROWS.    { A = TK_ROWS;  }

frame_bound(A) ::= UNBOUNDED PRECEDING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }
frame_bound(A) ::= expr(X) PRECEDING.   { A.eType = TK_PRECEDING; A.pExpr = X; }
frame_bound(A) ::= CURRENT ROW.         { A.eType = TK_CURRENT  ; A.pExpr = 0; }
frame_bound(A) ::= expr(X) FOLLOWING.   { A.eType = TK_FOLLOWING; A.pExpr = X; }
frame_bound(A) ::= UNBOUNDED FOLLOWING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }

%endif // SQLITE_OMIT_WINDOWFUNC

expr(A) ::= LP nexprlist(X) COMMA expr(Y) RP. {
  ExprList *pList = sqlite3ExprListAppend(pParse, X, Y);
  A = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
  if( A ){
    A->x.pList = pList;
  }else{
    sqlite3ExprListDelete(pParse->db, pList);
................................................................................
wqlist(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
  A = sqlite3WithAdd(pParse, 0, &X, Y, Z); /*A-overwrites-X*/
}
wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
  A = sqlite3WithAdd(pParse, A, &X, Y, Z);
}
%endif  SQLITE_OMIT_CTE












































































































<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1042
1043
1044
1045
1046
1047
1048
































































































1049
1050
1051
1052
1053
1054
1055
....
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
  A = sqlite3ExprFunction(pParse, 0, &X);
  sqlite3WindowAttach(pParse, A, Z);
}
term(A) ::= CTIME_KW(OP). {
  A = sqlite3ExprFunction(pParse, 0, &OP);
}

































































































expr(A) ::= LP nexprlist(X) COMMA expr(Y) RP. {
  ExprList *pList = sqlite3ExprListAppend(pParse, X, Y);
  A = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
  if( A ){
    A->x.pList = pList;
  }else{
    sqlite3ExprListDelete(pParse->db, pList);
................................................................................
wqlist(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
  A = sqlite3WithAdd(pParse, 0, &X, Y, Z); /*A-overwrites-X*/
}
wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
  A = sqlite3WithAdd(pParse, A, &X, Y, Z);
}
%endif  SQLITE_OMIT_CTE

//////////////////////// WINDOW FUNCTION EXPRESSIONS /////////////////////////
// These must be at the end of this file. Specifically, the windowdefn_opt
// rule must be the very last in the file. This causes the integer value
// assigned to the TK_WINDOW token to be larger than all other tokens that
// may be output by the tokenizer except TK_SPACE and TK_ILLEGAL.
//
%ifndef SQLITE_OMIT_WINDOWFUNC
%type windowdefn_list {Window*}
%destructor windowdefn_list {sqlite3WindowDelete(pParse->db, $$);}
windowdefn_list(A) ::= windowdefn(Z). { A = Z; }
windowdefn_list(A) ::= windowdefn_list(Y) COMMA windowdefn(Z). {
  if( Z ) Z->pNextWin = Y;
  A = Z;
}

%type windowdefn {Window*}
%destructor windowdefn {sqlite3WindowDelete(pParse->db, $$);}
windowdefn(A) ::= nm(X) AS window(Y). {
  if( Y ){
    Y->zName = sqlite3DbStrNDup(pParse->db, X.z, X.n);
  }
  A = Y;
}

%type over_opt {Window*}
%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);}

%type window {Window*}
%destructor window {sqlite3WindowDelete(pParse->db, $$);}

%type frame_opt {Window*}
%destructor frame_opt {sqlite3WindowDelete(pParse->db, $$);}

%type window_or_nm {Window*}
%destructor window_or_nm {
sqlite3WindowDelete(pParse->db, $$);}

%type part_opt {ExprList*}
%destructor part_opt {sqlite3ExprListDelete(pParse->db, $$);}

%type filter_opt {Expr*}
%destructor filter_opt {sqlite3ExprDelete(pParse->db, $$);}

%type range_or_rows {int}

%type frame_bound {struct FrameBound}
%destructor frame_bound {sqlite3ExprDelete(pParse->db, $$.pExpr);}

over_opt(A) ::= . { A = 0; }
over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). {
  A = Z;
  if( A ) A->pFilter = W;
}

window_or_nm(A) ::= window(Z). {A = Z;}
window_or_nm(A) ::= nm(Z). {
  A = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
  if( A ){
    A->zName = sqlite3DbStrNDup(pParse->db, Z.z, Z.n);
  }
}

window(A) ::= LP part_opt(X) orderby_opt(Y) frame_opt(Z) RP. {
  A = Z;
  if( A ){
    A->pPartition = X;
    A->pOrderBy = Y;
  }
}

part_opt(A) ::= PARTITION BY exprlist(X). { A = X; }
part_opt(A) ::= .                         { A = 0; }
filter_opt(A) ::= .                            { A = 0; }
filter_opt(A) ::= FILTER LP WHERE expr(X) RP.  { A = X; }

frame_opt(A) ::= .                             { 
  A = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0);
}
frame_opt(A) ::= range_or_rows(X) frame_bound(Y). { 
  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, TK_CURRENT, 0);
}
frame_opt(A) ::= range_or_rows(X) BETWEEN frame_bound(Y) AND frame_bound(Z). { 
  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, Z.eType, Z.pExpr);
}

range_or_rows(A) ::= RANGE.   { A = TK_RANGE; }
range_or_rows(A) ::= ROWS.    { A = TK_ROWS;  }

frame_bound(A) ::= UNBOUNDED PRECEDING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }
frame_bound(A) ::= expr(X) PRECEDING.   { A.eType = TK_PRECEDING; A.pExpr = X; }
frame_bound(A) ::= CURRENT ROW.         { A.eType = TK_CURRENT  ; A.pExpr = 0; }
frame_bound(A) ::= expr(X) FOLLOWING.   { A.eType = TK_FOLLOWING; A.pExpr = X; }
frame_bound(A) ::= UNBOUNDED FOLLOWING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }

%type windowdefn_opt {Window*}
%destructor windowdefn_opt {sqlite3WindowDelete(pParse->db, $$);}
windowdefn_opt(A) ::= . { A = 0; }
windowdefn_opt(A) ::= WINDOW windowdefn_list(B). { A = B; }
%endif // SQLITE_OMIT_WINDOWFUNC

Changes to src/resolve.c.

794
795
796
797
798
799
800

801



802
803
804
805
806
807
808
        pNC->nErr++;
      }else if( wrong_num_args ){
        sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()",
             nId, zId);
        pNC->nErr++;
      }
      if( is_agg ){

        pNC->ncFlags &= ~(pExpr->pWin ? NC_AllowWin : NC_AllowAgg);



      }
      sqlite3WalkExprList(pWalker, pList);
      if( is_agg ){
#ifndef SQLITE_OMIT_WINDOWFUNC
        if( pExpr->pWin ){
          Select *pSel = pNC->pWinSelect;
          sqlite3WalkExprList(pWalker, pExpr->pWin->pPartition);







>

>
>
>







794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
        pNC->nErr++;
      }else if( wrong_num_args ){
        sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()",
             nId, zId);
        pNC->nErr++;
      }
      if( is_agg ){
#ifndef SQLITE_OMIT_WINDOWFUNC
        pNC->ncFlags &= ~(pExpr->pWin ? NC_AllowWin : NC_AllowAgg);
#else
        pNC->ncFlags &= ~NC_AllowAgg;
#endif
      }
      sqlite3WalkExprList(pWalker, pList);
      if( is_agg ){
#ifndef SQLITE_OMIT_WINDOWFUNC
        if( pExpr->pWin ){
          Select *pSel = pNC->pWinSelect;
          sqlite3WalkExprList(pWalker, pExpr->pWin->pPartition);

Changes to src/tokenize.c.

184
185
186
187
188
189
190

191
192
193
194
195
196
197
...
203
204
205
206
207
208
209

210

211
212
213
214
215
216
217
...
234
235
236
237
238
239
240

241
242
243
244
245
246
247
...
561
562
563
564
565
566
567






568
569

570
571
572
573
574
575
576
...
582
583
584
585
586
587
588




589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
#endif

/* Make the IdChar function accessible from ctime.c */
#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
int sqlite3IsIdChar(u8 c){ return IdChar(c); }
#endif


/*
** Return the id of the next token in string (*pz). Before returning, set
** (*pz) to point to the byte following the parsed token.
**
** This function assumes that any keywords that start with "w" are 
** actually TK_ID.
*/
................................................................................
    ret = TK_ID;
  }else{
    z += sqlite3GetToken(z, &ret);
  }
  *pz = z;
  return ret;
}



/*
** The tokenizer has just parsed the keyword WINDOW. In this case the token
** may really be the keyword (TK_WINDOW), or may be an identifier (TK_ID).
** This function determines which it is by inspecting the next two tokens
** in the input stream. Specifically, the token is TK_WINDOW if the following
** two tokens are:
**
................................................................................
    while( (t = windowGetToken(&z))==TK_SPACE );
    if( t!=TK_AS ){
      ret = TK_ID;
    }
  }
  return ret;
}


/*
** Return the length (in bytes) of the token that begins at z[0]. 
** Store the token type in *tokenType before returning.
*/
int sqlite3GetToken(const unsigned char *z, int *tokenType){
  int i, c;
................................................................................
  while( 1 ){
    n = sqlite3GetToken((u8*)zSql, &tokenType);
    mxSqlLen -= n;
    if( mxSqlLen<0 ){
      pParse->rc = SQLITE_TOOBIG;
      break;
    }






    if( tokenType>=TK_SPACE ){
      assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );

      if( db->u1.isInterrupted ){
        pParse->rc = SQLITE_INTERRUPT;
        break;
      }
      if( tokenType==TK_SPACE ){
        zSql += n;
        continue;
................................................................................
          tokenType = 0;
        }else if( lastTokenParsed==0 ){
          break;
        }else{
          tokenType = TK_SEMI;
        }
        n = 0;




      }else{
        sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
        break;
      }
    }
    else if( tokenType==TK_WINDOW ){
      tokenType = analyzeWindowKeyword((const u8*)&zSql[6]);
    }
    pParse->sLastToken.z = zSql;
    pParse->sLastToken.n = n;
    sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
    lastTokenParsed = tokenType;
    zSql += n;
    if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
  }







>







 







>

>







 







>







 







>
>
>
>
>
>


>







 







>
>
>
>





<
<
<







184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
...
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
...
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
...
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
...
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608



609
610
611
612
613
614
615
#endif

/* Make the IdChar function accessible from ctime.c */
#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
int sqlite3IsIdChar(u8 c){ return IdChar(c); }
#endif

#ifndef SQLITE_OMIT_WINDOWFUNC
/*
** Return the id of the next token in string (*pz). Before returning, set
** (*pz) to point to the byte following the parsed token.
**
** This function assumes that any keywords that start with "w" are 
** actually TK_ID.
*/
................................................................................
    ret = TK_ID;
  }else{
    z += sqlite3GetToken(z, &ret);
  }
  *pz = z;
  return ret;
}
#endif // SQLITE_OMIT_WINDOWFUNC

#ifndef SQLITE_OMIT_WINDOWFUNC
/*
** The tokenizer has just parsed the keyword WINDOW. In this case the token
** may really be the keyword (TK_WINDOW), or may be an identifier (TK_ID).
** This function determines which it is by inspecting the next two tokens
** in the input stream. Specifically, the token is TK_WINDOW if the following
** two tokens are:
**
................................................................................
    while( (t = windowGetToken(&z))==TK_SPACE );
    if( t!=TK_AS ){
      ret = TK_ID;
    }
  }
  return ret;
}
#endif // SQLITE_OMIT_WINDOWFUNC

/*
** Return the length (in bytes) of the token that begins at z[0]. 
** Store the token type in *tokenType before returning.
*/
int sqlite3GetToken(const unsigned char *z, int *tokenType){
  int i, c;
................................................................................
  while( 1 ){
    n = sqlite3GetToken((u8*)zSql, &tokenType);
    mxSqlLen -= n;
    if( mxSqlLen<0 ){
      pParse->rc = SQLITE_TOOBIG;
      break;
    }
#ifndef SQLITE_OMIT_WINDOWFUNC
    if( tokenType>=TK_WINDOW ){
      assert( tokenType==TK_SPACE 
           || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW 
      );
#else
    if( tokenType>=TK_SPACE ){
      assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
#endif // SQLITE_OMIT_WINDOWFUNC
      if( db->u1.isInterrupted ){
        pParse->rc = SQLITE_INTERRUPT;
        break;
      }
      if( tokenType==TK_SPACE ){
        zSql += n;
        continue;
................................................................................
          tokenType = 0;
        }else if( lastTokenParsed==0 ){
          break;
        }else{
          tokenType = TK_SEMI;
        }
        n = 0;
#ifndef SQLITE_OMIT_WINDOWFUNC
      }else if( tokenType==TK_WINDOW ){
        tokenType = analyzeWindowKeyword((const u8*)&zSql[6]);
#endif // SQLITE_OMIT_WINDOWFUNC
      }else{
        sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
        break;
      }
    }



    pParse->sLastToken.z = zSql;
    pParse->sLastToken.n = n;
    sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
    lastTokenParsed = tokenType;
    zSql += n;
    if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
  }