Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Instead of using a lemon %fallback directive, have the tokenizer try to figure out whether an instance of "WINDOW" should be TK_WINDOW or TK_ID. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | weak-fallback |
Files: | files | file ages | folders |
SHA3-256: |
022079cb0d67be5ac0a50dd9a4d41ee5 |
User & Date: | dan 2018-06-29 17:44:52.692 |
Context
2018-06-29
| ||
19:54 | Improve on the previous checkin. Still a bit slow. (check-in: c1fb41aa7b user: dan tags: weak-fallback) | |
17:44 | Instead of using a lemon %fallback directive, have the tokenizer try to figure out whether an instance of "WINDOW" should be TK_WINDOW or TK_ID. (check-in: 022079cb0d user: dan tags: weak-fallback) | |
2018-06-28
| ||
20:05 | Modifications to parse.y to better support backwards compatibility for the "window" keyword. (check-in: 7c4b879bdb user: dan tags: weak-fallback) | |
Changes
Changes to src/parse.y.
︙ | ︙ | |||
213 214 215 216 217 218 219 | IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW ROWS ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT %ifdef SQLITE_OMIT_COMPOUND_SELECT EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT %ifndef SQLITE_OMIT_WINDOWFUNC | | | | 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 | IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW ROWS ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT %ifdef SQLITE_OMIT_COMPOUND_SELECT EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT %ifndef SQLITE_OMIT_WINDOWFUNC CURRENT FILTER FOLLOWING OVER PARTITION PRECEDING RANGE UNBOUNDED %endif SQLITE_OMIT_WINDOWFUNC REINDEX RENAME CTIME_KW IF . %wildcard ANY. // Define operator precedence early so that this is the first occurrence // of the operator tokens in the grammer. Keeping the operators together |
︙ | ︙ | |||
253 254 255 256 257 258 259 | // keywords. Any non-standard keyword can also be an identifier. // %token_class id ID|INDEXED. // And "ids" is an identifer-or-string. // | | < | | | 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 | // keywords. Any non-standard keyword can also be an identifier. // %token_class id ID|INDEXED. // And "ids" is an identifer-or-string. // %token_class ids ID|STRING. // The name of a column or table can be any of the following: // %type nm {Token} nm(A) ::= id(A). nm(A) ::= STRING(A). nm(A) ::= JOIN_KW(A). // A typetoken is really zero or more tokens that form a type name such // as can be found after the column name in a CREATE TABLE statement. // Multiple tokens are concatenated to form the value of the typetoken. // %type typetoken {Token} typetoken(A) ::= . {A.n = 0; A.z = 0;} typetoken(A) ::= typename(A). typetoken(A) ::= typename(A) LP signed RP(Y). { A.n = (int)(&Y.z[Y.n] - A.z); } typetoken(A) ::= typename(A) LP signed COMMA signed RP(Y). { A.n = (int)(&Y.z[Y.n] - A.z); } %type typename {Token} typename(A) ::= ids(A). typename(A) ::= typename(A) ids(Y). {A.n=Y.n+(int)(Y.z-A.z);} signed ::= plus_num. signed ::= minus_num. // The scanpt non-terminal takes a value which is a pointer to the // input text just past the last token that has been shifted into // the parser. By surrounding some phrase in the grammar with two // scanpt non-terminals, we can capture the input text for that phrase. |
︙ | ︙ | |||
339 340 341 342 343 344 345 | {sqlite3AddPrimaryKey(pParse,0,R,I,Z);} ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X);} ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R). {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} ccons ::= defer_subclause(D). {sqlite3DeferForeignKey(pParse,D);} | | | 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 | {sqlite3AddPrimaryKey(pParse,0,R,I,Z);} ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X);} ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R). {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} ccons ::= defer_subclause(D). {sqlite3DeferForeignKey(pParse,D);} ccons ::= COLLATE ids(C). {sqlite3AddCollateType(pParse, &C);} // The optional AUTOINCREMENT keyword %type autoinc {int} autoinc(X) ::= . {X = 0;} autoinc(X) ::= AUTOINCR. {X = 1;} // The next group of rules parses the arguments to a REFERENCES clause |
︙ | ︙ | |||
985 986 987 988 989 990 991 | Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1); Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &Z, 1); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); A = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= STRING(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} | < | 984 985 986 987 988 989 990 991 992 993 994 995 996 997 | Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1); Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &Z, 1); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); A = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= STRING(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= INTEGER(X). { A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); } expr(A) ::= VARIABLE(X). { if( !(X.z[0]=='#' && sqlite3Isdigit(X.z[1])) ){ u32 n = X.n; A = tokenExpr(pParse, TK_VARIABLE, X); |
︙ | ︙ | |||
1009 1010 1011 1012 1013 1014 1015 | A = 0; }else{ A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); if( A ) sqlite3GetInt32(&t.z[1], &A->iTable); } } } | | | | | 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 | A = 0; }else{ A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); if( A ) sqlite3GetInt32(&t.z[1], &A->iTable); } } } expr(A) ::= expr(A) COLLATE ids(C). { A = sqlite3ExprAddCollateToken(pParse, A, &C, 1); } %ifndef SQLITE_OMIT_CAST expr(A) ::= CAST LP expr(E) AS typetoken(T) RP. { A = sqlite3ExprAlloc(pParse->db, TK_CAST, &T, 1); sqlite3ExprAttachSubtrees(pParse->db, A, E, 0); } %endif SQLITE_OMIT_CAST expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP %ifndef SQLITE_OMIT_WINDOWFUNC over_opt(Z) %endif . { if( Y && Y->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &X); } A = sqlite3ExprFunction(pParse, Y, &X); sqlite3WindowAttach(pParse, A, Z); if( D==SF_Distinct && A ){ A->flags |= EP_Distinct; } } expr(A) ::= id(X) LP STAR RP %ifndef SQLITE_OMIT_WINDOWFUNC over_opt(Z) %endif . { A = sqlite3ExprFunction(pParse, 0, &X); sqlite3WindowAttach(pParse, A, Z); } |
︙ | ︙ | |||
1441 1442 1443 1444 1445 1446 1447 | } eidlist(A) ::= nm(Y) collate(C) sortorder(Z). { A = parserAddExprIdListTerm(pParse, 0, &Y, C, Z); /*A-overwrites-Y*/ } %type collate {int} collate(C) ::= . {C = 0;} | | | 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 | } eidlist(A) ::= nm(Y) collate(C) sortorder(Z). { A = parserAddExprIdListTerm(pParse, 0, &Y, C, Z); /*A-overwrites-Y*/ } %type collate {int} collate(C) ::= . {C = 0;} collate(C) ::= COLLATE ids. {C = 1;} ///////////////////////////// The DROP INDEX command ///////////////////////// // cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite3DropIndex(pParse, X, E);} ///////////////////////////// The VACUUM command ///////////////////////////// |
︙ | ︙ |
Changes to src/sqliteInt.h.
︙ | ︙ | |||
4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 | ** The interface to the LEMON-generated parser */ #ifndef SQLITE_AMALGAMATION void *sqlite3ParserAlloc(void*(*)(u64), Parse*); void sqlite3ParserFree(void*, void(*)(void*)); #endif void sqlite3Parser(void*, int, Token); #ifdef YYTRACKMAXSTACKDEPTH int sqlite3ParserStackPeak(void*); #endif void sqlite3AutoLoadExtensions(sqlite3*); #ifndef SQLITE_OMIT_LOAD_EXTENSION void sqlite3CloseExtensions(sqlite3*); | > | 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 | ** The interface to the LEMON-generated parser */ #ifndef SQLITE_AMALGAMATION void *sqlite3ParserAlloc(void*(*)(u64), Parse*); void sqlite3ParserFree(void*, void(*)(void*)); #endif void sqlite3Parser(void*, int, Token); int sqlite3ParserFallback(int); #ifdef YYTRACKMAXSTACKDEPTH int sqlite3ParserStackPeak(void*); #endif void sqlite3AutoLoadExtensions(sqlite3*); #ifndef SQLITE_OMIT_LOAD_EXTENSION void sqlite3CloseExtensions(sqlite3*); |
︙ | ︙ |
Changes to src/tokenize.c.
︙ | ︙ | |||
184 185 186 187 188 189 190 191 192 193 194 195 196 197 | #endif /* Make the IdChar function accessible from ctime.c */ #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS int sqlite3IsIdChar(u8 c){ return IdChar(c); } #endif /* ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ int sqlite3GetToken(const unsigned char *z, int *tokenType){ int i, c; | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | #endif /* Make the IdChar function accessible from ctime.c */ #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS int sqlite3IsIdChar(u8 c){ return IdChar(c); } #endif /* ** Return the id of the next token in string (*pz). Before returning, set ** (*pz) to point to the byte following the parsed token. ** ** This function assumes that any keywords that start with "w" are ** actually TK_ID. */ static int windowGetToken(const unsigned char **pz){ int ret; const unsigned char *z = *pz; if( z[0]=='w' || z[0]=='W' ){ do { z++; }while( IdChar(z[0]) ); ret = TK_ID; }else{ z += sqlite3GetToken(z, &ret); } *pz = z; return ret; } /* ** The tokenizer has just parsed the keyword WINDOW. In this case the token ** may really be the keyword (TK_WINDOW), or may be an identifier (TK_ID). ** This function determines which it is by inspecting the next two tokens ** in the input stream. Specifically, the token is TK_WINDOW if the following ** two tokens are: ** ** * TK_ID, or something else that can be used as a window name, and ** * TK_AS. ** ** Instead of using sqlite3GetToken() to parse tokens directly, this function ** uses windowGetToken(). This is to avoid recursion if the input is similar ** to "window window window window". */ static void analyzeWindowKeyword(const unsigned char *z, int *tokenType){ int t; assert( *tokenType==TK_WINDOW ); while( (t = windowGetToken(&z))==TK_SPACE ); if( t!=TK_ID && t!=TK_STRING && t!=TK_JOIN_KW && sqlite3ParserFallback(t)!=TK_ID ){ *tokenType = TK_ID; }else{ while( (t = windowGetToken(&z))==TK_SPACE ); if( t!=TK_AS ){ *tokenType = TK_ID; } } } /* ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ int sqlite3GetToken(const unsigned char *z, int *tokenType){ int i, c; |
︙ | ︙ | |||
429 430 431 432 433 434 435 | /* This token started out using characters that can appear in keywords, ** but z[i] is a character not allowed within keywords, so this must ** be an identifier instead */ i++; break; } *tokenType = TK_ID; | | > > > > > | 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 | /* This token started out using characters that can appear in keywords, ** but z[i] is a character not allowed within keywords, so this must ** be an identifier instead */ i++; break; } *tokenType = TK_ID; keywordCode((char*)z, i, tokenType); if( *tokenType==TK_WINDOW ){ assert( i==6 ); analyzeWindowKeyword(&z[6], tokenType); } return i; } case CC_X: { #ifndef SQLITE_OMIT_BLOB_LITERAL testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; for(i=2; sqlite3Isxdigit(z[i]); i++){} |
︙ | ︙ |
Added test/window6.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | # 2018 May 8 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. Specifically, # it tests the sqlite3_create_window_function() API. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix window6 ifcapable !windowfunc { finish_test return } set setup { CREATE TABLE %t1(%a, %b %typename); INSERT INTO %t1 VALUES(1, 'a'); INSERT INTO %t1 VALUES(2, 'b'); INSERT INTO %t1 VALUES(3, 'c'); INSERT INTO %t1 VALUES(4, 'd'); INSERT INTO %t1 VALUES(5, 'e'); } foreach {tn vars} { 1 {} 2 { set A(%t1) over } 3 { set A(%a) over } 4 { set A(%alias) over set A(%a) following set A(%b) over } 5 { set A(%t1) over set A(%a) following set A(%b) preceding set A(%w) current set A(%alias) filter set A(%typename) window } 6 { set A(%a) window } } { set A(%t1) t1 set A(%a) a set A(%b) b set A(%w) w set A(%alias) alias set A(%typename) integer eval $vars set MAP [array get A] set setup_sql [string map $MAP $setup] reset_db execsql $setup_sql do_execsql_test 1.$tn.1 [string map $MAP { SELECT group_concat(%a, '.') OVER (ORDER BY %b) FROM %t1 }] {1 1.2 1.2.3 1.2.3.4 1.2.3.4.5} do_execsql_test 1.$tn.2 [string map $MAP { SELECT sum(%a) OVER %w FROM %t1 WINDOW %w AS (ORDER BY %b) }] {1 3 6 10 15} do_execsql_test 1.$tn.3 [string map $MAP { SELECT sum(%alias.%a) OVER %w FROM %t1 %alias WINDOW %w AS (ORDER BY %b) }] {1 3 6 10 15} do_execsql_test 1.$tn.4 [string map $MAP { SELECT sum(%a) %alias FROM %t1 }] {15} } proc winproc {args} { return "window: $args" } db func window winproc do_execsql_test 2.0 { SELECT window('hello world'); } {{window: {hello world}}} proc wincmp {a b} { string compare $b $a } db collate window wincmp do_execsql_test 3.0 { CREATE TABLE window(x COLLATE window); INSERT INTO window VALUES('bob'), ('alice'), ('cate'); SELECT * FROM window ORDER BY x COLLATE window; } {cate bob alice} do_execsql_test 3.1 { DROP TABLE window; CREATE TABLE x1(x); INSERT INTO x1 VALUES('bob'), ('alice'), ('cate'); CREATE INDEX window ON x1(x COLLATE window); SELECT * FROM x1 ORDER BY x COLLATE window; } {cate bob alice} do_execsql_test 4.0 { CREATE TABLE t4(x, y); } # do_execsql_test 4.1 { PRAGMA parser_trace = 1 } do_execsql_test 4.1 { SELECT * FROM t4 window, t4; } finish_test |
Changes to tool/lempar.c.
︙ | ︙ | |||
1064 1065 1066 1067 1068 1069 1070 | cDiv = ' '; } fprintf(yyTraceFILE,"]\n"); } #endif return; } | > > > > > > > > > > > > > > | 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 | cDiv = ' '; } fprintf(yyTraceFILE,"]\n"); } #endif return; } /* ** Return the fallback token corresponding to canonical token iToken, or ** 0 if iToken has no fallback. */ int ParseFallback(int iToken){ #ifdef YYFALLBACK if( iToken<sizeof(yyFallback)/sizeof(yyFallback[0]) ){ return yyFallback[iToken]; } #endif return 0; } |