SQLite

Check-in [4f3c8a82fd]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a segfault caused by invoking a regular aggregate as a window-function. And some problems with count(*) when used as a window-function.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 4f3c8a82fd1c5b14d84f2301e34cfc8d52fe4b3a60840c39e895c11f2da529d9
User & Date: dan 2018-07-02 12:07:32.352
Context
2018-07-02
15:03
Fix a crash caused by a LIKE pattern that consists of a single escape character. Problem found by OSSFuzz. (check-in: bb9bfc3a12 user: dan tags: trunk)
12:07
Fix a segfault caused by invoking a regular aggregate as a window-function. And some problems with count(*) when used as a window-function. (check-in: 4f3c8a82fd user: dan tags: trunk)
2018-07-01
16:05
Quick patch to the Lemon parser template to avoid an array overread reported by OSSFuzz. A proper fix involves enhancements to the table generators in Lemon to make the overread impossible. That fix will take longer to implement. The current check-in is a stop-gap. (check-in: 3f6730be4c user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/func.c.
1568
1569
1570
1571
1572
1573
1574



1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600












1601
1602
1603
1604
1605
1606
1607
/*
** The following structure keeps track of state information for the
** count() aggregate function.
*/
typedef struct CountCtx CountCtx;
struct CountCtx {
  i64 n;



};

/*
** Routines to implement the count() aggregate function.
*/
static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){
  CountCtx *p;
  p = sqlite3_aggregate_context(context, sizeof(*p));
  if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){
    p->n++;
  }

#ifndef SQLITE_OMIT_DEPRECATED
  /* The sqlite3_aggregate_count() function is deprecated.  But just to make
  ** sure it still operates correctly, verify that its count agrees with our 
  ** internal count when using count(*) and when the total count can be
  ** expressed as a 32-bit integer. */
  assert( argc==1 || p==0 || p->n>0x7fffffff
          || p->n==sqlite3_aggregate_count(context) );
#endif
}   
static void countFinalize(sqlite3_context *context){
  CountCtx *p;
  p = sqlite3_aggregate_context(context, 0);
  sqlite3_result_int64(context, p ? p->n : 0);
}













/*
** Routines to implement min() and max() aggregate functions.
*/
static void minmaxStep(
  sqlite3_context *context, 
  int NotUsed, 







>
>
>

















|








>
>
>
>
>
>
>
>
>
>
>
>







1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
/*
** The following structure keeps track of state information for the
** count() aggregate function.
*/
typedef struct CountCtx CountCtx;
struct CountCtx {
  i64 n;
#ifdef SQLITE_DEBUG
  int bInverse;                   /* True if xInverse() ever called */
#endif
};

/*
** Routines to implement the count() aggregate function.
*/
static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){
  CountCtx *p;
  p = sqlite3_aggregate_context(context, sizeof(*p));
  if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){
    p->n++;
  }

#ifndef SQLITE_OMIT_DEPRECATED
  /* The sqlite3_aggregate_count() function is deprecated.  But just to make
  ** sure it still operates correctly, verify that its count agrees with our 
  ** internal count when using count(*) and when the total count can be
  ** expressed as a 32-bit integer. */
  assert( argc==1 || p==0 || p->n>0x7fffffff || p->bInverse
          || p->n==sqlite3_aggregate_count(context) );
#endif
}   
static void countFinalize(sqlite3_context *context){
  CountCtx *p;
  p = sqlite3_aggregate_context(context, 0);
  sqlite3_result_int64(context, p ? p->n : 0);
}
#ifndef SQLITE_OMIT_WINDOWFUNC
static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){
  CountCtx *p;
  p = sqlite3_aggregate_context(ctx, sizeof(*p));
  if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){
    p->n--;
#ifdef SQLITE_DEBUG
    p->bInverse = 1;
#endif
  }
}   
#endif

/*
** Routines to implement min() and max() aggregate functions.
*/
static void minmaxStep(
  sqlite3_context *context, 
  int NotUsed, 
1939
1940
1941
1942
1943
1944
1945
1946

1947

1948
1949
1950
1951
1952
1953
1954
    FUNCTION(replace,            3, 0, 0, replaceFunc      ),
    FUNCTION(zeroblob,           1, 0, 0, zeroblobFunc     ),
    FUNCTION(substr,             2, 0, 0, substrFunc       ),
    FUNCTION(substr,             3, 0, 0, substrFunc       ),
    WAGGREGATE(sum,   1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0),
    WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0),
    WAGGREGATE(avg,   1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0),
    AGGREGATE2(count, 0,0,0, countStep, countFinalize, SQLITE_FUNC_COUNT  ),

    WAGGREGATE(count, 1,0,0, countStep, countFinalize, 0, 0, 0 ),

    WAGGREGATE(group_concat, 1, 0, 0, groupConcatStep, 
        groupConcatFinalize, groupConcatValue, groupConcatInverse, 0),
    WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, 
        groupConcatFinalize, groupConcatValue, groupConcatInverse, 0),
  
    LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE),
#ifdef SQLITE_CASE_SENSITIVE_LIKE







|
>
|
>







1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
    FUNCTION(replace,            3, 0, 0, replaceFunc      ),
    FUNCTION(zeroblob,           1, 0, 0, zeroblobFunc     ),
    FUNCTION(substr,             2, 0, 0, substrFunc       ),
    FUNCTION(substr,             3, 0, 0, substrFunc       ),
    WAGGREGATE(sum,   1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0),
    WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0),
    WAGGREGATE(avg,   1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0),
    WAGGREGATE(count, 0,0,0, countStep, 
        countFinalize, countFinalize, countInverse, SQLITE_FUNC_COUNT  ),
    WAGGREGATE(count, 1,0,0, countStep, 
        countFinalize, countFinalize, countInverse, 0  ),
    WAGGREGATE(group_concat, 1, 0, 0, groupConcatStep, 
        groupConcatFinalize, groupConcatValue, groupConcatInverse, 0),
    WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, 
        groupConcatFinalize, groupConcatValue, groupConcatInverse, 0),
  
    LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE),
#ifdef SQLITE_CASE_SENSITIVE_LIKE
Changes to src/resolve.c.
754
755
756
757
758
759
760




761
762
763
764
765
766
767
768
          ** in an index. */
          notValid(pParse, pNC, "non-deterministic functions",
                   NC_IdxExpr|NC_PartIdx);
        }
      }

#ifndef SQLITE_OMIT_WINDOWFUNC




      if( is_agg==0 && pExpr->pWin ){
        sqlite3ErrorMsg(pParse, 
            "%.*s() may not be used as a window function", nId, zId
        );
        pNC->nErr++;
      }else if( 
            (is_agg && (pNC->ncFlags & NC_AllowAgg)==0)
         || (is_agg && (pDef->funcFlags & SQLITE_FUNC_WINDOW) && !pExpr->pWin)







>
>
>
>
|







754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
          ** in an index. */
          notValid(pParse, pNC, "non-deterministic functions",
                   NC_IdxExpr|NC_PartIdx);
        }
      }

#ifndef SQLITE_OMIT_WINDOWFUNC
      assert( is_agg==0 || (pDef->funcFlags & SQLITE_FUNC_MINMAX)
          || (pDef->xValue==0 && pDef->xInverse==0)
          || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize)
      );
      if( pDef && pDef->xValue==0 && pExpr->pWin ){
        sqlite3ErrorMsg(pParse, 
            "%.*s() may not be used as a window function", nId, zId
        );
        pNC->nErr++;
      }else if( 
            (is_agg && (pNC->ncFlags & NC_AllowAgg)==0)
         || (is_agg && (pDef->funcFlags & SQLITE_FUNC_WINDOW) && !pExpr->pWin)
Changes to src/test_window.c.
297
298
299
300
301
302
303


























304
305
306
307
308
309
310
311
312
313

314
315
316
317
318
319
320

  if( rc!=SQLITE_OK ){
    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
    return TCL_ERROR;
  }
  return TCL_OK;
}



























int Sqlitetest_window_Init(Tcl_Interp *interp){
  static struct {
     char *zName;
     Tcl_ObjCmdProc *xProc;
     int clientData;
  } aObjCmd[] = {
     { "sqlite3_create_window_function", test_create_window, 0 },
     { "test_create_window_function_misuse", test_create_window_misuse, 0 },
     { "test_create_sumint", test_create_sumint, 0 },

  };
  int i;
  for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){
    ClientData c = (ClientData)SQLITE_INT_TO_PTR(aObjCmd[i].clientData);
    Tcl_CreateObjCommand(interp, aObjCmd[i].zName, aObjCmd[i].xProc, c, 0);
  }
  return TCL_OK;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>










>







297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347

  if( rc!=SQLITE_OK ){
    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
    return TCL_ERROR;
  }
  return TCL_OK;
}

static int SQLITE_TCLAPI test_override_sum(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  sqlite3 *db;
  int rc;

  if( objc!=2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "DB");
    return TCL_ERROR;
  }
  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;

  rc = sqlite3_create_function(db, "sum", -1, SQLITE_UTF8, 0,
      0, sumintStep, sumintFinal
  );

  if( rc!=SQLITE_OK ){
    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
    return TCL_ERROR;
  }
  return TCL_OK;
}

int Sqlitetest_window_Init(Tcl_Interp *interp){
  static struct {
     char *zName;
     Tcl_ObjCmdProc *xProc;
     int clientData;
  } aObjCmd[] = {
     { "sqlite3_create_window_function", test_create_window, 0 },
     { "test_create_window_function_misuse", test_create_window_misuse, 0 },
     { "test_create_sumint", test_create_sumint, 0 },
     { "test_override_sum", test_override_sum, 0 },
  };
  int i;
  for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){
    ClientData c = (ClientData)SQLITE_INT_TO_PTR(aObjCmd[i].clientData);
    Tcl_CreateObjCommand(interp, aObjCmd[i].zName, aObjCmd[i].xProc, c, 0);
  }
  return TCL_OK;
Changes to src/window.c.
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601





1602
1603

1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
    sqlite3VdbeJumpHere(v, addrIfPos2);
  }

  if( pMWin->eStart==TK_CURRENT 
   || pMWin->eStart==TK_PRECEDING 
   || pMWin->eStart==TK_FOLLOWING 
  ){
    int addrJumpHere = 0;
    if( pMWin->eStart==TK_PRECEDING ){
      addrJumpHere = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0 , 1);
      VdbeCoverage(v);
    }





    sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1);
    VdbeCoverage(v);

    windowAggStep(pParse, pMWin, csrStart, 1, regArg, regSize);
    if( addrJumpHere ){
      sqlite3VdbeJumpHere(v, addrJumpHere);
    }
  }
  if( pMWin->eEnd==TK_FOLLOWING ){
    sqlite3VdbeJumpHere(v, addrIfPos1);
  }
  sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop);

  /* flush_partition_done: */







|

|


>
>
>
>
>
|
|
>

<
|
<







1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610

1611

1612
1613
1614
1615
1616
1617
1618
    sqlite3VdbeJumpHere(v, addrIfPos2);
  }

  if( pMWin->eStart==TK_CURRENT 
   || pMWin->eStart==TK_PRECEDING 
   || pMWin->eStart==TK_FOLLOWING 
  ){
    int lblSkipInverse = sqlite3VdbeMakeLabel(v);;
    if( pMWin->eStart==TK_PRECEDING ){
      sqlite3VdbeAddOp3(v, OP_IfPos, regStart, lblSkipInverse, 1);
      VdbeCoverage(v);
    }
    if( pMWin->eStart==TK_FOLLOWING ){
      sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+2);
      VdbeCoverage(v);
      sqlite3VdbeAddOp2(v, OP_Goto, 0, lblSkipInverse);
    }else{
      sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1);
      VdbeCoverage(v);
    }
    windowAggStep(pParse, pMWin, csrStart, 1, regArg, regSize);

    sqlite3VdbeResolveLabel(v, lblSkipInverse);

  }
  if( pMWin->eEnd==TK_FOLLOWING ){
    sqlite3VdbeJumpHere(v, addrIfPos1);
  }
  sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop);

  /* flush_partition_done: */
Changes to test/window3.tcl.
304
305
306
307
308
309
310
311
312

313
314
315
316
317

318
319
320
321
322

323
324
325
326
327

328

329
330
331
332
    SELECT string_agg(CAST(b AS TEXT), '.') OVER ( ORDER BY b%10,a $window ) FROM t2
  "
  execsql_test 1.$tn.14.6 "
    SELECT string_agg(CAST(b AS TEXT), '.') OVER (PARTITION BY b%2,a ORDER BY b%10 $window) FROM t2
  "

  execsql_test 1.$tn.15.1 "
    SELECT string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE a%2=0) OVER (ORDER BY a $window) FROM t2

  "

  execsql_test 1.$tn.15.2 "
    SELECT string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE 0=1) OVER (ORDER BY a $window) FROM t2

  "

  execsql_test 1.$tn.15.3 "
    SELECT string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE 1=0) OVER (PARTITION BY (a%10) ORDER BY a $window) FROM t2

  "

  execsql_test 1.$tn.15.4 "
    SELECT string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE a%2=0) OVER (PARTITION BY (a%10) ORDER BY a $window) FROM t2

  "

}

finish_test








|
|
>



|
|
>



|
|
>



|
|
>

>




304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
    SELECT string_agg(CAST(b AS TEXT), '.') OVER ( ORDER BY b%10,a $window ) FROM t2
  "
  execsql_test 1.$tn.14.6 "
    SELECT string_agg(CAST(b AS TEXT), '.') OVER (PARTITION BY b%2,a ORDER BY b%10 $window) FROM t2
  "

  execsql_test 1.$tn.15.1 "
    SELECT count(*) OVER win, string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE a%2=0) OVER win FROM t2
    WINDOW win AS (ORDER BY a $window)
  "

  execsql_test 1.$tn.15.2 "
    SELECT count(*) OVER win, string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE 0=1) OVER win FROM t2
    WINDOW win AS (ORDER BY a $window)
  "

  execsql_test 1.$tn.15.3 "
    SELECT count(*) OVER win, string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE 1=0) OVER win FROM t2
    WINDOW win AS (PARTITION BY (a%10) ORDER BY a $window)
  "

  execsql_test 1.$tn.15.4 "
    SELECT count(*) OVER win, string_agg(CAST(b AS TEXT), '.') 
    FILTER (WHERE a%2=0) OVER win FROM t2
    WINDOW win AS (PARTITION BY (a%10) ORDER BY a $window)
  "

}

finish_test

Changes to test/window3.test.

cannot compute difference between binary files

Changes to test/window5.test.
77
78
79
80
81
82
83










84
85
86
87
do_execsql_test 2.0 {
  SELECT sumint(a) OVER (ORDER BY rowid) FROM t1 ORDER BY rowid;
} {4 10 11 16 18 21}

do_execsql_test 2.1 {
  SELECT sumint(a) OVER (ORDER BY rowid ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM t1 ORDER BY rowid;
} {10 11 12 8 10 5}












finish_test








>
>
>
>
>
>
>
>
>
>




77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
do_execsql_test 2.0 {
  SELECT sumint(a) OVER (ORDER BY rowid) FROM t1 ORDER BY rowid;
} {4 10 11 16 18 21}

do_execsql_test 2.1 {
  SELECT sumint(a) OVER (ORDER BY rowid ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM t1 ORDER BY rowid;
} {10 11 12 8 10 5}

test_override_sum db
do_catchsql_test 3.0 {
  SELECT sum(a) OVER 
  (ORDER BY b ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) 
  FROM t1;
} {1 {sum() may not be used as a window function}}
do_execsql_test 3.1 {
  SELECT sum(a) FROM t1;
} {21}


finish_test

Changes to test/window6.test.
135
136
137
138
139
140
141














142
143
  SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over);
} {2 6 12}

do_execsql_test 5.4 {
  SELECT sum(window) OVER window window FROM window window window window AS (ORDER BY window);
} {2 6 12}















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>


135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
  SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over);
} {2 6 12}

do_execsql_test 5.4 {
  SELECT sum(window) OVER window window FROM window window window window AS (ORDER BY window);
} {2 6 12}

do_execsql_test 5.5 {
  SELECT count(*) OVER win FROM over
  WINDOW win AS (ORDER BY x ROWS BETWEEN +2 FOLLOWING AND +3 FOLLOWING)
} {1 0 0}

#-------------------------------------------------------------------------
#
do_execsql_test 6.0 {
  SELECT LIKE("!","","!")""WHeRE"";
} {1}
do_catchsql_test 6.1 {
  SELECT LIKE("!","","!")""window"";
} {1 {near "window": syntax error}}

finish_test